Merge 0.11->trunk

181 files changed, 7065 insertions, 2402 deletions

diff --git a/.busted b/.busted
index f5f9472e..0206c093 100644
--- a/.busted
+++ b/.busted
@@ -2,7 +2,7 @@ return {
   _all = {
   },
   default = {
-    ["exclude-tags"] = "mod_bosh,storage";
+    ["exclude-tags"] = "mod_bosh,storage,SLOW";
   };
   bosh = {
     tags = "mod_bosh";
diff --git a/.luacheckrc b/.luacheckrc
index ce3d377b..6e8481de 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -1,7 +1,8 @@
 cache = true
 codes = true
-ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", "113/unpack" }
+ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", }
 
+std = "lua53c"
 max_line_length = 150
 
 read_globals = {
@@ -33,7 +34,6 @@ files["plugins/"] = {
 		"module.name",
 		"module.host",
 		"module._log",
-		"module.log",
 		"module.event_handlers",
 		"module.reloading",
 		"module.saved_state",
@@ -64,12 +64,15 @@ files["plugins/"] = {
 		"module.get_option_scalar",
 		"module.get_option_set",
 		"module.get_option_string",
+		"module.get_status",
 		"module.handle_items",
 		"module.hook",
 		"module.hook_global",
 		"module.hook_object_event",
 		"module.hook_tag",
 		"module.load_resource",
+		"module.log",
+		"module.log_status",
 		"module.measure",
 		"module.measure_event",
 		"module.measure_global_event",
@@ -79,7 +82,9 @@ files["plugins/"] = {
 		"module.remove_item",
 		"module.require",
 		"module.send",
+		"module.send_iq",
 		"module.set_global",
+		"module.set_status",
 		"module.shared",
 		"module.unhook",
 		"module.unhook_object_event",
@@ -123,46 +128,40 @@ files["prosody.cfg.lua"] = {
 if os.getenv("PROSODY_STRICT_LINT") ~= "1" then
 	-- These files have not yet been brought up to standard
 	-- Do not add more files here, but do help us fix these!
-	unused_secondaries = false
 
 	local exclude_files = {
-	"doc/net.server.lua";
+		"doc/net.server.lua";
 
-	"fallbacks/bit.lua";
-	"fallbacks/lxp.lua";
+		"fallbacks/bit.lua";
+		"fallbacks/lxp.lua";
 
-	"net/adns.lua";
-	"net/cqueues.lua";
-	"net/dns.lua";
-	"net/server_select.lua";
+		"net/cqueues.lua";
+		"net/dns.lua";
+		"net/server_select.lua";
 
-	"plugins/mod_storage_sql1.lua";
+		"plugins/mod_storage_sql1.lua";
 
-	"spec/core_configmanager_spec.lua";
-	"spec/core_moduleapi_spec.lua";
-	"spec/net_http_parser_spec.lua";
-	"spec/util_events_spec.lua";
-	"spec/util_http_spec.lua";
-	"spec/util_ip_spec.lua";
-	"spec/util_multitable_spec.lua";
-	"spec/util_rfc6724_spec.lua";
-	"spec/util_throttle_spec.lua";
-	"spec/util_xmppstream_spec.lua";
+		"spec/core_moduleapi_spec.lua";
+		"spec/util_http_spec.lua";
+		"spec/util_ip_spec.lua";
+		"spec/util_multitable_spec.lua";
+		"spec/util_rfc6724_spec.lua";
+		"spec/util_throttle_spec.lua";
 
-	"tools/ejabberd2prosody.lua";
-	"tools/ejabberdsql2prosody.lua";
-	"tools/erlparse.lua";
-	"tools/jabberd14sql2prosody.lua";
-	"tools/migration/migrator.cfg.lua";
-	"tools/migration/migrator/jabberd14.lua";
-	"tools/migration/migrator/mtools.lua";
-	"tools/migration/migrator/prosody_files.lua";
-	"tools/migration/migrator/prosody_sql.lua";
-	"tools/migration/prosody-migrator.lua";
-	"tools/openfire2prosody.lua";
-	"tools/xep227toprosody.lua";
+		"tools/ejabberd2prosody.lua";
+		"tools/ejabberdsql2prosody.lua";
+		"tools/erlparse.lua";
+		"tools/jabberd14sql2prosody.lua";
+		"tools/migration/migrator.cfg.lua";
+		"tools/migration/migrator/jabberd14.lua";
+		"tools/migration/migrator/mtools.lua";
+		"tools/migration/migrator/prosody_files.lua";
+		"tools/migration/migrator/prosody_sql.lua";
+		"tools/migration/prosody-migrator.lua";
+		"tools/openfire2prosody.lua";
+		"tools/xep227toprosody.lua";
 
-	"util/sasl/digest-md5.lua";
+		"util/sasl/digest-md5.lua";
 	}
 	for _, file in ipairs(exclude_files) do
 		files[file] = { only = {} }
diff --git a/CHANGES b/CHANGES
index 136b7d2b..2e6caf0d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,20 @@
+TRUNK
+=====
+
+-   Module statuses
+-   SNI support (not completely finished)
+-   CORS handling now provided by mod\_http
+-   CSI improvements
+-   mod\_limits: Exempted JIDs
+-   Archive quotas
+-   mod\_mimicking
+-   Rewritten migrator
+-   SCRAM-SHA-256
+-   Bi-directional server-to-server (XEP-0288)
+-   Built-in HTTP server now handles HEAD requests
+-   MUC presence broadcast controls
+-   ALPN support in mod\_net\_multiplex
+
 0.11.0
 ======
 
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 00000000..2e732c73
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,9 @@
+Thanks for your interest in contributing to the project!
+
+There are many ways to contribute, such as helping improve the
+documentation, reporting bugs, spreading the word or testing the latest
+development version.
+
+You can find more information on how to contribute at <https://prosody.im/doc/contributing>
+
+See also the HACKERS and README files.
diff --git a/GNUmakefile b/GNUmakefile
index 6c134679..15362f5c 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -21,6 +21,7 @@ MKDIR_PRIVATE=$(MKDIR) -m750
 
 LUACHECK=luacheck
 BUSTED=busted
+SCANSION=scansion
 
 .PHONY: all test coverage clean install
 
@@ -71,6 +72,13 @@ clean:
 test:
 	$(BUSTED) --lua=$(RUNWITH)
 
+integration-test: all
+	$(MKDIR) data
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua start
+	$(SCANSION) -d ./spec/scansion; R=$$? \
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop \
+	exit $$R
+
 coverage:
 	-rm -- luacov.*
 	$(BUSTED) --lua=$(RUNWITH) -c
diff --git a/HACKERS b/HACKERS
index cae98091..240de63d 100644
--- a/HACKERS
+++ b/HACKERS
@@ -5,7 +5,7 @@ involved you can join us on our mailing list and discussion rooms. More
 information on these at https://prosody.im/discuss
 
 Patches are welcome, though before sending we would appreciate if you read 
-docs/coding_style.txt for guidelines on how to format your code, and other tips.
+docs/coding_style.md for guidelines on how to format your code, and other tips.
 
 Documentation for developers can be found at https://prosody.im/doc/developers
 
diff --git a/README b/README
index 797a2175..4b52186b 100644
--- a/README
+++ b/README
@@ -12,6 +12,7 @@ rapidly prototype new protocols.
 Homepage:        https://prosody.im/
 Download:        https://prosody.im/download
 Documentation:   https://prosody.im/doc/
+Issue tracker:   https://issues.prosody.im/
 
 Jabber/XMPP Chat:
                Address:
@@ -26,9 +27,6 @@ Mailing lists:
                Development discussion:
                  https://groups.google.com/group/prosody-dev
                
-               Issue tracker changes:
-                 https://groups.google.com/group/prosody-issues
-
 ## Installation
 
 See the accompanying INSTALL file for help on building Prosody from source. Alternatively 
diff --git a/TODO b/TODO
index d22d20f4..9ec820b2 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,4 @@
 == 1.0 ==
 - Roster providers
-- Statistics
 - Clustering
 - World domination
diff --git a/configure b/configure
index 0fe186b5..2b58efe5 100755
--- a/configure
+++ b/configure
@@ -23,7 +23,8 @@ EXCERTS="yes"
 PRNG=
 PRNGLIBS=
 
-CFLAGS="-fPIC -Wall -pedantic -std=c99"
+CFLAGS="-fPIC -std=c99"
+CFLAGS="$CFLAGS -Wall -pedantic -Wextra -Wshadow -Wformat=2"
 LDFLAGS="-shared"
 
 IDN_LIBRARY="idn"
@@ -152,74 +153,8 @@ do
       SYSCONFDIR_SET=yes
       ;;
    --ostype)
-	# TODO make this a switch?
       OSPRESET="$value"
-      if [ "$OSPRESET" = "debian" ]; then
-         if [ "$LUA_SUFFIX_SET" != "yes" ]; then
-            LUA_SUFFIX="5.1";
-            LUA_SUFFIX_SET=yes
-         fi
-         if [ "$RUNWITH_SET" != "yes" ]; then
-            RUNWITH="lua$LUA_SUFFIX";
-            RUNWITH_SET=yes
-         fi
-         LUA_INCDIR="/usr/include/lua$LUA_SUFFIX"
-         LUA_INCDIR_SET=yes
-         CFLAGS="$CFLAGS -ggdb"
-      fi
-      if [ "$OSPRESET" = "macosx" ]; then
-         LUA_INCDIR=/usr/local/include;
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR=/usr/local/lib
-         LUA_LIBDIR_SET=yes
-         CFLAGS="$CFLAGS -mmacosx-version-min=10.3"
-         LDFLAGS="-bundle -undefined dynamic_lookup"
-      fi
-      if [ "$OSPRESET" = "linux" ]; then
-         LUA_INCDIR=/usr/local/include;
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR=/usr/local/lib
-         LUA_LIBDIR_SET=yes
-         CFLAGS="$CFLAGS -ggdb"
-      fi
-      if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then
-         LUA_INCDIR="/usr/local/include/lua51"
-         LUA_INCDIR_SET=yes
-         CFLAGS="-Wall -fPIC -I/usr/local/include"
-         LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared"
-         LUA_SUFFIX="51"
-         LUA_SUFFIX_SET=yes
-         LUA_DIR=/usr/local
-         LUA_DIR_SET=yes
-         CC=cc
-         LD=ld
-      fi
-      if [ "$OSPRESET" = "openbsd" ]; then
-         LUA_INCDIR="/usr/local/include";
-         LUA_INCDIR_SET="yes"
-      fi
-      if [ "$OSPRESET" = "netbsd" ]; then
-         LUA_INCDIR="/usr/pkg/include/lua-5.1"
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR="/usr/pkg/lib/lua/5.1"
-         LUA_LIBDIR_SET=yes
-         CFLAGS="-Wall -fPIC -I/usr/pkg/include"
-         LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared"
-      fi
-      if [ "$OSPRESET" = "pkg-config" ]; then
-         if [ "$LUA_SUFFIX_SET" != "yes" ]; then
-            LUA_SUFFIX="5.1";
-            LUA_SUFFIX_SET=yes
-         fi
-         LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)"
-         LUA_CF="${LUA_CF#*-I}"
-         LUA_CF="${LUA_CF%% *}"
-         if [ "$LUA_CF" != "" ]; then
-            LUA_INCDIR="$LUA_CF"
-            LUA_INCDIR_SET=yes
-         fi
-         CFLAGS="$CFLAGS"
-      fi
+      OSPRESET_SET="yes"
       ;;
    --libdir)
       LIBDIR="$value"
@@ -237,7 +172,7 @@ do
    --lua-version|--with-lua-version)
       [ -n "$value" ] || die "Missing value in flag $key."
       LUA_VERSION="$value"
-      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || die "Invalid Lua version in flag $key."
+      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || [ "$LUA_VERSION" = "5.4" ] || die "Invalid Lua version in flag $key."
       LUA_VERSION_SET=yes
       ;;
    --with-lua)
@@ -318,6 +253,66 @@ do
    shift
 done
 
+if [ "$OSPRESET_SET" = "yes" ]; then
+	# TODO make this a switch?
+   if [ "$OSPRESET" = "debian" ]; then
+      CFLAGS="$CFLAGS -ggdb"
+   fi
+   if [ "$OSPRESET" = "macosx" ]; then
+      if [ "$LUA_INCDIR_SET" != "yes" ]; then
+         LUA_INCDIR=/usr/local/include;
+         LUA_INCDIR_SET=yes
+      fi
+      if [ "$LUA_LIBDIR_SET" != "yes" ]; then
+         LUA_LIBDIR=/usr/local/lib
+         LUA_LIBDIR_SET=yes
+      fi
+      CFLAGS="$CFLAGS -mmacosx-version-min=10.3"
+      LDFLAGS="-bundle -undefined dynamic_lookup"
+   fi
+   if [ "$OSPRESET" = "linux" ]; then
+      CFLAGS="$CFLAGS -ggdb"
+   fi
+   if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then
+      LUA_INCDIR="/usr/local/include/lua51"
+      LUA_INCDIR_SET=yes
+      CFLAGS="-Wall -fPIC -I/usr/local/include"
+      LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared"
+      LUA_SUFFIX="51"
+      LUA_SUFFIX_SET=yes
+      LUA_DIR=/usr/local
+      LUA_DIR_SET=yes
+      CC=cc
+      LD=ld
+   fi
+   if [ "$OSPRESET" = "openbsd" ]; then
+      LUA_INCDIR="/usr/local/include";
+      LUA_INCDIR_SET="yes"
+   fi
+   if [ "$OSPRESET" = "netbsd" ]; then
+      LUA_INCDIR="/usr/pkg/include/lua-5.1"
+      LUA_INCDIR_SET=yes
+      LUA_LIBDIR="/usr/pkg/lib/lua/5.1"
+      LUA_LIBDIR_SET=yes
+      CFLAGS="-Wall -fPIC -I/usr/pkg/include"
+      LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared"
+   fi
+   if [ "$OSPRESET" = "pkg-config" ]; then
+      if [ "$LUA_SUFFIX_SET" != "yes" ]; then
+         LUA_SUFFIX="5.1";
+         LUA_SUFFIX_SET=yes
+      fi
+      LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)"
+      LUA_CF="${LUA_CF#*-I}"
+      LUA_CF="${LUA_CF%% *}"
+      if [ "$LUA_CF" != "" ]; then
+         LUA_INCDIR="$LUA_CF"
+         LUA_INCDIR_SET=yes
+      fi
+      CFLAGS="$CFLAGS"
+   fi
+fi
+
 if [ "$PREFIX_SET" = "yes" ] && [ ! "$SYSCONFDIR_SET" = "yes" ]
 then
    if [ "$PREFIX" = "/usr" ]
@@ -340,7 +335,7 @@ then
 fi
 
 detect_lua_version() {
-   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[123])$"))' 2> /dev/null)
+   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[1234])$"))' 2> /dev/null)
    if [ "$detected_lua" != "nil" ]
    then
       if [ "$LUA_VERSION_SET" != "yes" ]
@@ -403,8 +398,14 @@ then
    elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.3" ]
    then
       suffixes="5.3 53 -5.3 -53"
+   elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.4" ]
+   then
+      suffixes="5.4 54 -5.4 -54"
    else
-      suffixes="5.1 51 -5.1 -51 5.2 52 -5.2 -52 5.3 53 -5.3 -53"
+      suffixes="5.1 51 -5.1 -51"
+      suffixes="$suffixes 5.2 52 -5.2 -52"
+      suffixes="$suffixes 5.3 53 -5.3 -53"
+      suffixes="$suffixes 5.4 54 -5.4 -54"
    fi
    for suffix in "" $suffixes
    do
@@ -464,30 +465,46 @@ then
    LUA_LIBDIR="$LUA_DIR/lib"
 fi
 
-echo_n "Checking Lua includes... "
 lua_h="$LUA_INCDIR/lua.h"
+echo_n "Looking for lua.h at $lua_h..."
 if [ -f "$lua_h" ]
 then
-   echo "lua.h found in $lua_h"
+   echo found
 else
-   v_dir="$LUA_INCDIR/lua/$LUA_VERSION"
-   lua_h="$v_dir/lua.h"
-   if [ -f "$lua_h" ]
-   then
-      echo "lua.h found in $lua_h"
+  echo "not found"
+  for postfix in "$LUA_VERSION" "$LUA_SUFFIX"; do
+    if ! [ "$postfix" = "" ]; then
+      v_dir="$LUA_INCDIR/lua/$postfix";
+    else
+      v_dir="$LUA_INCDIR/lua";
+    fi
+    lua_h="$v_dir/lua.h"
+    echo_n "Looking for lua.h at $lua_h..."
+    if [ -f "$lua_h" ]
+    then
       LUA_INCDIR="$v_dir"
-   else
-      d_dir="$LUA_INCDIR/lua$LUA_VERSION"
+      echo found
+      break;
+    else
+      echo "not found"
+      d_dir="$LUA_INCDIR/lua$postfix"
       lua_h="$d_dir/lua.h"
+      echo_n "Looking for lua.h at $lua_h..."
       if [ -f "$lua_h" ]
       then
-         echo "lua.h found in $lua_h (Debian/Ubuntu)"
-         LUA_INCDIR="$d_dir"
+        echo found
+        LUA_INCDIR="$d_dir"
+        break;
       else
-         echo "lua.h not found (looked in $LUA_INCDIR, $v_dir, $d_dir)"
-         die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+        echo "not found"
       fi
-   fi
+    fi
+  done
+  if [ ! -f "$lua_h" ]; then
+    echo "lua.h not found."
+    echo
+    die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+  fi
 fi
 
 if [ "$lua_interp_found" = "yes" ]
diff --git a/core/certmanager.lua b/core/certmanager.lua
index 5282a6f5..663cebb4 100644
--- a/core/certmanager.lua
+++ b/core/certmanager.lua
@@ -20,7 +20,6 @@ end
 local configmanager = require "core.configmanager";
 local log = require "util.logger".init("certmanager");
 local ssl_context = ssl.context or softreq"ssl.context";
-local ssl_x509 = ssl.x509 or softreq"ssl.x509";
 local ssl_newcontext = ssl.newcontext;
 local new_config = require"util.sslconfig".new;
 local stat = require "lfs".attributes;
@@ -106,7 +105,7 @@ local core_defaults = {
 	capath = "/etc/ssl/certs";
 	depth = 9;
 	protocol = "tlsv1+";
-	verify = (ssl_x509 and { "peer", "client_once", }) or "none";
+	verify = "none";
 	options = {
 		cipher_server_preference = luasec_has.options.cipher_server_preference;
 		no_ticket = luasec_has.options.no_ticket;
@@ -123,8 +122,8 @@ local core_defaults = {
 		"P-521",
 	};
 	ciphers = {      -- Enabled ciphers in order of preference:
-		"HIGH+kEDH",   -- Ephemeral Diffie-Hellman key exchange, if a 'dhparam' file is set
 		"HIGH+kEECDH", -- Ephemeral Elliptic curve Diffie-Hellman key exchange
+		"HIGH+kEDH",   -- Ephemeral Diffie-Hellman key exchange, if a 'dhparam' file is set
 		"HIGH",        -- Other "High strength" ciphers
 		               -- Disabled cipher suites:
 		"!PSK",        -- Pre-Shared Key - not used for XMPP
@@ -148,13 +147,6 @@ local path_options = { -- These we pass through resolve_path()
 	key = true, certificate = true, cafile = true, capath = true, dhparam = true
 }
 
-if luasec_version < 5 and ssl_x509 then
-	-- COMPAT mw/luasec-hg
-	for i=1,#core_defaults.verifyext do -- Remove lsec_ prefix
-		core_defaults.verify[#core_defaults.verify+1] = core_defaults.verifyext[i]:sub(6);
-	end
-end
-
 local function create_context(host, mode, ...)
 	local cfg = new_config();
 	cfg:apply(core_defaults);
@@ -177,8 +169,10 @@ local function create_context(host, mode, ...)
 	local user_ssl_config = cfg:final();
 
 	if mode == "server" then
-		if not user_ssl_config.certificate then return nil, "No certificate present in SSL/TLS configuration for "..host; end
-		if not user_ssl_config.key then return nil, "No key present in SSL/TLS configuration for "..host; end
+		if not user_ssl_config.certificate then
+			log("info", "No certificate present in SSL/TLS configuration for %s. SNI will be required.", host);
+		end
+		if user_ssl_config.certificate and not user_ssl_config.key then return nil, "No key present in SSL/TLS configuration for "..host; end
 	end
 
 	for option in pairs(path_options) do
@@ -258,4 +252,5 @@ return {
 	create_context = create_context;
 	reload_ssl_config = reload_ssl_config;
 	find_cert = find_cert;
+	find_host_cert = find_host_cert;
 };
diff --git a/core/configmanager.lua b/core/configmanager.lua
index 1e67da9b..ae0a274a 100644
--- a/core/configmanager.lua
+++ b/core/configmanager.lua
@@ -7,15 +7,16 @@
 --
 
 local _G = _G;
-local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs =
-      setmetatable, rawget, rawset, io, os, error, dofile, type, pairs;
-local format, math_max = string.format, math.max;
+local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs =
+      setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs;
+local format, math_max, t_insert = string.format, math.max, table.insert;
 
 local envload = require"util.envload".envload;
 local deps = require"util.dependencies";
 local resolve_relative_path = require"util.paths".resolve_relative_path;
 local glob_to_pattern = require"util.paths".glob_to_pattern;
 local path_sep = package.config:sub(1,1);
+local get_traceback_table = require "util.debug".get_traceback_table;
 
 local encodings = deps.softreq"util.encodings";
 local nameprep = encodings and encodings.stringprep.nameprep or function (host) return host:lower(); end
@@ -100,8 +101,18 @@ end
 -- Built-in Lua parser
 do
 	local pcall = _G.pcall;
+	local function get_line_number(config_file)
+		local tb = get_traceback_table(nil, 2);
+		for i = 1, #tb do
+			if tb[i].info.short_src == config_file then
+				return tb[i].info.currentline;
+			end
+		end
+	end
 	parser = {};
 	function parser.load(data, config_file, config_table)
+		local set_options = {}; -- set_options[host.."/"..option_name] = true (when the option has been set already in this file)
+		local warnings = {};
 		local env;
 		-- The ' = true' are needed so as not to set off __newindex when we assign the functions below
 		env = setmetatable({
@@ -115,13 +126,26 @@ do
 					return rawget(_G, k);
 				end,
 				__newindex = function (_, k, v)
+					local host = env.__currenthost or "*";
+					local option_path = host.."/"..k;
+					if set_options[option_path] then
+						t_insert(warnings, ("%s:%d: Duplicate option '%s'"):format(config_file, get_line_number(config_file), k));
+					end
+					set_options[option_path] = true;
 					set(config_table, env.__currenthost or "*", k, v);
 				end
 		});
 
 		rawset(env, "__currenthost", "*") -- Default is global
 		function env.VirtualHost(name)
-			name = nameprep(name);
+			if not name then
+				error("Host must have a name", 2);
+			end
+			local prepped_name = nameprep(name);
+			if not prepped_name then
+				error(format("Name of Host %q contains forbidden characters", name), 0);
+			end
+			name = prepped_name;
 			if rawget(config_table, name) and rawget(config_table[name], "component_module") then
 				error(format("Host %q clashes with previously defined %s Component %q, for services use a sub-domain like conference.%s",
 					name, config_table[name].component_module:gsub("^%a+$", { component = "external", muc = "MUC"}), name, name), 0);
@@ -139,7 +163,14 @@ do
 		env.Host, env.host = env.VirtualHost, env.VirtualHost;
 
 		function env.Component(name)
-			name = nameprep(name);
+			if not name then
+				error("Component must have a name", 2);
+			end
+			local prepped_name = nameprep(name);
+			if not prepped_name then
+				error(format("Name of Component %q contains forbidden characters", name), 0);
+			end
+			name = prepped_name;
 			if rawget(config_table, name) and rawget(config_table[name], "defined")
 				and not rawget(config_table[name], "component_module") then
 				error(format("Component %q clashes with previously defined Host %q, for services use a sub-domain like conference.%s",
@@ -195,6 +226,11 @@ do
 			if f then
 				local ret, err = parser.load(f:read("*a"), file, config_table);
 				if not ret then error(err:gsub("%[string.-%]", file), 0); end
+				if err then
+					for _, warning in ipairs(err) do
+						t_insert(warnings, warning);
+					end
+				end
 			end
 			if not f then error("Error loading included "..file..": "..err, 0); end
 			return f, err;
@@ -217,7 +253,7 @@ do
 			return nil, err;
 		end
 
-		return true;
+		return true, warnings;
 	end
 
 end
diff --git a/core/loggingmanager.lua b/core/loggingmanager.lua
index cfa8246a..85a6380b 100644
--- a/core/loggingmanager.lua
+++ b/core/loggingmanager.lua
@@ -18,6 +18,9 @@ local getstyle, getstring = require "util.termcolours".getstyle, require "util.t
 local config = require "core.configmanager";
 local logger = require "util.logger";
 
+local have_pposix, pposix = pcall(require, "util.pposix");
+have_pposix = have_pposix and pposix._VERSION == "0.4.0";
+
 local _ENV = nil;
 -- luacheck: std none
 
@@ -232,6 +235,22 @@ local function log_to_console(sink_config)
 end
 log_sink_types.console = log_to_console;
 
+if have_pposix then
+	local syslog_opened;
+	local function log_to_syslog(sink_config) -- luacheck: ignore 212/sink_config
+		if not syslog_opened then
+			local facility = sink_config.syslog_facility or config.get("*", "syslog_facility");
+			pposix.syslog_open(sink_config.syslog_name or "prosody", facility);
+			syslog_opened = true;
+		end
+		local syslog = pposix.syslog_log;
+		return function (name, level, message, ...)
+			syslog(level, name, format(message, ...));
+		end;
+	end
+	log_sink_types.syslog = log_to_syslog;
+end
+
 local function register_sink_type(name, sink_maker)
 	local old_sink_maker = log_sink_types[name];
 	log_sink_types[name] = sink_maker;
diff --git a/core/moduleapi.lua b/core/moduleapi.lua
index 10f9f04d..0a8adc36 100644
--- a/core/moduleapi.lua
+++ b/core/moduleapi.lua
@@ -14,13 +14,18 @@ local pluginloader = require "util.pluginloader";
 local timer = require "util.timer";
 local resolve_relative_path = require"util.paths".resolve_relative_path;
 local st = require "util.stanza";
+local cache = require "util.cache";
+local errutil = require "util.error";
+local promise = require "util.promise";
+local time_now = require "util.time".now;
+local format = require "util.format".format;
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 local error, setmetatable, type = error, setmetatable, type;
 local ipairs, pairs, select = ipairs, pairs, select;
 local tonumber, tostring = tonumber, tostring;
 local require = require;
-local pack = table.pack or function(...) return {n=select("#",...), ...}; end -- table.pack is only in 5.2
+local pack = table.pack or require "util.table".pack; -- table.pack is only in 5.2
 local unpack = table.unpack or unpack; --luacheck: ignore 113 -- renamed in 5.2
 
 local prosody = prosody;
@@ -361,6 +366,91 @@ function api:send(stanza, origin)
 	return core_post_stanza(origin or hosts[self.host], stanza);
 end
 
+function api:send_iq(stanza, origin, timeout)
+	local iq_cache = self._iq_cache;
+	if not iq_cache then
+		iq_cache = cache.new(256, function (_, iq)
+			iq.reject(errutil.new({
+				type = "wait", condition = "resource-constraint",
+				text = "evicted from iq tracking cache"
+			}));
+		end);
+		self._iq_cache = iq_cache;
+	end
+
+	local event_type;
+	if stanza.attr.from == self.host then
+		event_type = "host";
+	else -- assume bare since we can't hook full jids
+		event_type = "bare";
+	end
+	local result_event = "iq-result/"..event_type.."/"..stanza.attr.id;
+	local error_event = "iq-error/"..event_type.."/"..stanza.attr.id;
+	local cache_key = event_type.."/"..stanza.attr.id;
+
+	local p = promise.new(function (resolve, reject)
+		local function result_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				resolve(event);
+				return true;
+			end
+		end
+
+		local function error_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				reject(errutil.from_stanza(event.stanza), event);
+				return true;
+			end
+		end
+
+		if iq_cache:get(cache_key) then
+			reject(errutil.new({
+				type = "modify", condition = "conflict",
+				text = "IQ stanza id attribute already used",
+			}));
+			return;
+		end
+
+		self:hook(result_event, result_handler);
+		self:hook(error_event, error_handler);
+
+		local timeout_handle = self:add_timer(timeout or 120, function ()
+			reject(errutil.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "IQ stanza timed out",
+			}));
+		end);
+
+		local ok = iq_cache:set(cache_key, {
+			reject = reject, resolve = resolve,
+			timeout_handle = timeout_handle,
+			result_handler = result_handler, error_handler = error_handler;
+		});
+
+		if not ok then
+			reject(errutil.new({
+				type = "wait", condition = "internal-server-error",
+				text = "Could not store IQ tracking data"
+			}));
+			return;
+		end
+
+		self:send(stanza, origin);
+	end);
+
+	p:finally(function ()
+		local iq = iq_cache:get(cache_key);
+		if iq then
+			self:unhook(result_event, iq.result_handler);
+			self:unhook(error_event, iq.error_handler);
+			iq.timeout_handle:stop();
+			iq_cache:set(cache_key, nil);
+		end
+	end);
+
+	return p;
+end
+
 function api:broadcast(jids, stanza, iter)
 	for jid in (iter or it.values)(jids) do
 		local new_stanza = st.clone(stanza);
@@ -432,4 +522,32 @@ function api:measure_global_event(event_name, stat_name)
 	return self:measure_object_event(prosody.events.wrappers, event_name, stat_name);
 end
 
+local status_priorities = { error = 3, warn = 2, info = 1, core = 0 };
+
+function api:set_status(status_type, status_message, override)
+	local priority = status_priorities[status_type];
+	if not priority then
+		self:log("error", "set_status: Invalid status type '%s', assuming 'info'");
+		status_type, priority = "info", status_priorities.info;
+	end
+	local current_priority = status_priorities[self.status_type] or 0;
+	-- By default an 'error' status can only be overwritten by another 'error' status
+	if (current_priority >= status_priorities.error and priority < current_priority and override ~= true)
+	or (override == false and current_priority > priority) then
+		self:log("debug", "moduleapi: ignoring status [prio %d override %s]: %s", priority, override, status_message);
+		return;
+	end
+	self.status_type, self.status_message, self.status_time = status_type, status_message, time_now();
+	self:fire_event("module-status/updated", { name = self.name });
+end
+
+function api:log_status(level, msg, ...)
+	self:set_status(level, format(msg, ...));
+	return self:log(level, msg, ...);
+end
+
+function api:get_status()
+	return self.status_type, self.status_message, self.status_time;
+end
+
 return api;
diff --git a/core/modulemanager.lua b/core/modulemanager.lua
index a824d36a..e23f1e55 100644
--- a/core/modulemanager.lua
+++ b/core/modulemanager.lua
@@ -23,8 +23,24 @@ local debug_traceback = debug.traceback;
 local setmetatable, rawget = setmetatable, rawget;
 local ipairs, pairs, type, t_insert = ipairs, pairs, type, table.insert;
 
-local autoload_modules = {prosody.platform, "presence", "message", "iq", "offline", "c2s", "s2s", "s2s_auth_certs"};
-local component_inheritable_modules = {"tls", "saslauth", "dialback", "iq", "s2s"};
+local autoload_modules = {
+	prosody.platform,
+	"presence",
+	"message",
+	"iq",
+	"offline",
+	"c2s",
+	"s2s",
+	"s2s_auth_certs",
+};
+local component_inheritable_modules = {
+	"tls",
+	"saslauth",
+	"dialback",
+	"iq",
+	"s2s",
+	"s2s_bidi",
+};
 
 -- We need this to let modules access the real global namespace
 local _G = _G;
@@ -174,6 +190,7 @@ local function do_load_module(host, module_name, state)
 	local mod, err = pluginloader.load_code(module_name, nil, pluginenv);
 	if not mod then
 		log("error", "Unable to load module '%s': %s", module_name or "nil", err or "nil");
+		api_instance:set_status("error", "Failed to load (see log)");
 		return nil, err;
 	end
 
@@ -187,6 +204,7 @@ local function do_load_module(host, module_name, state)
 			ok, err = call_module_method(pluginenv, "load");
 			if not ok then
 				log("warn", "Error loading module '%s' on '%s': %s", module_name, host, err or "nil");
+				api_instance:set_status("warn", "Error during load (see log)");
 			end
 		end
 		api_instance.reloading, api_instance.saved_state = nil, nil;
@@ -209,6 +227,9 @@ local function do_load_module(host, module_name, state)
 	if not ok then
 		modulemap[api_instance.host][module_name] = nil;
 		log("error", "Error initializing module '%s' on '%s': %s", module_name, host, err or "nil");
+		api_instance:set_status("warn", "Error during load (see log)");
+	else
+		api_instance:set_status("core", "Loaded", false);
 	end
 	return ok and pluginenv, err;
 end
@@ -225,7 +246,8 @@ local function do_reload_module(host, name)
 
 	local saved;
 	if module_has_method(mod, "save") then
-		local ok, ret, err = call_module_method(mod, "save");
+		-- FIXME What goes in 'err' here?
+		local ok, ret, err = call_module_method(mod, "save"); -- luacheck: ignore 211/err
 		if ok then
 			saved = ret;
 		else
diff --git a/core/portmanager.lua b/core/portmanager.lua
index bed5eca5..0712f5ac 100644
--- a/core/portmanager.lua
+++ b/core/portmanager.lua
@@ -9,7 +9,8 @@ local set = require "util.set";
 
 local table = table;
 local setmetatable, rawset, rawget = setmetatable, rawset, rawget;
-local type, tonumber, tostring, ipairs = type, tonumber, tostring, ipairs;
+local type, tonumber, ipairs = type, tonumber, ipairs;
+local pairs = pairs;
 
 local prosody = prosody;
 local fire_event = prosody.events.fire_event;
@@ -95,25 +96,25 @@ local function activate(service_name)
 		   }
 	bind_ports = set.new(type(bind_ports) ~= "table" and { bind_ports } or bind_ports );
 
-	local mode, ssl = listener.default_mode or default_mode;
+	local mode = listener.default_mode or default_mode;
 	local hooked_ports = {};
 
 	for interface in bind_interfaces do
 		for port in bind_ports do
 			local port_number = tonumber(port);
 			if not port_number then
-				log("error", "Invalid port number specified for service '%s': %s", service_info.name, tostring(port));
+				log("error", "Invalid port number specified for service '%s': %s", service_info.name, port);
 			elseif #active_services:search(nil, interface, port_number) > 0 then
 				log("error", "Multiple services configured to listen on the same port ([%s]:%d): %s, %s", interface, port,
 					active_services:search(nil, interface, port)[1][1].service.name or "<unnamed>", service_name or "<unnamed>");
 			else
-				local err;
+				local ssl, cfg, err;
 				-- Create SSL context for this service/port
 				if service_info.encryption == "ssl" then
 					local global_ssl_config = config.get("*", "ssl") or {};
 					local prefix_ssl_config = config.get("*", config_prefix.."ssl") or global_ssl_config;
 					log("debug", "Creating context for direct TLS service %s on port %d", service_info.name, port);
-					ssl, err = certmanager.create_context(service_info.name.." port "..port, "server",
+					ssl, err, cfg = certmanager.create_context(service_info.name.." port "..port, "server",
 						prefix_ssl_config[interface],
 						prefix_ssl_config[port],
 						prefix_ssl_config,
@@ -127,7 +128,12 @@ local function activate(service_name)
 				end
 				if not err then
 					-- Start listening on interface+port
-					local handler, err = server.addserver(interface, port_number, listener, mode, ssl);
+					local handler, err = server.listen(interface, port_number, listener, {
+						read_size = mode,
+						tls_ctx = ssl,
+						tls_direct = service_info.encryption == "ssl";
+						sni_hosts = {},
+					});
 					if not handler then
 						log("error", "Failed to open server port %d on %s, %s", port_number, interface,
 							error_to_friendly_message(service_name, port_number, err));
@@ -137,6 +143,7 @@ local function activate(service_name)
 						active_services:add(service_name, interface, port_number, {
 							server = handler;
 							service = service_info;
+							tls_cfg = cfg;
 						});
 					end
 				end
@@ -222,15 +229,46 @@ end
 
 -- Event handlers
 
+local function add_sni_host(host, service)
+	for name, interface, port, n, active_service --luacheck: ignore 213
+		in active_services:iter(service, nil, nil, nil) do
+		if active_service.server.hosts and active_service.tls_cfg then
+			local config_prefix = (active_service.config_prefix or name).."_";
+			if config_prefix == "_" then config_prefix = ""; end
+			local prefix_ssl_config = config.get(host, config_prefix.."ssl");
+			local autocert = certmanager.find_host_cert(host);
+			-- luacheck: ignore 211/cfg
+			local ssl, err, cfg = certmanager.create_context(host, "server", prefix_ssl_config, autocert, active_service.tls_cfg);
+			if ssl then
+				active_service.server.hosts[host] = ssl;
+			else
+				log("error", "Error creating TLS context for SNI host %s: %s", host, err);
+			end
+		end
+	end
+end
 prosody.events.add_handler("item-added/net-provider", function (event)
 	local item = event.item;
 	register_service(item.name, item);
+	for host in pairs(prosody.hosts) do
+		add_sni_host(host, item.name);
+	end
 end);
 prosody.events.add_handler("item-removed/net-provider", function (event)
 	local item = event.item;
 	unregister_service(item.name, item);
 end);
 
+prosody.events.add_handler("host-activated", add_sni_host);
+prosody.events.add_handler("host-deactivated", function (host)
+	for name, interface, port, n, active_service --luacheck: ignore 213
+		in active_services:iter(nil, nil, nil, nil) do
+		if active_service.tls_cfg then
+			active_service.server.hosts[host] = nil;
+		end
+	end
+end);
+
 return {
 	activate = activate;
 	deactivate = deactivate;
diff --git a/core/rostermanager.lua b/core/rostermanager.lua
index 61b08002..7b104339 100644
--- a/core/rostermanager.lua
+++ b/core/rostermanager.lua
@@ -12,6 +12,7 @@
 local log = require "util.logger".init("rostermanager");
 
 local new_id = require "util.id".short;
+local new_cache = require "util.cache".new;
 
 local pairs = pairs;
 local tostring = tostring;
@@ -111,6 +112,23 @@ local function load_roster(username, host)
 	else -- Attempt to load roster for non-loaded user
 		log("debug", "load_roster: loading for offline user: %s", jid);
 	end
+	local roster_cache = hosts[host] and hosts[host].roster_cache;
+	if not roster_cache then
+		if hosts[host] then
+			roster_cache = new_cache(1024);
+			hosts[host].roster_cache = roster_cache;
+		end
+	else
+		roster = roster_cache:get(jid);
+		if roster then
+			log("debug", "load_roster: cache hit");
+			roster_cache:set(jid, roster);
+			if user then user.roster = roster; end
+			return roster;
+		else
+			log("debug", "load_roster: cache miss, loading from storage");
+		end
+	end
 	local roster_store = storagemanager.open(host, "roster", "keyval");
 	local data, err = roster_store:get(username);
 	roster = data or {};
@@ -134,6 +152,10 @@ local function load_roster(username, host)
 	if not err then
 		hosts[host].events.fire_event("roster-load", { username = username, host = host, roster = roster });
 	end
+	if roster_cache and not user then
+		log("debug", "load_roster: caching loaded roster");
+		roster_cache:set(jid, roster);
+	end
 	return roster, err;
 end
 
@@ -263,15 +285,15 @@ end
 
 function is_contact_pending_in(username, host, jid)
 	local roster = load_roster(username, host);
-	return roster[false].pending[jid];
+	return roster[false].pending[jid] ~= nil;
 end
-local function set_contact_pending_in(username, host, jid)
+local function set_contact_pending_in(username, host, jid, stanza)
 	local roster = load_roster(username, host);
 	local item = roster[jid];
 	if item and (item.subscription == "from" or item.subscription == "both") then
 		return; -- false
 	end
-	roster[false].pending[jid] = true;
+	roster[false].pending[jid] = st.is_stanza(stanza) and st.preserialize(stanza) or true;
 	return save_roster(username, host, roster, jid);
 end
 function is_contact_pending_out(username, host, jid)
@@ -279,6 +301,11 @@ function is_contact_pending_out(username, host, jid)
 	local item = roster[jid];
 	return item and item.ask;
 end
+local function is_contact_preapproved(username, host, jid)
+	local roster = load_roster(username, host);
+	local item = roster[jid];
+	return item and (item.approved == "true");
+end
 local function set_contact_pending_out(username, host, jid) -- subscribe
 	local roster = load_roster(username, host);
 	local item = roster[jid];
@@ -309,9 +336,10 @@ local function unsubscribe(username, host, jid)
 	return save_roster(username, host, roster, jid);
 end
 local function subscribed(username, host, jid)
+	local roster = load_roster(username, host);
+	local item = roster[jid];
+
 	if is_contact_pending_in(username, host, jid) then
-		local roster = load_roster(username, host);
-		local item = roster[jid];
 		if not item then -- FIXME should roster item be auto-created?
 			item = {subscription = "none", groups = {}};
 			roster[jid] = item;
@@ -323,7 +351,17 @@ local function subscribed(username, host, jid)
 		end
 		roster[false].pending[jid] = nil;
 		return save_roster(username, host, roster, jid);
-	end -- TODO else implement optional feature pre-approval (ask = subscribed)
+	elseif not item or item.subscription == "none" or item.subscription == "to" then
+		-- Contact is not subscribed and has not sent a subscription request.
+		-- We store a pre-approval as per RFC6121 3.4
+		if not item then
+			item = {subscription = "none", groups = {}};
+			roster[jid] = item;
+		end
+		item.approved = "true";
+		log("debug", "Storing preapproval for %s", jid);
+		return save_roster(username, host, roster, jid);
+	end
 end
 local function unsubscribed(username, host, jid)
 	local roster = load_roster(username, host);
@@ -381,6 +419,7 @@ return {
 	set_contact_pending_in = set_contact_pending_in;
 	is_contact_pending_out = is_contact_pending_out;
 	set_contact_pending_out = set_contact_pending_out;
+	is_contact_preapproved = is_contact_preapproved;
 	unsubscribe = unsubscribe;
 	subscribed = subscribed;
 	unsubscribed = unsubscribed;
diff --git a/core/s2smanager.lua b/core/s2smanager.lua
index 58269c49..7f156f12 100644
--- a/core/s2smanager.lua
+++ b/core/s2smanager.lua
@@ -9,10 +9,10 @@
 
 
 local hosts = prosody.hosts;
-local tostring, pairs, setmetatable
-    = tostring, pairs, setmetatable;
+local pairs, setmetatable = pairs, setmetatable;
 
 local logger_init = require "util.logger".init;
+local sessionlib = require "util.session";
 
 local log = logger_init("s2smanager");
 
@@ -26,18 +26,29 @@ local _ENV = nil;
 -- luacheck: std none
 
 local function new_incoming(conn)
-	local session = { conn = conn, type = "s2sin_unauthed", direction = "incoming", hosts = {} };
-	session.log = logger_init("s2sin"..tostring(session):match("[a-f0-9]+$"));
-	incoming_s2s[session] = true;
-	return session;
+	local host_session = sessionlib.new("s2sin");
+	sessionlib.set_id(host_session);
+	sessionlib.set_logger(host_session);
+	sessionlib.set_conn(host_session, conn);
+	host_session.direction = "incoming";
+	host_session.incoming = true;
+	host_session.hosts = {};
+	incoming_s2s[host_session] = true;
+	return host_session;
 end
 
 local function new_outgoing(from_host, to_host)
-	local host_session = { to_host = to_host, from_host = from_host, host = from_host,
-		               notopen = true, type = "s2sout_unauthed", direction = "outgoing" };
+	local host_session = sessionlib.new("s2sout");
+	sessionlib.set_id(host_session);
+	sessionlib.set_logger(host_session);
+	host_session.to_host = to_host;
+	host_session.from_host = from_host;
+	host_session.host = from_host;
+	host_session.notopen = true;
+	host_session.direction = "outgoing";
+	host_session.outgoing = true;
+	host_session.hosts = {};
 	hosts[from_host].s2sout[to_host] = host_session;
-	local conn_name = "s2sout"..tostring(host_session):match("[a-f0-9]*$");
-	host_session.log = logger_init(conn_name);
 	return host_session;
 end
 
@@ -50,6 +61,9 @@ local resting_session = { -- Resting, not dead
 		close = function (session)
 			session.log("debug", "Attempt to close already-closed session");
 		end;
+		reset_stream = function (session)
+			session.log("debug", "Attempt to reset stream of already-closed session");
+		end;
 		filter = function (type, data) return data; end; --luacheck: ignore 212/type
 	}; resting_session.__index = resting_session;
 
@@ -63,23 +77,25 @@ local function retire_session(session, reason)
 
 	session.destruction_reason = reason;
 
-	function session.send(data) log("debug", "Discarding data sent to resting session: %s", tostring(data)); end
-	function session.data(data) log("debug", "Discarding data received from resting session: %s", tostring(data)); end
+	function session.send(data) log("debug", "Discarding data sent to resting session: %s", data); end
+	function session.data(data) log("debug", "Discarding data received from resting session: %s", data); end
 	session.thread = { run = function (_, data) return session.data(data) end };
 	session.sends2s = session.send;
 	return setmetatable(session, resting_session);
 end
 
-local function destroy_session(session, reason)
+local function destroy_session(session, reason, bounce_reason)
 	if session.destroyed then return; end
-	(session.log or log)("debug", "Destroying "..tostring(session.direction)
-		.." session "..tostring(session.from_host).."->"..tostring(session.to_host)
-		..(reason and (": "..reason) or ""));
+	local log = session.log or log;
+	log("debug", "Destroying %s session %s->%s%s%s", session.direction, session.from_host, session.to_host, reason and ": " or "", reason or "");
 
 	if session.direction == "outgoing" then
 		hosts[session.from_host].s2sout[session.to_host] = nil;
-		session:bounce_sendq(reason);
+		session:bounce_sendq(bounce_reason or reason);
 	elseif session.direction == "incoming" then
+		if session.outgoing then
+			hosts[session.to_host].s2sout[session.from_host] = nil;
+		end
 		incoming_s2s[session] = nil;
 	end
 
diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua
index 2843001a..6c005fcd 100644
--- a/core/sessionmanager.lua
+++ b/core/sessionmanager.lua
@@ -21,6 +21,7 @@ local config_get = require "core.configmanager".get;
 local resourceprep = require "util.encodings".stringprep.resourceprep;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
 local generate_identifier = require "util.id".short;
+local sessionlib = require "util.session";
 
 local initialize_filters = require "util.filters".initialize;
 local gettime = require "socket".gettime;
@@ -29,23 +30,34 @@ local _ENV = nil;
 -- luacheck: std none
 
 local function new_session(conn)
-	local session = { conn = conn, type = "c2s_unauthed", conntime = gettime() };
+	local session = sessionlib.new("c2s");
+	sessionlib.set_id(session);
+	sessionlib.set_logger(session);
+	sessionlib.set_conn(session, conn);
+
+	session.conntime = gettime();
 	local filter = initialize_filters(session);
 	local w = conn.write;
+
+	function session.rawsend(t)
+		t = filter("bytes/out", tostring(t));
+		if t then
+			local ret, err = w(conn, t);
+			if not ret then
+				session.log("debug", "Error writing to connection: %s", err);
+				return false, err;
+			end
+		end
+		return true;
+	end
+
 	session.send = function (t)
 		session.log("debug", "Sending[%s]: %s", session.type, t.top_tag and t:top_tag() or t:match("^[^>]*>?"));
 		if t.name then
 			t = filter("stanzas/out", t);
 		end
 		if t then
-			t = filter("bytes/out", tostring(t));
-			if t then
-				local ret, err = w(conn, t);
-				if not ret then
-					session.log("debug", "Error writing to connection: %s", tostring(err));
-					return false, err;
-				end
-			end
+			return session.rawsend(t);
 		end
 		return true;
 	end
@@ -73,8 +85,8 @@ local function retire_session(session)
 		end
 	end
 
-	function session.send(data) log("debug", "Discarding data sent to resting session: %s", tostring(data)); return false; end
-	function session.data(data) log("debug", "Discarding data received from resting session: %s", tostring(data)); end
+	function session.send(data) log("debug", "Discarding data sent to resting session: %s", data); return false; end
+	function session.data(data) log("debug", "Discarding data received from resting session: %s", data); end
 	session.thread = { run = function (_, data) return session.data(data) end };
 	return setmetatable(session, resting_session);
 end
@@ -117,7 +129,7 @@ local function make_authenticated(session, username)
 	if session.type == "c2s_unauthed" then
 		session.type = "c2s_unbound";
 	end
-	session.log("info", "Authenticated as %s@%s", username or "(unknown)", session.host or "(unknown)");
+	session.log("info", "Authenticated as %s@%s", username, session.host or "(unknown)");
 	return true;
 end
 
@@ -138,7 +150,7 @@ local function bind_resource(session, resource)
 		resource = event_payload.resource;
 	end
 
-	resource = resourceprep(resource);
+	resource = resourceprep(resource or "", true);
 	resource = resource ~= "" and resource or generate_identifier();
 	--FIXME: Randomly-generated resources must be unique per-user, and never conflict with existing
 
diff --git a/core/stanza_router.lua b/core/stanza_router.lua
index f5a34f59..1d8db3e7 100644
--- a/core/stanza_router.lua
+++ b/core/stanza_router.lua
@@ -12,6 +12,7 @@ local hosts = _G.prosody.hosts;
 local tostring = tostring;
 local st = require "util.stanza";
 local jid_split = require "util.jid".split;
+local jid_host = require "util.jid".host;
 local jid_prepped_split = require "util.jid".prepped_split;
 
 local full_sessions = _G.prosody.full_sessions;
@@ -27,7 +28,7 @@ local function handle_unhandled_stanza(host, origin, stanza) --luacheck: ignore
 		local st_type = stanza.attr.type;
 		if st_type == "error" or (name == "iq" and st_type == "result") then
 			if st_type == "error" then
-				local err_type, err_condition, err_message = stanza:get_error();
+				local err_type, err_condition, err_message = stanza:get_error(); -- luacheck: ignore 211/err_message
 				log("debug", "Discarding unhandled error %s (%s, %s) from %s: %s",
 					name, err_type, err_condition or "unknown condition", origin_type, stanza:top_tag());
 			else
@@ -81,7 +82,7 @@ function core_process_stanza(origin, stanza)
 	local to_bare, from_bare;
 	if to then
 		if full_sessions[to] or bare_sessions[to] or hosts[to] then
-			node, host = jid_split(to); -- TODO only the host is needed, optimize
+			host = jid_host(to);
 		else
 			node, host, resource = jid_prepped_split(to);
 			if not host then
@@ -111,8 +112,8 @@ function core_process_stanza(origin, stanza)
 		stanza.attr.from = from;
 	end
 
-	if (origin.type == "s2sin" or origin.type == "c2s" or origin.type == "component") and xmlns == nil then
-		if origin.type == "s2sin" and not origin.dummy then
+	if (origin.type == "s2sin" or origin.type == "s2sout" or origin.type == "c2s" or origin.type == "component") and xmlns == nil then
+		if (origin.type == "s2sin" or origin.type == "s2sout") and not origin.dummy then
 			local host_status = origin.hosts[from_host];
 			if not host_status or not host_status.authed then -- remote server trying to impersonate some other server?
 				log("warn", "Received a stanza claiming to be from %s, over a stream authed for %s!", from_host, origin.from_host);
@@ -186,8 +187,8 @@ function core_post_stanza(origin, stanza, preevents)
 end
 
 function core_route_stanza(origin, stanza)
-	local node, host, resource = jid_split(stanza.attr.to);
-	local from_node, from_host, from_resource = jid_split(stanza.attr.from);
+	local host = jid_host(stanza.attr.to);
+	local from_host = jid_host(stanza.attr.from);
 
 	-- Auto-detect origin if not specified
 	origin = origin or hosts[from_host];
@@ -199,7 +200,7 @@ function core_route_stanza(origin, stanza)
 	else
 		local host_session = hosts[from_host];
 		if not host_session then
-			log("error", "No hosts[from_host] (please report): %s", tostring(stanza));
+			log("error", "No hosts[from_host] (please report): %s", stanza);
 		else
 			local xmlns = stanza.attr.xmlns;
 			stanza.attr.xmlns = nil;
diff --git a/core/statsmanager.lua b/core/statsmanager.lua
index 237b1dd5..d3f1d4f5 100644
--- a/core/statsmanager.lua
+++ b/core/statsmanager.lua
@@ -79,6 +79,7 @@ if stats then
 			if stats.get_stats then
 				changed_stats, stats_extra = {}, {};
 				for stat_name, getter in pairs(stats.get_stats()) do
+					-- luacheck: ignore 211/type
 					local type, value, extra = getter();
 					local old_value = latest_stats[stat_name];
 					latest_stats[stat_name] = value;
@@ -97,6 +98,7 @@ if stats then
 		end
 		timer.add_task(stats_interval, collect);
 		prosody.events.add_handler("server-started", function () collect() end, -1);
+		prosody.events.add_handler("server-stopped", function () collect() end, -1);
 	else
 		log("debug", "Statistics enabled using %s provider, collection is disabled", stats_provider_name);
 	end
diff --git a/doc/coding_style.md b/doc/coding_style.md
new file mode 100644
index 00000000..f9a10ece
--- /dev/null
+++ b/doc/coding_style.md
@@ -0,0 +1,804 @@
+
+# Prosody Coding Style Guide
+
+This style guides lists the coding conventions used in the
+[Prosody](https://prosody.im/) project. It is based heavily on the [style guide used by the LuaRocks project](https://github.com/luarocks/lua-style-guide).
+
+## Indentation and formatting
+
+* Prosody code is indented with tabs at the start of the line, a single
+  tab per logical indent level:
+
+```lua
+for i, pkg in ipairs(packages) do
+    for name, version in pairs(pkg) do
+        if name == searched then
+            print(version);
+        end
+    end
+end
+```
+
+Tab width is configurable in editors, so never assume a particular width.
+Specically this means you should not mix tabs and spaces, or use tabs for
+alignment of items at different indentation levels.
+
+* Use LF (Unix) line endings.
+
+## Comments
+
+* Comments are encouraged where necessary to explain non-obvious code.
+
+* In general comments should be used to explain 'why', not 'how'
+
+### Comment tags
+
+A comment may be prefixed with one of the following tags:
+
+* **FIXME**: Indicates a serious problem with the code that should be addressed
+* **TODO**: Indicates an open task, feature request or code restructuring that
+  is primarily of interest to developers (otherwise it should be in the
+  issue tracker).
+* **COMPAT**: Must be used on all code that is present only for backwards-compatibility,
+  and may be removed one day. For example code that is added to support old
+  or buggy third-party software or dependencies.
+
+**Example:**
+
+```lua
+-- TODO: implement method
+local function something()
+   -- FIXME: check conditions
+end
+
+```
+
+## Variable names
+
+* Variable names with larger scope should be more descriptive than those with
+smaller scope. One-letter variable names should be avoided except for very
+small scopes (less than ten lines) or for iterators.
+
+* `i` should be used only as a counter variable in for loops (either numeric for
+or `ipairs`).
+
+* Prefer more descriptive names than `k` and `v` when iterating with `pairs`,
+unless you are writing a function that operates on generic tables.
+
+* Use `_` for ignored variables (e.g. in for loops:)
+
+```lua
+for _, item in ipairs(items) do
+   do_something_with_item(item);
+end
+```
+
+* Generally all identifiers (variables and function names) should use `snake_case`,
+  i.e. lowercase words joined by `_`.
+
+```lua
+-- bad
+local OBJEcttsssss = {}
+local thisIsMyObject = {}
+local c = function()
+   -- ...stuff...
+end
+
+-- good
+local this_is_my_object = {};
+
+local function do_that_thing()
+   -- ...stuff...
+end
+```
+
+> **Rationale:** The standard library uses lowercase APIs, with `joinedlowercase`
+names, but this does not scale too well for more complex APIs. `snake_case`
+tends to look good enough and not too out-of-place along side the standard
+APIs.
+
+```lua
+for _, name in pairs(names) do
+   -- ...stuff...
+end
+```
+
+* Prefer using `is_` when naming boolean functions:
+
+```lua
+-- bad
+local function evil(alignment)
+   return alignment < 100
+end
+
+-- good
+local function is_evil(alignment)
+   return alignment < 100;
+end
+```
+
+* `UPPER_CASE` is to be used sparingly, with "constants" only.
+
+> **Rationale:** "Sparingly", since Lua does not have real constants. This
+notation is most useful in libraries that bind C libraries, when bringing over
+constants from C.
+
+* Do not use uppercase names starting with `_`, they are reserved by Lua.
+
+## Tables
+
+* When creating a table, prefer populating its fields all at once, if possible:
+
+```lua
+local player = { name = "Jack", class = "Rogue" };
+```
+
+* Items should be separated by commas. If there are many items, put each
+  key/value on a separate line and use a semi-colon after each item (including
+  the last one):
+
+```lua
+local player = {
+   name = "Jack";
+   class = "Rogue";
+}
+```
+
+> **Rationale:** This makes the structure of your tables more evident at a glance.
+Trailing semi-colons make it quicker to add new fields and produces shorter diffs.
+
+* Use plain `key` syntax whenever possible, use `["key"]` syntax when using names
+that can't be represented as identifiers and avoid mixing representations in
+a declaration:
+
+```lua
+local mytable = {
+   ["1394-E"] = val1;
+   ["UTF-8"] = val2;
+   ["and"] = val2;
+}
+```
+
+## Strings
+
+* Use `"double quotes"` for strings; use `'single quotes'` when writing strings
+that contain double quotes.
+
+```lua
+local name = "Prosody";
+local sentence = 'The name of the program is "Prosody"';
+```
+
+> **Rationale:** Double quotes are used as string delimiters in a larger number of
+programming languages. Single quotes are useful for avoiding escaping when
+using double quotes in literals.
+
+## Line lengths
+
+* There are no hard or soft limits on line lengths. Line lengths are naturally
+limited by using one statement per line. If that still produces lines that are
+too long (e.g. an expression that produces a line over 256-characters long,
+for example), this means the expression is too complex and would do better
+split into subexpressions with reasonable names.
+
+> **Rationale:** No one works on VT100 terminals anymore. If line lengths are a proxy
+for code complexity, we should address code complexity instead of using line
+breaks to fit mind-bending statements over multiple lines.
+
+## Function declaration syntax
+
+* Prefer function syntax over variable syntax. This helps differentiate between
+named and anonymous functions.
+
+```lua
+-- bad
+local nope = function(name, options)
+   -- ...stuff...
+end
+
+-- good
+local function yup(name, options)
+   -- ...stuff...
+end
+```
+
+* Perform validation early and return as early as possible.
+
+```lua
+-- bad
+local function is_good_name(name, options, arg)
+   local is_good = #name > 3
+   is_good = is_good and #name < 30
+
+   -- ...stuff...
+
+   return is_good
+end
+
+-- good
+local function is_good_name(name, options, args)
+   if #name < 3 or #name > 30 then
+      return false;
+   end
+
+   -- ...stuff...
+
+   return true;
+end
+```
+
+## Function calls
+
+* Even though Lua allows it, generally you should not omit parentheses
+  for functions that take a unique string literal argument.
+
+```lua
+-- bad
+local data = get_data"KRP"..tostring(area_number)
+-- good
+local data = get_data("KRP"..tostring(area_number));
+local data = get_data("KRP")..tostring(area_number);
+```
+
+> **Rationale:** It is not obvious at a glace what the precedence rules are
+when omitting the parentheses in a function call. Can you quickly tell which
+of the two "good" examples in equivalent to the "bad" one? (It's the second
+one).
+
+* You should not omit parenthesis for functions that take a unique table
+argument on a single line. You may do so for table arguments that span several
+lines.
+
+```lua
+local an_instance = a_module.new {
+   a_parameter = 42;
+   another_parameter = "yay";
+}
+```
+
+> **Rationale:** The use as in `a_module.new` above occurs alone in a statement,
+so there are no precedence issues.
+
+## Table attributes
+
+* Use dot notation when accessing known properties.
+
+```lua
+local luke = {
+   jedi = true;
+   age = 28;
+}
+
+-- bad
+local is_jedi = luke["jedi"]
+
+-- good
+local is_jedi = luke.jedi;
+```
+
+* Use subscript notation `[]` when accessing properties with a variable or if using a table as a list.
+
+```lua
+local vehicles = load_vehicles_from_disk("vehicles.dat")
+
+if vehicles["Porsche"] then
+   porsche_handler(vehicles["Porsche"]);
+   vehicles["Porsche"] = nil;
+end
+for name, cars in pairs(vehicles) do
+   regular_handler(cars);
+end
+```
+
+> **Rationale:** Using dot notation makes it clearer that the given key is meant
+to be used as a record/object field.
+
+## Functions in tables
+
+* When declaring modules and classes, declare functions external to the table definition:
+
+```lua
+local my_module = {};
+
+function my_module.a_function(x)
+   -- code
+end
+```
+
+* When declaring metatables, declare function internal to the table definition.
+
+```lua
+local version_mt = {
+   __eq = function(a, b)
+      -- code
+   end;
+   __lt = function(a, b)
+      -- code
+   end;
+}
+```
+
+> **Rationale:** Metatables contain special behavior that affect the tables
+they're assigned (and are used implicitly at the call site), so it's good to
+be able to get a view of the complete behavior of the metatable at a glance.
+
+This is not as important for objects and modules, which usually have way more
+code, and which don't fit in a single screen anyway, so nesting them inside
+the table does not gain much: when scrolling a longer file, it is more evident
+that `check_version` is a method of `Api` if it says `function Api:check_version()`
+than if it says `check_version = function()` under some indentation level.
+
+## Variable declaration
+
+* Always use `local` to declare variables.
+
+```lua
+-- bad
+superpower = get_superpower()
+
+-- good
+local superpower = get_superpower();
+```
+
+> **Rationale:** Not doing so will result in global variables to avoid polluting
+the global namespace.
+
+## Variable scope
+
+* Assign variables with the smallest possible scope.
+
+```lua
+-- bad
+local function good()
+   local name = get_name()
+
+   test()
+   print("doing stuff..")
+
+   --...other stuff...
+
+   if name == "test" then
+      return false
+   end
+
+   return name
+end
+
+-- good
+local bad = function()
+   test();
+   print("doing stuff..");
+
+   --...other stuff...
+
+   local name = get_name();
+
+   if name == "test" then
+      return false;
+   end
+
+   return name;
+end
+```
+
+> **Rationale:** Lua has proper lexical scoping. Declaring the function later means that its
+scope is smaller, so this makes it easier to check for the effects of a variable.
+
+## Conditional expressions
+
+* False and nil are falsy in conditional expressions. Use shortcuts when you
+can, unless you need to know the difference between false and nil.
+
+```lua
+-- bad
+if name ~= nil then
+   -- ...stuff...
+end
+
+-- good
+if name then
+   -- ...stuff...
+end
+```
+
+* Avoid designing APIs which depend on the difference between `nil` and `false`.
+
+* Use the `and`/`or` idiom for the pseudo-ternary operator when it results in
+more straightforward code. When nesting expressions, use parentheses to make it
+easier to scan visually:
+
+```lua
+local function default_name(name)
+   -- return the default "Waldo" if name is nil
+   return name or "Waldo";
+end
+
+local function brew_coffee(machine)
+   return (machine and machine.is_loaded) and "coffee brewing" or "fill your water";
+end
+```
+
+Note that the `x and y or z` as a substitute for `x ? y : z` does not work if
+`y` may be `nil` or `false` so avoid it altogether for returning booleans or
+values which may be nil.
+
+## Blocks
+
+* Use single-line blocks only for `then return`, `then break` and `function return` (a.k.a "lambda") constructs:
+
+```lua
+-- good
+if test then break end
+
+-- good
+if not ok then return nil, "this failed for this reason: " .. reason end
+
+-- good
+use_callback(x, function(k) return k.last end);
+
+-- good
+if test then
+  return false
+end
+
+-- bad
+if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then do_other_complicated_function() end
+
+-- good
+if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then
+   do_other_complicated_function();
+   return false;
+end
+```
+
+* Separate statements onto multiple lines. Use semicolons as statement terminators.
+
+```lua
+-- bad
+local whatever = "sure"
+a = 1 b = 2
+
+-- good
+local whatever = "sure";
+a = 1;
+b = 2;
+```
+
+## Spacing
+
+* Use a space after `--`.
+
+```lua
+--bad
+-- good
+```
+
+* Always put a space after commas and between operators and assignment signs:
+
+```lua
+-- bad
+local x = y*9
+local numbers={1,2,3}
+numbers={1 , 2 , 3}
+numbers={1 ,2 ,3}
+local strings = { "hello"
+                , "Lua"
+                , "world"
+                }
+dog.set( "attr",{
+  age="1 year",
+  breed="Bernese Mountain Dog"
+})
+
+-- good
+local x = y * 9;
+local numbers = {1, 2, 3};
+local strings = {
+    "hello";
+    "Lua";
+    "world";
+}
+dog.set("attr", {
+   age = "1 year";
+   breed = "Bernese Mountain Dog";
+});
+```
+
+* Indent tables and functions according to the start of the line, not the construct:
+
+```lua
+-- bad
+local my_table = {
+                    "hello",
+                    "world",
+                 }
+using_a_callback(x, function(...)
+                       print("hello")
+                    end)
+
+-- good
+local my_table = {
+    "hello";
+    "world";
+}
+using_a_callback(x, function(...)
+   print("hello");
+end)
+```
+
+> **Rationale:** This keep indentation levels aligned at predictable places. You don't
+need to realign the entire block if something in the first line changes (such as
+replacing `x` with `xy` in the `using_a_callback` example above).
+
+* The concatenation operator gets a pass for avoiding spaces:
+
+```lua
+-- okay
+local message = "Hello, "..user.."! This is your day # "..day.." in our platform!";
+```
+
+> **Rationale:** Being at the baseline, the dots already provide some visual spacing.
+
+* No spaces after the name of a function in a declaration or in its arguments:
+
+```lua
+-- bad
+local function hello ( name, language )
+   -- code
+end
+
+-- good
+local function hello(name, language)
+   -- code
+end
+```
+
+* Add blank lines between functions:
+
+```lua
+-- bad
+local function foo()
+   -- code
+end
+local function bar()
+   -- code
+end
+
+-- good
+local function foo()
+   -- code
+end
+
+local function bar()
+   -- code
+end
+```
+
+* Avoid aligning variable declarations:
+
+```lua
+-- bad
+local a               = 1
+local long_identifier = 2
+
+-- good
+local a = 1;
+local long_identifier = 2;
+```
+
+> **Rationale:** This produces extra diffs which add noise to `git blame`.
+
+* Alignment is occasionally useful when logical correspondence is to be highlighted:
+
+```lua
+-- okay
+sys_command(form, UI_FORM_UPDATE_NODE, "a",      FORM_NODE_HIDDEN,  false);
+sys_command(form, UI_FORM_UPDATE_NODE, "sample", FORM_NODE_VISIBLE, false);
+```
+
+## Typing
+
+* In non-performance critical code, it can be useful to add type-checking assertions
+for function arguments:
+
+```lua
+function manif.load_manifest(repo_url, lua_version)
+   assert(type(repo_url) == "string");
+   assert(type(lua_version) == "string" or not lua_version);
+
+   -- ...
+end
+```
+
+* Use the standard functions for type conversion, avoid relying on coercion:
+
+```lua
+-- bad
+local total_score = review_score .. ""
+
+-- good
+local total_score = tostring(review_score);
+```
+
+## Errors
+
+* Functions that can fail for reasons that are expected (e.g. I/O) should
+return `nil` and a (string) error message on error, possibly followed by other
+return values such as an error code.
+
+* On errors such as API misuse, an error should be thrown, either with `error()`
+or `assert()`.
+
+## Modules
+
+Follow [these guidelines](http://hisham.hm/2014/01/02/how-to-write-lua-modules-in-a-post-module-world/) for writing modules. In short:
+
+* Always require a module into a local variable named after the last component of the module’s full name.
+
+```lua
+local bar = require("foo.bar"); -- requiring the module
+
+bar.say("hello"); -- using the module
+```
+
+* Don’t rename modules arbitrarily:
+
+```lua
+-- bad
+local skt = require("socket")
+```
+
+> **Rationale:** Code is much harder to read if we have to keep going back to the top
+to check how you chose to call a module.
+
+* Start a module by declaring its table using the same all-lowercase local
+name that will be used to require it. You may use an LDoc comment to identify
+the whole module path.
+
+```lua
+--- @module foo.bar
+local bar = {};
+```
+
+* Try to use names that won't clash with your local variables. For instance, don't
+name your module something like “size”.
+
+* Use `local function` to declare _local_ functions only: that is, functions
+that won’t be accessible from outside the module.
+
+That is, `local function helper_foo()` means that `helper_foo` is really local.
+
+* Public functions are declared in the module table, with dot syntax:
+
+```lua
+function bar.say(greeting)
+   print(greeting);
+end
+```
+
+> **Rationale:** Visibility rules are made explicit through syntax.
+
+* Do not set any globals in your module and always return a table in the end.
+
+* If you would like your module to be used as a function, you may set the
+`__call` metamethod on the module table instead.
+
+> **Rationale:** Modules should return tables in order to be amenable to have their
+contents inspected via the Lua interactive interpreter or other tools.
+
+* Requiring a module should cause no side-effect other than loading other
+modules and returning the module table.
+
+* A module should not have state. If a module needs configuration, turn
+  it into a factory. For example, do not make something like this:
+
+```lua
+-- bad
+local mp = require "MessagePack"
+mp.set_integer("unsigned")
+```
+
+and do something like this instead:
+
+```lua
+-- good
+local messagepack = require("messagepack");
+local mpack = messagepack.new({integer = "unsigned"});
+```
+
+* The invocation of require may omit parentheses around the module name:
+
+```lua
+local bla = require "bla";
+```
+
+## Metatables, classes and objects
+
+If creating a new type of object that has a metatable and methods, the
+metatable and methods table should be separate, and the metatable name
+should end with `_mt`.
+
+```lua
+local mytype_methods = {};
+local mytype_mt = { __index = mytype_methods };
+
+function mytype_methods:add_new_thing(thing)
+end
+
+local function new()
+    return setmetatable({}, mytype_mt);
+end
+
+return { new = new };
+```
+
+* Use the method notation when invoking methods:
+
+```
+-- bad
+my_object.my_method(my_object)
+
+-- good
+my_object:my_method();
+```
+
+> **Rationale:** This makes it explicit that the intent is to use the function as a method.
+
+* Do not rely on the `__gc` metamethod to release resources other than memory.
+If your object manage resources such as files, add a `close` method to their
+APIs and do not auto-close via `__gc`. Auto-closing via `__gc` would entice
+users of your module to not close resources as soon as possible. (Note that
+the standard `io` library does not follow this recommendation, and users often
+forget that not closing files immediately can lead to "too many open files"
+errors when the program runs for a while.)
+
+> **Rationale:** The garbage collector performs automatic *memory* management,
+dealing with memory only. There is no guarantees as to when the garbage
+collector will be invoked, and memory pressure does not correlate to pressure
+on other resources.
+
+## File structure
+
+* Lua files should be named in all lowercase.
+
+* Tests should be in a top-level `spec` directory. Prosody uses
+[Busted](http://olivinelabs.com/busted/) for testing.
+
+## Static checking
+
+All code should pass [luacheck](https://github.com/mpeterv/luacheck) using
+the `.luacheckrc` provided in the Prosody repository, and using miminal
+inline exceptions.
+
+* luacheck warnings of class 211, 212, 213 (unused variable, argument or loop
+variable) may be ignored, if the unused variable was added explicitly: for
+example, sometimes it is useful, for code understandability, to spell out what
+the keys and values in a table are, even if you're only using one of them.
+Another example is a function that needs to follow a given signature for API
+reasons (e.g. a callback that follows a given format) but doesn't use some of
+its arguments; it's better to spell out in the argument what the API the
+function implements is, instead of adding `_` variables.
+
+```
+local foo, bar = some_function(); --luacheck: ignore 212/foo
+print(bar);
+```
+
+* luacheck warning 542 (empty if branch) can also be ignored, when a sequence
+of `if`/`elseif`/`else` blocks implements a "switch/case"-style list of cases,
+and one of the cases is meant to mean "pass". For example:
+
+```lua
+if warning >= 600 and warning <= 699 then
+   print("no whitespace warnings");
+elseif warning == 542 then --luacheck: ignore 542
+   -- pass
+else
+   print("got a warning: "..warning);
+end
+```
+
+> **Rationale:** This avoids writing negated conditions in the final fallback
+case, and it's easy to add another case to the construct without having to
+edit the fallback.
+
diff --git a/doc/coding_style.txt b/doc/coding_style.txt
deleted file mode 100644
index af44da1a..00000000
--- a/doc/coding_style.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-This file describes some coding styles to try and adhere to when contributing to this project.
-Please try to follow, and feel free to fix code you see not following this standard.
-
-== Indentation ==
-
-	1 tab indentation for all blocks
-
-== Spacing ==
-
-No space between function names and parenthesis and parenthesis and parameters:
-
-		function foo(bar, baz)
-
-Single space between braces and key/value pairs in table constructors:
-
-		{ foo = "bar", bar = "foo" }
-
-== Local variable naming ==
-
-In this project there are many places where use of globals is restricted, and locals used for faster access.
-
-Local versions of standard functions should follow the below form:
-
-	math.random -> m_random
-	string.char -> s_char	
-
-== Miscellaneous ==
-
-Single-statement blocks may be written on one line when short
-	
-	if foo then bar(); end
-
-'do' and 'then' keywords should be placed at the end of the line, and never on a line by themself.
diff --git a/doc/doap.xml b/doc/doap.xml
new file mode 100644
index 00000000..20686e03
--- /dev/null
+++ b/doc/doap.xml
@@ -0,0 +1,534 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:xmpp="https://linkmauve.fr/ns/xmpp-doap#" xml:lang="en">
+  <Project xmlns="http://usefulinc.com/ns/doap#">
+    <name>Prosody IM</name>
+    <shortdesc>Lightweight XMPP server</shortdesc>
+    <description>Prosody is a server for Jabber/XMPP written in Lua. It aims to be easy to use and light on resources. For developers, it aims to give a flexible system on which to rapidly develop added functionality or rapidly prototype new protocols.</description>
+    <created>2008-08-22</created>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-xmpp"/>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-jabber"/>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-server"/>
+    <homepage rdf:resource="https://prosody.im/"/>
+    <download-page rdf:resource="https://prosody.im/download/"/>
+    <license rdf:resource="https://hg.prosody.im/trunk/file/tip/COPYING"/>
+    <bug-database rdf:resource="https://issues.prosody.im/"/>
+    <support-forum rdf:resource="xmpp:prosody@conference.prosody.im?join"/>
+    <repository>
+      <HgRepository>
+        <location rdf:resource="https://hg.prosody.im/trunk/"/>
+        <browse rdf:location="https://hg.prosody.im/trunk/"/>
+      </HgRepository>
+    </repository>
+    <programming-langauge>Lua</programming-langauge>
+    <programming-langauge>C</programming-langauge>
+    <os>Linux</os>
+    <os>macOS</os>
+    <os>FreeBSD</os>
+    <os>OpenBSD</os>
+    <os>NetBSD</os>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Matthew Wild</foaf:name>
+        <foaf:nick>MattJ</foaf:nick>
+        <foaf:homepage>https://matthewwild.co.uk/</foaf:homepage>
+      </foaf:Person>
+    </maintainer>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Waqas Hussain</foaf:name>
+        <foaf:nick>waqas</foaf:nick>
+      </foaf:Person>
+    </maintainer>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Kim Alvefur</foaf:name>
+        <foaf:nick>Zash</foaf:nick>
+        <foaf:homepage>https://www.zash.se/</foaf:homepage>
+      </foaf:Person>
+    </maintainer>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc5802"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6120"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6121"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6122"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6455"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc7301"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc7395"/>
+    <!-- Added in hg:0bbbc9042361 released in 0.6.0 -->
+    <implements rdf:resource="https://datatracker.ietf.org/doc/draft-cridland-xmpp-session/"/>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0004.html"/>
+        <xmpp:version>2.9</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0009.html"/>
+        <xmpp:since>0.4</xmpp:since>
+        <xmpp:until>0.7</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0012.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:note>mod_lastactivity and mod_uptime</xmpp:note>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0016.html"/>
+        <xmpp:since>0.7</xmpp:since>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0030.html"/>
+        <xmpp:since>0.10</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0045.html"/>
+        <xmpp:version>1.32.0</xmpp:version>
+        <xmpp:since>0.3</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0049.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0050.html"/>
+        <xmpp:since>0.8</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0054.html"/>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0060.html"/>
+        <xmpp:version>1.15.8</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0065.html"/>
+        <xmpp:version>1.8.1</xmpp:version>
+        <xmpp:since>0.7</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0068.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0077.html"/>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:version>2.4</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0078.html"/>
+        <xmpp:version>2.5</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0080.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0082.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0084.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0090.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0091.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0092.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0106.html"/>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0107.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0108.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0114.html"/>
+        <xmpp:version>1.6</xmpp:version>
+        <xmpp:since>0.4</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0115.html"/>
+        <xmpp:since>0.8</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0118.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0122.html"/>
+        <xmpp:version>1.0.2</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0124.html"/>
+        <xmpp:since>0.2</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0126.html"/>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0128.html"/>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0133.html"/>
+        <xmpp:since>0.7</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0138.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:since>0.6</xmpp:since>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0153.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>via XEP-0398</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0157.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0160.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0163.html"/>
+        <xmpp:version>1.2.1</xmpp:version>
+        <xmpp:since>0.5</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0170.html"/>
+        <xmpp:version>1.0</xmpp:version>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0172.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0175.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.4</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0178.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0182.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0185.html"/>
+        <xmpp:version>1.0</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.9.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0189.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0191.html"/>
+        <xmpp:version>1.3</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0194.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0195.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0196.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0197.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0199.html"/>
+        <xmpp:version>2.0.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0202.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0203.html"/>
+        <xmpp:version>2.0</xmpp:version>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0206.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.2</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0212.html"/>
+        <xmpp:note>required level</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0220.html"/>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0227.html"/>
+        <xmpp:since>0.7</xmpp:since>
+        <xmpp:note>Used in migrator tools</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0237.html"/>
+        <xmpp:since>0.4</xmpp:since>
+        <xmpp:note>implied by rfc6121</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0280.html"/>
+        <xmpp:version>0.12.1</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0286.html"/>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>mod_csi_simple</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0288.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.12</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0292.html"/>
+        <xmpp:version>0.10</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>mod_vcard4, mod_vcard_legacy</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0302.html"/>
+        <xmpp:note>Core Server</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0313.html"/>
+        <xmpp:version>0.6.3</xmpp:version>
+        <xmpp:since>0.10</xmpp:since>
+        <xmpp:note>mod_mam, mod_muc_mam</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0318.html"/>
+        <xmpp:version>0.2</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+        <xmpp:note>refers to inclusion of delay stamp in presence</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0352.html"/>
+        <xmpp:version>0.3.0</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0368.html"/>
+        <xmpp:version>1.1.0</xmpp:version>
+        <xmpp:status>partial</xmpp:status>
+        <xmpp:since>0.2</xmpp:since>
+        <xmpp:note>legacy_ssl_ports</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0380.html"/>
+        <xmpp:version>0.3.0</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>Used in context of XEP-0352</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0384.html"/>
+        <xmpp:note>via XEP-0163, XEP-0222</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0398.html"/>
+        <xmpp:version>0.2.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:note>mod_vcard_legacy</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0410.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:note>Server Optimization</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+  </Project>
+</rdf:RDF>
diff --git a/doc/net.server.lua b/doc/net.server.lua
index 7342c549..aa9a4a9d 100644
--- a/doc/net.server.lua
+++ b/doc/net.server.lua
@@ -160,6 +160,26 @@ Returns:
 local function addserver(address, port, listeners, pattern, sslctx)
 end
 
+--[[ Binds and listens on the given address and port
+Mostly the same as addserver but with all optional arguments in a table
+
+Arguments:
+  - address: address to bind to, may be "*" to bind all addresses. will be resolved if it is a string.
+  - port: port to bind (as number)
+  - listeners: a table of listeners
+	- config: table of extra settings
+		- read_size: the amount of bytes to read or a read pattern
+		- tls_ctx: is a valid luasec constructor
+		- tls_direct: boolean true for direct TLS, false (or nil) for starttls
+
+Returns:
+  - handle
+  - nil, "an error message": on failure (e.g. out of file descriptors)
+]]
+local function listen(address, port, listeners, config)
+end
+
+
 --[[ Wraps a lua-socket socket client socket in a handle.
 The socket must be already connected to the remote end.
 If `sslctx` is given, a SSL session will be negotiated before listeners are called.
@@ -255,4 +275,5 @@ return {
 	closeall = closeall;
 	hook_signal = hook_signal;
 	watchfd = watchfd;
+	listen = listen;
 }
diff --git a/doc/storage.tld b/doc/storage.tld
index f1d33e58..057649a4 100644
--- a/doc/storage.tld
+++ b/doc/storage.tld
@@ -47,6 +47,9 @@ interface archive_store
 
 	-- Array of dates which do have messages (Optional?)
 	dates  : ( self, string? ) -> ({ string }) | (nil, string)
+
+	-- Map of counts per "with" field
+	summary : ( self, string?, archive_query? ) -> ( { string : integer } ) | (nil, string)
 end
 
 -- This represents moduleapi
diff --git a/makefile b/makefile
index d19ec24d..0b1c8788 100644
--- a/makefile
+++ b/makefile
@@ -19,6 +19,9 @@ INSTALL_EXEC=$(INSTALL) -m755
 MKDIR=install -d
 MKDIR_PRIVATE=$(MKDIR) -m750
 
+LUACHECK=luacheck
+BUSTED=busted
+
 .PHONY: all test clean install
 
 all: prosody.install prosodyctl.install prosody.cfg.lua.install prosody.version
@@ -68,8 +71,13 @@ clean:
 	rm -f prosody.version
 	$(MAKE) clean -C util-src
 
+lint:
+	$(LUACHECK) -q $$(HGPLAIN= hg files -I '**.lua') prosody prosodyctl
+	@echo $$(sed -n '/^\tlocal exclude_files/,/^}/p;' .luacheckrc | sed '1d;$d' | wc -l) files ignored
+	shellcheck configure
+
 test:
-	busted --lua=$(RUNWITH)
+	$(BUSTED) --lua=$(RUNWITH)
 
 
 prosody.install: prosody
diff --git a/net/adns.lua b/net/adns.lua
index 560e4b53..d7266958 100644
--- a/net/adns.lua
+++ b/net/adns.lua
@@ -11,10 +11,10 @@ local new_resolver = require "net.dns".resolver;
 
 local log = require "util.logger".init("adns");
 
-local coroutine, tostring, pcall = coroutine, tostring, pcall;
+local coroutine, pcall = coroutine, pcall;
 local setmetatable = setmetatable;
 
-local function dummy_send(sock, data, i, j) return (j-i)+1; end
+local function dummy_send(sock, data, i, j) return (j-i)+1; end -- luacheck: ignore 212
 
 local _ENV = nil;
 -- luacheck: std none
@@ -29,8 +29,7 @@ local function new_async_socket(sock, resolver)
 	local peername = "<unknown>";
 	local listener = {};
 	local handler = {};
-	local err;
-	function listener.onincoming(conn, data)
+	function listener.onincoming(conn, data) -- luacheck: ignore 212/conn
 		if data then
 			resolver:feed(handler, data);
 		end
@@ -46,9 +45,12 @@ local function new_async_socket(sock, resolver)
 			resolver:servfail(conn); -- Let the magic commence
 		end
 	end
-	handler, err = server.wrapclient(sock, "dns", 53, listener);
-	if not handler then
-		return nil, err;
+	do
+		local err;
+		handler, err = server.wrapclient(sock, "dns", 53, listener);
+		if not handler then
+			return nil, err;
+		end
 	end
 
 	handler.settimeout = function () end
@@ -71,11 +73,11 @@ function async_resolver_methods:lookup(handler, qname, qtype, qclass)
 					handler(peek);
 					return;
 				end
-				log("debug", "Records for %s not in cache, sending query (%s)...", qname, tostring(coroutine.running()));
+				log("debug", "Records for %s not in cache, sending query (%s)...", qname, coroutine.running());
 				local ok, err = resolver:query(qname, qtype, qclass);
 				if ok then
 					coroutine.yield(setmetatable({ resolver, qclass or "IN", qtype or "A", qname, coroutine.running()}, query_mt)); -- Wait for reply
-					log("debug", "Reply for %s (%s)", qname, tostring(coroutine.running()));
+					log("debug", "Reply for %s (%s)", qname, coroutine.running());
 				end
 				if ok then
 					ok, err = pcall(handler, resolver:peek(qname, qtype, qclass));
@@ -84,13 +86,13 @@ function async_resolver_methods:lookup(handler, qname, qtype, qclass)
 					ok, err = pcall(handler, nil, err);
 				end
 				if not ok then
-					log("error", "Error in DNS response handler: %s", tostring(err));
+					log("error", "Error in DNS response handler: %s", err);
 				end
 			end)(resolver:peek(qname, qtype, qclass));
 end
 
-function query_methods:cancel(call_handler, reason)
-	log("warn", "Cancelling DNS lookup for %s", tostring(self[4]));
+function query_methods:cancel(call_handler, reason) -- luacheck: ignore 212/reason
+	log("warn", "Cancelling DNS lookup for %s", self[4]);
 	self[1].cancel(self[2], self[3], self[4], self[5], call_handler);
 end
 
diff --git a/net/connect.lua b/net/connect.lua
index b812ffcd..6d399dda 100644
--- a/net/connect.lua
+++ b/net/connect.lua
@@ -2,6 +2,13 @@ local server = require "net.server";
 local log = require "util.logger".init("net.connect");
 local new_id = require "util.id".short;
 
+-- TODO Respect use_ipv4, use_ipv6
+-- TODO #1246 Happy Eyeballs
+-- FIXME RFC 6724
+-- FIXME Error propagation from resolvers doesn't work
+-- FIXME #1428 Reuse DNS resolver object between service and basic resolver
+-- FIXME #1429 Close DNS resolver object when done
+
 local pending_connection_methods = {};
 local pending_connection_mt = {
 	__name = "pending_connection";
@@ -38,7 +45,7 @@ local function attempt_connection(p)
 		p:log("debug", "Next target to try is %s:%d", ip, port);
 		local conn, err = server.addclient(ip, port, pending_connection_listeners, p.options.pattern or "*a", p.options.sslctx, conn_type, extra);
 		if not conn then
-			log("debug", "Connection attempt failed immediately: %s", tostring(err));
+			log("debug", "Connection attempt failed immediately: %s", err);
 			p.last_error = err or "unknown reason";
 			return attempt_connection(p);
 		end
diff --git a/net/connlisteners.lua b/net/connlisteners.lua
deleted file mode 100644
index 9b8f88c3..00000000
--- a/net/connlisteners.lua
+++ /dev/null
@@ -1,18 +0,0 @@
--- COMPAT w/pre-0.9
-local log = require "util.logger".init("net.connlisteners");
-local traceback = debug.traceback;
-
-local _ENV = nil;
--- luacheck: std none
-
-local function fail()
-	log("error", "Attempt to use legacy connlisteners API. For more info see https://prosody.im/doc/developers/network");
-	log("error", "Legacy connlisteners API usage, %s", traceback("", 2));
-end
-
-return {
-	register = fail;
-	get = fail;
-	start = fail;
-	-- epic fail
-};
diff --git a/net/http.lua b/net/http.lua
index fe5250ac..8ca30db2 100644
--- a/net/http.lua
+++ b/net/http.lua
@@ -40,7 +40,7 @@ local listener = { default_port = 80, default_mode = "*a" };
 local function handleerr(err) log("error", "Traceback[http]: %s", traceback(tostring(err), 2)); return err; end
 local function log_if_failed(req, ret, ...)
 	if not ret then
-		log("error", "Request '%s': error in callback: %s", req.id, tostring((...)));
+		log("error", "Request '%s': error in callback: %s", req.id, (...));
 		if not req.suppress_errors then
 			error(...);
 		end
@@ -150,7 +150,7 @@ function listener.onincoming(conn, data)
 	local request = requests[conn];
 
 	if not request then
-		log("warn", "Received response from connection %s with no request attached!", tostring(conn));
+		log("warn", "Received response from connection %s with no request attached!", conn);
 		return;
 	end
 
@@ -260,7 +260,7 @@ local function request(self, u, ex, callback)
 		sslctx = ex and ex.sslctx or self.options and self.options.sslctx;
 	end
 
-	local http_service = basic_resolver.new(host, port_number);
+	local http_service = basic_resolver.new(host, port_number, "tcp", { servername = req.host });
 	connect(http_service, listener, { sslctx = sslctx }, req);
 
 	self.events.fire_event("request", { http = self, request = req, url = u });
@@ -285,7 +285,7 @@ local function new(options)
 end
 
 local default_http = new({
-	sslctx = { mode = "client", protocol = "sslv23", options = { "no_sslv2", "no_sslv3" } };
+	sslctx = { mode = "client", protocol = "sslv23", options = { "no_sslv2", "no_sslv3" }, alpn = "http/1.1" };
 	suppress_errors = true;
 });
 
diff --git a/net/http/codes.lua b/net/http/codes.lua
index 8098b5c3..4327f151 100644
--- a/net/http/codes.lua
+++ b/net/http/codes.lua
@@ -82,5 +82,5 @@ local response_codes = {
 	-- [512-599] = "Unassigned";
 };
 
-for k,v in pairs(response_codes) do response_codes[k] = k.." "..v; end
+for k,v in pairs(response_codes) do response_codes[k] = ("%03d %s"):format(k, v); end
 return setmetatable(response_codes, { __index = function(_, k) return k.." Unassigned"; end })
diff --git a/net/http/files.lua b/net/http/files.lua
new file mode 100644
index 00000000..650c6f47
--- /dev/null
+++ b/net/http/files.lua
@@ -0,0 +1,149 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local server = require"net.http.server";
+local lfs = require "lfs";
+local new_cache = require "util.cache".new;
+local log = require "util.logger".init("net.http.files");
+
+local os_date = os.date;
+local open = io.open;
+local stat = lfs.attributes;
+local build_path = require"socket.url".build_path;
+local path_sep = package.config:sub(1,1);
+
+
+local forbidden_chars_pattern = "[/%z]";
+if package.config:sub(1,1) == "\\" then
+	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+end
+
+local urldecode = require "util.http".urldecode;
+local function sanitize_path(path) --> util.paths or util.http?
+	if not path then return end
+	local out = {};
+
+	local c = 0;
+	for component in path:gmatch("([^/]+)") do
+		component = urldecode(component);
+		if component:find(forbidden_chars_pattern) then
+			return nil;
+		elseif component == ".." then
+			if c <= 0 then
+				return nil;
+			end
+			out[c] = nil;
+			c = c - 1;
+		elseif component ~= "." then
+			c = c + 1;
+			out[c] = component;
+		end
+	end
+	if path:sub(-1,-1) == "/" then
+		out[c+1] = "";
+	end
+	return "/"..table.concat(out, "/");
+end
+
+local function serve(opts)
+	if type(opts) ~= "table" then -- assume path string
+		opts = { path = opts };
+	end
+	local mime_map = opts.mime_map or { html = "text/html" };
+	local cache = new_cache(opts.cache_size or 256);
+	local cache_max_file_size = tonumber(opts.cache_max_file_size) or 1024
+	-- luacheck: ignore 431
+	local base_path = opts.path;
+	local dir_indices = opts.index_files or { "index.html", "index.htm" };
+	local directory_index = opts.directory_index;
+	local function serve_file(event, path)
+		local request, response = event.request, event.response;
+		local sanitized_path = sanitize_path(path);
+		if path and not sanitized_path then
+			return 400;
+		end
+		path = sanitized_path;
+		local orig_path = sanitize_path(request.path);
+		local full_path = base_path .. (path or ""):gsub("/", path_sep);
+		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
+		if not attr then
+			return 404;
+		end
+
+		local request_headers, response_headers = request.headers, response.headers;
+
+		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
+		response_headers.last_modified = last_modified;
+
+		local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0);
+		response_headers.etag = etag;
+
+		local if_none_match = request_headers.if_none_match
+		local if_modified_since = request_headers.if_modified_since;
+		if etag == if_none_match
+		or (not if_none_match and last_modified == if_modified_since) then
+			return 304;
+		end
+
+		local data = cache:get(orig_path);
+		if data and data.etag == etag then
+			response_headers.content_type = data.content_type;
+			data = data.data;
+			cache:set(orig_path, data);
+		elseif attr.mode == "directory" and path then
+			if full_path:sub(-1) ~= "/" then
+				local dir_path = { is_absolute = true, is_directory = true };
+				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
+				response_headers.location = build_path(dir_path);
+				return 301;
+			end
+			for i=1,#dir_indices do
+				if stat(full_path..dir_indices[i], "mode") == "file" then
+					return serve_file(event, path..dir_indices[i]);
+				end
+			end
+
+			if directory_index then
+				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
+			end
+			if not data then
+				return 403;
+			end
+			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
+			response_headers.content_type = mime_map.html;
+
+		else
+			local f, err = open(full_path, "rb");
+			if not f then
+				log("debug", "Could not open %s. Error was %s", full_path, err);
+				return 403;
+			end
+			local ext = full_path:match("%.([^./]+)$");
+			local content_type = ext and mime_map[ext];
+			response_headers.content_type = content_type;
+			if attr.size > cache_max_file_size then
+				response_headers.content_length = ("%d"):format(attr.size);
+				log("debug", "%d > cache_max_file_size", attr.size);
+				return response:send_file(f);
+			else
+				data = f:read("*a");
+				f:close();
+			end
+			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
+		end
+
+		return response:send(data);
+	end
+
+	return serve_file;
+end
+
+return {
+	serve = serve;
+}
+
diff --git a/net/http/parser.lua b/net/http/parser.lua
index 4e4ae9fb..62c09aef 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -63,7 +63,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 					if buftable then buf, buftable = t_concat(buf), false; end
 					local index = buf:find("\r\n\r\n", nil, true);
 					if not index then return; end -- not enough data
-					local method, path, httpversion, status_code, reason_phrase;
+					-- FIXME was reason_phrase meant to be passed on somewhere?
+					local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase
 					local first_line;
 					local headers = {};
 					for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request
diff --git a/net/http/server.lua b/net/http/server.lua
index 9b63d516..f4f67d18 100644
--- a/net/http/server.lua
+++ b/net/http/server.lua
@@ -13,6 +13,8 @@ local traceback = debug.traceback;
 local tostring = tostring;
 local cache = require "util.cache";
 local codes = require "net.http.codes";
+local promise = require "util.promise";
+local errors = require "util.error";
 local blocksize = 2^16;
 
 local _M = {};
@@ -170,6 +172,47 @@ local headerfix = setmetatable({}, {
 	end
 });
 
+local function handle_result(request, response, result)
+	if result == nil then
+		result = 404;
+	end
+
+	if result == true then
+		return;
+	end
+
+	local body;
+	local result_type = type(result);
+	if result_type == "number" then
+		response.status_code = result;
+		if result >= 400 then
+			body = events.fire_event("http-error", { request = request, response = response, code = result });
+		end
+	elseif result_type == "string" then
+		body = result;
+	elseif errors.is_err(result) then
+		body = events.fire_event("http-error", { request = request, response = response, code = result.code or 500, error = result });
+	elseif promise.is_promise(result) then
+		result:next(function (ret)
+			handle_result(request, response, ret);
+		end, function (err)
+			handle_result(request, response, err or 500);
+		end);
+		return true;
+	elseif result_type == "table" then
+		for k, v in pairs(result) do
+			if k ~= "headers" then
+				response[k] = v;
+			else
+				for header_name, header_value in pairs(v) do
+					response.headers[header_name] = header_value;
+				end
+			end
+		end
+	end
+	return response:send(body);
+end
+
 function _M.hijack_response(response, listener) -- luacheck: ignore
 	error("TODO");
 end
@@ -194,8 +237,11 @@ function handle_request(conn, request, finish_cb)
 		response_conn_header = httpversion == "1.1" and "close" or nil
 	end
 
+	local is_head_request = request.method == "HEAD";
+
 	local response = {
 		request = request;
+		is_head_request = is_head_request;
 		status_code = 200;
 		headers = { date = date_header, connection = response_conn_header };
 		persistent = persistent;
@@ -226,6 +272,11 @@ function handle_request(conn, request, finish_cb)
 	local payload = { request = request, response = response };
 	log("debug", "Firing event: %s", global_event);
 	local result = events.fire_event(global_event, payload);
+	if result == nil and is_head_request then
+		local global_head_event = "GET "..request.path:match("[^?]*");
+		log("debug", "Firing event: %s", global_head_event);
+		result = events.fire_event(global_head_event, payload);
+	end
 	if result == nil then
 		if not hosts[host] then
 			if hosts[default_host] then
@@ -246,40 +297,17 @@ function handle_request(conn, request, finish_cb)
 		local host_event = request.method.." "..host..request.path:match("[^?]*");
 		log("debug", "Firing event: %s", host_event);
 		result = events.fire_event(host_event, payload);
-	end
-	if result ~= nil then
-		if result ~= true then
-			local body;
-			local result_type = type(result);
-			if result_type == "number" then
-				response.status_code = result;
-				if result >= 400 then
-					payload.code = result;
-					body = events.fire_event("http-error", payload);
-				end
-			elseif result_type == "string" then
-				body = result;
-			elseif result_type == "table" then
-				for k, v in pairs(result) do
-					if k ~= "headers" then
-						response[k] = v;
-					else
-						for header_name, header_value in pairs(v) do
-							response.headers[header_name] = header_value;
-						end
-					end
-				end
-			end
-			response:send(body);
+
+		if result == nil and is_head_request then
+			local host_head_event = "GET "..host..request.path:match("[^?]*");
+			log("debug", "Firing event: %s", host_head_event);
+			result = events.fire_event(host_head_event, payload);
 		end
-		return;
 	end
 
-	-- if handler not called, return 404
-	response.status_code = 404;
-	payload.code = 404;
-	response:send(events.fire_event("http-error", payload));
+	return handle_result(request, response, result);
 end
+
 local function prepare_header(response)
 	local status_line = "HTTP/"..response.request.httpversion.." "..(response.status or codes[response.status_code]);
 	local headers = response.headers;
@@ -291,16 +319,29 @@ local function prepare_header(response)
 	return output;
 end
 _M.prepare_header = prepare_header;
+function _M.send_head_response(response)
+	if response.finished then return; end
+	local output = prepare_header(response);
+	response.conn:write(t_concat(output));
+	response:done();
+end
 function _M.send_response(response, body)
 	if response.finished then return; end
 	body = body or response.body or "";
-	response.headers.content_length = #body;
+	response.headers.content_length = ("%d"):format(#body);
+	if response.is_head_request then
+		return _M.send_head_response(response)
+	end
 	local output = prepare_header(response);
 	t_insert(output, body);
 	response.conn:write(t_concat(output));
 	response:done();
 end
 function _M.send_file(response, f)
+	if response.is_head_request then
+		if f.close then f:close(); end
+		return _M.send_head_response(response);
+	end
 	if response.finished then return; end
 	local chunked = not response.headers.content_length;
 	if chunked then response.headers.transfer_encoding = "chunked"; end
diff --git a/net/resolvers/basic.lua b/net/resolvers/basic.lua
index 08c71ef5..41ebda80 100644
--- a/net/resolvers/basic.lua
+++ b/net/resolvers/basic.lua
@@ -2,10 +2,16 @@ local adns = require "net.adns";
 local inet_pton = require "util.net".pton;
 local inet_ntop = require "util.net".ntop;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local methods = {};
 local resolver_mt = { __index = methods };
 
+-- TODO Respect use_ipv4, use_ipv6
+-- FIXME RFC 6724
+-- FIXME #1428 Reuse DNS resolver object (from service resolver)
+-- FIXME #1429 Close DNS resolver object when done
+
 -- Find the next target to connect to, and
 -- pass it to cb()
 function methods:next(cb)
diff --git a/net/resolvers/manual.lua b/net/resolvers/manual.lua
index c0d4e5d5..dbc40256 100644
--- a/net/resolvers/manual.lua
+++ b/net/resolvers/manual.lua
@@ -1,5 +1,6 @@
 local methods = {};
 local resolver_mt = { __index = methods };
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 -- Find the next target to connect to, and
 -- pass it to cb()
diff --git a/net/resolvers/service.lua b/net/resolvers/service.lua
index 34f14cba..b4300d08 100644
--- a/net/resolvers/service.lua
+++ b/net/resolvers/service.lua
@@ -1,6 +1,11 @@
 local adns = require "net.adns";
 local basic = require "net.resolvers.basic";
+local inet_pton = require "util.net".pton;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
+
+-- FIXME #1428 Reuse DNS resolver object (pass to basic resorver)
+-- FIXME #1429 Close DNS resolver object when done
 
 local methods = {};
 local resolver_mt = { __index = methods };
@@ -39,7 +44,11 @@ function methods:next(cb)
 
 	-- Resolve DNS to target list
 	local dns_resolver = adns.resolver();
-	dns_resolver:lookup(function (answer)
+	dns_resolver:lookup(function (answer, err)
+		if not answer and not err then
+			-- net.adns returns nil if there are zero records or nxdomain
+			answer = {};
+		end
 		if answer then
 			if #answer == 0 then
 				if self.extra and self.extra.default_port then
@@ -64,6 +73,14 @@ function methods:next(cb)
 end
 
 local function new(hostname, service, conn_type, extra)
+	local is_ip = inet_pton(hostname);
+	if not is_ip and hostname:sub(1,1) == '[' then
+		is_ip = inet_pton(hostname:sub(2,-2));
+	end
+	if is_ip and extra and extra.default_port then
+		return basic.new(hostname, extra.default_port, conn_type, extra);
+	end
+
 	return setmetatable({
 		hostname = idna_to_ascii(hostname);
 		service = service;
diff --git a/net/server_epoll.lua b/net/server_epoll.lua
index 0c03ae15..e2c5f977 100644
--- a/net/server_epoll.lua
+++ b/net/server_epoll.lua
@@ -9,20 +9,22 @@
 local t_insert = table.insert;
 local t_concat = table.concat;
 local setmetatable = setmetatable;
-local tostring = tostring;
 local pcall = pcall;
 local type = type;
 local next = next;
 local pairs = pairs;
-local log = require "util.logger".init("server_epoll");
+local logger = require "util.logger";
+local log = logger.init("server_epoll");
 local socket = require "socket";
 local luasec = require "ssl";
-local gettime = require "util.time".now;
+local realtime = require "util.time".now;
+local monotonic = require "util.time".monotonic;
 local indexedbheap = require "util.indexedbheap";
 local createtable = require "util.table".create;
 local inet = require "util.net";
 local inet_pton = inet.pton;
 local _SOCKETINVALID = socket._SOCKETINVALID or -1;
+local new_id = require "util.id".medium;
 
 local poller = require "util.poll"
 local EEXIST = poller.EEXIST;
@@ -38,7 +40,10 @@ local default_config = { __index = {
 	read_timeout = 14 * 60;
 
 	-- How long to wait for a socket to become writable after queuing data to send
-	send_timeout = 60;
+	send_timeout = 180;
+
+	-- How long to wait for a socket to become writable after creation
+	connect_timeout = 20;
 
 	-- Some number possibly influencing how many pending connections can be accepted
 	tcp_backlog = 128;
@@ -58,6 +63,13 @@ local default_config = { __index = {
 	-- Maximum and minimum amount of time to sleep waiting for events (adjusted for pending timers)
 	max_wait = 86400;
 	min_wait = 1e-06;
+
+	-- EXPERIMENTAL
+	-- Whether to kill connections in case of callback errors.
+	fatal_errors = false;
+
+	-- Attempt writes instantly
+	opportunistic_writes = false;
 }};
 local cfg = default_config.__index;
 
@@ -75,45 +87,43 @@ local function closetimer(t)
 end
 
 local function reschedule(t, time)
+	time = monotonic() + time;
 	t[1] = time;
 	timers:reprioritize(t.id, time);
 end
 
--- Add absolute timer
-local function at(time, f)
+-- Add relative timer
+local function addtimer(timeout, f)
+	local time = monotonic() + timeout;
 	local timer = { time, f, close = closetimer, reschedule = reschedule, id = nil };
 	timer.id = timers:insert(timer, time);
 	return timer;
 end
 
--- Add relative timer
-local function addtimer(timeout, f)
-	return at(gettime() + timeout, f);
-end
-
 -- Run callbacks of expired timers
 -- Return time until next timeout
 local function runtimers(next_delay, min_wait)
 	-- Any timers at all?
-	local now = gettime();
+	local elapsed = monotonic();
+	local now = realtime();
 	local peek = timers:peek();
 	while peek do
 
-		if peek > now then
-			next_delay = peek - now;
+		if peek > elapsed then
+			next_delay = peek - elapsed;
 			break;
-	end
+		end
 
-		local _, timer, id = timers:pop();
+		local _, timer = timers:pop();
 		local ok, ret = pcall(timer[2], now);
 		if ok and type(ret) == "number"  then
-			local next_time = now+ret;
+			local next_time = elapsed+ret;
 			timer[1] = next_time;
 			timers:insert(timer, next_time);
-	end
+		end
 
 		peek = timers:peek();
-			end
+	end
 	if peek == nil then
 		return next_delay;
 	end
@@ -138,6 +148,15 @@ function interface_mt:__tostring()
 	return ("FD %d"):format(self:getfd());
 end
 
+interface.log = log;
+function interface:debug(msg, ...) --luacheck: ignore 212/self
+	self.log("debug", msg, ...);
+end
+
+function interface:error(msg, ...) --luacheck: ignore 212/self
+	self.log("error", msg, ...);
+end
+
 -- Replace the listener and tell the old one
 function interface:setlistener(listeners, data)
 	self:on("detach");
@@ -148,21 +167,32 @@ end
 -- Call a listener callback
 function interface:on(what, ...)
 	if not self.listeners then
-		log("error", "%s has no listeners", self);
+		self:debug("Interface is missing listener callbacks");
 		return;
 	end
 	local listener = self.listeners["on"..what];
 	if not listener then
-		-- log("debug", "Missing listener 'on%s'", what); -- uncomment for development and debugging
+		-- self:debug("Missing listener 'on%s'", what); -- uncomment for development and debugging
 		return;
 	end
 	local ok, err = pcall(listener, self, ...);
 	if not ok then
-		log("error", "Error calling on%s: %s", what, err);
+		if cfg.fatal_errors then
+			self:debug("Closing due to error calling on%s: %s", what, err);
+			self:destroy();
+		else
+			self:debug("Error calling on%s: %s", what, err);
+		end
+		return nil, err;
 	end
 	return err;
 end
 
+-- Allow this one to be overridden
+function interface:onincoming(...)
+	return self:on("incoming", ...);
+end
+
 -- Return the file descriptor number
 function interface:getfd()
 	if self.conn then
@@ -226,12 +256,14 @@ function interface:setreadtimeout(t)
 	end
 	t = t or cfg.read_timeout;
 	if self._readtimeout then
-		self._readtimeout:reschedule(gettime() + t);
+		self._readtimeout:reschedule(t);
 	else
 		self._readtimeout = addtimer(t, function ()
 			if self:on("readtimeout") then
+				self:debug("Read timeout handled");
 				return cfg.read_timeout;
 			else
+				self:debug("Read timeout not handled, disconnecting");
 				self:on("disconnect", "read timeout");
 				self:destroy();
 			end
@@ -250,9 +282,10 @@ function interface:setwritetimeout(t)
 	end
 	t = t or cfg.send_timeout;
 	if self._writetimeout then
-		self._writetimeout:reschedule(gettime() + t);
+		self._writetimeout:reschedule(t);
 	else
 		self._writetimeout = addtimer(t, function ()
+			self:debug("Write timeout");
 			self:on("disconnect", "write timeout");
 			self:destroy();
 		end);
@@ -269,15 +302,15 @@ function interface:add(r, w)
 	local ok, err, errno = poll:add(fd, r, w);
 	if not ok then
 		if errno == EEXIST then
-			log("debug", "%s already registered!", self);
+			self:debug("FD already registered in poller! (EEXIST)");
 			return self:set(r, w); -- So try to change its flags
 		end
-		log("error", "Could not register %s: %s(%d)", self, err, errno);
+		self:debug("Could not register in poller: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = r, w;
 	fds[fd] = self;
-	log("debug", "Watching %s", self);
+	self:debug("Registered in poller");
 	return true;
 end
 
@@ -290,7 +323,7 @@ function interface:set(r, w)
 	if w == nil then w = self._wantwrite; end
 	local ok, err, errno = poll:set(fd, r, w);
 	if not ok then
-		log("error", "Could not update poller state %s: %s(%d)", self, err, errno);
+		self:debug("Could not update poller state: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = r, w;
@@ -307,12 +340,12 @@ function interface:del()
 	end
 	local ok, err, errno = poll:del(fd);
 	if not ok and errno ~= ENOENT then
-		log("error", "Could not unregister %s: %s(%d)", self, err, errno);
+		self:debug("Could not unregister: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = nil, nil;
 	fds[fd] = nil;
-	log("debug", "Unwatched %s", self);
+	self:debug("Unregistered from poller");
 	return true;
 end
 
@@ -334,7 +367,7 @@ function interface:onreadable()
 	local data, err, partial = self.conn:receive(self.read_size or cfg.read_size);
 	if data then
 		self:onconnect();
-		self:on("incoming", data);
+		self:onincoming(data);
 	else
 		if err == "wantread" then
 			self:set(true, nil);
@@ -345,7 +378,7 @@ function interface:onreadable()
 		end
 		if partial and partial ~= "" then
 			self:onconnect();
-			self:on("incoming", partial, err);
+			self:onincoming(partial, err);
 		end
 		if err ~= "timeout" then
 			self:on("disconnect", err);
@@ -354,6 +387,14 @@ function interface:onreadable()
 		end
 	end
 	if not self.conn then return; end
+	if self._limit and (data or partial) then
+		local cost = self._limit * #(data or partial);
+		if cost > cfg.min_wait then
+			self:setreadtimeout(false);
+			self:pausefor(cost);
+			return;
+		end
+	end
 	if self._wantread and self.conn:dirty() then
 		self:setreadtimeout(false);
 		self:pausefor(cfg.read_retry_delay);
@@ -378,10 +419,12 @@ function interface:onwritable()
 		self:ondrain(); -- Be aware of writes in ondrain
 		return;
 	elseif partial then
+		self:debug("Sent %d out of %d buffered bytes", partial, #data);
 		buffer[1] = data:sub(partial+1);
 		for i = #buffer, 2, -1 do
 			buffer[i] = nil;
 		end
+		self:set(nil, true);
 		self:setwritetimeout();
 	end
 	if err == "wantwrite" or err == "timeout" then
@@ -407,8 +450,14 @@ function interface:write(data)
 	else
 		self.writebuffer = { data };
 	end
-	self:setwritetimeout();
-	self:set(nil, true);
+	if not self._write_lock then
+		if cfg.opportunistic_writes then
+			self:onwritable();
+			return #data;
+		end
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
 	return #data;
 end
 interface.send = interface.write;
@@ -418,10 +467,10 @@ function interface:close()
 	if self.writebuffer and self.writebuffer[1] then
 		self:set(false, true); -- Flush final buffer contents
 		self.write, self.send = noop, noop; -- No more writing
-		log("debug", "Close %s after writing", self);
+		self:debug("Close after writing remaining buffered data");
 		self.ondrain = interface.close;
 	else
-		log("debug", "Close %s now", self);
+		self:debug("Closing now");
 		self.write, self.send = noop, noop;
 		self.close = noop;
 		self:on("disconnect");
@@ -450,7 +499,7 @@ function interface:starttls(tls_ctx)
 	if tls_ctx then self.tls_ctx = tls_ctx; end
 	self.starttls = false;
 	if self.writebuffer and self.writebuffer[1] then
-		log("debug", "Start TLS on %s after write", self);
+		self:debug("Start TLS after write");
 		self.ondrain = interface.starttls;
 		self:set(nil, true); -- make sure wantwrite is set
 	else
@@ -460,7 +509,7 @@ function interface:starttls(tls_ctx)
 		self.onwritable = interface.tlshandskake;
 		self.onreadable = interface.tlshandskake;
 		self:set(true, true);
-		log("debug", "Prepare to start TLS on %s", self);
+		self:debug("Prepared to start TLS");
 	end
 end
 
@@ -469,12 +518,13 @@ function interface:tlshandskake()
 	self:setreadtimeout(false);
 	if not self._tls then
 		self._tls = true;
-		log("debug", "Start TLS on %s now", self);
+		self:debug("Starting TLS now");
 		self:del();
+		self:updatenames(); -- Can't getpeer/sockname after wrap()
 		local ok, conn, err = pcall(luasec.wrap, self.conn, self.tls_ctx);
 		if not ok then
 			conn, err = ok, conn;
-			log("error", "Failed to initialize TLS: %s", err);
+			self:debug("Failed to initialize TLS: %s", err);
 		end
 		if not conn then
 			self:on("disconnect", err);
@@ -483,6 +533,13 @@ function interface:tlshandskake()
 		end
 		conn:settimeout(0);
 		self.conn = conn;
+		if conn.sni then
+			if self.servername then
+				conn:sni(self.servername);
+			elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+				conn:sni(self._server.hosts, true);
+			end
+		end
 		self:on("starttls");
 		self.ondrain = nil;
 		self.onwritable = interface.tlshandskake;
@@ -491,40 +548,55 @@ function interface:tlshandskake()
 	end
 	local ok, err = self.conn:dohandshake();
 	if ok then
-		log("debug", "TLS handshake on %s complete", self);
+		local info = self.conn.info and self.conn:info();
+		if type(info) == "table" then
+			self:debug("TLS handshake complete (%s with %s)", info.protocol, info.cipher);
+		else
+			self:debug("TLS handshake complete");
+		end
 		self.onwritable = nil;
 		self.onreadable = nil;
 		self:on("status", "ssl-handshake-complete");
 		self:setwritetimeout();
 		self:set(true, true);
 	elseif err == "wantread" then
-		log("debug", "TLS handshake on %s to wait until readable", self);
+		self:debug("TLS handshake to wait until readable");
 		self:set(true, false);
 		self:setreadtimeout(cfg.ssl_handshake_timeout);
 	elseif err == "wantwrite" then
-		log("debug", "TLS handshake on %s to wait until writable", self);
+		self:debug("TLS handshake to wait until writable");
 		self:set(false, true);
 		self:setwritetimeout(cfg.ssl_handshake_timeout);
 	else
-		log("debug", "TLS handshake error on %s: %s", self, err);
+		self:debug("TLS handshake error: %s", err);
 		self:on("disconnect", err);
 		self:destroy();
 	end
 end
 
-local function wrapsocket(client, server, read_size, listeners, tls_ctx) -- luasocket object -> interface object
+local function wrapsocket(client, server, read_size, listeners, tls_ctx, extra) -- luasocket object -> interface object
 	client:settimeout(0);
+	local conn_id = ("conn%s"):format(new_id());
 	local conn = setmetatable({
 		conn = client;
 		_server = server;
-		created = gettime();
+		created = realtime();
 		listeners = listeners;
 		read_size = read_size or (server and server.read_size);
 		writebuffer = {};
 		tls_ctx = tls_ctx or (server and server.tls_ctx);
 		tls_direct = server and server.tls_direct;
+		id = conn_id;
+		log = logger.init(conn_id);
+		extra = extra;
 	}, interface_mt);
 
+	if extra then
+		if extra.servername then
+			conn.servername = extra.servername;
+		end
+	end
+
 	conn:updatenames();
 	return conn;
 end
@@ -532,11 +604,11 @@ end
 function interface:updatenames()
 	local conn = self.conn;
 	local ok, peername, peerport = pcall(conn.getpeername, conn);
-	if ok then
+	if ok and peername then
 		self.peername, self.peerport = peername, peerport;
 	end
 	local ok, sockname, sockport = pcall(conn.getsockname, conn);
-	if ok then
+	if ok and sockname then
 		self.sockname, self.sockport = sockname, sockport;
 	end
 end
@@ -546,34 +618,39 @@ end
 function interface:onacceptable()
 	local conn, err = self.conn:accept();
 	if not conn then
-		log("debug", "Error accepting new client: %s, server will be paused for %ds", err, cfg.accept_retry_interval);
+		self:debug("Error accepting new client: %s, server will be paused for %ds", err, cfg.accept_retry_interval);
 		self:pausefor(cfg.accept_retry_interval);
 		return;
 	end
 	local client = wrapsocket(conn, self, nil, self.listeners);
-	log("debug", "New connection %s", tostring(client));
+	client:debug("New connection %s on server %s", client, self);
 	client:init();
 	if self.tls_direct then
 		client:starttls(self.tls_ctx);
+	else
+		client:onconnect();
 	end
 end
 
 -- Initialization
 function interface:init()
-	self:setwritetimeout();
+	self:setwritetimeout(cfg.connect_timeout);
 	return self:add(true, true);
 end
 
 function interface:pause()
+	self:debug("Pause reading");
 	return self:set(false);
 end
 
 function interface:resume()
+	self:debug("Resume reading");
 	return self:set(true);
 end
 
 -- Pause connection for some time
 function interface:pausefor(t)
+	self:debug("Pause for %fs", t);
 	if self._pausefor then
 		self._pausefor:close();
 	end
@@ -582,43 +659,85 @@ function interface:pausefor(t)
 	self._pausefor = addtimer(t, function ()
 		self._pausefor = nil;
 		self:set(true);
+		self:debug("Resuming after pause, connection is %s", not self.conn and "missing" or self.conn:dirty() and "dirty" or "clean");
 		if self.conn and self.conn:dirty() then
 			self:onreadable();
 		end
 	end);
 end
 
+function interface:setlimit(Bps)
+	if Bps > 0 then
+		self._limit = 1/Bps;
+	else
+		self._limit = nil;
+	end
+end
+
+function interface:pause_writes()
+	if self._write_lock then
+		return
+	end
+	self:debug("Pause writes");
+	self._write_lock = true;
+	self:setwritetimeout(false);
+	self:set(nil, false);
+end
+
+function interface:resume_writes()
+	if not self._write_lock then
+		return
+	end
+	self:debug("Resume writes");
+	self._write_lock = nil;
+	if self.writebuffer[1] then
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
+end
+
 -- Connected!
 function interface:onconnect()
-	if self.conn and not self.peername and self.conn.getpeername then
-		self.peername, self.peerport = self.conn:getpeername();
-	end
+	self:updatenames();
+	self:debug("Connected (%s)", self);
 	self.onconnect = noop;
 	self:on("connect");
 end
 
-local function addserver(addr, port, listeners, read_size, tls_ctx)
+local function listen(addr, port, listeners, config)
 	local conn, err = socket.bind(addr, port, cfg.tcp_backlog);
 	if not conn then return conn, err; end
 	conn:settimeout(0);
 	local server = setmetatable({
 		conn = conn;
-		created = gettime();
+		created = realtime();
 		listeners = listeners;
-		read_size = read_size;
+		read_size = config and config.read_size;
 		onreadable = interface.onacceptable;
-		tls_ctx = tls_ctx;
-		tls_direct = tls_ctx and true or false;
+		tls_ctx = config and config.tls_ctx;
+		tls_direct = config and config.tls_direct;
+		hosts = config and config.sni_hosts;
 		sockname = addr;
 		sockport = port;
+		log = logger.init(("serv%s"):format(new_id()));
 	}, interface_mt);
+	server:debug("Server %s created", server);
 	server:add(true, false);
 	return server;
 end
 
 -- COMPAT
-local function wrapclient(conn, addr, port, listeners, read_size, tls_ctx)
-	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx);
+local function addserver(addr, port, listeners, read_size, tls_ctx)
+	return listen(addr, port, listeners, {
+		read_size = read_size;
+		tls_ctx = tls_ctx;
+		tls_direct = tls_ctx and true or false;
+	});
+end
+
+-- COMPAT
+local function wrapclient(conn, addr, port, listeners, read_size, tls_ctx, extra)
+	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx, extra);
 	if not client.peername then
 		client.peername, client.peerport = addr, port;
 	end
@@ -631,7 +750,7 @@ local function wrapclient(conn, addr, port, listeners, read_size, tls_ctx)
 end
 
 -- New outgoing TCP connection
-local function addclient(addr, port, listeners, read_size, tls_ctx, typ)
+local function addclient(addr, port, listeners, read_size, tls_ctx, typ, extra)
 	local create;
 	if not typ then
 		local n = inet_pton(addr);
@@ -649,13 +768,19 @@ local function addclient(addr, port, listeners, read_size, tls_ctx, typ)
 		return nil, "invalid socket type";
 	end
 	local conn, err = create();
+	if not conn then return conn, err; end
 	local ok, err = conn:settimeout(0);
 	if not ok then return ok, err; end
 	local ok, err = conn:setpeername(addr, port);
 	if not ok and err ~= "timeout" then return ok, err; end
-	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx)
+	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx, extra)
 	local ok, err = client:init();
+	if not client.peername then
+		-- otherwise not set until connected
+		client.peername, client.peerport = addr, port;
+	end
 	if not ok then return ok, err; end
+	client:debug("Client %s created", client);
 	if tls_ctx then
 		client:starttls(tls_ctx);
 	end
@@ -677,23 +802,23 @@ local function watchfd(fd, onreadable, onwritable)
 		end;
 		-- Otherwise it'll need to be something LuaSocket-compatible
 	end
+	conn.id = new_id();
+	conn.log = logger.init(("fdwatch%s"):format(conn.id));
 	conn:add(onreadable, onwritable);
 	return conn;
 end;
 
 -- Dump all data from one connection into another
-local function link(from, to)
-	from.listeners = setmetatable({
-		onincoming = function (_, data)
-			from:pause();
-			to:write(data);
-		end,
-	}, {__index=from.listeners});
-	to.listeners = setmetatable({
-		ondrain = function ()
-			from:resume();
-		end,
-	}, {__index=to.listeners});
+local function link(from, to, read_size)
+	from:debug("Linking to %s", to.id);
+	function from:onincoming(data)
+		self:pause();
+		to:write(data);
+	end
+	function to:ondrain() -- luacheck: ignore 212/self
+		from:resume();
+	end
+	from:set_mode(read_size);
 	from:set(true, nil);
 	to:set(nil, true);
 end
@@ -752,7 +877,7 @@ return {
 	addserver = addserver;
 	addclient = addclient;
 	add_task = addtimer;
-	at = at;
+	listen = listen;
 	loop = loop;
 	closeall = closeall;
 	setquitting = setquitting;
@@ -766,6 +891,7 @@ return {
 	-- libevent emulation
 	event = { EV_READ = "r", EV_WRITE = "w", EV_READWRITE = "rw", EV_LEAVE = -1 };
 	addevent = function (fd, mode, callback)
+		log("warn", "Using deprecated libevent emulation, please update code to use watchfd API instead");
 		local function onevent(self)
 			local ret = self:callback();
 			if ret == -1 then
@@ -785,6 +911,8 @@ return {
 				fds[fd] = nil;
 			end;
 		}, interface_mt);
+		conn.id = conn:getfd();
+		conn.log = logger.init(("fdwatch%d"):format(conn.id));
 		local ok, err = conn:add(mode == "r" or mode == "rw", mode == "w" or mode == "rw");
 		if not ok then return ok, err; end
 		return conn;
diff --git a/net/server_event.lua b/net/server_event.lua
index 11bd6a29..f7e1f448 100644
--- a/net/server_event.lua
+++ b/net/server_event.lua
@@ -164,6 +164,15 @@ function interface_mt:_start_ssl(call_onconnect) -- old socket will be destroyed
 		debug( "fatal error while ssl wrapping:", err )
 		return false
 	end
+
+	if self.conn.sni then
+		if self.servername then
+			self.conn:sni(self.servername);
+		elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+			self.conn:sni(self._server.hosts, true);
+		end
+	end
+
 	self.conn:settimeout( 0 )  -- set non blocking
 	local handshakecallback = coroutine_wrap(function( event )
 		local _, err
@@ -253,6 +262,7 @@ end
 
 --TODO: Deprecate
 function interface_mt:lock_read(switch)
+	log("warn", ":lock_read is deprecated, use :pasue() and :resume()");
 	if switch then
 		return self:pause();
 	else
@@ -272,6 +282,19 @@ function interface_mt:resume()
 	end
 end
 
+function interface_mt:pause_writes()
+	return self:_lock(self.nointerface, self.noreading, true);
+end
+
+function interface_mt:resume_writes()
+	self:_lock(self.nointerface, self.noreading, false);
+	if self.writecallback and not self.eventwrite then
+		self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT );  -- register callback
+		return true;
+	end
+end
+
+
 function interface_mt:counter(c)
 	if c then
 		self._connections = self._connections + c
@@ -281,7 +304,7 @@ end
 
 -- Public methods
 function interface_mt:write(data)
-	if self.nowriting then return nil, "locked" end
+	if self.nointerface then return nil, "locked"; end
 	--vdebug( "try to send data to client, id/data:", self.id, data )
 	data = tostring( data )
 	local len = #data
@@ -293,7 +316,7 @@ function interface_mt:write(data)
 	end
 	t_insert(self.writebuffer, data) -- new buffer
 	self.writebufferlen = total
-	if not self.eventwrite then  -- register new write event
+	if not self.eventwrite and not self.nowriting  then  -- register new write event
 		--vdebug( "register new write event" )
 		self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT )
 	end
@@ -440,10 +463,6 @@ end
 function interface_mt:ontimeout()
 end
 function interface_mt:onreadtimeout()
-	self.fatalerror = "timeout during receiving"
-	debug( "connection failed:", self.fatalerror )
-	self:_close()
-	self.eventread = nil
 end
 function interface_mt:ondrain()
 end
@@ -456,7 +475,7 @@ end
 
 -- End of client interface methods
 
-local function handleclient( client, ip, port, server, pattern, listener, sslctx )  -- creates an client interface
+local function handleclient( client, ip, port, server, pattern, listener, sslctx, extra )  -- creates an client interface
 	--vdebug("creating client interfacce...")
 	local interface = {
 		type = "client";
@@ -492,6 +511,8 @@ local function handleclient( client, ip, port, server, pattern, listener, sslctx
 		_serverport = (server and server:port() or nil),
 		_sslctx = sslctx; -- parameters
 		_usingssl = false;  -- client is using ssl;
+		extra = extra;
+		servername = extra and extra.servername;
 	}
 	if not has_luasec then interface.starttls = false; end
 	interface.id = tostring(interface):match("%x+$");
@@ -635,7 +656,7 @@ local function handleclient( client, ip, port, server, pattern, listener, sslctx
 	return interface
 end
 
-local function handleserver( server, addr, port, pattern, listener, sslctx )  -- creates an server interface
+local function handleserver( server, addr, port, pattern, listener, sslctx, startssl )  -- creates a server interface
 	debug "creating server interface..."
 	local interface = {
 		_connections = 0;
@@ -651,6 +672,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 
 		_ip = addr, _port = port, _pattern = pattern,
 		_sslctx = sslctx;
+		hosts = {};
 	}
 	interface.id = tostring(interface):match("%x+$");
 	interface.readcallback = function( event )  -- server handler, called on incoming connections
@@ -670,6 +692,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 			end
 		end
 		--vdebug("max connection check ok, accepting...")
+		-- luacheck: ignore 231/err
 		local client, err = server:accept()    -- try to accept; TODO: check err
 		while client do
 			if interface._connections >= cfg.MAX_CONNECTIONS then
@@ -681,7 +704,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 			interface._connections = interface._connections + 1  -- increase connection count
 			local clientinterface = handleclient( client, client_ip, client_port, interface, pattern, listener, sslctx )
 			--vdebug( "client id:", clientinterface, "startssl:", startssl )
-			if has_luasec and sslctx then
+			if has_luasec and startssl then
 				clientinterface:starttls(sslctx, true)
 			else
 				clientinterface:_start_session( true )
@@ -700,9 +723,9 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 	return interface
 end
 
-local function addserver( addr, port, listener, pattern, sslctx, startssl )  -- TODO: check arguments
-	--vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil")
-	if sslctx and not has_luasec then
+local function listen(addr, port, listener, config)
+	config = config or {}
+	if config.sslctx and not has_luasec then
 		debug "fatal error: luasec not found"
 		return nil, "luasec not found"
 	end
@@ -711,19 +734,28 @@ local function addserver( addr, port, listener, pattern, sslctx, startssl )  --
 		debug( "creating server socket on "..addr.." port "..port.." failed:", err )
 		return nil, err
 	end
-	local interface = handleserver( server, addr, port, pattern, listener, sslctx, startssl )  -- new server handler
+	local interface = handleserver( server, addr, port, config.read_size, listener, config.tls_ctx, config.tls_direct)  -- new server handler
 	debug( "new server created with id:", tostring(interface))
 	return interface
 end
 
-local function wrapclient( client, ip, port, listeners, pattern, sslctx )
-	local interface = handleclient( client, ip, port, nil, pattern, listeners, sslctx )
+local function addserver( addr, port, listener, pattern, sslctx )  -- TODO: check arguments
+	--vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil")
+	return listen( addr, port, listener, {
+		read_size = pattern,
+		tls_ctx = sslctx,
+		tls_direct = not not sslctx,
+	});
+end
+
+local function wrapclient( client, ip, port, listeners, pattern, sslctx, extra )
+	local interface = handleclient( client, ip, port, nil, pattern, listeners, sslctx, extra )
 	interface:_start_connection(sslctx)
 	return interface, client
 	--function handleclient( client, ip, port, server, pattern, listener, _, sslctx )  -- creates an client interface
 end
 
-local function addclient( addr, serverport, listener, pattern, sslctx, typ )
+local function addclient( addr, serverport, listener, pattern, sslctx, typ, extra )
 	if sslctx and not has_luasec then
 		debug "need luasec, but not available"
 		return nil, "luasec not found"
@@ -749,8 +781,9 @@ local function addclient( addr, serverport, listener, pattern, sslctx, typ )
 	client:settimeout( 0 )  -- set nonblocking
 	local res, err = client:setpeername( addr, serverport )  -- connect
 	if res or ( err == "timeout" ) then
+		-- luacheck: ignore 211/port
 		local ip, port = client:getsockname( )
-		local interface = wrapclient( client, ip, serverport, listener, pattern, sslctx )
+		local interface = wrapclient( client, ip, serverport, listener, pattern, sslctx, extra )
 		debug( "new connection id:", interface.id )
 		return interface, err
 	else
@@ -876,6 +909,7 @@ return {
 	event_base = base,
 	addevent = newevent,
 	addserver = addserver,
+	listen = listen,
 	addclient = addclient,
 	wrapclient = wrapclient,
 	setquitting = setquitting,
diff --git a/net/server_select.lua b/net/server_select.lua
index 1a40a6d3..9cd3463e 100644
--- a/net/server_select.lua
+++ b/net/server_select.lua
@@ -68,6 +68,7 @@ local idfalse
 local closeall
 local addsocket
 local addserver
+local listen
 local addtimer
 local getserver
 local wrapserver
@@ -123,7 +124,7 @@ local _maxsslhandshake
 
 _server = { } -- key = port, value = table; list of listening servers
 _readlist = { } -- array with sockets to read from
-_sendlist = { } -- arrary with sockets to write to
+_sendlist = { } -- array with sockets to write to
 _timerlist = { } -- array of timer functions
 _socketlist = { } -- key = socket, value = wrapped socket (handlers)
 _readtimes = { } -- key = handler, value = timestamp of last data reading
@@ -149,7 +150,7 @@ _checkinterval = 30 -- interval in secs to check idle clients
 _sendtimeout = 60000 -- allowed send idle time in secs
 _readtimeout = 14 * 60 -- allowed read idle time in secs
 
-local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to detemine whether this is Windows
+local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to determine whether this is Windows
 _maxfd = (is_windows and math.huge) or luasocket._SETSIZE or 1024 -- max fd number, limit to 1024 by default to prevent glibc buffer overflow, but not on Windows
 _maxselectlen = luasocket._SETSIZE or 1024 -- But this still applies on Windows
 
@@ -157,7 +158,7 @@ _maxsslhandshake = 30 -- max handshake round-trips
 
 ----------------------------------// PRIVATE //--
 
-wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- this function wraps a server -- FIXME Make sure FD < _maxfd
+wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx, ssldirect ) -- this function wraps a server -- FIXME Make sure FD < _maxfd
 
 	if socket:getfd() >= _maxfd then
 		out_error("server.lua: Disallowed FD number: "..socket:getfd())
@@ -183,6 +184,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 	handler.sslctx = function( )
 		return sslctx
 	end
+	handler.hosts = {} -- sni
 	handler.remove = function( )
 		connections = connections - 1
 		if handler then
@@ -244,13 +246,13 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 		local client, err = accept( socket )	-- try to accept
 		if client then
 			local ip, clientport = client:getpeername( )
-			local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx ) -- wrap new client socket
+			local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx, ssldirect ) -- wrap new client socket
 			if err then -- error while wrapping ssl socket
 				return false
 			end
 			connections = connections + 1
 			out_put( "server.lua: accepted new client connection from ", tostring(ip), ":", tostring(clientport), " to ", tostring(serverport))
-			if dispatch and not sslctx then -- SSL connections will notify onconnect when handshake completes
+			if dispatch and not ssldirect then -- SSL connections will notify onconnect when handshake completes
 				return dispatch( handler );
 			end
 			return;
@@ -264,7 +266,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 	return handler
 end
 
-wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx ) -- this function wraps a client to a handler object
+wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx, ssldirect, extra ) -- this function wraps a client to a handler object
 
 	if socket:getfd() >= _maxfd then
 		out_error("server.lua: Disallowed FD number: "..socket:getfd()) -- PROTIP: Switch to libevent
@@ -314,6 +316,11 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 
 	local handler = bufferqueue -- saves a table ^_^
 
+	handler.extra = extra
+	if extra then
+		handler.servername = extra.servername
+	end
+
 	handler.dispatch = function( )
 		return dispatch
 	end
@@ -424,9 +431,8 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 		bufferlen = bufferlen + #data
 		if bufferlen > maxsendlen then
 			_closelist[ handler ] = "send buffer exceeded"	 -- cannot close the client at the moment, have to wait to the end of the cycle
-			handler.write = idfalse -- don't write anymore
 			return false
-		elseif socket and not _sendlist[ socket ] then
+		elseif not nosend and socket and not _sendlist[ socket ] then
 			_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
 		end
 		bufferqueuelen = bufferqueuelen + 1
@@ -456,49 +462,55 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 		maxreadlen = readlen or maxreadlen
 		return bufferlen, maxreadlen, maxsendlen
 	end
-	--TODO: Deprecate
 	handler.lock_read = function (self, switch)
+		out_error( "server.lua, lock_read() is deprecated, use pause() and resume()" )
 		if switch == true then
-			local tmp = _readlistlen
-			_readlistlen = removesocket( _readlist, socket, _readlistlen )
-			_readtimes[ handler ] = nil
-			if _readlistlen ~= tmp then
-				noread = true
-			end
+			return self:pause()
 		elseif switch == false then
-			if noread then
-				noread = false
-				_readlistlen = addsocket(_readlist, socket, _readlistlen)
-				_readtimes[ handler ] = _currenttime
-			end
+			return self:resume()
 		end
 		return noread
 	end
 	handler.pause = function (self)
-		return self:lock_read(true);
+		local tmp = _readlistlen
+		_readlistlen = removesocket( _readlist, socket, _readlistlen )
+		_readtimes[ handler ] = nil
+		if _readlistlen ~= tmp then
+			noread = true
+		end
+		return noread;
 	end
 	handler.resume = function (self)
-		return self:lock_read(false);
+		if noread then
+			noread = false
+			_readlistlen = addsocket(_readlist, socket, _readlistlen)
+			_readtimes[ handler ] = _currenttime
+		end
+		return noread;
 	end
 	handler.lock = function( self, switch )
-		handler.lock_read (switch)
+		out_error( "server.lua, lock() is deprecated" )
+		handler.lock_read (self, switch)
 		if switch == true then
-			handler.write = idfalse
-			local tmp = _sendlistlen
-			_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
-			_writetimes[ handler ] = nil
-			if _sendlistlen ~= tmp then
-				nosend = true
-			end
+			handler.pause_writes (self)
 		elseif switch == false then
-			handler.write = write
-			if nosend then
-				nosend = false
-				write( "" )
-			end
+			handler.resume_writes (self)
 		end
 		return noread, nosend
 	end
+	handler.pause_writes = function (self)
+		local tmp = _sendlistlen
+		_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
+		_writetimes[ handler ] = nil
+		nosend = true
+	end
+	handler.resume_writes = function (self)
+		nosend = false
+		if bufferlen > 0 then
+			_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
+		end
+	end
+
 	local _readbuffer = function( ) -- this function reads data
 		local buffer, err, part = receive( socket, pattern )	-- receive buffer with "pattern"
 		if not err or (err == "wantread" or err == "timeout") then -- received something
@@ -599,7 +611,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 						coroutine_yield( ) -- handshake not finished
 					end
 				end
-				err = "ssl handshake error: " .. ( err or "handshake too long" );
+				err = ( err or "handshake too long" );
 				out_put( "server.lua: ", err );
 				_ = handler and handler:force_close(err)
 				return false, err -- handshake failed
@@ -619,11 +631,20 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 			out_put( "server.lua: attempting to start tls on " .. tostring( socket ) )
 			local oldsocket, err = socket
 			socket, err = ssl_wrap( socket, sslctx )	-- wrap socket
+
 			if not socket then
 				out_put( "server.lua: error while starting tls on client: ", tostring(err or "unknown error") )
 				return nil, err -- fatal error
 			end
 
+			if socket.sni then
+				if self.servername then
+					socket:sni(self.servername);
+				elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+					socket:sni(self.server().hosts, true);
+				end
+			end
+
 			socket:settimeout( 0 )
 
 			-- add the new socket to our system
@@ -659,7 +680,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 	_socketlist[ socket ] = handler
 	_readlistlen = addsocket(_readlist, socket, _readlistlen)
 
-	if sslctx and has_luasec then
+	if sslctx and ssldirect and has_luasec then
 		out_put "server.lua: auto-starting ssl negotiation..."
 		handler.autostart_ssl = true;
 		local ok, err = handler:starttls(sslctx);
@@ -734,9 +755,13 @@ end
 
 ----------------------------------// PUBLIC //--
 
-addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server
+listen = function ( addr, port, listeners, config )
 	addr = addr or "*"
+	config = config or {}
 	local err
+	local sslctx = config.tls_ctx;
+	local ssldirect = config.tls_direct;
+	local pattern = config.read_size;
 	if type( listeners ) ~= "table" then
 		err = "invalid listener table"
 	elseif type ( addr ) ~= "string" then
@@ -757,7 +782,7 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function
 		out_error( "server.lua, [", addr, "]:", port, ": ", err )
 		return nil, err
 	end
-	local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx ) -- wrap new server socket
+	local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx, ssldirect ) -- wrap new server socket
 	if not handler then
 		server:close( )
 		return nil, err
@@ -770,6 +795,14 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function
 	return handler
 end
 
+addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server
+	return listen(addr, port, listeners, {
+		read_size = pattern;
+		tls_ctx = sslctx;
+		tls_direct = sslctx and true or false;
+	});
+end
+
 getserver = function ( addr, port )
 	return _server[ addr..":"..port ];
 end
@@ -977,8 +1010,8 @@ end
 
 --// EXPERIMENTAL //--
 
-local wrapclient = function( socket, ip, serverport, listeners, pattern, sslctx )
-	local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx )
+local wrapclient = function( socket, ip, serverport, listeners, pattern, sslctx, extra )
+	local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx, sslctx, extra)
 	if not handler then return nil, err end
 	_socketlist[ socket ] = handler
 	if not sslctx then
@@ -997,7 +1030,7 @@ local wrapclient = function( socket, ip, serverport, listeners, pattern, sslctx
 	return handler, socket
 end
 
-local addclient = function( address, port, listeners, pattern, sslctx, typ )
+local addclient = function( address, port, listeners, pattern, sslctx, typ, extra )
 	local err
 	if type( listeners ) ~= "table" then
 		err = "invalid listener table"
@@ -1034,7 +1067,7 @@ local addclient = function( address, port, listeners, pattern, sslctx, typ )
 	client:settimeout( 0 )
 	local ok, err = client:setpeername( address, port )
 	if ok or err == "timeout" or err == "Operation already in progress" then
-		return wrapclient( client, address, port, listeners, pattern, sslctx )
+		return wrapclient( client, address, port, listeners, pattern, sslctx, extra )
 	else
 		return nil, err
 	end
@@ -1114,6 +1147,7 @@ return {
 	stats = stats,
 	closeall = closeall,
 	addserver = addserver,
+	listen = listen,
 	getserver = getserver,
 	setlogger = setlogger,
 	getsettings = getsettings,
diff --git a/net/websocket.lua b/net/websocket.lua
index 469c6a58..193cd556 100644
--- a/net/websocket.lua
+++ b/net/websocket.lua
@@ -23,6 +23,7 @@ local websockets = {};
 local websocket_listeners = {};
 function websocket_listeners.ondisconnect(conn, err)
 	local s = websockets[conn];
+	if not s then return; end
 	websockets[conn] = nil;
 	if s.close_timer then
 		timer.stop(s.close_timer);
@@ -113,7 +114,7 @@ function websocket_listeners.onincoming(conn, buffer, err) -- luacheck: ignore 2
 				frame.MASK = true; -- RFC 6455 6.1.5: If the data is being sent by the client, the frame(s) MUST be masked
 				conn:write(frames.build(frame));
 			elseif frame.opcode == 0xA then -- Pong frame
-				log("debug", "Received unexpected pong frame: " .. tostring(frame.data));
+				log("debug", "Received unexpected pong frame: %s", frame.data);
 			else
 				return fail(s, 1002, "Reserved opcode");
 			end
@@ -131,7 +132,7 @@ end
 function websocket_methods:close(code, reason)
 	if self.readyState < 2 then
 		code = code or 1000;
-		log("debug", "closing WebSocket with code %i: %s" , code , tostring(reason));
+		log("debug", "closing WebSocket with code %i: %s" , code , reason);
 		self.readyState = 2;
 		local conn = self.conn;
 		conn:write(frames.build_close(code, reason, true));
@@ -245,7 +246,7 @@ local function connect(url, ex, listeners)
 		   or (protocol and not protocol[r.headers["sec-websocket-protocol"]])
 		   then
 			s.readyState = 3;
-			log("warn", "WebSocket connection to %s failed: %s", url, tostring(b));
+			log("warn", "WebSocket connection to %s failed: %s", url, b);
 			if s.onerror then s:onerror("connecting-failed"); end
 			return;
 		end
diff --git a/net/websocket/frames.lua b/net/websocket/frames.lua
index ba25d261..5e17df07 100644
--- a/net/websocket/frames.lua
+++ b/net/websocket/frames.lua
@@ -9,20 +9,20 @@
 local softreq = require "util.dependencies".softreq;
 local random_bytes = require "util.random".bytes;
 
-local bit = assert(softreq"bit" or softreq"bit32",
-	"No bit module found. See https://prosody.im/doc/depends#bitop");
+local bit = require "util.bitcompat";
 local band = bit.band;
 local bor = bit.bor;
 local bxor = bit.bxor;
 local lshift = bit.lshift;
 local rshift = bit.rshift;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local t_concat = table.concat;
 local s_byte = string.byte;
 local s_char= string.char;
 local s_sub = string.sub;
-local s_pack = string.pack; -- luacheck: ignore 143
-local s_unpack = string.unpack; -- luacheck: ignore 143
+local s_pack = string.pack;
+local s_unpack = string.unpack;
 
 if not s_pack and softreq"struct" then
 	s_pack = softreq"struct".pack;
diff --git a/plugins/adhoc/mod_adhoc.lua b/plugins/adhoc/mod_adhoc.lua
index bf1775b4..f6553773 100644
--- a/plugins/adhoc/mod_adhoc.lua
+++ b/plugins/adhoc/mod_adhoc.lua
@@ -8,7 +8,7 @@
 local it = require "util.iterators";
 local st = require "util.stanza";
 local is_admin = require "core.usermanager".is_admin;
-local jid_split = require "util.jid".split;
+local jid_host = require "util.jid".host;
 local adhoc_handle_cmd = module:require "adhoc".handle_cmd;
 local xmlns_cmd = "http://jabber.org/protocol/commands";
 local commands = {};
@@ -21,7 +21,7 @@ module:hook("host-disco-info-node", function (event)
 		local from = stanza.attr.from;
 		local privileged = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		local command = commands[node];
 		if (command.permission == "admin" and privileged)
 		    or (command.permission == "global_admin" and global_admin)
@@ -52,7 +52,7 @@ module:hook("host-disco-items-node", function (event)
 	local from = stanza.attr.from;
 	local admin = is_admin(from, stanza.attr.to);
 	local global_admin = is_admin(from);
-	local username, hostname = jid_split(from);
+	local hostname = jid_host(from);
 	for node, command in it.sorted_pairs(commands) do
 		if (command.permission == "admin" and admin)
 		    or (command.permission == "global_admin" and global_admin)
@@ -74,7 +74,7 @@ module:hook("iq-set/host/"..xmlns_cmd..":command", function (event)
 		local from = stanza.attr.from;
 		local admin = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		if (command.permission == "admin" and not admin)
 		    or (command.permission == "global_admin" and not global_admin)
 		    or (command.permission == "local_user" and hostname ~= module.host) then
diff --git a/plugins/mod_admin_adhoc.lua b/plugins/mod_admin_adhoc.lua
index 37e77ab0..674b3339 100644
--- a/plugins/mod_admin_adhoc.lua
+++ b/plugins/mod_admin_adhoc.lua
@@ -59,7 +59,7 @@ local add_user_command_handler = adhoc_simple(add_user_layout, function(fields,
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return { status = "completed", error = { message = "Trying to add a user on " .. host .. " but command was sent to " .. module_host}};
 	end
@@ -94,7 +94,7 @@ local change_user_password_command_handler = adhoc_simple(change_user_password_l
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return {
 			status = "completed",
@@ -136,7 +136,7 @@ local delete_user_command_handler = adhoc_simple(delete_user_layout, function(fi
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and usermanager_delete_user(username, host) then
 			module:log("debug", "User %s has been deleted", aJID);
 			succeeded[#succeeded+1] = aJID;
@@ -180,7 +180,7 @@ local end_user_session_handler = adhoc_simple(end_user_session_layout, function(
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and disconnect_user(aJID) then
 			succeeded[#succeeded+1] = aJID;
 		else
@@ -212,7 +212,7 @@ local get_user_password_handler = adhoc_simple(get_user_password_layout, functio
 	if err then
 		return generate_error_message(err);
 	end
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	local accountjid;
 	local password;
 	if host ~= module_host then
@@ -243,7 +243,7 @@ local get_user_roster_handler = adhoc_simple(get_user_roster_layout, function(fi
 		return generate_error_message(err);
 	end
 
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	if host ~= module_host then
 		return { status = "completed", error = { message = "Tried to get roster for a user on " .. host .. " but command was sent to " .. module_host } };
 	elseif not usermanager_user_exists(user, host) then
@@ -392,6 +392,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		flags[#flags+1] = "authenticated";
 	end
+	if session.dialback_key then
+		flags[#flags+1] = "dialback";
+	end
+	if session.external_auth then
+		flags[#flags+1] = "SASL";
+	end
 	if session.secure then
 		flags[#flags+1] = "encrypted";
 	end
@@ -404,6 +410,12 @@ local function session_flags(session, line)
 	if session.ip and session.ip:match(":") then
 		flags[#flags+1] = "IPv6";
 	end
+	if session.incoming and session.outgoing then
+		flags[#flags+1] = "bidi";
+	elseif session.is_bidi or session.bidi_session then
+		flags[#flags+1] = "bidi";
+	end
+
 	line[#line+1] = "("..t_concat(flags, ", ")..")";
 
 	return t_concat(line, " ");
diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua
index 1cbe27a4..7df63da9 100644
--- a/plugins/mod_admin_telnet.lua
+++ b/plugins/mod_admin_telnet.lua
@@ -22,6 +22,7 @@ local prosody = _G.prosody;
 
 local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" };
 
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local iterators = require "util.iterators";
 local keys, values = iterators.keys, iterators.values;
 local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
@@ -30,6 +31,9 @@ local cert_verify_identity = require "util.x509".verify_identity;
 local envload = require "util.envload".envload;
 local envloadfile = require "util.envload".envloadfile;
 local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local serialize = require "util.serialization".new({ fatal = false, unquoted = true});
+local time = require "util.time";
 
 local commands = module:shared("commands")
 local def_env = module:shared("env");
@@ -47,6 +51,24 @@ end
 
 console = {};
 
+local runner_callbacks = {};
+
+function runner_callbacks:ready()
+	self.data.conn:resume();
+end
+
+function runner_callbacks:waiting()
+	self.data.conn:pause();
+end
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[telnet]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+
 function console:new_session(conn)
 	local w = function(s) conn:write(s:gsub("\n", "\r\n")); end;
 	local session = { conn = conn;
@@ -62,6 +84,11 @@ function console:new_session(conn)
 			};
 	session.env = setmetatable({}, default_env_mt);
 
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+		session.send(string.char(0));
+	end, runner_callbacks, session);
+
 	-- Load up environment with helper objects
 	for name, t in pairs(def_env) do
 		if type(t) == "table" then
@@ -91,8 +118,14 @@ function console:process_line(session, line)
 
 	session.env._ = line;
 
+	if not useglobalenv and commands[line:lower()] then
+		commands[line:lower()](session, line);
+		return;
+	end
+
 	local chunkname = "=console";
 	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
+	-- luacheck: ignore 311/err
 	local chunk, err = envload("return "..line, chunkname, env);
 	if not chunk then
 		chunk, err = envload(line, chunkname, env);
@@ -105,18 +138,7 @@ function console:process_line(session, line)
 		end
 	end
 
-	local ranok, taskok, message = pcall(chunk);
-
-	if not (ranok or message or useglobalenv) and commands[line:lower()] then
-		commands[line:lower()](session, line);
-		return;
-	end
-
-	if not ranok then
-		session.print("Fatal error while running command, it did not complete");
-		session.print("Error: "..taskok);
-		return;
-	end
+	local taskok, message = chunk();
 
 	if not message then
 		session.print("Result: "..tostring(taskok));
@@ -150,8 +172,7 @@ function console_listener.onincoming(conn, data)
 
 	for line in data:gmatch("[^\n]*[\n\004]") do
 		if session.closed then return end
-		console:process_line(session, line);
-		session.send(string.char(0));
+		session.thread:run(line);
 	end
 	session.partial_data = data:match("[^\n]+$");
 end
@@ -220,6 +241,7 @@ function commands.help(session, data)
 		print [[server - Uptime, version, shutting down, etc.]]
 		print [[port - Commands to manage ports the server is listening on]]
 		print [[dns - Commands to manage and inspect the internal DNS resolver]]
+		print [[xmpp - Commands for sending XMPP stanzas]]
 		print [[config - Reloading the configuration, etc.]]
 		print [[console - Help regarding the console itself]]
 	elseif section == "c2s" then
@@ -227,7 +249,9 @@ function commands.help(session, data)
 		print [[c2s:show_insecure() - Show all unencrypted client connections]]
 		print [[c2s:show_secure() - Show all encrypted client connections]]
 		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
+		print [[c2s:count() - Count sessions without listing them]]
 		print [[c2s:close(jid) - Close all sessions for the specified JID]]
+		print [[c2s:closeall() - Close all active c2s connections ]]
 	elseif section == "s2s" then
 		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
 		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
@@ -261,8 +285,11 @@ function commands.help(session, data)
 		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
 		print [[dns:purge() - Clear the DNS cache]]
 		print [[dns:cache() - Show cached records]]
+	elseif section == "xmpp" then
+		print [[xmpp:ping(localhost, remotehost) -- Sends a ping to a remote XMPP server and reports the response]]
 	elseif section == "config" then
 		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
+		print [[config:get([host,] option) - Show the value of a config option.]]
 	elseif section == "console" then
 		print [[Hey! Welcome to Prosody's admin console.]]
 		print [[First thing, if you're ever wondering how to get out, simply type 'quit'.]]
@@ -339,7 +366,7 @@ end
 
 def_env.module = {};
 
-local function get_hosts_set(hosts, module)
+local function get_hosts_set(hosts)
 	if type(hosts) == "table" then
 		if hosts[1] then
 			return set.new(hosts);
@@ -349,17 +376,23 @@ local function get_hosts_set(hosts, module)
 	elseif type(hosts) == "string" then
 		return set.new { hosts };
 	elseif hosts == nil then
-		local hosts_set = set.new(array.collect(keys(prosody.hosts)))
-			/ function (host) return (prosody.hosts[host].type == "local" or module and modulemanager.is_loaded(host, module)) and host or nil; end;
-		if module and modulemanager.get_module("*", module) then
-			hosts_set:add("*");
-		end
-		return hosts_set;
+		return set.new(array.collect(keys(prosody.hosts)));
 	end
 end
 
+-- Hosts with a module or all virtualhosts if no module given
+-- matching modules_enabled in the global section
+local function get_hosts_with_module(hosts, module)
+	local hosts_set = get_hosts_set(hosts)
+	/ function (host) return (prosody.hosts[host].type == "local" or module and modulemanager.is_loaded(host, module)) and host or nil; end;
+	if module and modulemanager.get_module("*", module) then
+		hosts_set:add("*");
+	end
+	return hosts_set;
+end
+
 function def_env.module:load(name, hosts, config)
-	hosts = get_hosts_set(hosts);
+	hosts = get_hosts_with_module(hosts);
 
 	-- Load the module for each host
 	local ok, err, count, mod = true, nil, 0;
@@ -386,7 +419,7 @@ function def_env.module:load(name, hosts, config)
 end
 
 function def_env.module:unload(name, hosts)
-	hosts = get_hosts_set(hosts, name);
+	hosts = get_hosts_with_module(hosts, name);
 
 	-- Unload the module for each host
 	local ok, err, count = true, nil, 0;
@@ -408,11 +441,11 @@ end
 local function _sort_hosts(a, b)
 	if a == "*" then return true
 	elseif b == "*" then return false
-	else return a < b; end
+	else return a:gsub("[^.]+", string.reverse):reverse() < b:gsub("[^.]+", string.reverse):reverse(); end
 end
 
 function def_env.module:reload(name, hosts)
-	hosts = array.collect(get_hosts_set(hosts, name)):sort(_sort_hosts)
+	hosts = array.collect(get_hosts_with_module(hosts, name)):sort(_sort_hosts)
 
 	-- Reload the module for each host
 	local ok, err, count = true, nil, 0;
@@ -435,16 +468,7 @@ function def_env.module:reload(name, hosts)
 end
 
 function def_env.module:list(hosts)
-	if hosts == nil then
-		hosts = array.collect(keys(prosody.hosts));
-		table.insert(hosts, 1, "*");
-	end
-	if type(hosts) == "string" then
-		hosts = { hosts };
-	end
-	if type(hosts) ~= "table" then
-		return false, "Please supply a host or a list of hosts you would like to see";
-	end
+	hosts = array.collect(set.new({ not hosts and "*" or nil }) + get_hosts_set(hosts)):sort(_sort_hosts);
 
 	local print = self.session.print;
 	for _, host in ipairs(hosts) do
@@ -458,7 +482,12 @@ function def_env.module:list(hosts)
 			end
 		else
 			for _, name in ipairs(modules) do
-				print("    "..name);
+				local status, status_text = modulemanager.get_module(host, name).module:get_status();
+				local status_summary = "";
+				if status == "warn" or status == "error" then
+					status_summary = (" (%s: %s)"):format(status, status_text);
+				end
+				print(("    %s%s"):format(name, status_summary));
 			end
 		end
 	end
@@ -474,9 +503,12 @@ function def_env.config:load(filename, format)
 	return true, "Config loaded";
 end
 
-function def_env.config:get(host, section, key)
+function def_env.config:get(host, key)
+	if key == nil then
+		host, key = "*", host;
+	end
 	local config_get = require "core.configmanager".get
-	return true, tostring(config_get(host, section, key));
+	return true, serialize(config_get(host, key));
 end
 
 function def_env.config:reload()
@@ -505,6 +537,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		line[#line+1] = "(authenticated)";
 	end
+	if session.dialback_key then
+		line[#line+1] = "(dialback)";
+	end
+	if session.external_auth then
+		line[#line+1] = "(SASL)";
+	end
 	if session.secure then
 		line[#line+1] = "(encrypted)";
 	end
@@ -520,6 +558,17 @@ local function session_flags(session, line)
 	if session.remote then
 		line[#line+1] = "(remote)";
 	end
+	if session.incoming and session.outgoing then
+		line[#line+1] = "(bidi)";
+	elseif session.is_bidi or session.bidi_session then
+		line[#line+1] = "(bidi)";
+	end
+	if session.bosh_version then
+		line[#line+1] = "(bosh)";
+	end
+	if session.websocket_request then
+		line[#line+1] = "(websocket)";
+	end
 	return table.concat(line, " ");
 end
 
@@ -534,6 +583,18 @@ local function tls_info(session, line)
 		else
 			line[#line+1] = "(cipher info unavailable)";
 		end
+		if sock.getsniname then
+			local name = sock:getsniname();
+			if name then
+				line[#line+1] = ("(SNI:%q)"):format(name);
+			end
+		end
+		if sock.getalpn then
+			local proto = sock:getalpn();
+			if proto then
+				line[#line+1] = ("(ALPN:%q)"):format(proto);
+			end
+		end
 	else
 		line[#line+1] = "(insecure)";
 	end
@@ -555,23 +616,31 @@ local function get_jid(session)
 	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
 end
 
+local function get_c2s()
+	local c2s = array.collect(values(prosody.full_sessions));
+	c2s:append(array.collect(values(module:shared"/*/c2s/sessions")));
+	c2s:append(array.collect(values(module:shared"/*/bosh/sessions")));
+	c2s:unique();
+	return c2s;
+end
+
 local function show_c2s(callback)
-	local c2s = array.collect(values(module:shared"/*/c2s/sessions"));
-	c2s:sort(function(a, b)
+	get_c2s():sort(function(a, b)
 		if a.host == b.host then
 			if a.username == b.username then
 				return (a.resource or "") > (b.resource or "");
 			end
 			return (a.username or "") > (b.username or "");
 		end
-		return (a.host or "") > (b.host or "");
+		return _sort_hosts(a.host or "", b.host or "");
 	end):map(function (session)
 		callback(get_jid(session), session)
 	end);
 end
 
 function def_env.c2s:count()
-	return true, "Total: "..  iterators.count(values(module:shared"/*/c2s/sessions")) .." clients";
+	local c2s = get_c2s();
+	return true, "Total: "..  #c2s .." clients";
 end
 
 function def_env.c2s:show(match_jid, annotate)
@@ -617,17 +686,36 @@ function def_env.c2s:show_tls(match_jid)
 	return self:show(match_jid, tls_info);
 end
 
-function def_env.c2s:close(match_jid)
+local function build_reason(text, condition)
+	if text or condition then
+		return {
+			text = text,
+			condition = condition or "undefined-condition",
+		};
+	end
+end
+
+function def_env.c2s:close(match_jid, text, condition)
 	local count = 0;
 	show_c2s(function (jid, session)
 		if jid == match_jid or jid_bare(jid) == match_jid then
 			count = count + 1;
-			session:close();
+			session:close(build_reason(text, condition));
 		end
 	end);
 	return true, "Total: "..count.." sessions closed";
 end
 
+function def_env.c2s:closeall(text, condition)
+	local count = 0;
+	--luacheck: ignore 212/jid
+	show_c2s(function (jid, session)
+		count = count + 1;
+		session:close(build_reason(text, condition));
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
 
 def_env.s2s = {};
 function def_env.s2s:show(match_jid, annotate)
@@ -695,8 +783,8 @@ function def_env.s2s:show(match_jid, annotate)
 
 	-- Sort by local host, then remote host
 	table.sort(s2s_list, function(a,b)
-		if a.l == b.l then return a.r < b.r; end
-		return a.l < b.l;
+		if a.l == b.l then return _sort_hosts(a.r, b.r); end
+		return _sort_hosts(a.l, b.l);
 	end);
 	local lasthost;
 	for _, sess_lines in ipairs(s2s_list) do
@@ -828,7 +916,7 @@ function def_env.s2s:showcert(domain)
 		.." presented by "..domain..".");
 end
 
-function def_env.s2s:close(from, to)
+function def_env.s2s:close(from, to, text, condition)
 	local print, count = self.session.print, 0;
 	local s2s_sessions = module:shared"/*/s2s/sessions";
 
@@ -842,23 +930,23 @@ function def_env.s2s:close(from, to)
 	end
 
 	for _, session in pairs(s2s_sessions) do
-		local id = session.type..tostring(session):match("[a-f0-9]+$");
+		local id = session.id or (session.type..tostring(session):match("[a-f0-9]+$"));
 		if (match_id and match_id == id)
 		or (session.from_host == from and session.to_host == to) then
 			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
-			(session.close or s2smanager.destroy_session)(session);
+			(session.close or s2smanager.destroy_session)(session, build_reason(text, condition));
 			count = count + 1 ;
 		end
 	end
 	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
 end
 
-function def_env.s2s:closeall(host)
+function def_env.s2s:closeall(host, text, condition)
 	local count = 0;
 	local s2s_sessions = module:shared"/*/s2s/sessions";
 	for _,session in pairs(s2s_sessions) do
 		if not host or session.from_host == host or session.to_host == host then
-			session:close();
+			session:close(build_reason(text, condition));
 			count = count + 1;
 		end
 	end
@@ -879,7 +967,7 @@ function def_env.host:list()
 	local print = self.session.print;
 	local i = 0;
 	local type;
-	for host, host_session in iterators.sorted_pairs(prosody.hosts) do
+	for host, host_session in iterators.sorted_pairs(prosody.hosts, _sort_hosts) do
 		i = i + 1;
 		type = host_session.type;
 		if type == "local" then
@@ -1062,13 +1150,28 @@ end
 def_env.xmpp = {};
 
 local st = require "util.stanza";
-function def_env.xmpp:ping(localhost, remotehost)
-	if prosody.hosts[localhost] then
-		module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" }
-				:tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]);
-		return true, "Sent ping";
+local new_id = require "util.id".medium;
+function def_env.xmpp:ping(localhost, remotehost, timeout)
+	localhost = select(2, jid_split(localhost));
+	remotehost = select(2, jid_split(remotehost));
+	if not localhost then
+		return nil, "Invalid sender hostname";
+	elseif not prosody.hosts[localhost] then
+		return nil, "No such local host";
+	end
+	if not remotehost then
+		return nil, "Invalid destination hostname";
+	elseif prosody.hosts[remotehost] then
+		return nil, "Both hosts are local";
+	end
+	local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()}
+			:tag("ping", {xmlns="urn:xmpp:ping"});
+	local time_start = time.now();
+	local ret, err = async.wait(module:context(localhost):send_iq(iq, nil, timeout));
+	if ret then
+		return true, ("pong from %s in %gs"):format(ret.stanza.attr.from, time.now() - time_start);
 	else
-		return nil, "No such host";
+		return false, tostring(err);
 	end
 end
 
@@ -1157,7 +1260,6 @@ function def_env.debug:events(host, event)
 end
 
 function def_env.debug:timers()
-	local socket = require "socket";
 	local print = self.session.print;
 	local add_task = require"util.timer".add_task;
 	local h, params = add_task.h, add_task.params;
@@ -1185,7 +1287,7 @@ function def_env.debug:timers()
 	if h then
 		local next_time = h:peek();
 		if next_time then
-			return true, os.date("Next event at %F %T (in %%.6fs)", next_time):format(next_time - socket.gettime());
+			return true, os.date("Next event at %F %T (in %%.6fs)", next_time):format(next_time - time.now());
 		end
 	end
 	return true;
@@ -1207,7 +1309,7 @@ local function format_stat(type, value, ref_value)
 	--do return tostring(value) end
 	if type == "duration" then
 		if ref_value < 0.001 then
-			return ("%d µs"):format(value*1000000);
+			return ("%g µs"):format(value*1000000);
 		elseif ref_value < 0.9 then
 			return ("%0.2f ms"):format(value*1000);
 		end
@@ -1326,7 +1428,7 @@ end
 
 function stats_methods:cfgraph()
 	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
 		local function print(s)
 			table.insert(stat_info.output, s);
 		end
@@ -1392,7 +1494,7 @@ end
 
 function stats_methods:histogram()
 	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
 		local function print(s)
 			table.insert(stat_info.output, s);
 		end
@@ -1492,10 +1594,11 @@ local function new_stats_context(self)
 end
 
 function def_env.stats:show(filter)
+	-- luacheck: ignore 211/changed
 	local stats, changed, extra = require "core.statsmanager".get_stats();
 	local available, displayed = 0, 0;
 	local displayed_stats = new_stats_context(self);
-	for name, value in pairs(stats) do
+	for name, value in iterators.sorted_pairs(stats) do
 		available = available + 1;
 		if not filter or name:match(filter) then
 			displayed = displayed + 1;
diff --git a/plugins/mod_announce.lua b/plugins/mod_announce.lua
index 970a273a..c742ebb8 100644
--- a/plugins/mod_announce.lua
+++ b/plugins/mod_announce.lua
@@ -38,6 +38,7 @@ end
 -- Old <message>-based jabberd-style announcement sending
 function handle_announcement(event)
 	local stanza = event.stanza;
+	-- luacheck: ignore 211/node
 	local node, host, resource = jid.split(stanza.attr.to);
 
 	if resource ~= "announce/online" then
diff --git a/plugins/mod_auth_internal_hashed.lua b/plugins/mod_auth_internal_hashed.lua
index 083f648b..c81830de 100644
--- a/plugins/mod_auth_internal_hashed.lua
+++ b/plugins/mod_auth_internal_hashed.lua
@@ -9,7 +9,7 @@
 
 local max = math.max;
 
-local getAuthenticationDatabaseSHA1 = require "util.sasl.scram".getAuthenticationDatabaseSHA1;
+local scram_hashers = require "util.sasl.scram".hashers;
 local usermanager = require "core.usermanager";
 local generate_uuid = require "util.uuid".generate;
 local new_sasl = require "util.sasl".new;
@@ -21,7 +21,9 @@ local host = module.host;
 
 local accounts = module:open_store("accounts");
 
-
+local hash_name = module:get_option_string("password_hash", "SHA-1");
+local get_auth_db = assert(scram_hashers[hash_name], "SCRAM-"..hash_name.." not supported by SASL library");
+local scram_name = "scram_"..hash_name:gsub("%-","_"):lower();
 
 -- Default; can be set per-user
 local default_iteration_count = 4096;
@@ -49,7 +51,7 @@ function provider.test_password(username, password)
 		return nil, "Auth failed. Stored salt and iteration count information is not complete.";
 	end
 
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, credentials.salt, credentials.iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, credentials.salt, credentials.iteration_count);
 
 	local stored_key_hex = to_hex(stored_key);
 	local server_key_hex = to_hex(server_key);
@@ -67,7 +69,7 @@ function provider.set_password(username, password)
 	if account then
 		account.salt = generate_uuid();
 		account.iteration_count = max(account.iteration_count or 0, default_iteration_count);
-		local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, account.salt, account.iteration_count);
+		local valid, stored_key, server_key = get_auth_db(password, account.salt, account.iteration_count);
 		if not valid then
 			return valid, stored_key;
 		end
@@ -101,7 +103,7 @@ function provider.create_user(username, password)
 		return accounts:set(username, {});
 	end
 	local salt = generate_uuid();
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, default_iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, salt, default_iteration_count);
 	if not valid then
 		return valid, stored_key;
 	end
@@ -122,7 +124,7 @@ function provider.get_sasl_handler()
 		plain_test = function(_, username, password, realm)
 			return usermanager.test_password(username, realm, password), true;
 		end,
-		scram_sha_1 = function(_, username)
+		[scram_name] = function(_, username)
 			local credentials = accounts:get(username);
 			if not credentials then return; end
 			if credentials.password then
diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua
index ee48ffad..dad06b62 100644
--- a/plugins/mod_blocklist.lua
+++ b/plugins/mod_blocklist.lua
@@ -67,7 +67,7 @@ local function migrate_privacy_list(username)
 		if item.type == "jid" and item.action == "deny" then
 			local jid = jid_prep(item.value);
 			if not jid then
-				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, tostring(item.value));
+				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, item.value);
 			else
 				migrated_data[jid] = true;
 			end
@@ -162,7 +162,7 @@ local function edit_blocklist(event)
 	local blocklist = cache[username] or get_blocklist(username);
 
 	local new_blocklist = {
-		-- We set the [false] key to someting as a signal not to migrate privacy lists
+		-- We set the [false] key to something as a signal not to migrate privacy lists
 		[false] = blocklist[false] or { created = now; };
 	};
 	if type(blocklist[false]) == "table" then
@@ -189,6 +189,7 @@ local function edit_blocklist(event)
 
 	if is_blocking then
 		for jid in pairs(send_unavailable) do
+			-- Check that this JID isn't already blocked, i.e. this is not a change
 			if not blocklist[jid] then
 				for _, session in pairs(sessions[username].sessions) do
 					if session.presence then
diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua
index d4701148..b45a9dc2 100644
--- a/plugins/mod_bosh.lua
+++ b/plugins/mod_bosh.lua
@@ -44,19 +44,41 @@ local bosh_max_polling = module:get_option_number("bosh_max_polling", 5);
 local bosh_max_wait = module:get_option_number("bosh_max_wait", 120);
 
 local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure");
-local cross_domain = module:get_option("cross_domain_bosh", false);
+local cross_domain = module:get_option("cross_domain_bosh");
 
-if cross_domain == true then cross_domain = "*"; end
-if type(cross_domain) == "table" then cross_domain = table.concat(cross_domain, ", "); end
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_bosh' option has been deprecated");
+end
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 
 -- All sessions, and sessions that have no requests open
 local sessions = module:shared("sessions");
 
+local measure_active = module:measure("active_sessions", "amount");
+local measure_inactive = module:measure("inactive_sessions", "amount");
+local report_bad_host = module:measure("bad_host", "rate");
+local report_bad_sid = module:measure("bad_sid", "rate");
+local report_new_sid = module:measure("new_sid", "rate");
+local report_timeout = module:measure("timeout", "rate");
+
+module:hook("stats-update", function ()
+	local active = 0;
+	local inactive = 0;
+	for _, session in pairs(sessions) do
+		if #session.requests > 0 then
+			active = active + 1;
+		else
+			inactive = inactive + 1;
+		end
+	end
+	measure_active(active);
+	measure_inactive(inactive);
+end);
+
 -- Used to respond to idle sessions (those with waiting requests)
 function on_destroy_request(request)
-	log("debug", "Request destroyed: %s", tostring(request));
+	log("debug", "Request destroyed: %s", request);
 	local session = sessions[request.context.sid];
 	if session then
 		local requests = session.requests;
@@ -73,7 +95,7 @@ function on_destroy_request(request)
 			if session.inactive_timer then
 				session.inactive_timer:stop();
 			end
-			session.inactive_timer = module:add_timer(max_inactive, check_inactive, session, request.context,
+			session.inactive_timer = module:add_timer(max_inactive, session_timeout, session, request.context,
 				"BOSH client silent for over "..max_inactive.." seconds");
 			(session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive);
 		end
@@ -84,31 +106,16 @@ function on_destroy_request(request)
 	end
 end
 
-function check_inactive(now, session, context, reason) -- luacheck: ignore 212/now
+function session_timeout(now, session, context, reason) -- luacheck: ignore 212/now
 	if not session.destroyed then
+		report_timeout();
 		sessions[context.sid] = nil;
 		sm_destroy_session(session, reason);
 	end
 end
 
-local function set_cross_domain_headers(response)
-	local headers = response.headers;
-	headers.access_control_allow_methods = "GET, POST, OPTIONS";
-	headers.access_control_allow_headers = "Content-Type";
-	headers.access_control_max_age = "7200";
-	headers.access_control_allow_origin = cross_domain;
-	return response;
-end
-
-function handle_OPTIONS(event)
-	if cross_domain and event.request.headers.origin then
-		set_cross_domain_headers(event.response);
-	end
-	return "";
-end
-
 function handle_POST(event)
-	log("debug", "Handling new request %s: %s\n----------", tostring(event.request), tostring(event.request.body));
+	log("debug", "Handling new request %s: %s\n----------", event.request, event.request.body);
 
 	local request, response = event.request, event.response;
 	response.on_destroy = on_destroy_request;
@@ -121,10 +128,6 @@ function handle_POST(event)
 	local headers = response.headers;
 	headers.content_type = "text/xml; charset=utf-8";
 
-	if cross_domain and request.headers.origin then
-		set_cross_domain_headers(response);
-	end
-
 	-- stream:feed() calls the stream_callbacks, so all stanzas in
 	-- the body are processed in this next line before it returns.
 	-- In particular, the streamopened() stream callback is where
@@ -205,6 +208,7 @@ function handle_POST(event)
 		return;
 	end
 	module:log("warn", "Unable to associate request with a session (incomplete request?)");
+	report_bad_sid();
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams, condition = "item-not-found" });
 	return tostring(close_reply) .. "\n";
@@ -220,7 +224,7 @@ local function bosh_reset_stream(session) session.notopen = true; end
 
 local stream_xmlns_attr = { xmlns = "urn:ietf:params:xml:ns:xmpp-streams" };
 local function bosh_close_stream(session, reason)
-	(session.log or log)("info", "BOSH client disconnected: %s", tostring((reason and reason.condition or reason) or "session close"));
+	(session.log or log)("info", "BOSH client disconnected: %s", (reason and reason.condition or reason) or "session close");
 
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams });
@@ -245,7 +249,7 @@ local function bosh_close_stream(session, reason)
 				close_reply = reason;
 			end
 		end
-		log("info", "Disconnecting client, <stream:error> is: %s", tostring(close_reply));
+		log("info", "Disconnecting client, <stream:error> is: %s", close_reply);
 	end
 
 	local response_body = tostring(close_reply);
@@ -268,17 +272,27 @@ function stream_callbacks.streamopened(context, attr)
 		-- New session request
 		context.notopen = nil; -- Signals that we accept this opening tag
 
+		if not attr.to then
+			log("debug", "BOSH client tried to connect without specifying a host");
+			report_bad_host();
+			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
+				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
+			response:send(tostring(close_reply));
+			return;
+		end
+
 		local to_host = nameprep(attr.to);
 		local wait = tonumber(attr.wait);
 		if not to_host then
-			log("debug", "BOSH client tried to connect to invalid host: %s", tostring(attr.to));
+			log("debug", "BOSH client tried to connect to invalid host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
 			return;
 		end
 		if not rid or (not attr.wait or not wait or wait < 0 or wait % 1 ~= 0) then
-			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", tostring(attr.rid), tostring(attr.wait));
+			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", attr.rid, attr.wait);
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "bad-request" });
 			response:send(tostring(close_reply));
@@ -309,6 +323,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		session.log("debug", "BOSH session created for request from %s", session.ip);
 		log("info", "New BOSH session, assigned it sid '%s'", sid);
+		report_new_sid();
 
 		module:fire_event("bosh-session", { session = session, request = request });
 
@@ -323,7 +338,7 @@ function stream_callbacks.streamopened(context, attr)
 				s.attr.xmlns = "jabber:client";
 			end
 			s = filter("stanzas/out", s);
-			--log("debug", "Sending BOSH data: %s", tostring(s));
+			--log("debug", "Sending BOSH data: %s", s);
 			if not s then return true end
 			t_insert(session.send_buffer, tostring(s));
 
@@ -363,6 +378,7 @@ function stream_callbacks.streamopened(context, attr)
 	if not session then
 		-- Unknown sid
 		log("info", "Client tried to use sid '%s' which we don't know about", sid);
+		report_bad_sid();
 		response:send(tostring(st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", condition = "item-not-found" })));
 		context.notopen = nil;
 		return;
@@ -425,7 +441,7 @@ function stream_callbacks.streamopened(context, attr)
 	end
 end
 
-local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(err, 2)); end
 
 function runner_callbacks:error(err) -- luacheck: ignore 212/self
 	return handleerr(err);
@@ -511,8 +527,6 @@ module:provides("http", {
 	route = {
 		["GET"] = GET_response;
 		["GET /"] = GET_response;
-		["OPTIONS"] = handle_OPTIONS;
-		["OPTIONS /"] = handle_OPTIONS;
 		["POST"] = handle_POST;
 		["POST /"] = handle_POST;
 	};
diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua
index 15d3a9be..aecf2210 100644
--- a/plugins/mod_c2s.lua
+++ b/plugins/mod_c2s.lua
@@ -56,6 +56,11 @@ local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
 
 function stream_callbacks.streamopened(session, attr)
 	local send = session.send;
+	if not attr.to then
+		session:close{ condition = "improper-addressing",
+			text = "A 'to' attribute is required on stream headers" };
+		return;
+	end
 	local host = nameprep(attr.to);
 	if not host then
 		session:close{ condition = "improper-addressing",
@@ -97,7 +102,6 @@ function stream_callbacks.streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -106,7 +110,13 @@ function stream_callbacks.streamopened(session, attr)
 	if features.tags[1] or session.full_jid then
 		send(features);
 	else
-		(session.log or log)("warn", "No stream features to offer");
+		if session.secure then
+			-- Normally STARTTLS would be offered
+			(session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings.");
+		else
+			-- Here SASL should be offered
+			(session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings.");
+		end
 		session:close{ condition = "undefined-condition", text = "No stream features to proceed with" };
 	end
 end
@@ -121,7 +131,7 @@ function stream_callbacks.error(session, error, data)
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		(session.log or log)("debug", "Client XML parse error: %s", tostring(data));
+		(session.log or log)("debug", "Client XML parse error: %s", data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -251,8 +261,6 @@ function listener.onconnect(conn)
 		local sock = conn:socket();
 		if sock.info then
 			session.compressed = sock:info"compression";
-		elseif sock.compression then
-			session.compressed = sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -283,7 +291,7 @@ function listener.onconnect(conn)
 			if data then
 				local ok, err = stream:feed(data);
 				if not ok then
-					log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+					log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 					session:close("not-well-formed");
 				end
 			end
@@ -327,6 +335,13 @@ function listener.onreadtimeout(conn)
 	end
 end
 
+function listener.ondrain(conn)
+	local session = sessions[conn];
+	if session then
+		return (hosts[session.host] or prosody).events.fire_event("c2s-ondrain", { session = session });
+	end
+end
+
 local function keepalive(event)
 	local session = event.session;
 	if not session.notopen then
@@ -359,6 +374,7 @@ module:provides("net", {
 	default_port = 5222;
 	encryption = "starttls";
 	multiplex = {
+		protocol = "xmpp-client";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:client%1.*>";
 	};
 });
diff --git a/plugins/mod_carbons.lua b/plugins/mod_carbons.lua
index 1dcd4a07..0f8c7c60 100644
--- a/plugins/mod_carbons.lua
+++ b/plugins/mod_carbons.lua
@@ -74,17 +74,7 @@ local function message_handler(event, c2s)
 		return
 	end
 
-	-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
-	local copy = st.clone(stanza);
-	if c2s and not orig_to then
-		stanza.attr.to = bare_from;
-	end
-	copy.attr.xmlns = "jabber:client";
-	local carbon = st.message{ from = bare_jid, type = orig_type, }
-		:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
-			:tag("forwarded", { xmlns = xmlns_forward })
-				:add_child(copy):reset();
-
+	local carbon;
 	user_sessions = user_sessions and user_sessions.sessions;
 	for _, session in pairs(user_sessions) do
 		-- Carbons are sent to resources that have enabled it
@@ -93,6 +83,20 @@ local function message_handler(event, c2s)
 		and session ~= target_session
 		-- and isn't among the top resources that would receive the message per standard routing rules
 		and (c2s or session.priority ~= top_priority) then
+			if not carbon then
+				-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
+				local copy = st.clone(stanza);
+				if c2s and not orig_to then
+					stanza.attr.to = bare_from;
+				end
+				copy.attr.xmlns = "jabber:client";
+				carbon = st.message{ from = bare_jid, type = orig_type, }
+					:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
+						:tag("forwarded", { xmlns = xmlns_forward })
+							:add_child(copy):reset();
+
+			end
+
 			carbon.attr.to = session.full_jid;
 			module:log("debug", "Sending carbon to %s", session.full_jid);
 			session.send(carbon);
diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua
index b41204a2..afcfc68c 100644
--- a/plugins/mod_component.lua
+++ b/plugins/mod_component.lua
@@ -49,6 +49,7 @@ function module.add_host(module)
 	local send;
 
 	local function on_destroy(session, err) --luacheck: ignore 212/err
+		module:set_status("warn", err and ("Disconnected: "..err) or "Disconnected");
 		env.connected = false;
 		env.session = false;
 		send = nil;
@@ -102,6 +103,7 @@ function module.add_host(module)
 		module:log("info", "External component successfully authenticated");
 		session.send(st.stanza("handshake"));
 		module:fire_event("component-authenticated", { session = session });
+		module:set_status("info", "Connected");
 
 		return true;
 	end
@@ -165,11 +167,11 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.error(session, error, data)
 	if session.destroyed then return; end
-	module:log("warn", "Error processing component stream: %s", tostring(error));
+	module:log("warn", "Error processing component stream: %s", error);
 	if error == "no-stream" then
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		session.log("warn", "External component %s XML parse error: %s", tostring(session.host), tostring(data));
+		session.log("warn", "External component %s XML parse error: %s", session.host, data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -206,7 +208,7 @@ function stream_callbacks.streamclosed(session)
 	session:close();
 end
 
-local function handleerr(err) log("error", "Traceback[component]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[component]: %s", traceback(err, 2)); end
 function stream_callbacks.handlestanza(session, stanza)
 	-- Namespaces are icky.
 	if not stanza.attr.xmlns and stanza.name == "handshake" then
@@ -266,10 +268,10 @@ local function session_close(session, reason)
 					if reason.extra then
 						stanza:add_child(reason.extra);
 					end
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(stanza));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", stanza);
 					session.send(stanza);
 				elseif reason.name then -- a stanza
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(reason));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", reason);
 					session.send(reason);
 				end
 			end
@@ -310,7 +312,7 @@ function listener.onconnect(conn)
 	function session.data(_, data)
 		local ok, err = stream:feed(data);
 		if ok then return; end
-		module:log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+		log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 		session:close("not-well-formed");
 	end
 
@@ -325,7 +327,7 @@ end
 function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
-		(session.log or log)("info", "component disconnected: %s (%s)", tostring(session.host), tostring(err));
+		(session.log or log)("info", "component disconnected: %s (%s)", session.host, err);
 		if session.host then
 			module:context(session.host):fire_event("component-disconnected", { session = session, reason = err });
 		end
diff --git a/plugins/mod_csi.lua b/plugins/mod_csi.lua
index 84476cac..458ff491 100644
--- a/plugins/mod_csi.lua
+++ b/plugins/mod_csi.lua
@@ -2,8 +2,9 @@ local st = require "util.stanza";
 local xmlns_csi = "urn:xmpp:csi:0";
 local csi_feature = st.stanza("csi", { xmlns = xmlns_csi });
 
+local csi_handler_available = nil;
 module:hook("stream-features", function (event)
-	if event.origin.username then
+	if event.origin.username and csi_handler_available then
 		event.features:add_child(csi_feature);
 	end
 end);
@@ -21,3 +22,14 @@ end
 module:hook("stanza/"..xmlns_csi..":active", refire_event("csi-client-active"));
 module:hook("stanza/"..xmlns_csi..":inactive", refire_event("csi-client-inactive"));
 
+function module.load()
+	if prosody.hosts[module.host].events._handlers["csi-client-active"] then
+		csi_handler_available = true;
+		module:set_status("core", "CSI handler module loaded");
+	else
+		csi_handler_available = false;
+		module:set_status("warn", "No CSI handler module loaded");
+	end
+end
+module:hook("module-loaded", module.load);
+module:hook("module-unloaded", module.load);
diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua
index da2dd953..4a87f06c 100644
--- a/plugins/mod_csi_simple.lua
+++ b/plugins/mod_csi_simple.lua
@@ -9,40 +9,7 @@ module:depends"csi"
 local jid = require "util.jid";
 local st = require "util.stanza";
 local dt = require "util.datetime";
-local new_queue = require "util.queue".new;
-
-local function new_pump(output, ...)
-	-- luacheck: ignore 212/self
-	local q = new_queue(...);
-	local flush = true;
-	function q:pause()
-		flush = false;
-	end
-	function q:resume()
-		flush = true;
-		return q:flush();
-	end
-	local push = q.push;
-	function q:push(item)
-		local ok = push(self, item);
-		if not ok then
-			q:flush();
-			output(item, self);
-		elseif flush then
-			return q:flush();
-		end
-		return true;
-	end
-	function q:flush()
-		local item = self:pop();
-		while item do
-			output(item, self);
-			item = self:pop();
-		end
-		return true;
-	end
-	return q;
-end
+local filters = require "util.filters";
 
 local queue_size = module:get_option_number("csi_queue_size", 256);
 
@@ -84,37 +51,98 @@ module:hook("csi-is-stanza-important", function (event)
 	return true;
 end, -1);
 
-module:hook("csi-client-inactive", function (event)
-	local session = event.origin;
-	if session.pump then
-		session.pump:pause();
+local function with_timestamp(stanza, from)
+	if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
+		stanza = st.clone(stanza);
+		stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = from, stamp = dt.datetime()}));
+	end
+	return stanza;
+end
+
+local function manage_buffer(stanza, session)
+	local ctr = session.csi_counter or 0;
+	if ctr >= queue_size then
+		session.log("debug", "Queue size limit hit, flushing buffer (queue size is %d)", session.csi_counter);
+		session.conn:resume_writes();
+	elseif module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
+		session.log("debug", "Important stanza, flushing buffer (queue size is %d)", session.csi_counter);
+		session.conn:resume_writes();
 	else
-		local bare_jid = jid.join(session.username, session.host);
-		local send = session.send;
-		session._orig_send = send;
-		local pump = new_pump(session.send, queue_size);
-		pump:pause();
-		session.pump = pump;
-		function session.send(stanza)
-			if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
-				pump:flush();
-				send(stanza);
-			else
-				if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
-					stanza = st.clone(stanza);
-					stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = bare_jid, stamp = dt.datetime()}));
-				end
-				pump:push(stanza);
-			end
-			return true;
-		end
+		stanza = with_timestamp(stanza, jid.join(session.username, session.host))
+	end
+	session.csi_counter = ctr + 1;
+	return stanza;
+end
+
+local function flush_buffer(data, session)
+	if session.csi_flushing then
+		return data;
+	end
+	session.csi_flushing = true;
+	session.log("debug", "Client sent something, flushing buffer once (queue size is %d)", session.csi_counter);
+	session.conn:resume_writes();
+	return data;
+end
+
+function enable_optimizations(session)
+	if session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		filters.add_filter(session, "stanzas/out", manage_buffer);
+		filters.add_filter(session, "bytes/in", flush_buffer);
+	else
+		session.log("warn", "Session connection does not support write pausing");
+	end
+end
+
+function disable_optimizations(session)
+	session.csi_flushing = nil;
+	filters.remove_filter(session, "stanzas/out", manage_buffer);
+	filters.remove_filter(session, "bytes/in", flush_buffer);
+	if session.conn and session.conn.resume_writes then
+		session.conn:resume_writes();
 	end
+end
+
+module:hook("csi-client-inactive", function (event)
+	local session = event.origin;
+	enable_optimizations(session);
 end);
 
 module:hook("csi-client-active", function (event)
 	local session = event.origin;
-	if session.pump then
-		session.pump:resume();
+	disable_optimizations(session);
+end);
+
+module:hook("pre-resource-unbind", function (event)
+	local session = event.session;
+	disable_optimizations(session);
+end, 1);
+
+module:hook("c2s-ondrain", function (event)
+	local session = event.session;
+	if session.state == "inactive" and session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.log("debug", "Buffer flushed, resuming inactive mode (queue size was %d)", session.csi_counter);
+		session.csi_counter = 0;
 	end
 end);
 
+function module.load()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				enable_optimizations(session);
+			end
+		end
+	end
+end
+
+function module.unload()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				disable_optimizations(session);
+			end
+		end
+	end
+end
diff --git a/plugins/mod_dialback.lua b/plugins/mod_dialback.lua
index eddc3209..f580d948 100644
--- a/plugins/mod_dialback.lua
+++ b/plugins/mod_dialback.lua
@@ -93,6 +93,11 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 		-- he wants to be identified through dialback
 		-- We need to check the key with the Authoritative server
 		local attr = stanza.attr;
+		if not attr.to or not attr.from then
+			origin.log("debug", "Missing Dialback addressing (from=%q, to=%q)", attr.from, attr.to);
+			origin:close("improper-addressing");
+			return true;
+		end
 		local to, from = nameprep(attr.to), nameprep(attr.from);
 
 		if not hosts[to] then
@@ -102,6 +107,7 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 			return true;
 		elseif not from then
 			origin:close("improper-addressing");
+			return true;
 		end
 
 		if dwd and origin.secure then
diff --git a/plugins/mod_groups.lua b/plugins/mod_groups.lua
index 646b7408..0c44f481 100644
--- a/plugins/mod_groups.lua
+++ b/plugins/mod_groups.lua
@@ -25,7 +25,7 @@ function inject_roster_contacts(event)
 	local function import_jids_to_roster(group_name)
 		for jid in pairs(groups[group_name]) do
 			-- Add them to roster
-			--module:log("debug", "processing jid %s in group %s", tostring(jid), tostring(group_name));
+			--module:log("debug", "processing jid %s in group %s", jid, group_name);
 			if jid ~= bare_jid then
 				if not roster[jid] then roster[jid] = {}; end
 				roster[jid].subscription = "both";
@@ -99,7 +99,7 @@ function module.load()
 				end
 				members[false][#members[false]+1] = curr_group; -- Is a public group
 			end
-			module:log("debug", "New group: %s", tostring(curr_group));
+			module:log("debug", "New group: %s", curr_group);
 			groups[curr_group] = groups[curr_group] or {};
 		else
 			-- Add JID
@@ -108,7 +108,7 @@ function module.load()
 			local jid;
 			jid = jid_prep(entryjid:match("%S+"));
 			if jid then
-				module:log("debug", "New member of %s: %s", tostring(curr_group), tostring(jid));
+				module:log("debug", "New member of %s: %s", curr_group, jid);
 				groups[curr_group][jid] = name or false;
 				members[jid] = members[jid] or {};
 				members[jid][#members[jid]+1] = curr_group;
diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua
index a1d409bd..c3e19bb3 100644
--- a/plugins/mod_http.lua
+++ b/plugins/mod_http.lua
@@ -7,13 +7,16 @@
 --
 
 module:set_global();
-module:depends("http_errors");
+pcall(function ()
+	module:depends("http_errors");
+end);
 
 local portmanager = require "core.portmanager";
 local moduleapi = require "core.moduleapi";
 local url_parse = require "socket.url".parse;
 local url_build = require "socket.url".build;
 local normalize_path = require "util.http".normalize_path;
+local set = require "util.set";
 
 local server = require "net.http.server";
 
@@ -22,6 +25,12 @@ server.set_default_host(module:get_option_string("http_default_host"));
 server.set_option("body_size_limit", module:get_option_number("http_max_content_size"));
 server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size"));
 
+-- CORS settigs
+local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "OPTIONS" });
+local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" });
+local opt_credentials = module:get_option_boolean("access_control_allow_credentials", false);
+local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60);
+
 local function get_http_event(host, app_path, key)
 	local method, path = key:match("^(%S+)%s+(.+)$");
 	if not method then -- No path specified, default to "" (base path)
@@ -83,6 +92,16 @@ function moduleapi.http_url(module, app_name, default_path)
 	return "http://disabled.invalid/";
 end
 
+local function apply_cors_headers(response, methods, headers, max_age, allow_credentials, origin)
+	response.headers.access_control_allow_methods = tostring(methods);
+	response.headers.access_control_allow_headers = tostring(headers);
+	response.headers.access_control_max_age = tostring(max_age)
+	response.headers.access_control_allow_origin = origin or "*";
+	if allow_credentials then
+		response.headers.access_control_allow_credentials = "true";
+	end
+end
+
 function module.add_host(module)
 	local host = module.host;
 	if host ~= "*" then
@@ -101,9 +120,27 @@ function module.add_host(module)
 		end
 		apps[app_name] = apps[app_name] or {};
 		local app_handlers = apps[app_name];
+
+		local app_methods = opt_methods;
+
+		local function cors_handler(event_data)
+			local request, response = event_data.request, event_data.response;
+			apply_cors_headers(response, app_methods, opt_headers, opt_max_age, opt_credentials, request.headers.origin);
+		end
+
+		local function options_handler(event_data)
+			cors_handler(event_data);
+			return "";
+		end
+
 		for key, handler in pairs(event.item.route or {}) do
 			local event_name = get_http_event(host, app_path, key);
 			if event_name then
+				local method = event_name:match("^%S+");
+				if not app_methods:contains(method) then
+					app_methods = app_methods + set.new{ method };
+				end
+				local options_event_name = event_name:gsub("^%S+", "OPTIONS");
 				if type(handler) ~= "function" then
 					local data = handler;
 					handler = function () return data; end
@@ -119,8 +156,14 @@ function module.add_host(module)
 					module:hook_object_event(server, event_name:sub(1, -2), redir_handler, -1);
 				end
 				if not app_handlers[event_name] then
-					app_handlers[event_name] = handler;
+					app_handlers[event_name] = {
+						main = handler;
+						cors = cors_handler;
+						options = options_handler;
+					};
 					module:hook_object_event(server, event_name, handler);
+					module:hook_object_event(server, event_name, cors_handler, 1);
+					module:hook_object_event(server, options_event_name, options_handler, -1);
 				else
 					module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name);
 				end
@@ -130,7 +173,7 @@ function module.add_host(module)
 		end
 		local services = portmanager.get_active_services();
 		if services:get("https") or services:get("http") then
-			module:log("debug", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
+			module:log("info", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
 		else
 			module:log("warn", "Not listening on any ports, '%s' will be unreachable", app_name);
 		end
@@ -139,8 +182,11 @@ function module.add_host(module)
 	local function http_app_removed(event)
 		local app_handlers = apps[event.item.name];
 		apps[event.item.name] = nil;
-		for event_name, handler in pairs(app_handlers) do
-			module:unhook_object_event(server, event_name, handler);
+		for event_name, handlers in pairs(app_handlers) do
+			module:unhook_object_event(server, event_name, handlers.main);
+			module:unhook_object_event(server, event_name, handlers.cors);
+			local options_event_name = event_name:gsub("^%S+", "OPTIONS");
+			module:unhook_object_event(server, options_event_name, handlers.options);
 		end
 	end
 
@@ -195,10 +241,8 @@ module:provides("net", {
 	listener = server.listener;
 	default_port = 5281;
 	encryption = "ssl";
-	ssl_config = {
-		verify = "none";
-	};
 	multiplex = {
+		protocol = "http/1.1";
 		pattern = "^[A-Z]";
 	};
 });
diff --git a/plugins/mod_http_errors.lua b/plugins/mod_http_errors.lua
index 13473219..2158b403 100644
--- a/plugins/mod_http_errors.lua
+++ b/plugins/mod_http_errors.lua
@@ -26,21 +26,24 @@ local html = [[
 <meta charset="utf-8">
 <title>{title}</title>
 <style>
-body{
-	margin-top:14%;
-	text-align:center;
-	background-color:#F8F8F8;
-	font-family:sans-serif;
+body {
+	margin-top : 14%;
+	text-align : center;
+	background-color : #F8F8F8;
+	font-family : sans-serif
 }
-h1{
-	font-size:xx-large;
+
+h1 {
+	font-size : xx-large
 }
-p{
-	font-size:x-large;
+
+p {
+	font-size : x-large
 }
+
 p+p {
-	font-size:large;
-	font-family:courier;
+	font-size : large;
+	font-family : courier
 }
 </style>
 </head>
@@ -72,3 +75,15 @@ module:hook_object_event(server, "http-error", function (event)
 	end
 	return get_page(event.code, (show_private and event.private_message) or event.message);
 end);
+
+module:hook_object_event(server, "http-error", function (event)
+	local request, response = event.request, event.response;
+	if request and response and request.path == "/" and response.status_code == 404 then
+		response.headers.content_type = "text/html; charset=utf-8";
+		return render(html, {
+				title = "Prosody is running!";
+				message = "Welcome to the XMPP world!";
+			});
+	end
+end, 1);
+
diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua
index 1dae0d6d..4d0b14cd 100644
--- a/plugins/mod_http_files.lua
+++ b/plugins/mod_http_files.lua
@@ -7,14 +7,9 @@
 --
 
 module:depends("http");
-local server = require"net.http.server";
-local lfs = require "lfs";
 
-local os_date = os.date;
 local open = io.open;
-local stat = lfs.attributes;
-local build_path = require"socket.url".build_path;
-local path_sep = package.config:sub(1,1);
+local fileserver = require"net.http.files";
 
 local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path"));
 local cache_size = module:get_option_number("http_files_cache_size", 128);
@@ -38,7 +33,9 @@ if not mime_map then
 	module:shared("/*/http_files/mime").types = mime_map;
 
 	local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r");
-	if mime_types then
+	if not mime_types then
+		module:log("debug", "Could not open MIME database: %s", err);
+	else
 		local mime_data = mime_types:read("*a");
 		mime_types:close();
 		setmetatable(mime_map, {
@@ -51,148 +48,56 @@ if not mime_map then
 	end
 end
 
-local forbidden_chars_pattern = "[/%z]";
-if prosody.platform == "windows" then
-	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+local function get_calling_module()
+	local info = debug.getinfo(3, "S");
+	if not info then return "An unknown module"; end
+	return info.source:match"mod_[^/\\.]+" or info.short_src;
 end
 
-local urldecode = require "util.http".urldecode;
-function sanitize_path(path)
-	if not path then return end
-	local out = {};
-
-	local c = 0;
-	for component in path:gmatch("([^/]+)") do
-		component = urldecode(component);
-		if component:find(forbidden_chars_pattern) then
-			return nil;
-		elseif component == ".." then
-			if c <= 0 then
-				return nil;
-			end
-			out[c] = nil;
-			c = c - 1;
-		elseif component ~= "." then
-			c = c + 1;
-			out[c] = component;
-		end
-	end
-	if path:sub(-1,-1) == "/" then
-		out[c+1] = "";
-	end
-	return "/"..table.concat(out, "/");
-end
-
-local cache = require "util.cache".new(cache_size);
-
+-- COMPAT -- TODO deprecate
 function serve(opts)
 	if type(opts) ~= "table" then -- assume path string
 		opts = { path = opts };
 	end
-	-- luacheck: ignore 431
-	local base_path = opts.path;
-	local dir_indices = opts.index_files or dir_indices;
-	local directory_index = opts.directory_index;
-	local function serve_file(event, path)
-		local request, response = event.request, event.response;
-		local sanitized_path = sanitize_path(path);
-		if path and not sanitized_path then
-			return 400;
-		end
-		path = sanitized_path;
-		local orig_path = sanitize_path(request.path);
-		local full_path = base_path .. (path or ""):gsub("/", path_sep);
-		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
-		if not attr then
-			return 404;
-		end
-
-		local request_headers, response_headers = request.headers, response.headers;
-
-		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
-		response_headers.last_modified = last_modified;
-
-		local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0);
-		response_headers.etag = etag;
-
-		local if_none_match = request_headers.if_none_match
-		local if_modified_since = request_headers.if_modified_since;
-		if etag == if_none_match
-		or (not if_none_match and last_modified == if_modified_since) then
-			return 304;
-		end
-
-		local data = cache:get(orig_path);
-		if data and data.etag == etag then
-			response_headers.content_type = data.content_type;
-			data = data.data;
-		elseif attr.mode == "directory" and path then
-			if full_path:sub(-1) ~= "/" then
-				local dir_path = { is_absolute = true, is_directory = true };
-				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
-				response_headers.location = build_path(dir_path);
-				return 301;
-			end
-			for i=1,#dir_indices do
-				if stat(full_path..dir_indices[i], "mode") == "file" then
-					return serve_file(event, path..dir_indices[i]);
-				end
-			end
-
-			if directory_index then
-				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
-			end
-			if not data then
-				return 403;
-			end
-			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
-			response_headers.content_type = mime_map.html;
-
-		else
-			local f, err = open(full_path, "rb");
-			if not f then
-				module:log("debug", "Could not open %s. Error was %s", full_path, err);
-				return 403;
-			end
-			local ext = full_path:match("%.([^./]+)$");
-			local content_type = ext and mime_map[ext];
-			response_headers.content_type = content_type;
-			if attr.size > cache_max_file_size then
-				response_headers.content_length = attr.size;
-				module:log("debug", "%d > cache_max_file_size", attr.size);
-				return response:send_file(f);
-			else
-				data = f:read("*a");
-				f:close();
-			end
-			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
-		end
-
-		return response:send(data);
+	if opts.directory_index == nil then
+		opts.directory_index = directory_index;
 	end
-
-	return serve_file;
+	if opts.mime_map == nil then
+		opts.mime_map = mime_map;
+	end
+	if opts.cache_size == nil then
+		opts.cache_size = cache_size;
+	end
+	if opts.cache_max_file_size == nil then
+		opts.cache_max_file_size = cache_max_file_size;
+	end
+	if opts.index_files == nil then
+		opts.index_files = dir_indices;
+	end
+	-- TODO Crank up to warning
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
+	return fileserver.serve(opts);
 end
 
 function wrap_route(routes)
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
 	for route,handler in pairs(routes) do
 		if type(handler) ~= "function" then
-			routes[route] = serve(handler);
+			routes[route] = fileserver.serve(handler);
 		end
 	end
 	return routes;
 end
 
-if base_path then
-	module:provides("http", {
-		route = {
-			["GET /*"] = serve {
-				path = base_path;
-				directory_index = directory_index;
-			}
-		};
-	});
-else
-	module:log("debug", "http_files_dir not set, assuming use by some other module");
-end
-
+module:provides("http", {
+	route = {
+		["GET /*"] = fileserver.serve({
+			path = base_path;
+			directory_index = directory_index;
+			mime_map = mime_map;
+			cache_size = cache_size;
+			cache_max_file_size = cache_max_file_size;
+			index_files = dir_indices;
+		});
+	};
+});
diff --git a/plugins/mod_legacyauth.lua b/plugins/mod_legacyauth.lua
index 0f41d3e7..941806d3 100644
--- a/plugins/mod_legacyauth.lua
+++ b/plugins/mod_legacyauth.lua
@@ -78,8 +78,10 @@ module:hook("stanza/iq/jabber:iq:auth:query", function(event)
 					session:close(); -- FIXME undo resource bind and auth instead of closing the session?
 					return true;
 				end
+				session.send(st.reply(stanza));
+			else
+				session.send(st.error_reply(stanza, "auth", "not-authorized", err));
 			end
-			session.send(st.reply(stanza));
 		else
 			session.send(st.error_reply(stanza, "auth", "not-authorized"));
 		end
diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua
index 914d5c44..024ab686 100644
--- a/plugins/mod_limits.lua
+++ b/plugins/mod_limits.lua
@@ -32,7 +32,7 @@ local function parse_burst(burst, sess_type)
 	end
 	local n_burst = tonumber(burst);
 	if not n_burst then
-		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, tostring(burst), default_burst);
+		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, burst, default_burst);
 	end
 	return n_burst or default_burst;
 end
@@ -51,18 +51,18 @@ end
 local default_filter_set = {};
 
 function default_filter_set.bytes_in(bytes, session)
-  local sess_throttle = session.throttle;
-  if sess_throttle then
-    local ok, balance, outstanding = sess_throttle:poll(#bytes, true);
+	local sess_throttle = session.throttle;
+	if sess_throttle then
+		local ok, _, outstanding = sess_throttle:poll(#bytes, true);
 		if not ok then
-      session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
+			session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
 			outstanding = ceil(outstanding);
 			session.conn:pause(); -- Read no more data from the connection until there is no outstanding data
 			local outstanding_data = bytes:sub(-outstanding);
 			bytes = bytes:sub(1, #bytes-outstanding);
 			timer.add_task(limits_resolution, function ()
 				if not session.conn then return; end
-        if sess_throttle:peek(#outstanding_data) then
+				if sess_throttle:peek(#outstanding_data) then
 					session.log("debug", "Resuming paused session");
 					session.conn:resume();
 				end
@@ -84,8 +84,13 @@ local function filter_hook(session)
 	local session_type = session.type:match("^[^_]+");
 	local filter_set, opts = type_filters[session_type], limits[session_type];
 	if opts then
-		session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
-		filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		if session.conn and session.conn.setlimit then
+			session.conn:setlimit(opts.bytes_per_second);
+			-- Currently no burst support
+		else
+			session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
+			filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		end
 	end
 end
 
@@ -96,3 +101,25 @@ end
 function module.unload()
 	filters.remove_filter_hook(filter_hook);
 end
+
+function module.add_host(module)
+	local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {});
+
+	if not unlimited_jids:empty() then
+		module:hook("authentication-success", function (event)
+			local session = event.session;
+			local session_type = session.type:match("^[^_]+");
+			local jid = session.username .. "@" .. session.host;
+			if unlimited_jids:contains(jid) then
+				if session.conn and session.conn.setlimit then
+					session.conn:setlimit(0);
+					-- Currently no burst support
+				else
+					local filter_set = type_filters[session_type];
+					filters.remove_filter(session, "bytes/in", filter_set.bytes_in);
+					session.throttle = nil;
+				end
+			end
+		end);
+	end
+end
diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index fd319486..b67635c9 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -25,6 +25,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prepped_split = require "util.jid".prepped_split;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 local host = module.host;
 
 local rm_load_roster = require "core.rostermanager".load_roster;
@@ -40,6 +41,9 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja
 local archive_store = module:get_option_string("archive_store", "archive");
 local archive = module:open_store(archive_store, "archive");
 
+local cleanup_after = module:get_option_string("archive_expires_after", "1w");
+local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -98,7 +102,14 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local qwith, qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -117,10 +128,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query, id %s with %s from %s until %s",
-		tostring(qid), qwith or "anyone",
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qwith or "*",
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -128,6 +141,9 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local reverse = qset and qset.before or false;
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(origin.username, {
@@ -140,7 +156,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -189,13 +210,13 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
-
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -213,13 +234,13 @@ local function shall_store(user, who)
 	end
 	local prefs = get_prefs(user);
 	local rule = prefs[who];
-	module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
+	module:log("debug", "%s's rule for %s is %s", user, who, rule);
 	if rule ~= nil then
 		return rule;
 	end
 	-- Below could be done by a metatable
 	local default = prefs[false];
-	module:log("debug", "%s's default rule is %s", user, tostring(default));
+	module:log("debug", "%s's default rule is %s", user, default);
 	if default == "roster" then
 		return has_in_roster(user, who);
 	end
@@ -297,7 +318,28 @@ local function message_handler(event, c2s)
 		log("debug", "Archiving stanza: %s", stanza:top_tag());
 
 		-- And stash it
-		local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with);
+		local time = time_now();
+		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+		if not ok and err == "quota-limit" then
+			if type(cleanup_after) == "number" then
+				module:log("debug", "User '%s' over quota, cleaning archive", store_user);
+				local cleaned = archive:delete(store_user, {
+					["end"] = (os.time() - cleanup_after);
+				});
+				if cleaned then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+			if not ok and (archive.caps and archive.caps.truncate) then
+				module:log("debug", "User '%s' over quota, truncating archive", store_user);
+				local truncated = archive:delete(store_user, {
+					truncate = archive_item_limit - 1;
+				});
+				if truncated then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+		end
 		if ok then
 			local clone_for_other_handlers = st.clone(stanza);
 			local id = ok;
@@ -325,8 +367,6 @@ end
 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 
-local cleanup_after = module:get_option_string("archive_expires_after", "1w");
-local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("archive_cleanup");
 	local cleanup_map = module:open_store("archive_cleanup", "map");
@@ -361,8 +401,10 @@ if cleanup_after ~= "never" then
 			last_date:set(username, date);
 		end
 	end
+	local cleanup_time = module:measure("cleanup", "times");
 
 	cleanup_runner = require "util.async".runner(function ()
+		local cleanup_done = cleanup_time();
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -393,6 +435,7 @@ if cleanup_after ~= "never" then
 			end
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua
new file mode 100644
index 00000000..b586a70c
--- /dev/null
+++ b/plugins/mod_mimicking.lua
@@ -0,0 +1,85 @@
+-- Prosody IM
+-- Copyright (C) 2012 Florian Zeitz
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local encodings = require "util.encodings";
+assert(encodings.confusable, "This module requires that Prosody be built with ICU");
+local skeleton = encodings.confusable.skeleton;
+
+local usage = require "util.prosodyctl".show_usage;
+local usermanager = require "core.usermanager";
+local storagemanager = require "core.storagemanager";
+
+local skeletons
+function module.load()
+	if module.host ~= "*" then
+		skeletons = module:open_store("skeletons");
+	end
+end
+
+module:hook("user-registered", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, { username = user.username });
+	if not ok then
+		module:log("error", "Unable to store mimicry data (%q => %q): %s", user.username, skel, err);
+	end
+end);
+
+module:hook("user-deleted", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, nil);
+	if not ok and err then
+		module:log("error", "Unable to clear mimicry data (%q): %s", skel, err);
+	end
+end);
+
+module:hook("user-registering", function(user)
+	local existing, err = skeletons:get(skeleton(user.username));
+	if existing then
+		module:log("debug", "Attempt to register username '%s' which could be confused with '%s'", user.username, existing.username);
+		user.allowed = false;
+	elseif err then
+		module:log("error", "Unable to check if new username '%s' can be confused with any existing user: %s", err);
+	end
+end);
+
+function module.command(arg)
+	if (arg[1] ~= "bootstrap" or not arg[2]) then
+		usage("mod_mimicking bootstrap <host>", "Initialize username mimicry database");
+		return;
+	end
+
+	local host = arg[2];
+
+	local host_session = prosody.hosts[host];
+	if not host_session then
+		return "No such host";
+	end
+
+	storagemanager.initialize_host(host);
+	usermanager.initialize_host(host);
+
+	skeletons = storagemanager.open(host, "skeletons");
+
+	local count = 0;
+	for user in usermanager.users(host) do
+		local skel = skeleton(user);
+		local existing, err = skeletons:get(skel);
+		if existing and existing.username ~= user then
+			module:log("warn", "Existing usernames '%s' and '%s' are confusable", existing.username, user);
+		elseif err then
+			module:log("error", "Error checking for existing mimicry data (%q = %q): %s", user, skel, err);
+		end
+		local ok, err = skeletons:set(skel, { username = user });
+		if ok then
+			count = count + 1;
+		elseif err then
+			module:log("error", "Unable to store mimicry data (%q => %q): %s", user, skel, err);
+		end
+	end
+	module:log("info", "%d usernames indexed", count);
+end
diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua
index a9287224..fee75e33 100644
--- a/plugins/mod_muc_mam.lua
+++ b/plugins/mod_muc_mam.lua
@@ -4,7 +4,7 @@
 -- This file is MIT/X11 licensed.
 
 if module:get_host_type() ~= "component" then
-	module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
+	module:log_status("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
 	return;
 end
 
@@ -21,6 +21,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prep = require "util.jid".prep;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 
 local mod_muc = module:depends"muc";
 local get_room_from_jid = mod_muc.get_room_from_jid;
@@ -32,6 +33,9 @@ local m_min = math.min;
 local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 
+local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
+local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
+
 local default_history_length = 20;
 local max_history_length = module:get_option_number("max_history_messages", math.huge);
 
@@ -49,6 +53,8 @@ local log_by_default = module:get_option_boolean("muc_log_by_default", true);
 local archive_store = "muc_log";
 local archive = module:open_store(archive_store, "archive");
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+
 if archive.name == "null" or not archive.find then
 	if not archive.find then
 		module:log("error", "Attempt to open archive storage returned a driver without archive API support");
@@ -63,12 +69,15 @@ end
 
 local function archiving_enabled(room)
 	if log_all_rooms then
+		module:log("debug", "Archiving all rooms");
 		return true;
 	end
 	local enabled = room._data.archiving;
 	if enabled == nil then
+		module:log("debug", "Default is %s (for %s)", log_by_default, room.jid);
 		return log_by_default;
 	end
+	module:log("debug", "Logging in room %s is %s", room.jid, enabled);
 	return enabled;
 end
 
@@ -135,7 +144,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	local qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -153,10 +169,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query id %s from %s until %s)",
-		tostring(qid),
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -165,6 +182,9 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(room_node, {
@@ -176,7 +196,12 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -233,13 +258,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -274,7 +300,7 @@ module:hook("muc-get-history", function (event)
 	local data, err = archive:find(jid_split(room_jid), query);
 
 	if not data then
-		module:log("error", "Could not fetch history: %s", tostring(err));
+		module:log("error", "Could not fetch history: %s", err);
 		return
 	end
 
@@ -300,7 +326,7 @@ module:hook("muc-get-history", function (event)
 			maxchars = maxchars - chars;
 		end
 		history[i], i = item, i+1;
-		-- module:log("debug", tostring(item));
+		-- module:log("debug", item);
 	end
 	function event.next_stanza()
 		i = i - 1;
@@ -328,7 +354,7 @@ end, 0);
 
 -- Handle messages
 local function save_to_history(self, stanza)
-	local room_node, room_host = jid_split(self.jid);
+	local room_node = jid_split(self.jid);
 
 	local stored_stanza = stanza;
 
@@ -352,7 +378,29 @@ local function save_to_history(self, stanza)
 	end
 
 	-- And stash it
-	local id = archive:append(room_node, nil, stored_stanza, time_now(), with);
+	local time = time_now();
+	local id, err = archive:append(room_node, nil, stored_stanza, time, with);
+
+	if not id and err == "quota-limit" then
+		if type(cleanup_after) == "number" then
+			module:log("debug", "Room '%s' over quota, cleaning archive", room_node);
+			local cleaned = archive:delete(room_node, {
+				["end"] = (os.time() - cleanup_after);
+			});
+			if cleaned then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+		if not id and (archive.caps and archive.caps.truncate) then
+			module:log("debug", "User '%s' over quota, truncating archive", room_node);
+			local truncated = archive:delete(room_node, {
+				truncate = archive_item_limit - 1;
+			});
+			if truncated then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+	end
 
 	if id then
 		schedule_cleanup(room_node);
@@ -391,14 +439,13 @@ end
 module:add_feature(xmlns_mam);
 
 module:hook("muc-disco#info", function(event)
-	event.reply:tag("feature", {var=xmlns_mam}):up();
+	if archiving_enabled(event.room) then
+		event.reply:tag("feature", {var=xmlns_mam}):up();
+	end
 end);
 
 -- Cleanup
 
-local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
-local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
-
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("muc_log_cleanup");
 	local cleanup_map = module:open_store("muc_log_cleanup", "map");
diff --git a/plugins/mod_net_multiplex.lua b/plugins/mod_net_multiplex.lua
index 8ef77883..849b22ee 100644
--- a/plugins/mod_net_multiplex.lua
+++ b/plugins/mod_net_multiplex.lua
@@ -1,22 +1,37 @@
 module:set_global();
 
+local array = require "util.array";
 local max_buffer_len = module:get_option_number("multiplex_buffer_size", 1024);
 
 local portmanager = require "core.portmanager";
 
 local available_services = {};
+local service_by_protocol = {};
+local available_protocols = array();
 
 local function add_service(service)
 	local multiplex_pattern = service.multiplex and service.multiplex.pattern;
+	local protocol_name = service.multiplex and service.multiplex.protocol;
+	if protocol_name then
+		module:log("debug", "Adding multiplex service %q with protocol %q", service.name, protocol_name);
+		service_by_protocol[protocol_name] = service;
+		available_protocols:push(protocol_name);
+	end
 	if multiplex_pattern then
 		module:log("debug", "Adding multiplex service %q with pattern %q", service.name, multiplex_pattern);
 		available_services[service] = multiplex_pattern;
-	else
+	elseif not protocol_name then
 		module:log("debug", "Service %q is not multiplex-capable", service.name);
 	end
 end
 module:hook("service-added", function (event) add_service(event.service); end);
-module:hook("service-removed", function (event)	available_services[event.service] = nil; end);
+module:hook("service-removed", function (event)
+	available_services[event.service] = nil;
+	if event.service.multiplex and event.service.multiplex.protocol then
+		available_protocols:filter(function (p) return p ~= event.service.multiplex.protocol end);
+		service_by_protocol[event.service.multiplex.protocol] = nil;
+	end
+end);
 
 for _, services in pairs(portmanager.get_registered_services()) do
 	for _, service in ipairs(services) do
@@ -28,7 +43,19 @@ local buffers = {};
 
 local listener = { default_mode = "*a" };
 
-function listener.onconnect()
+function listener.onconnect(conn)
+	local sock = conn:socket();
+	if sock.getalpn then
+		local selected_proto = sock:getalpn();
+		local service = service_by_protocol[selected_proto];
+		if service then
+			module:log("debug", "Routing incoming connection to %s based on ALPN %q", service.name, selected_proto);
+			local next_listener = service.listener;
+			conn:setlistener(next_listener);
+			local onconnect = next_listener.onconnect;
+			if onconnect then return onconnect(conn) end
+		end
+	end
 end
 
 function listener.onincoming(conn, data)
@@ -68,5 +95,10 @@ module:provides("net", {
 	name = "multiplex_ssl";
 	config_prefix = "ssl";
 	encryption = "ssl";
+	ssl_config = {
+		alpn = function ()
+			return available_protocols;
+		end;
+	};
 	listener = listener;
 });
diff --git a/plugins/mod_offline.lua b/plugins/mod_offline.lua
index 487098d1..dffe8357 100644
--- a/plugins/mod_offline.lua
+++ b/plugins/mod_offline.lua
@@ -24,11 +24,16 @@ module:hook("message/offline/handle", function(event)
 		node = origin.username;
 	end
 
-	return offline_messages:append(node, nil, stanza, os.time(), "");
+	local ok = offline_messages:append(node, nil, stanza, os.time(), "");
+	if ok then
+		module:log("debug", "Saved to offline storage: %s", stanza:top_tag());
+	end
+	return ok;
 end, -1);
 
 module:hook("message/offline/broadcast", function(event)
 	local origin = event.origin;
+	origin.log("debug", "Broadcasting offline messages");
 
 	local node, host = origin.username, origin.host;
 
@@ -38,6 +43,9 @@ module:hook("message/offline/broadcast", function(event)
 		stanza:tag("delay", {xmlns = "urn:xmpp:delay", from = host, stamp = datetime.datetime(when)}):up(); -- XEP-0203
 		origin.send(stanza);
 	end
-	offline_messages:delete(node);
+	local ok = offline_messages:delete(node);
+	if type(ok) == "number" and ok > 0 then
+		origin.log("debug", "%d offline messages consumed");
+	end
 	return true;
 end, -1);
diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua
index 12be41a2..7110649f 100644
--- a/plugins/mod_pep.lua
+++ b/plugins/mod_pep.lua
@@ -8,6 +8,7 @@ local calculate_hash = require "util.caps".calculate_hash;
 local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed;
 local cache = require "util.cache";
 local set = require "util.set";
+local new_id = require "util.id".medium;
 local storagemanager = require "core.storagemanager";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
@@ -123,9 +124,6 @@ local function get_broadcaster(username)
 		if kind == "retract" then
 			kind = "items"; -- XEP-0060 signals retraction in an <items> container
 		end
-		local message = st.message({ from = user_bare, type = "headline" })
-			:tag("event", { xmlns = xmlns_pubsub_event })
-				:tag(kind, { node = node });
 		if item then
 			item = st.clone(item);
 			item.attr.xmlns = nil; -- Clear the pubsub namespace
@@ -134,10 +132,19 @@ local function get_broadcaster(username)
 					item:maptags(function () return nil; end);
 				end
 			end
+		end
+
+		local id = new_id();
+		local message = st.message({ from = user_bare, type = "headline", id = id })
+			:tag("event", { xmlns = xmlns_pubsub_event })
+				:tag(kind, { node = node });
+
+		if item then
 			message:add_child(item);
 		end
+
 		for jid in pairs(jids) do
-			module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item));
+			module:log("debug", "Sending notification to %s from %s for node %s", jid, user_bare, node);
 			message.attr.to = jid;
 			module:send(message);
 		end
@@ -166,12 +173,12 @@ local function get_subscriber_filter(username)
 end
 
 function get_pep_service(username)
-	module:log("debug", "get_pep_service(%q)", username);
 	local user_bare = jid_join(username, host);
 	local service = services[username];
 	if service then
 		return service;
 	end
+	module:log("debug", "Creating pubsub service for user %q", username);
 	service = pubsub.new({
 		pep_username = username;
 		node_defaults = {
@@ -238,8 +245,6 @@ end
 module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq);
 module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq);
 
-module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody"));
-module:add_feature("http://jabber.org/protocol/pubsub#publish");
 
 local function get_caps_hash_from_presence(stanza, current)
 	local t = stanza.attr.type;
diff --git a/plugins/mod_pep_simple.lua b/plugins/mod_pep_simple.lua
index f0b5d7ef..e686b99b 100644
--- a/plugins/mod_pep_simple.lua
+++ b/plugins/mod_pep_simple.lua
@@ -14,6 +14,7 @@ local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed
 local pairs = pairs;
 local next = next;
 local type = type;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local calculate_hash = require "util.caps".calculate_hash;
 local core_post_stanza = prosody.core_post_stanza;
 local bare_sessions = prosody.bare_sessions;
@@ -84,6 +85,7 @@ local function publish_all(user, recipient, session)
 	if d and notify then
 		for node in pairs(notify) do
 			if d[node] then
+				-- luacheck: ignore id
 				local id, item = unpack(d[node]);
 				session.send(st.message({from=user, to=recipient, type='headline'})
 					:tag('event', {xmlns='http://jabber.org/protocol/pubsub#event'})
@@ -229,13 +231,13 @@ module:hook("iq/bare/http://jabber.org/protocol/pubsub:pubsub", function(event)
 				return true;
 			else --invalid request
 				session.send(st.error_reply(stanza, 'modify', 'bad-request'));
-				module:log("debug", "Invalid request: %s", tostring(payload));
+				module:log("debug", "Invalid request: %s", payload);
 				return true;
 			end
 		else --no presence subscription
 			session.send(st.error_reply(stanza, 'auth', 'not-authorized')
 				:tag('presence-subscription-required', {xmlns='http://jabber.org/protocol/pubsub#errors'}));
-			module:log("debug", "Unauthorized request: %s", tostring(payload));
+			module:log("debug", "Unauthorized request: %s", payload);
 			return true;
 		end
 	end
diff --git a/plugins/mod_ping.lua b/plugins/mod_ping.lua
index 5fff58d1..df24c495 100644
--- a/plugins/mod_ping.lua
+++ b/plugins/mod_ping.lua
@@ -16,18 +16,3 @@ end
 
 module:hook("iq-get/bare/urn:xmpp:ping:ping", ping_handler);
 module:hook("iq-get/host/urn:xmpp:ping:ping", ping_handler);
-
--- Ad-hoc command
-
-local datetime = require "util.datetime".datetime;
-
-function ping_command_handler (self, data, state) -- luacheck: ignore 212
-	local now = datetime();
-	return { info = "Pong\n"..now, status = "completed" };
-end
-
-module:depends "adhoc";
-local adhoc_new = module:require "adhoc".new;
-local descriptor = adhoc_new("Ping", "ping", ping_command_handler);
-module:provides("adhoc", descriptor);
-
diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua
index 23df4d23..a2a60dd0 100644
--- a/plugins/mod_posix.lua
+++ b/plugins/mod_posix.lua
@@ -20,7 +20,6 @@ if not have_signal then
 	module:log("warn", "Couldn't load signal library, won't respond to SIGTERM");
 end
 
-local format = require "util.format".format;
 local lfs = require "lfs";
 local stat = lfs.attributes;
 
@@ -113,19 +112,6 @@ local function write_pidfile()
 	end
 end
 
-local syslog_opened;
-function syslog_sink_maker(config) -- luacheck: ignore 212/config
-	if not syslog_opened then
-		pposix.syslog_open("prosody", module:get_option_string("syslog_facility"));
-		syslog_opened = true;
-	end
-	local syslog = pposix.syslog_log;
-	return function (name, level, message, ...)
-		syslog(level, name, format(message, ...));
-	end;
-end
-require "core.loggingmanager".register_sink_type("syslog", syslog_sink_maker);
-
 local daemonize = module:get_option("daemonize", prosody.installed);
 
 local function remove_log_sinks()
diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua
index 268a2f0c..e69c31a5 100644
--- a/plugins/mod_presence.lua
+++ b/plugins/mod_presence.lua
@@ -81,8 +81,14 @@ function handle_normal_presence(origin, stanza)
 				res.presence.attr.to = nil;
 			end
 		end
-		for jid in pairs(roster[false].pending) do -- resend incoming subscription requests
-			origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original?
+		for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests
+			if type(pending_request) == "table" then
+				local subscribe = st.deserialize(pending_request);
+				subscribe.attr.type, subscribe.attr.from = "subscribe", jid;
+				origin.send(subscribe);
+			else
+				origin.send(st.presence({type="subscribe", from=jid}));
+			end
 		end
 		local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host});
 		for jid, item in pairs(roster) do -- resend outgoing subscription requests
@@ -175,8 +181,10 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		if rostermanager.subscribed(node, host, to_bare) then
 			rostermanager.roster_push(node, host, to_bare);
 		end
-		core_post_stanza(origin, stanza);
-		send_presence_of_available_resources(node, host, to_bare, origin);
+		if rostermanager.is_contact_subscribed(node, host, to_bare) then
+			core_post_stanza(origin, stanza);
+			send_presence_of_available_resources(node, host, to_bare, origin);
+		end
 		if rostermanager.is_user_subscribed(node, host, to_bare) then
 			core_post_stanza(origin, st.presence({ type = "probe", from = from_bare, to = to_bare }));
 		end
@@ -184,6 +192,8 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		-- 1. send unavailable
 		-- 2. route stanza
 		-- 3. roster push (subscription = from or both)
+		-- luacheck: ignore 211/pending_in
+		-- Is pending_in meant to be used?
 		local success, pending_in, subscribed = rostermanager.unsubscribed(node, host, to_bare);
 		if success then
 			if subscribed then
@@ -223,10 +233,16 @@ function handle_inbound_presence_subscriptions_and_probes(origin, stanza, from_b
 			if 0 == send_presence_of_available_resources(node, host, from_bare, origin) then
 				core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- TODO send last activity
 			end
+		elseif rostermanager.is_contact_preapproved(node, host, from_bare) then
+			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
+					core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="subscribed"}), true);
+				end -- TODO else return error, unable to save
+			end
 		else
 			core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt
 			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
-				if rostermanager.set_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
 					sessionmanager.send_to_available_resources(node, host, stanza);
 				end -- TODO else return error, unable to save
 			end
diff --git a/plugins/mod_proxy65.lua b/plugins/mod_proxy65.lua
index 00833772..29c821e2 100644
--- a/plugins/mod_proxy65.lua
+++ b/plugins/mod_proxy65.lua
@@ -117,7 +117,7 @@ function module.add_host(module)
 				if jid_compare(jid, acl) then allow = true; break; end
 			end
 			if allow then break; end
-			module:log("warn", "Denying use of proxy for %s", tostring(stanza.attr.from));
+			module:log("warn", "Denying use of proxy for %s", stanza.attr.from);
 			origin.send(st.error_reply(stanza, "auth", "forbidden"));
 			return true;
 		end
diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua
index 855c5fd2..faf08cb2 100644
--- a/plugins/mod_pubsub/mod_pubsub.lua
+++ b/plugins/mod_pubsub/mod_pubsub.lua
@@ -75,14 +75,13 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 	local msg_type = node_obj and node_obj.config.message_type or "headline";
 	local message = st.message({ from = module.host, type = msg_type, id = id })
 		:tag("event", { xmlns = xmlns_pubsub_event })
-			:tag(kind, { node = node })
+			:tag(kind, { node = node });
 
 	if item then
 		message:add_child(item);
 	end
 
 	local summary;
-	-- Compose a sensible textual representation of at least Atom payloads
 	if item and item.tags[1] then
 		local payload = item.tags[1];
 		summary = module:fire_event("pubsub-summary/"..payload.attr.xmlns, {
@@ -101,11 +100,12 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 end
 
 local max_max_items = module:get_option_number("pubsub_max_items", 256);
-function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node
+function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor
 	if (new_config["max_items"] or 1) > max_max_items then
 		return false;
 	end
-	if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then
+	if new_config["access_model"] ~= "whitelist"
+	and new_config["access_model"] ~= "open" then
 		return false;
 	end
 	return true;
@@ -115,6 +115,7 @@ function is_item_stanza(item)
 	return st.is_stanza(item) and item.attr.xmlns == xmlns_pubsub and item.name == "item";
 end
 
+-- Compose a textual representation of Atom payloads
 module:hook("pubsub-summary/http://www.w3.org/2005/Atom", function (event)
 	local payload = event.payload;
 	local title = payload:get_child_text("title");
diff --git a/plugins/mod_pubsub/pubsub.lib.lua b/plugins/mod_pubsub/pubsub.lib.lua
index 50ef7ddf..0938dbbc 100644
--- a/plugins/mod_pubsub/pubsub.lib.lua
+++ b/plugins/mod_pubsub/pubsub.lib.lua
@@ -7,6 +7,7 @@ local st = require "util.stanza";
 local it = require "util.iterators";
 local uuid_generate = require "util.uuid".generate;
 local dataform = require"util.dataforms".new;
+local errors = require "util.error";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
 local xmlns_pubsub_errors = "http://jabber.org/protocol/pubsub#errors";
@@ -34,6 +35,9 @@ local pubsub_errors = {
 };
 local function pubsub_error_reply(stanza, error)
 	local e = pubsub_errors[error];
+	if not e and errors.is_err(error) then
+		e = { error.type, error.condition, error.text, error.pubsub_condition };
+	end
 	local reply = st.error_reply(stanza, t_unpack(e, 1, 3));
 	if e[4] then
 		reply:tag(e[4], { xmlns = xmlns_pubsub_errors }):up();
@@ -185,6 +189,14 @@ local node_metadata_form = dataform {
 		type = "text-single";
 		name = "pubsub#type";
 	};
+	{
+		type = "text-single";
+		name = "pubsub#access_model";
+	};
+	{
+		type = "text-single";
+		name = "pubsub#publish_model";
+	};
 };
 
 local service_method_feature_map = {
@@ -258,6 +270,8 @@ function _M.handle_disco_info_node(event, service)
 			["pubsub#title"] = node_obj.config.title;
 			["pubsub#description"] = node_obj.config.description;
 			["pubsub#type"] = node_obj.config.payload_type;
+			["pubsub#access_model"] = node_obj.config.access_model;
+			["pubsub#publish_model"] = node_obj.config.publish_model;
 		}, "result"));
 	end
 end
@@ -318,14 +332,9 @@ function handlers.get_items(origin, stanza, items, service)
 	for _, id in ipairs(results) do
 		data:add_child(results[id]);
 	end
-	local reply;
-	if data then
-		reply = st.reply(stanza)
-			:tag("pubsub", { xmlns = xmlns_pubsub })
-				:add_child(data);
-	else
-		reply = pubsub_error_reply(stanza, "item-not-found");
-	end
+	local reply = st.reply(stanza)
+		:tag("pubsub", { xmlns = xmlns_pubsub })
+			:add_child(data);
 	origin.send(reply);
 	return true;
 end
@@ -633,14 +642,13 @@ function handlers.set_retract(origin, stanza, retract, service)
 end
 
 function handlers.owner_set_purge(origin, stanza, purge, service)
-	local node, notify = purge.attr.node, purge.attr.notify;
-	notify = (notify == "1") or (notify == "true");
+	local node = purge.attr.node;
 	local reply;
 	if not node then
 		origin.send(pubsub_error_reply(stanza, "nodeid-required"));
 		return true;
 	end
-	local ok, ret = service:purge(node, stanza.attr.from, notify);
+	local ok, ret = service:purge(node, stanza.attr.from, true);
 	if ok then
 		reply = st.reply(stanza);
 	else
@@ -802,6 +810,7 @@ local function archive_itemstore(archive, config, user, node)
 		end
 		module:log("debug", "Listed items %s", data);
 		return it.reverse(function()
+			-- luacheck: ignore 211/when
 			local id, payload, when, publisher = data();
 			if id == nil then
 				return;
diff --git a/plugins/mod_register_ibr.lua b/plugins/mod_register_ibr.lua
index bbe7581d..6de9bc33 100644
--- a/plugins/mod_register_ibr.lua
+++ b/plugins/mod_register_ibr.lua
@@ -25,6 +25,7 @@ end);
 local account_details = module:open_store("account_details");
 
 local field_map = {
+	FORM_TYPE = { name = "FORM_TYPE", type = "hidden", value = "jabber:iq:register" };
 	username = { name = "username", type = "text-single", label = "Username", required = true };
 	password = { name = "password", type = "text-private", label = "Password", required = true };
 	nick = { name = "nick", type = "text-single", label = "Nickname" };
@@ -50,6 +51,7 @@ local registration_form = dataform_new{
 	title = title;
 	instructions = instructions;
 
+	field_map.FORM_TYPE;
 	field_map.username;
 	field_map.password;
 };
@@ -153,7 +155,7 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	local username, password = nodeprep(data.username), data.password;
+	local username, password = nodeprep(data.username, true), data.password;
 	data.username, data.password = nil, nil;
 	local host = module.host;
 	if not username or username == "" then
@@ -166,7 +168,15 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 	module:fire_event("user-registering", user);
 	if not user.allowed then
 		log("debug", "Registration disallowed by module: %s", user.reason or "no reason given");
-		session.send(st.error_reply(stanza, "modify", "not-acceptable", user.reason));
+		local error_type, error_condition, reason;
+		local err = user.error;
+		if err then
+			error_type, error_condition, reason = err.type, err.condition, err.text;
+		else
+			-- COMPAT pre-util.error
+			error_type, error_condition, reason = user.error_type, user.error_condition, user.reason;
+		end
+		session.send(st.error_reply(stanza, error_type or "modify", error_condition or "not-acceptable", reason));
 		return true;
 	end
 
@@ -176,14 +186,13 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	-- TODO unable to write file, file may be locked, etc, what's the correct error?
-	local error_reply = st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk.");
-	if usermanager_create_user(username, password, host) then
+	local created, err = usermanager_create_user(username, password, host);
+	if created then
 		data.registered = os.time();
 		if not account_details:set(username, data) then
 			log("debug", "Could not store extra details");
 			usermanager_delete_user(username, host);
-			session.send(error_reply);
+			session.send(st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk."));
 			return true;
 		end
 		session.send(st.reply(stanza)); -- user created!
@@ -192,8 +201,8 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 			username = username, host = host, source = "mod_register",
 			session = session });
 	else
-		log("debug", "Could not create user");
-		session.send(error_reply);
+		log("debug", "Could not create user", err);
+		session.send(st.error_reply(stanza, "cancel", "feature-not-implemented", err));
 	end
 	return true;
 end);
diff --git a/plugins/mod_register_limits.lua b/plugins/mod_register_limits.lua
index 736282a5..fc9bf27a 100644
--- a/plugins/mod_register_limits.lua
+++ b/plugins/mod_register_limits.lua
@@ -13,6 +13,7 @@ local ip_util = require "util.ip";
 local new_ip = ip_util.new_ip;
 local match_ip = ip_util.match;
 local parse_cidr = ip_util.parse_cidr;
+local errors = require "util.error";
 
 local min_seconds_between_registrations = module:get_option_number("min_seconds_between_registrations");
 local whitelist_only = module:get_option_boolean("whitelist_registration_only");
@@ -54,6 +55,24 @@ local function ip_in_set(set, ip)
 	return false;
 end
 
+local err_registry = {
+	blacklisted = {
+		text = "Your IP address is blacklisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	not_whitelisted = {
+		text = "Your IP address is not whitelisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	throttled = {
+		reason = "Too many registrations from this IP address recently";
+		type = "wait";
+		condition = "policy-violation";
+	};
+}
+
 module:hook("user-registering", function (event)
 	local session = event.session;
 	local ip = event.ip or session and session.ip;
@@ -63,16 +82,22 @@ module:hook("user-registering", function (event)
 	elseif ip_in_set(blacklisted_ips, ip) then
 		log("debug", "Registration disallowed by blacklist");
 		event.allowed = false;
-		event.reason = "Your IP address is blacklisted";
+		event.error = errors.new("blacklisted", err_registry, event);
 	elseif (whitelist_only and not ip_in_set(whitelisted_ips, ip)) then
 		log("debug", "Registration disallowed by whitelist");
 		event.allowed = false;
-		event.reason = "Your IP address is not whitelisted";
+		event.error = errors.new("not_whitelisted", err_registry, event);
 	elseif throttle_max and not ip_in_set(whitelisted_ips, ip) then
 		if not check_throttle(ip) then
 			log("debug", "Registrations over limit for ip %s", ip or "?");
 			event.allowed = false;
-			event.reason = "Too many registrations from this IP address recently";
+			event.error = errors.new("throttle", err_registry, event);
 		end
 	end
+	if event.error then
+		-- COMPAT pre-util.error
+		event.reason = event.error.text;
+		event.error_type = event.error.type;
+		event.error_condition = event.error.condition;
+	end
 end);
diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s/mod_s2s.lua
index aae37b7f..00d8f5d9 100644
--- a/plugins/mod_s2s/mod_s2s.lua
+++ b/plugins/mod_s2s/mod_s2s.lua
@@ -27,8 +27,10 @@ local s2s_destroy_session = require "core.s2smanager".destroy_session;
 local uuid_gen = require "util.uuid".generate;
 local fire_global_event = prosody.events.fire_event;
 local runner = require "util.async".runner;
-
-local s2sout = module:require("s2sout");
+local connect = require "net.connect".connect;
+local service = require "net.resolvers.service";
+local errors = require "util.error";
+local set = require "util.set";
 
 local connect_timeout = module:get_option_number("s2s_timeout", 90);
 local stream_close_timeout = module:get_option_number("s2s_close_timeout", 5);
@@ -45,6 +47,8 @@ local sessions = module:shared("sessions");
 
 local runner_callbacks = {};
 
+local listener = {};
+
 local log = module._log;
 
 module:hook("stats-update", function ()
@@ -77,15 +81,28 @@ local function bounce_sendq(session, reason)
 			(session.log or log)("error", "Attempting to close the dummy origin of s2s error replies, please report this! Traceback: %s", traceback());
 		end;
 	};
+	-- FIXME Allow for more specific error conditions
+	-- TODO use util.error ?
+	local error_type = "cancel";
+	local condition = "remote-server-not-found";
+	local reason_text;
+	if session.had_stream then -- set when a stream is opened by the remote
+		error_type, condition = "wait", "remote-server-timeout";
+	end
+	if errors.is_err(reason) then
+		error_type, condition, reason_text = reason.type, reason.condition, reason.text;
+	elseif type(reason) == "string" then
+		reason_text = reason;
+	end
 	for i, data in ipairs(sendq) do
 		local reply = data[2];
 		if reply and not(reply.attr.xmlns) and bouncy_stanzas[reply.name] then
 			reply.attr.type = "error";
-			reply:tag("error", {type = "cancel", by = session.from_host})
-				:tag("remote-server-not-found", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
-			if reason then
+			reply:tag("error", {type = error_type, by = session.from_host})
+				:tag(condition, {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
+			if reason_text then
 				reply:tag("text", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"})
-					:text("Server-to-server connection failed: "..reason):up();
+					:text("Server-to-server connection failed: "..reason_text):up();
 			end
 			core_process_stanza(dummy, reply);
 		end
@@ -106,38 +123,33 @@ function route_to_existing_session(event)
 		return false;
 	end
 	local host = hosts[from_host].s2sout[to_host];
-	if host then
-		-- We have a connection to this host already
-		if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
-			(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
-
-			-- Queue stanza until we are able to send it
-			local queued_item = {
-				tostring(stanza),
-				stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
-			};
-			if host.sendq then
-				t_insert(host.sendq, queued_item);
-			else
-				-- luacheck: ignore 122
-				host.sendq = { queued_item };
-			end
-			host.log("debug", "stanza [%s] queued ", stanza.name);
-			return true;
-		elseif host.type == "local" or host.type == "component" then
-			log("error", "Trying to send a stanza to ourselves??")
-			log("error", "Traceback: %s", traceback());
-			log("error", "Stanza: %s", tostring(stanza));
-			return false;
+	if not host then return end
+
+	-- We have a connection to this host already
+	if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
+		(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
+
+		-- Queue stanza until we are able to send it
+		local queued_item = {
+			tostring(stanza),
+			stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
+		};
+		if host.sendq then
+			t_insert(host.sendq, queued_item);
 		else
-			-- FIXME
-			if host.from_host ~= from_host then
-				log("error", "WARNING! This might, possibly, be a bug, but it might not...");
-				log("error", "We are going to send from %s instead of %s", host.from_host, from_host);
-			end
-			if host.sends2s(stanza) then
-				return true;
-			end
+			-- luacheck: ignore 122
+			host.sendq = { queued_item };
+		end
+		host.log("debug", "stanza [%s] queued ", stanza.name);
+		return true;
+	elseif host.type == "local" or host.type == "component" then
+		log("error", "Trying to send a stanza to ourselves??")
+		log("error", "Traceback: %s", traceback());
+		log("error", "Stanza: %s", stanza);
+		return false;
+	else
+		if host.sends2s(stanza) then
+			return true;
 		end
 	end
 end
@@ -147,17 +159,13 @@ function route_to_new_session(event)
 	local from_host, to_host, stanza = event.from_host, event.to_host, event.stanza;
 	log("debug", "opening a new outgoing connection for this stanza");
 	local host_session = s2s_new_outgoing(from_host, to_host);
+	host_session.version = 1;
 
 	-- Store in buffer
 	host_session.bounce_sendq = bounce_sendq;
 	host_session.sendq = { {tostring(stanza), stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza)} };
-	log("debug", "stanza [%s] queued until connection complete", tostring(stanza.name));
-	s2sout.initiate_connection(host_session);
-	if (not host_session.connecting) and (not host_session.conn) then
-		log("warn", "Connection to %s failed already, destroying session...", to_host);
-		s2s_destroy_session(host_session, "Connection failed");
-		return false;
-	end
+	log("debug", "stanza [%s] queued until connection complete", stanza.name);
+	connect(service.new(to_host, "xmpp-server", "tcp", { default_port = 5269 }), listener, nil, { session = host_session });
 	return true;
 end
 
@@ -182,10 +190,20 @@ function module.add_host(module)
 			-- so the stream is ready for stanzas.  RFC 6120 Section 4.3
 			mark_connected(session);
 			return true;
+		elseif require_encryption and not session.secure then
+			session.log("warn", "Encrypted server-to-server communication is required but was not offered by %s", session.to_host);
+			session:close({
+					condition = "policy-violation",
+					text = "Encrypted server-to-server communication is required but was not offered",
+				}, nil, "Could not establish encrypted connection to remote server");
+			return true;
 		elseif not session.dialback_verifying then
 			session.log("warn", "No SASL EXTERNAL offer and Dialback doesn't seem to be enabled, giving up");
-			session:close();
-			return false;
+			session:close({
+					condition = "unsupported-feature",
+					text = "No viable authentication method offered",
+				}, nil, "No viable authentication method offered by remote server");
+			return true;
 		end
 	end, -1);
 end
@@ -203,7 +221,18 @@ function mark_connected(session)
 	if session.type == "s2sout" then
 		fire_global_event("s2sout-established", event_data);
 		hosts[from].events.fire_event("s2sout-established", event_data);
+
+		if session.incoming then
+			session.send = function(stanza)
+				return hosts[from].events.fire_event("route/remote", { from_host = from, to_host = to, stanza = stanza });
+			end;
+		end
+
 	else
+		if session.outgoing and not hosts[to].s2sout[from] then
+			session.log("debug", "Setting up to handle route from %s to %s", to, from);
+			hosts[to].s2sout[from] = session; -- luacheck: ignore 122
+		end
 		local host_session = hosts[to];
 		session.send = function(stanza)
 			return host_session.events.fire_event("route/remote", { from_host = to, to_host = from, stanza = stanza });
@@ -223,13 +252,6 @@ function mark_connected(session)
 			end
 			session.sendq = nil;
 		end
-
-		if session.resolver then
-			session.resolver._resolver:closeall()
-		end
-		session.resolver = nil;
-		session.ip_hosts = nil;
-		session.srv_hosts = nil;
 	end
 end
 
@@ -241,7 +263,7 @@ function make_authenticated(event)
 				condition = "policy-violation",
 				text = "Encrypted server-to-server communication is required but was not "
 				       ..((session.direction == "outgoing" and "offered") or "used")
-			});
+			}, nil, "Could not establish encrypted connection to remote server");
 		end
 	end
 	if hosts[host] then
@@ -251,15 +273,13 @@ function make_authenticated(event)
 		session.type = "s2sout";
 	elseif session.type == "s2sin_unauthed" then
 		session.type = "s2sin";
-		if host then
-			if not session.hosts[host] then session.hosts[host] = {}; end
-			session.hosts[host].authed = true;
-		end
-	elseif session.type == "s2sin" and host then
+	elseif session.type ~= "s2sin" and session.type ~= "s2sout" then
+		return false;
+	end
+
+	if session.incoming and host then
 		if not session.hosts[host] then session.hosts[host] = {}; end
 		session.hosts[host].authed = true;
-	else
-		return false;
 	end
 	session.log("debug", "connection %s->%s is now authenticated for %s", session.from_host, session.to_host, host);
 
@@ -301,6 +321,7 @@ end
 
 function stream_callbacks._streamopened(session, attr)
 	session.version = tonumber(attr.version) or 0;
+	session.had_stream = true; -- Had a stream opened at least once
 
 	-- TODO: Rename session.secure to session.encrypted
 	if session.secure == false then
@@ -314,7 +335,6 @@ function stream_callbacks._streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -322,7 +342,9 @@ function stream_callbacks._streamopened(session, attr)
 		-- Send a reply stream header
 
 		-- Validate to/from
-		local to, from = nameprep(attr.to), nameprep(attr.from);
+		local to, from = attr.to, attr.from;
+		if to then to = nameprep(attr.to); end
+		if from then from = nameprep(attr.from); end
 		if not to and attr.to then -- COMPAT: Some servers do not reliably set 'to' (especially on stream restarts)
 			session:close({ condition = "improper-addressing", text = "Invalid 'to' address" });
 			return;
@@ -416,20 +438,6 @@ function stream_callbacks._streamopened(session, attr)
 			end
 		end
 
-		-- Send unauthed buffer
-		-- (stanzas which are fine to send before dialback)
-		-- Note that this is *not* the stanza queue (which
-		-- we can only send if auth succeeds) :)
-		local send_buffer = session.send_buffer;
-		if send_buffer and #send_buffer > 0 then
-			log("debug", "Sending s2s send_buffer now...");
-			for i, data in ipairs(send_buffer) do
-				session.sends2s(tostring(data));
-				send_buffer[i] = nil;
-			end
-		end
-		session.send_buffer = nil;
-
 		-- If server is pre-1.0, don't wait for features, just do dialback
 		if session.version < 1.0 then
 			if not session.dialback_verifying then
@@ -471,11 +479,9 @@ function stream_callbacks.error(session, error, data)
 	end
 end
 
-local listener = {};
-
 --- Session methods
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
-local function session_close(session, reason, remote_reason)
+local function session_close(session, reason, remote_reason, bounce_reason)
 	local log = session.log or log;
 	if session.conn then
 		if session.notopen then
@@ -486,28 +492,24 @@ local function session_close(session, reason, remote_reason)
 			end
 		end
 		if reason then -- nil == no err, initiated by us, false == initiated by remote
+			local stream_error;
 			if type(reason) == "string" then -- assume stream error
-				log("debug", "Disconnecting %s[%s], <stream:error> is: %s", session.host or session.ip or "(unknown host)", session.type, reason);
-				session.sends2s(st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' }));
-			elseif type(reason) == "table" then
-				if reason.condition then
-					local stanza = st.stanza("stream:error"):tag(reason.condition, stream_xmlns_attr):up();
-					if reason.text then
-						stanza:tag("text", stream_xmlns_attr):text(reason.text):up();
-					end
-					if reason.extra then
-						stanza:add_child(reason.extra);
-					end
-					log("debug", "Disconnecting %s[%s], <stream:error> is: %s",
-					session.host or session.ip or "(unknown host)", session.type, stanza);
-					session.sends2s(stanza);
-				elseif reason.name then -- a stanza
-					log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
-						session.from_host or "(unknown host)", session.to_host or "(unknown host)",
-						session.type, reason);
-					session.sends2s(reason);
+				stream_error = st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' });
+			elseif type(reason) == "table" and not st.is_stanza(reason) then
+				stream_error = st.stanza("stream:error"):tag(reason.condition or "undefined-condition", stream_xmlns_attr):up();
+				if reason.text then
+					stream_error:tag("text", stream_xmlns_attr):text(reason.text):up();
+				end
+				if reason.extra then
+					stream_error:add_child(reason.extra);
 				end
 			end
+			if st.is_stanza(stream_error) then
+				-- to and from are never unknown on outgoing connections
+				log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
+					session.from_host or "(unknown host)" or session.ip, session.to_host or "(unknown host)", session.type, reason);
+				session.sends2s(stream_error);
+			end
 		end
 
 		session.sends2s("</stream:stream>");
@@ -521,16 +523,16 @@ local function session_close(session, reason, remote_reason)
 
 		-- Authenticated incoming stream may still be sending us stanzas, so wait for </stream:stream> from remote
 		local conn = session.conn;
-		if reason == nil and not session.notopen and session.type == "s2sin" then
+		if reason == nil and not session.notopen and session.incoming then
 			add_task(stream_close_timeout, function ()
 				if not session.destroyed then
 					session.log("warn", "Failed to receive a stream close response, closing connection anyway...");
-					s2s_destroy_session(session, reason);
+					s2s_destroy_session(session, reason, bounce_reason);
 					conn:close();
 				end
 			end);
 		else
-			s2s_destroy_session(session, reason);
+			s2s_destroy_session(session, reason, bounce_reason);
 			conn:close(); -- Close immediately, as this is an outgoing connection or is not authed
 		end
 	end
@@ -595,9 +597,8 @@ local function initialize_session(session)
 		if data then
 			local ok, err = stream:feed(data);
 			if ok then return; end
-			log("warn", "Received invalid XML: %s", data);
-			log("warn", "Problem was: %s", err);
-			session:close("not-well-formed");
+			log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+			session:close("not-well-formed", nil, "Received invalid XML from remote server");
 		end
 	end
 
@@ -672,11 +673,20 @@ function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
 		sessions[conn] = nil;
+		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
+		if session.secure == false and err then
+			-- TODO util.error-ify this
+			err = "Error during negotiation of encrypted connection: "..err;
+		end
+		s2s_destroy_session(session, err);
+	end
+end
+
+function listener.onfail(data, err)
+	local session = data and data.session;
+	if session then
 		if err and session.direction == "outgoing" and session.notopen then
 			(session.log or log)("debug", "s2s connection attempt failed: %s", err);
-			if s2sout.attempt_connection(session, err) then
-				return; -- Session lives for now
-			end
 		end
 		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
 		s2s_destroy_session(session, err);
@@ -700,6 +710,34 @@ function listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
+function listener.onattach(conn, data)
+	local session = data and data.session;
+	if session then
+		session.conn = conn;
+		sessions[conn] = session;
+		initialize_session(session);
+	end
+end
+
+-- Complete the sentence "Your certificate " with what's wrong
+local function friendly_cert_error(session) --> string
+	if session.cert_chain_status == "invalid" then
+		if session.cert_chain_errors then
+			local cert_errors = set.new(session.cert_chain_errors[1]);
+			if cert_errors:contains("certificate has expired") then
+				return "has expired";
+			elseif cert_errors:contains("self signed certificate") then
+				return "is self-signed";
+			end
+		end
+		return "is not trusted"; -- for some other reason
+	elseif session.cert_identity_status == "invalid" then
+		return "is not valid for this name";
+	end
+	-- this should normally be unreachable except if no s2s auth module was loaded
+	return "could not be validated";
+end
+
 function check_auth_policy(event)
 	local host, session = event.host, event.session;
 	local must_secure = secure_auth;
@@ -711,20 +749,21 @@ function check_auth_policy(event)
 	end
 
 	if must_secure and (session.cert_chain_status ~= "valid" or session.cert_identity_status ~= "valid") then
-		module:log("warn", "Forbidding insecure connection to/from %s", host or session.ip or "(unknown host)");
-		if session.direction == "incoming" then
-			session:close({ condition = "not-authorized", text = "Your server's certificate is invalid, expired, or not trusted by "..session.to_host });
-		else -- Close outgoing connections without warning
-			session:close(false);
-		end
+		local reason = friendly_cert_error(session);
+		session.log("warn", "Forbidding insecure connection to/from %s because its certificate %s", host or session.ip or "(unknown host)", reason);
+		-- XEP-0178 recommends closing outgoing connections without warning
+		-- but does not give a rationale for this.
+		-- In practice most cases are configuration mistakes or forgotten
+		-- certificate renewals. We think it's better to let the other party
+		-- know about the problem so that they can fix it.
+		session:close({ condition = "not-authorized", text = "Your server's certificate "..reason },
+			nil, "Remote server's certificate "..reason);
 		return false;
 	end
 end
 
 module:hook("s2s-check-certificate", check_auth_policy, -1);
 
-s2sout.set_listener(listener);
-
 module:hook("server-stopping", function(event)
 	local reason = event.reason;
 	for _, session in pairs(sessions) do
@@ -739,7 +778,11 @@ module:provides("net", {
 	listener = listener;
 	default_port = 5269;
 	encryption = "starttls";
+	ssl_config = { -- FIXME This is not used atm, see mod_tls
+		verify = { "peer", "client_once", };
+	};
 	multiplex = {
+		protocol = "xmpp-server";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>";
 	};
 });
diff --git a/plugins/mod_s2s/s2sout.lib.lua b/plugins/mod_s2s/s2sout.lib.lua
deleted file mode 100644
index 5f765da8..00000000
--- a/plugins/mod_s2s/s2sout.lib.lua
+++ /dev/null
@@ -1,349 +0,0 @@
--- Prosody IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
-
---- Module containing all the logic for connecting to a remote server
-
--- luacheck: ignore 432/err
-
-local portmanager = require "core.portmanager";
-local wrapclient = require "net.server".wrapclient;
-local initialize_filters = require "util.filters".initialize;
-local idna_to_ascii = require "util.encodings".idna.to_ascii;
-local new_ip = require "util.ip".new_ip;
-local rfc6724_dest = require "util.rfc6724".destination;
-local socket = require "socket";
-local adns = require "net.adns";
-local t_insert, t_sort, ipairs = table.insert, table.sort, ipairs;
-local local_addresses = require "util.net".local_addresses;
-
-local s2s_destroy_session = require "core.s2smanager".destroy_session;
-
-local default_mode = module:get_option("network_default_read_size", 4096);
-
-local log = module._log;
-
-local sources = {};
-local has_ipv4, has_ipv6;
-
-local dns_timeout = module:get_option_number("dns_timeout", 15);
-local resolvers = module:get_option_set("s2s_dns_resolvers")
-
-local s2sout = {};
-
-local s2s_listener;
-
-
-function s2sout.set_listener(listener)
-	s2s_listener = listener;
-end
-
-local function compare_srv_priorities(a,b)
-	return a.priority < b.priority or (a.priority == b.priority and a.weight > b.weight);
-end
-
-function s2sout.initiate_connection(host_session)
-	local log = host_session.log or log;
-
-	initialize_filters(host_session);
-	host_session.version = 1;
-
-	host_session.resolver = adns.resolver();
-	host_session.resolver._resolver:settimeout(dns_timeout);
-	if resolvers then
-		for resolver in resolvers do
-			host_session.resolver._resolver:addnameserver(resolver);
-		end
-	end
-
-	-- Kick the connection attempting machine into life
-	if not s2sout.attempt_connection(host_session) then
-		-- Intentionally not returning here, the
-		-- session is needed, connected or not
-		s2s_destroy_session(host_session);
-	end
-
-	if not host_session.sends2s then
-		-- A sends2s which buffers data (until the stream is opened)
-		-- note that data in this buffer will be sent before the stream is authed
-		-- and will not be ack'd in any way, successful or otherwise
-		local buffer;
-		function host_session.sends2s(data)
-			if not buffer then
-				buffer = {};
-				host_session.send_buffer = buffer;
-			end
-			log("debug", "Buffering data on unconnected s2sout to %s", host_session.to_host);
-			buffer[#buffer+1] = data;
-			log("debug", "Buffered item %d: %s", #buffer, data);
-		end
-	end
-end
-
-function s2sout.attempt_connection(host_session, err)
-	local to_host = host_session.to_host;
-	local connect_host, connect_port = to_host and idna_to_ascii(to_host), 5269;
-	local log = host_session.log or log;
-
-	if not connect_host then
-		return false;
-	end
-
-	if not err then -- This is our first attempt
-		log("debug", "First attempt to connect to %s, starting with SRV lookup...", to_host);
-		host_session.connecting = true;
-		host_session.resolver:lookup(function (answer)
-			local srv_hosts = { answer = answer };
-			host_session.srv_hosts = srv_hosts;
-			host_session.srv_choice = 0;
-			host_session.connecting = nil;
-			if answer and #answer > 0 then
-				log("debug", "%s has SRV records, handling...", to_host);
-				for _, record in ipairs(answer) do
-					t_insert(srv_hosts, record.srv);
-				end
-				if #srv_hosts == 1 and srv_hosts[1].target == "." then
-					log("debug", "%s does not provide a XMPP service", to_host);
-					s2s_destroy_session(host_session, err); -- Nothing to see here
-					return;
-				end
-				t_sort(srv_hosts, compare_srv_priorities);
-
-				local srv_choice = srv_hosts[1];
-				host_session.srv_choice = 1;
-				if srv_choice then
-					connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-					log("debug", "Best record found, will connect to %s:%d", connect_host, connect_port);
-				end
-			else
-				log("debug", "%s has no SRV records, falling back to A/AAAA", to_host);
-			end
-			-- Try with SRV, or just the plain hostname if no SRV
-			local ok, err = s2sout.try_connect(host_session, connect_host, connect_port);
-			if not ok then
-				if not s2sout.attempt_connection(host_session, err) then
-					-- No more attempts will be made
-					s2s_destroy_session(host_session, err);
-				end
-			end
-		end, "_xmpp-server._tcp."..connect_host..".", "SRV");
-
-		return true; -- Attempt in progress
-	elseif host_session.ip_hosts then
-		return s2sout.try_connect(host_session, connect_host, connect_port, err);
-	elseif host_session.srv_hosts and #host_session.srv_hosts > host_session.srv_choice then -- Not our first attempt, and we also have SRV
-		host_session.srv_choice = host_session.srv_choice + 1;
-		local srv_choice = host_session.srv_hosts[host_session.srv_choice];
-		connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-		host_session.log("info", "Connection failed (%s). Attempt #%d: This time to %s:%d", err, host_session.srv_choice, connect_host, connect_port);
-	else
-		host_session.log("info", "Failed in all attempts to connect to %s", host_session.to_host);
-		-- We're out of options
-		return false;
-	end
-
-	if not (connect_host and connect_port) then
-		-- Likely we couldn't resolve DNS
-		log("warn", "Hmm, we're without a host (%s) and port (%s) to connect to for %s, giving up :(", connect_host, connect_port, to_host);
-		return false;
-	end
-
-	return s2sout.try_connect(host_session, connect_host, connect_port);
-end
-
-function s2sout.try_next_ip(host_session)
-	host_session.connecting = nil;
-	host_session.ip_choice = host_session.ip_choice + 1;
-	local ip = host_session.ip_hosts[host_session.ip_choice];
-	local ok, err= s2sout.make_connect(host_session, ip.ip, ip.port);
-	if not ok then
-		if not s2sout.attempt_connection(host_session, err or "closed") then
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connection failed"..err);
-		end
-	end
-end
-
-function s2sout.try_connect(host_session, connect_host, connect_port, err)
-	host_session.connecting = true;
-	local log = host_session.log or log;
-
-	if not err then
-		local IPs = {};
-		host_session.ip_hosts = IPs;
-		-- luacheck: ignore 231/handle4 231/handle6
-		local handle4, handle6;
-		local have_other_result = not(has_ipv4) or not(has_ipv6) or false;
-
-		if has_ipv4 then
-			handle4 = host_session.resolver:lookup(function (reply, err)
-				handle4 = nil;
-
-				if reply and reply[#reply] and reply[#reply].a then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.a);
-						IPs[#IPs+1] = new_ip(ip.a, "IPv4");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "A", "IN");
-		else
-			have_other_result = true;
-		end
-
-		if has_ipv6 then
-			handle6 = host_session.resolver:lookup(function (reply, err)
-				handle6 = nil;
-
-				if reply and reply[#reply] and reply[#reply].aaaa then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.aaaa);
-						IPs[#IPs+1] = new_ip(ip.aaaa, "IPv6");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "AAAA", "IN");
-		else
-			have_other_result = true;
-		end
-		return true;
-	elseif host_session.ip_hosts and #host_session.ip_hosts > host_session.ip_choice then -- Not our first attempt, and we also have IPs left to try
-		s2sout.try_next_ip(host_session);
-	else
-		log("debug", "Out of IP addresses, trying next SRV record (if any)");
-		host_session.ip_hosts = nil;
-		if not s2sout.attempt_connection(host_session, "out of IP addresses") then -- Retry if we can
-			log("debug", "No other records to try for %s - destroying", host_session.to_host);
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connecting failed"..err); -- End of the line, we can't
-			return false;
-		end
-	end
-
-	return true;
-end
-
-function s2sout.make_connect(host_session, connect_host, connect_port)
-	local log = host_session.log or log;
-	log("debug", "Beginning new connection attempt to %s ([%s]:%d)", host_session.to_host, connect_host.addr, connect_port);
-
-	-- Reset secure flag in case this is another
-	-- connection attempt after a failed STARTTLS
-	host_session.secure = nil;
-	host_session.encrypted = nil;
-
-	local conn, handler;
-	local proto = connect_host.proto;
-	if proto == "IPv4" then
-		conn, handler = socket.tcp();
-	elseif proto == "IPv6" and socket.tcp6 then
-		conn, handler = socket.tcp6();
-	else
-		handler = "Unsupported protocol: "..tostring(proto);
-	end
-
-	if not conn then
-		log("warn", "Failed to create outgoing connection, system error: %s", handler);
-		return false, handler;
-	end
-
-	conn:settimeout(0);
-	local success, err = conn:connect(connect_host.addr, connect_port);
-	if not success and err ~= "timeout" then
-		log("warn", "s2s connect() to %s (%s:%d) failed: %s", host_session.to_host, connect_host.addr, connect_port, err);
-		return false, err;
-	end
-
-	conn = wrapclient(conn, connect_host.addr, connect_port, s2s_listener, default_mode);
-	host_session.conn = conn;
-
-	-- Register this outgoing connection so that xmppserver_listener knows about it
-	-- otherwise it will assume it is a new incoming connection
-	s2s_listener.register_outgoing(conn, host_session);
-
-	log("debug", "Connection attempt in progress...");
-	return true;
-end
-
-module:hook_global("service-added", function (event)
-	if event.name ~= "s2s" then return end
-
-	local s2s_sources = portmanager.get_active_services():get("s2s");
-	if not s2s_sources then
-		module:log("warn", "s2s not listening on any ports, outgoing connections may fail");
-		return;
-	end
-	for source, _ in pairs(s2s_sources) do
-		if source == "*" or source == "0.0.0.0" then
-			for _, addr in ipairs(local_addresses("ipv4", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv4");
-			end
-		elseif source == "::" then
-			for _, addr in ipairs(local_addresses("ipv6", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv6");
-			end
-		else
-			sources[#sources + 1] = new_ip(source, (source:find(":") and "IPv6") or "IPv4");
-		end
-	end
-	for i = 1,#sources do
-		if sources[i].proto == "IPv6" then
-			has_ipv6 = true;
-		elseif sources[i].proto == "IPv4" then
-			has_ipv4 = true;
-		end
-	end
-	if not (has_ipv4 or has_ipv6)  then
-		module:log("warn", "No local IPv4 or IPv6 addresses detected, outgoing connections may fail");
-	end
-end);
-
-return s2sout;
diff --git a/plugins/mod_s2s_auth_certs.lua b/plugins/mod_s2s_auth_certs.lua
index dd0eb3cb..37519aa1 100644
--- a/plugins/mod_s2s_auth_certs.lua
+++ b/plugins/mod_s2s_auth_certs.lua
@@ -17,9 +17,6 @@ module:hook("s2s-check-certificate", function(event)
 	local chain_valid, errors;
 	if conn.getpeerverification then
 		chain_valid, errors = conn:getpeerverification();
-	elseif conn.getpeerchainvalid then -- COMPAT mw/luasec-hg
-		chain_valid, errors = conn:getpeerchainvalid();
-		errors = (not chain_valid) and { { errors } } or nil;
 	else
 		chain_valid, errors = false, { { "Chain verification not supported by this version of LuaSec" } };
 	end
@@ -30,6 +27,7 @@ module:hook("s2s-check-certificate", function(event)
 			log("debug", "certificate error(s) at depth %d: %s", depth-1, table.concat(t, ", "))
 		end
 		session.cert_chain_status = "invalid";
+		session.cert_chain_errors = errors;
 	else
 		log("debug", "certificate chain validation result: valid");
 		session.cert_chain_status = "valid";
diff --git a/plugins/mod_s2s_bidi.lua b/plugins/mod_s2s_bidi.lua
new file mode 100644
index 00000000..28e047de
--- /dev/null
+++ b/plugins/mod_s2s_bidi.lua
@@ -0,0 +1,40 @@
+-- Prosody IM
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local xmlns_bidi_feature = "urn:xmpp:features:bidi"
+local xmlns_bidi = "urn:xmpp:bidi";
+
+local require_encryption = module:get_option_boolean("s2s_require_encryption", false);
+
+module:hook("s2s-stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.type == "s2sin_unauthed" and (not require_encryption or origin.secure) then
+		features:tag("bidi", { xmlns = xmlns_bidi_feature }):up();
+	end
+end);
+
+module:hook_tag("http://etherx.jabber.org/streams", "features", function (session, stanza)
+	if session.type == "s2sout_unauthed" and (not require_encryption or session.secure) then
+		local bidi = stanza:get_child("bidi", xmlns_bidi_feature);
+		if bidi then
+			session.incoming = true;
+			session.log("debug", "Requesting bidirectional stream");
+			session.sends2s(st.stanza("bidi", { xmlns = xmlns_bidi }));
+		end
+	end
+end, 200);
+
+module:hook_tag("urn:xmpp:bidi", "bidi", function(session)
+	if session.type == "s2sin_unauthed" and (not require_encryption or session.secure) then
+		session.log("debug", "Requested bidirectional stream");
+		session.outgoing = true;
+		return true;
+	end
+end);
+
diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua
index fba84ef8..ecce8361 100644
--- a/plugins/mod_saslauth.lua
+++ b/plugins/mod_saslauth.lua
@@ -12,9 +12,10 @@ local st = require "util.stanza";
 local sm_bind_resource = require "core.sessionmanager".bind_resource;
 local sm_make_authenticated = require "core.sessionmanager".make_authenticated;
 local base64 = require "util.encodings".base64;
+local set = require "util.set";
+local errors = require "util.error";
 
 local usermanager_get_sasl_handler = require "core.usermanager".get_sasl_handler;
-local tostring = tostring;
 
 local secure_auth_only = module:get_option_boolean("c2s_require_encryption", module:get_option_boolean("require_encryption", false));
 local allow_unencrypted_plain_auth = module:get_option_boolean("allow_unencrypted_plain_auth", false)
@@ -67,7 +68,6 @@ local function sasl_process_cdata(session, stanza)
 	local text = stanza[1];
 	if text then
 		text = base64.decode(text);
-		--log("debug", "AUTH: %s", text:gsub("[%z\001-\008\011\012\014-\031]", " "));
 		if not text then
 			session.sasl_handler = nil;
 			session.send(build_reply("failure", "incorrect-encoding"));
@@ -77,7 +77,6 @@ local function sasl_process_cdata(session, stanza)
 	local status, ret, err_msg = session.sasl_handler:process(text);
 	status, ret, err_msg = handle_status(session, status, ret, err_msg);
 	local s = build_reply(status, ret, err_msg);
-	log("debug", "sasl reply: %s", tostring(s));
 	session.send(s);
 	return true;
 end
@@ -104,18 +103,27 @@ module:hook_tag(xmlns_sasl, "failure", function (session, stanza)
 			break;
 		end
 	end
-	if text and condition then
-		condition = condition .. ": " .. text;
-	end
-	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, condition);
+	local err = errors.new({
+			-- TODO type = what?
+			text = text,
+			condition = condition,
+		}, {
+			session = session,
+			stanza = stanza,
+		});
+
+	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, err);
 
 	session.external_auth = "failed"
-	session.external_auth_failure_reason = condition;
+	session.external_auth_failure_reason = err;
 end, 500)
 
 module:hook_tag(xmlns_sasl, "failure", function (session, stanza) -- luacheck: ignore 212/stanza
 	session.log("debug", "No fallback from SASL EXTERNAL failure, giving up");
-	session:close(nil, session.external_auth_failure_reason);
+	session:close(nil, session.external_auth_failure_reason, errors.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "Could not authenticate to remote server",
+		}, { session = session, sasl_failure = session.external_auth_failure_reason, }));
 	return true;
 end, 90)
 
@@ -248,37 +256,72 @@ module:hook("stream-features", function(event)
 		local sasl_handler = usermanager_get_sasl_handler(module.host, origin)
 		origin.sasl_handler = sasl_handler;
 		if origin.encrypted then
-			-- check wether LuaSec has the nifty binding to the function needed for tls-unique
+			-- check whether LuaSec has the nifty binding to the function needed for tls-unique
 			-- FIXME: would be nice to have this check only once and not for every socket
 			if sasl_handler.add_cb_handler then
 				local socket = origin.conn:socket();
 				if socket.getpeerfinished then
+					log("debug", "Channel binding 'tls-unique' supported");
 					sasl_handler:add_cb_handler("tls-unique", tls_unique);
+				else
+					log("debug", "Channel binding 'tls-unique' not supported (by LuaSec?)");
 				end
 				sasl_handler["userdata"] = {
 					["tls-unique"] = socket;
 				};
+			else
+				log("debug", "Channel binding not supported by SASL handler");
 			end
 		end
 		local mechanisms = st.stanza("mechanisms", mechanisms_attr);
 		local sasl_mechanisms = sasl_handler:mechanisms()
+		local available_mechanisms = set.new();
 		for mechanism in pairs(sasl_mechanisms) do
-			if disabled_mechanisms:contains(mechanism) then
-				log("debug", "Not offering disabled mechanism %s", mechanism);
-			elseif not origin.secure and insecure_mechanisms:contains(mechanism) then
-				log("debug", "Not offering mechanism %s on insecure connection", mechanism);
-			else
-				log("debug", "Offering mechanism %s", mechanism);
+			available_mechanisms:add(mechanism);
+		end
+		log("debug", "SASL mechanisms supported by handler: %s", available_mechanisms);
+
+		local usable_mechanisms = available_mechanisms - disabled_mechanisms;
+
+		local available_disabled = set.intersection(available_mechanisms, disabled_mechanisms);
+		if not available_disabled:empty() then
+			log("debug", "Not offering disabled mechanisms: %s", available_disabled);
+		end
+
+		local available_insecure = set.intersection(available_mechanisms, insecure_mechanisms);
+		if not origin.secure and not available_insecure:empty() then
+			log("debug", "Session is not secure, not offering insecure mechanisms: %s", available_insecure);
+			usable_mechanisms = usable_mechanisms - insecure_mechanisms;
+		end
+
+		if not usable_mechanisms:empty() then
+			log("debug", "Offering usable mechanisms: %s", usable_mechanisms);
+			for mechanism in usable_mechanisms do
 				mechanisms:tag("mechanism"):text(mechanism):up();
 			end
-		end
-		if mechanisms[1] then
 			features:add_child(mechanisms);
-		elseif not next(sasl_mechanisms) then
-			log("warn", "No available SASL mechanisms, verify that the configured authentication module is working");
-		else
-			log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection");
+			return;
+		end
+
+		local authmod = module:get_option_string("authentication", "internal_plain");
+		if available_mechanisms:empty() then
+			log("warn", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod);
+			return;
 		end
+
+		if not origin.secure and not available_insecure:empty() then
+			if not available_disabled:empty() then
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s) or disabled (%s)",
+					authmod, available_insecure, available_disabled);
+			else
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s)",
+					authmod, available_insecure);
+			end
+		elseif not available_disabled:empty() then
+			log("warn", "All SASL mechanisms provided by authentication module '%s' are disabled (%s)",
+				authmod, available_disabled);
+		end
+
 	else
 		features:tag("bind", bind_attr):tag("required"):up():up();
 		features:tag("session", xmpp_session_attr):tag("optional"):up():up();
diff --git a/plugins/mod_stanza_debug.lua b/plugins/mod_stanza_debug.lua
index 6dedb6f7..af98670c 100644
--- a/plugins/mod_stanza_debug.lua
+++ b/plugins/mod_stanza_debug.lua
@@ -1,18 +1,17 @@
 module:set_global();
 
-local tostring = tostring;
 local filters = require "util.filters";
 
 local function log_send(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "SEND: %s", tostring(t));
+		session.log("debug", "SEND: %s", t);
 	end
 	return t;
 end
 
 local function log_recv(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "RECV: %s", tostring(t));
+		session.log("debug", "RECV: %s", t);
 	end
 	return t;
 end
diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua
index 42b451bd..c8b902cf 100644
--- a/plugins/mod_storage_internal.lua
+++ b/plugins/mod_storage_internal.lua
@@ -1,12 +1,17 @@
+local cache = require "util.cache";
 local datamanager = require "core.storagemanager".olddm;
 local array = require "util.array";
 local datetime = require "util.datetime";
 local st = require "util.stanza";
 local now = require "util.time".now;
 local id = require "util.id".medium;
+local jid_join = require "util.jid".join;
 
 local host = module.host;
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
 local driver = {};
 
 function driver:open(store, typ)
@@ -43,6 +48,12 @@ end
 local archive = {};
 driver.archive = { __index = archive };
 
+archive.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive:append(username, key, value, when, with)
 	when = when or now();
 	if not st.is_stanza(value) then
@@ -54,28 +65,57 @@ function archive:append(username, key, value, when, with)
 	value.attr.stamp = datetime.datetime(when);
 	value.attr.stamp_legacy = datetime.legacy(when);
 
+	local cache_key = jid_join(username, host, self.store);
+	local item_count = archive_item_count_cache:get(cache_key);
+
 	if key then
 		local items, err = datamanager.list_load(username, host, self.store);
 		if not items and err then return items, err; end
+
+		-- Check the quota
+		item_count = items and #items or 0;
+		archive_item_count_cache:set(cache_key, item_count);
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
+
 		if items then
+			-- Filter out any item with the same key as the one being added
 			items = array(items);
 			items:filter(function (item)
 				return item.key ~= key;
 			end);
+
 			value.key = key;
 			items:push(value);
 			local ok, err = datamanager.list_store(username, host, self.store, items);
 			if not ok then return ok, err; end
+			archive_item_count_cache:set(cache_key, #items);
 			return key;
 		end
 	else
+		if not item_count then -- Item count not cached?
+			-- We need to load the list to get the number of items currently stored
+			local items, err = datamanager.list_load(username, host, self.store);
+			if not items and err then return items, err; end
+			item_count = items and #items or 0;
+			archive_item_count_cache:set(cache_key, item_count);
+		end
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
 		key = id();
 	end
 
+	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
+
 	value.key = key;
 
 	local ok, err = datamanager.list_append(username, host, self.store, value);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, item_count+1);
 	return key;
 end
 
@@ -84,11 +124,17 @@ function archive:find(username, query)
 	if not items then
 		if err then
 			return items, err;
-		else
-			return function () end, 0;
+		elseif query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
 		end
+		return function () end;
 	end
-	local count = #items;
+	local count = nil;
 	local i = 0;
 	if query then
 		items = array(items);
@@ -112,24 +158,36 @@ function archive:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -156,8 +214,37 @@ function archive:dates(username)
 	return array(items):pluck("when"):map(datetime.date):unique();
 end
 
+function archive:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	local body = {};
+	for _, stanza, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+		body[with] = stanza:get_child_text("body") or body[with];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+		body = body;
+	};
+end
+
+function archive:users()
+	return datamanager.users(host, self.store, "list");
+end
+
 function archive:delete(username, query)
+	local cache_key = jid_join(username, host, self.store);
 	if not query or next(query) == nil then
+		archive_item_count_cache:set(cache_key, nil);
 		return datamanager.list_store(username, host, self.store, nil);
 	end
 	local items, err = datamanager.list_load(username, host, self.store);
@@ -165,6 +252,7 @@ function archive:delete(username, query)
 		if err then
 			return items, err;
 		end
+		archive_item_count_cache:set(cache_key, 0);
 		-- Store is empty
 		return 0;
 	end
@@ -214,6 +302,7 @@ function archive:delete(username, query)
 	end
 	local ok, err = datamanager.list_store(username, host, self.store, items);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, #items);
 	return count;
 end
 
diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua
index 745e394b..8beb8c01 100644
--- a/plugins/mod_storage_memory.lua
+++ b/plugins/mod_storage_memory.lua
@@ -8,6 +8,8 @@ local new_id = require "util.id".medium;
 local auto_purge_enabled = module:get_option_boolean("storage_memory_temporary", false);
 local auto_purge_stores = module:get_option_set("storage_memory_temporary_stores", {});
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000);
+
 local memory = setmetatable({}, {
 	__index = function(t, k)
 		local store = module:shared(k)
@@ -51,6 +53,12 @@ archive_store.__index = archive_store;
 
 archive_store.users = _users;
 
+archive_store.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive_store:append(username, key, value, when, with)
 	if is_stanza(value) then
 		value = st.preserialize(value);
@@ -70,6 +78,8 @@ function archive_store:append(username, key, value, when, with)
 	end
 	if a[key] then
 		table.remove(a, a[key]);
+	elseif #a >= archive_item_limit then
+		return nil, "quota-limit";
 	end
 	local i = #a+1;
 	a[i] = v;
@@ -80,9 +90,17 @@ end
 function archive_store:find(username, query)
 	local items = self.store[username or NULL];
 	if not items then
-		return function () end, 0;
+		if query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
+		end
+		return function () end;
 	end
-	local count = #items;
+	local count = nil;
 	local i = 0;
 	if query then
 		items = array():append(items);
@@ -106,24 +124,36 @@ function archive_store:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -137,6 +167,26 @@ function archive_store:find(username, query)
 	end, count;
 end
 
+function archive_store:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	for _, _, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 
 function archive_store:delete(username, query)
 	if not query or next(query) == nil then
diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua
index a449091e..666cee41 100644
--- a/plugins/mod_storage_sql.lua
+++ b/plugins/mod_storage_sql.lua
@@ -1,17 +1,19 @@
 
 -- luacheck: ignore 212/self
 
+local cache = require "util.cache";
 local json = require "util.json";
 local sql = require "util.sql";
 local xml_parse = require "util.xml".parse;
 local uuid = require "util.uuid";
 local resolve_relative_path = require "util.paths".resolve_relative_path;
+local jid_join = require "util.jid".join;
 
 local is_stanza = require"util.stanza".is_stanza;
 local t_concat = table.concat;
 
 local noop = function() end
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local function iterator(result)
 	return function(result_)
 		local row = result_();
@@ -148,7 +150,10 @@ end
 
 --- Archive store API
 
--- luacheck: ignore 512 431/user 431/store
+local archive_item_limit = module:get_option_number("storage_archive_item_limit");
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
+-- luacheck: ignore 512 431/user 431/store 431/err
 local map_store = {};
 map_store.__index = map_store;
 map_store.remove = {};
@@ -228,10 +233,41 @@ end
 local archive_store = {}
 archive_store.caps = {
 	total = true;
+	quota = archive_item_limit;
+	truncate = true;
 };
 archive_store.__index = archive_store
 function archive_store:append(username, key, value, when, with)
 	local user,store = username,self.store;
+	local cache_key = jid_join(username, host, store);
+	local item_count = archive_item_count_cache:get(cache_key);
+	if not item_count then
+		local ok, ret = engine:transaction(function()
+			local count_sql = [[
+			SELECT COUNT(*) FROM "prosodyarchive"
+			WHERE "host"=? AND "user"=? AND "store"=?;
+			]];
+			local result = engine:select(count_sql, host, user, store);
+			if result then
+				for row in result do
+					item_count = row[1];
+				end
+			end
+		end);
+		if not ok or not item_count then
+			module:log("error", "Failed while checking quota for %s: %s", username, ret);
+			return nil, "Failure while checking quota";
+		end
+		archive_item_count_cache:set(cache_key, item_count);
+	end
+
+	if archive_item_limit then
+		module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit);
+		if item_count >= archive_item_limit then
+			return nil, "quota-limit";
+		end
+	end
+
 	when = when or os.time();
 	with = with or "";
 	local ok, ret = engine:transaction(function()
@@ -245,12 +281,16 @@ function archive_store:append(username, key, value, when, with)
 		VALUES (?,?,?,?,?,?,?,?);
 		]];
 		if key then
-			engine:delete(delete_sql, host, user or "", store, key);
+			local result, err = engine:delete(delete_sql, host, user or "", store, key);
+			if result then
+				item_count = item_count - result:affected();
+			end
 		else
 			key = uuid.generate();
 		end
 		local t, encoded_value = assert(serialize(value));
 		engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value);
+		archive_item_count_cache:set(cache_key, item_count+1);
 		return key;
 	end);
 	if not ok then return ok, ret; end
@@ -287,45 +327,47 @@ local function archive_where(query, args, where)
 	end
 end
 local function archive_where_id_range(query, args, where)
-	local args_len = #args
 	-- Before or after specific item, exclusive
+	local id_lookup_sql = [[
+	SELECT "sort_id"
+	FROM "prosodyarchive"
+	WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
+	LIMIT 1;
+	]];
 	if query.after then  -- keys better be unique!
-		where[#where+1] = [[
-		"sort_id" > COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			), 0)
-		]];
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.after, args[1], args[2], args[3];
-		args_len = args_len + 4
+		local after_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			after_id = row[1];
+		end
+		if not after_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" > ?';
+		args[#args+1] = after_id;
 	end
 	if query.before then
-		where[#where+1] = [[
-		"sort_id" < COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			),
-			(
-				SELECT MAX("sort_id")+1
-				FROM "prosodyarchive"
-			)
-		)
-		]]
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.before, args[1], args[2], args[3];
+		local before_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			before_id = row[1];
+		end
+		if not before_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" < ?';
+		args[#args+1] = before_id;
 	end
+	return true;
 end
 
 function archive_store:find(username, query)
 	query = query or {};
 	local user,store = username,self.store;
-	local total;
-	local ok, result = engine:transaction(function()
+	local cache_key = jid_join(username, host, self.store);
+	local total = archive_item_count_cache:get(cache_key);
+	if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+		return noop, total;
+	end
+	local ok, result, err = engine:transaction(function()
 		local sql_query = [[
 		SELECT "key", "type", "value", "when", "with"
 		FROM "prosodyarchive"
@@ -346,12 +388,16 @@ function archive_store:find(username, query)
 					total = row[1];
 				end
 			end
+			if query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+				archive_item_count_cache:set(cache_key, total);
+			end
 			if query.limit == 0 then -- Skip the real query
 				return noop, total;
 			end
 		end
 
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 
 		if query.limit then
 			args[#args+1] = query.limit;
@@ -361,7 +407,8 @@ function archive_store:find(username, query)
 			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
 		return engine:select(sql_query, unpack(args));
 	end);
-	if not ok then return ok, result end
+	if not ok then return ok, result; end
+	if not result then return nil, err; end
 	return function()
 		local row = result();
 		if row ~= nil then
@@ -372,6 +419,48 @@ function archive_store:find(username, query)
 	end, total;
 end
 
+function archive_store:summary(username, query)
+	query = query or {};
+	local user,store = username,self.store;
+	local ok, result = engine:transaction(function()
+		local sql_query = [[
+		SELECT DISTINCT "with", COUNT(*), MIN("when"), MAX("when")
+		FROM "prosodyarchive"
+		WHERE %s
+		GROUP BY "with"
+		ORDER BY "sort_id" %s%s;
+		]];
+		local args = { host, user or "", store, };
+		local where = { "\"host\" = ?", "\"user\" = ?", "\"store\" = ?", };
+
+		archive_where(query, args, where);
+
+		archive_where_id_range(query, args, where);
+
+		if query.limit then
+			args[#args+1] = query.limit;
+		end
+
+		sql_query = sql_query:format(t_concat(where, " AND "), query.reverse
+			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
+		return engine:select(sql_query, unpack(args));
+	end);
+	if not ok then return ok, result end
+	local counts = {};
+	local earliest, latest = {}, {};
+	for row in result do
+		local with, count = row[1], row[2];
+		counts[with] = count;
+		earliest[with] = row[3];
+		latest[with] = row[4];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 function archive_store:delete(username, query)
 	query = query or {};
 	local user,store = username,self.store;
@@ -384,7 +473,8 @@ function archive_store:delete(username, query)
 			table.remove(where, 2);
 		end
 		archive_where(query, args, where);
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 		if query.truncate == nil then
 			sql_query = sql_query:format(t_concat(where, " AND "));
 		else
@@ -423,9 +513,24 @@ function archive_store:delete(username, query)
 		end
 		return engine:delete(sql_query, unpack(args));
 	end);
+	local cache_key = jid_join(username, host, self.store);
+	archive_item_count_cache:set(cache_key, nil);
 	return ok and stmt:affected(), stmt;
 end
 
+function archive_store:users()
+	local ok, result = engine:transaction(function()
+		local select_sql = [[
+		SELECT DISTINCT "user"
+		FROM "prosodyarchive"
+		WHERE "host"=? AND "store"=?;
+		]];
+		return engine:select(select_sql, host, self.store);
+	end);
+	if not ok then error(result); end
+	return iterator(result);
+end
+
 local stores = {
 	keyval = keyval_store;
 	map = map_store;
diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua
index eb208e28..b16acd09 100644
--- a/plugins/mod_tls.lua
+++ b/plugins/mod_tls.lua
@@ -35,9 +35,10 @@ local host = hosts[module.host];
 
 local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin;
 local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin;
+local err_c2s, err_s2sin, err_s2sout;
 
 function module.load()
-	local NULL, err = {};
+	local NULL = {};
 	local modhost = module.host;
 	local parent = modhost:match("%.(.*)$");
 
@@ -53,16 +54,20 @@ function module.load()
 	local host_s2s   = rawgetopt(modhost, "s2s_ssl") or parent_s2s;
 
 	module:log("debug", "Creating context for c2s");
-	ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
-	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end
+	local request_client_certs = { verify = { "peer", "client_once", }; };
 
 	module:log("debug", "Creating context for s2sout");
-	ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
-	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end
+	ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
+	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end
 
 	module:log("debug", "Creating context for s2sin");
-	ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
-	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end
+	-- for outgoing server connections
+	ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end
+
+	-- for incoming server connections
+	ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end
 end
 
 module:hook_global("config-reloaded", module.load);
@@ -77,12 +82,21 @@ local function can_do_tls(session)
 		return session.ssl_ctx;
 	end
 	if session.type == "c2s_unauthed" then
+		if not ssl_ctx_c2s and c2s_require_encryption then
+			session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s);
+		end
 		session.ssl_ctx = ssl_ctx_c2s;
 		session.ssl_cfg = ssl_cfg_c2s;
 	elseif session.type == "s2sin_unauthed" and allow_s2s_tls then
+		if not ssl_ctx_s2sin and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin);
+		end
 		session.ssl_ctx = ssl_ctx_s2sin;
 		session.ssl_cfg = ssl_cfg_s2sin;
 	elseif session.direction == "outgoing" and allow_s2s_tls then
+		if not ssl_ctx_s2sout and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout);
+		end
 		session.ssl_ctx = ssl_ctx_s2sout;
 		session.ssl_cfg = ssl_cfg_s2sout;
 	else
diff --git a/plugins/mod_user_account_management.lua b/plugins/mod_user_account_management.lua
index 615c1ed6..130ed089 100644
--- a/plugins/mod_user_account_management.lua
+++ b/plugins/mod_user_account_management.lua
@@ -53,9 +53,10 @@ local function handle_registration_stanza(event)
 			log("info", "User removed their account: %s@%s", username, host);
 			module:fire_event("user-deregistered", { username = username, host = host, source = "mod_register", session = session });
 		else
-			local username = nodeprep(query:get_child_text("username"));
+			local username = query:get_child_text("username");
 			local password = query:get_child_text("password");
 			if username and password then
+				username = nodeprep(username);
 				if username == session.username then
 					if usermanager_set_password(username, password, session.host, session.resource) then
 						session.send(st.reply(stanza));
diff --git a/plugins/mod_vcard.lua b/plugins/mod_vcard.lua
index b1a4c6e8..c3d6fb8b 100644
--- a/plugins/mod_vcard.lua
+++ b/plugins/mod_vcard.lua
@@ -19,7 +19,7 @@ local function handle_vcard(event)
 	if stanza.attr.type == "get" then
 		local vCard;
 		if to then
-			local node, host = jid_split(to);
+			local node = jid_split(to);
 			vCard = st.deserialize(vcards:get(node)); -- load vCard for user or server
 		else
 			vCard = st.deserialize(vcards:get(session.username));-- load user's own vCard
diff --git a/plugins/mod_vcard_legacy.lua b/plugins/mod_vcard_legacy.lua
index 5e75947a..a6ff47d0 100644
--- a/plugins/mod_vcard_legacy.lua
+++ b/plugins/mod_vcard_legacy.lua
@@ -38,7 +38,7 @@ local simple_map = {
 module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 	local origin, stanza = event.origin, event.stanza;
 	local pep_service = mod_pep.get_pep_service(jid_split(stanza.attr.to) or origin.username);
-	local ok, id, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
+	local ok, _, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
 
 	local vcard_temp = st.stanza("vCard", { xmlns = "vcard-temp" });
 	if ok and vcard4_item then
@@ -105,6 +105,23 @@ module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 					vcard_temp:tag("WORK"):up();
 				end
 				vcard_temp:up();
+			elseif tag.name == "impp" then
+				local uri = tag:get_child_text("uri");
+				if uri and uri:sub(1, 5) == "xmpp:" then
+					vcard_temp:text_tag("JABBERID", uri:sub(6))
+				end
+			elseif tag.name == "org" then
+				vcard_temp:tag("ORG")
+					:text_tag("ORGNAME", tag:get_child_text("text"))
+				:up();
+			end
+		end
+	else
+		local ok, _, nick_item = pep_service:get_last_item("http://jabber.org/protocol/nick", stanza.attr.from);
+		if ok and nick_item then
+			local nickname = nick_item:get_child_text("nick", "http://jabber.org/protocol/nick");
+			if nickname then
+				vcard_temp:text_tag("NICKNAME", nickname);
 			end
 		end
 	end
@@ -216,6 +233,10 @@ function vcard_to_pep(vcard_temp)
 				vcard4:text_tag("text", "work");
 			end
 			vcard4:up():up():up();
+		elseif tag.name == "JABBERID" then
+			vcard4:tag("impp")
+				:text_tag("uri", "xmpp:" .. tag:get_text())
+			:up();
 		elseif tag.name == "PHOTO" then
 			local avatar_type = tag:get_child_text("TYPE");
 			local avatar_payload = tag:get_child_text("BINVAL");
diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua
index 686a8981..386a4d60 100644
--- a/plugins/mod_websocket.lua
+++ b/plugins/mod_websocket.lua
@@ -29,18 +29,10 @@ local t_concat = table.concat;
 
 local stream_close_timeout = module:get_option_number("c2s_close_timeout", 5);
 local consider_websocket_secure = module:get_option_boolean("consider_websocket_secure");
-local cross_domain = module:get_option_set("cross_domain_websocket", {});
-if cross_domain:contains("*") or cross_domain:contains(true) then
-	cross_domain = true;
+local cross_domain = module:get_option("cross_domain_websocket");
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_websocket' option has been deprecated");
 end
-
-local function check_origin(origin)
-	if cross_domain == true then
-		return true;
-	end
-	return cross_domain:contains(origin);
-end
-
 local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing";
 local xmlns_streams = "http://etherx.jabber.org/streams";
 local xmlns_client = "jabber:client";
@@ -88,7 +80,7 @@ local function session_close(session, reason)
 					stream_error = reason;
 				end
 			end
-			log("debug", "Disconnecting client, <stream:error> is: %s", tostring(stream_error));
+			log("debug", "Disconnecting client, <stream:error> is: %s", stream_error);
 			session.send(stream_error);
 		end
 
@@ -144,7 +136,7 @@ function handle_request(event)
 
 	conn.starttls = false; -- Prevent mod_tls from believing starttls can be done
 
-	if not request.headers.sec_websocket_key then
+	if not request.headers.sec_websocket_key or request.method ~= "GET" then
 		response.headers.content_type = "text/html";
 		return [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
 			<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
@@ -158,11 +150,6 @@ function handle_request(event)
 		return 501;
 	end
 
-	if not check_origin(request.headers.origin or "") then
-		module:log("debug", "Origin %s is not allowed by 'cross_domain_websocket' [ %s ]", request.headers.origin or "(missing header)", cross_domain);
-		return 403;
-	end
-
 	local function websocket_close(code, message)
 		conn:write(build_close(code, message));
 		conn:close();
@@ -330,27 +317,4 @@ module:provides("http", {
 
 function module.add_host(module)
 	module:hook("c2s-read-timeout", keepalive, -0.9);
-
-	if cross_domain ~= true then
-		local url = require "socket.url";
-		local ws_url = module:http_url("websocket", "xmpp-websocket");
-		local url_components = url.parse(ws_url);
-		-- The 'Origin' consists of the base URL without path
-		url_components.path = nil;
-		local this_origin = url.build(url_components);
-		local local_cross_domain = module:get_option_set("cross_domain_websocket", { this_origin });
-		if local_cross_domain:contains(true) then
-			module:log("error", "cross_domain_websocket = true only works in the global section");
-			return;
-		end
-
-		-- Don't add / remove something added by another host
-		-- This might be weird with random load order
-		local_cross_domain:exclude(cross_domain);
-		cross_domain:include(local_cross_domain);
-		module:log("debug", "cross_domain = %s", tostring(cross_domain));
-		function module.unload()
-			cross_domain:exclude(local_cross_domain);
-		end
-	end
 end
diff --git a/plugins/muc/history.lib.lua b/plugins/muc/history.lib.lua
index 0d69c97d..f9ddabbf 100644
--- a/plugins/muc/history.lib.lua
+++ b/plugins/muc/history.lib.lua
@@ -48,16 +48,18 @@ module:hook("muc-config-form", function(event)
 	table.insert(event.form, {
 		name = "muc#roomconfig_historylength";
 		type = "text-single";
+		datatype = "xs:integer";
 		label = "Maximum number of history messages returned by room";
 		desc = "Specify the maximum number of previous messages that should be sent to users when they join the room";
-		value = tostring(get_historylength(event.room));
+		value = get_historylength(event.room);
 	});
 	table.insert(event.form, {
 		name = 'muc#roomconfig_defaulthistorymessages',
 		type = 'text-single',
+		datatype = "xs:integer";
 		label = 'Default number of history messages returned by room',
 		desc = "Specify the number of previous messages sent to new users when they join the room";
-		value = tostring(get_defaulthistorymessages(event.room))
+		value = get_defaulthistorymessages(event.room);
 	});
 end, 70-5);
 
diff --git a/plugins/muc/language.lib.lua b/plugins/muc/language.lib.lua
index ee80806b..2ee2ba0f 100644
--- a/plugins/muc/language.lib.lua
+++ b/plugins/muc/language.lib.lua
@@ -32,6 +32,7 @@ local function add_form_option(event)
 		label = "Language tag for room (e.g. 'en', 'de', 'fr' etc.)";
 		type = "text-single";
 		desc = "Indicate the primary language spoken in this room";
+		datatype = "xs:language";
 		value = get_language(event.room) or "";
 	});
 end
diff --git a/plugins/muc/lock.lib.lua b/plugins/muc/lock.lib.lua
index 062ab615..32f2647b 100644
--- a/plugins/muc/lock.lib.lua
+++ b/plugins/muc/lock.lib.lua
@@ -43,7 +43,7 @@ end
 module:hook("muc-occupant-pre-join", function(event)
 	if not event.is_new_room and is_locked(event.room) then -- Deny entry
 		module:log("debug", "Room is locked, denying entry");
-		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found"));
+		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found", nil, module.host));
 		return true;
 	end
 end, -30);
diff --git a/plugins/muc/members_only.lib.lua b/plugins/muc/members_only.lib.lua
index 4194c5c7..79077153 100644
--- a/plugins/muc/members_only.lib.lua
+++ b/plugins/muc/members_only.lib.lua
@@ -113,7 +113,7 @@ module:hook("muc-occupant-pre-join", function(event)
 		local stanza = event.stanza;
 		local affiliation = room:get_affiliation(stanza.attr.from);
 		if valid_affiliations[affiliation or "none"] <= valid_affiliations.none then
-			local reply = st.error_reply(stanza, "auth", "registration-required"):up();
+			local reply = st.error_reply(stanza, "auth", "registration-required", nil, room.jid):up();
 			reply.tags[1].attr.code = "407";
 			event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 			return true;
@@ -131,7 +131,7 @@ module:hook("muc-pre-invite", function(event)
 		local inviter_affiliation = room:get_affiliation(stanza.attr.from) or "none";
 		local required_affiliation = room._data.allow_member_invites and "member" or "admin";
 		if valid_affiliations[inviter_affiliation] < valid_affiliations[required_affiliation] then
-			event.origin.send(st.error_reply(stanza, "auth", "forbidden"));
+			event.origin.send(st.error_reply(stanza, "auth", "forbidden", nil, room.jid));
 			return true;
 		end
 	end
diff --git a/plugins/muc/mod_muc.lua b/plugins/muc/mod_muc.lua
index 954bae92..fc39d89f 100644
--- a/plugins/muc/mod_muc.lua
+++ b/plugins/muc/mod_muc.lua
@@ -86,7 +86,14 @@ room_mt.get_registered_nick = register.get_registered_nick;
 room_mt.get_registered_jid = register.get_registered_jid;
 room_mt.handle_register_iq = register.handle_register_iq;
 
+local presence_broadcast = module:require "muc/presence_broadcast";
+room_mt.get_presence_broadcast = presence_broadcast.get;
+room_mt.set_presence_broadcast = presence_broadcast.set;
+room_mt.get_valid_broadcast_roles = presence_broadcast.get_valid_broadcast_roles;
+
+
 local jid_split = require "util.jid".split;
+local jid_prep = require "util.jid".prep;
 local jid_bare = require "util.jid".bare;
 local st = require "util.stanza";
 local cache = require "util.cache";
@@ -184,7 +191,7 @@ end
 
 local function handle_broken_room(room, origin, stanza)
 	module:log("debug", "Returning error from broken room %s", room.jid);
-	origin.send(st.error_reply(stanza, "wait", "internal-server-error"));
+	origin.send(st.error_reply(stanza, "wait", "internal-server-error", nil, room.jid));
 	return true;
 end
 
@@ -263,9 +270,13 @@ local function set_room_defaults(room, lang)
 	room:set_changesubject(module:get_option_boolean("muc_room_default_change_subject", room:get_changesubject()));
 	room:set_historylength(module:get_option_number("muc_room_default_history_length", room:get_historylength()));
 	room:set_language(lang or module:get_option_string("muc_room_default_language"));
+	room:set_presence_broadcast(module:get_option("muc_room_default_presence_broadcast", room:get_presence_broadcast()));
 end
 
 function create_room(room_jid, config)
+	if jid_bare(room_jid) ~= room_jid or not jid_prep(room_jid, true) then
+		return nil, "invalid-jid";
+	end
 	local exists = get_room_from_jid(room_jid);
 	if exists then
 		return nil, "room-exists";
@@ -344,7 +355,7 @@ end, 1);
 module:hook("muc-room-pre-create", function(event)
 	local origin, stanza = event.origin, event.stanza;
 	if not track_room(event.room) then
-		origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+		origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 		return true;
 	end
 end, -1000);
@@ -395,7 +406,7 @@ do
 				restrict_room_creation == "local" and
 				select(2, jid_split(user_jid)) == host_suffix
 			) then
-				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted"));
+				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted", module.host));
 				return true;
 			end
 		end);
@@ -440,7 +451,7 @@ for event_name, method in pairs {
 				room = nil;
 			else
 				if stanza.attr.type ~= "error" then
-					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason)
+					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason, module.host)
 					if room._data.newjid then
 						local uri = "xmpp:"..room._data.newjid.."?join";
 						reply:get_child("error"):child_with_name("gone"):text(uri);
@@ -453,17 +464,21 @@ for event_name, method in pairs {
 
 		if room == nil then
 			-- Watch presence to create rooms
-			if stanza.attr.type == nil and stanza.name == "presence" then
+			if not jid_prep(room_jid, true) then
+				origin.send(st.error_reply(stanza, "modify", "jid-malformed", nil, module.host));
+				return true;
+			end
+			if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then
 				room = muclib.new_room(room_jid);
 				return room:handle_first_presence(origin, stanza);
 			elseif stanza.attr.type ~= "error" then
-				origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+				origin.send(st.error_reply(stanza, "cancel", "item-not-found", nil, module.host));
 				return true;
 			else
 				return;
 			end
 		elseif room == false then -- Error loading room
-			origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+			origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 			return true;
 		end
 		return room[method](room, origin, stanza);
diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua
index 639ecc38..399b090e 100644
--- a/plugins/muc/muc.lib.lua
+++ b/plugins/muc/muc.lib.lua
@@ -23,6 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep;
 local st = require "util.stanza";
 local base64 = require "util.encodings".base64;
 local md5 = require "util.hashes".md5;
+local new_id = require "util.id".medium;
 
 local log = module._log;
 
@@ -39,7 +40,7 @@ function room_mt:__tostring()
 end
 
 function room_mt.save()
-	-- overriden by mod_muc.lua
+	-- overridden by mod_muc.lua
 end
 
 function room_mt:get_occupant_jid(real_jid)
@@ -217,13 +218,13 @@ end
 
 -- Broadcasts an occupant's presence to the whole room
 -- Takes the x element that goes into the stanzas
-function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
+function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason, prev_role, force_unavailable)
 	local base_x = x.base or x;
 	-- Build real jid and (optionally) occupant jid template presences
 	local base_presence do
 		-- Try to use main jid's presence
 		local pr = occupant:get_presence();
-		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") then
+		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") and not force_unavailable then
 			base_presence = st.clone(pr);
 		else -- user is leaving but didn't send a leave presence. make one for them
 			base_presence = st.presence {from = occupant.nick; type = "unavailable";};
@@ -279,7 +280,9 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		self_p = st.clone(base_presence):add_child(self_x);
 	end
 
-	-- General populance
+	local broadcast_roles = self:get_presence_broadcast();
+
+	-- General populace
 	for occupant_nick, n_occupant in self:each_occupant() do
 		if occupant_nick ~= occupant.nick then
 			local pr;
@@ -290,7 +293,13 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 			else
 				pr = get_anon_p();
 			end
-			self:route_to_occupant(n_occupant, pr);
+			if broadcast_roles[occupant.role or "none"] or force_unavailable then
+				self:route_to_occupant(n_occupant, pr);
+			elseif prev_role and broadcast_roles[prev_role] then
+				pr.attr.type = 'unavailable';
+				self:route_to_occupant(n_occupant, pr);
+			end
+
 		end
 	end
 
@@ -314,6 +323,7 @@ function room_mt:send_occupant_list(to, filter)
 	local to_bare = jid_bare(to);
 	local is_anonymous = false;
 	local whois = self:get_whois();
+	local broadcast_roles = self:get_presence_broadcast();
 	if whois ~= "anyone" then
 		local affiliation = self:get_affiliation(to);
 		if affiliation ~= "admin" and affiliation ~= "owner" then
@@ -330,7 +340,9 @@ function room_mt:send_occupant_list(to, filter)
 			local pres = st.clone(occupant:get_presence());
 			pres.attr.to = to;
 			pres:add_child(x);
-			self:route_stanza(pres);
+			if to_bare == occupant.bare_jid or broadcast_roles[occupant.role or "none"] then
+				self:route_stanza(pres);
+			end
 		end
 	end
 end
@@ -373,7 +385,7 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	local real_jid = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(real_jid);
 	if occupant == nil then return nil; end
-	local type, condition, text = stanza:get_error();
+	local _, condition, text = stanza:get_error();
 	local error_message = "Kicked: "..(condition and condition:gsub("%-", " ") or "presence error");
 	if text and self:get_whois() == "anyone" then
 		error_message = error_message..": "..text;
@@ -391,7 +403,11 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	end
 	self:publicise_occupant_status(new_occupant or occupant, x);
 	if is_last_session then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -406,7 +422,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	local room, stanza = event.room, event.stanza;
 	local affiliation = room:get_affiliation(stanza.attr.from);
 	if affiliation == "outcast" then
-		local reply = st.error_reply(stanza, "auth", "forbidden"):up();
+		local reply = st.error_reply(stanza, "auth", "forbidden", nil, room.jid):up();
 		reply.tags[1].attr.code = "403";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -414,28 +430,41 @@ module:hook("muc-occupant-pre-join", function(event)
 end, -10);
 
 module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
 	local nick = jid_resource(event.occupant.nick);
 	if not nick:find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
 module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
 	if not jid_resource(event.dest_occupant.nick):find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
-function room_mt:handle_first_presence(origin, stanza)
-	if not stanza:get_child("x", "http://jabber.org/protocol/muc") then
-		module:log("debug", "Room creation without <x>, possibly desynced");
+module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
+		return true;
+	end
+end, 2);
 
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.dest_occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
 		return true;
 	end
+end, 2);
 
+function room_mt:handle_first_presence(origin, stanza)
 	local real_jid = stanza.attr.from;
 	local dest_jid = stanza.attr.to;
 	local bare_jid = jid_bare(real_jid);
@@ -505,7 +534,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	if orig_occupant == nil and not muc_x and stanza.attr.type == nil then
 		module:log("debug", "Attempted join without <x>, possibly desynced");
 		origin.send(st.error_reply(stanza, "cancel", "item-not-found",
-			"You must join the room before sending presence updates"));
+			"You are not currently connected to this chat", self.jid));
 		return true;
 	end
 
@@ -567,7 +596,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 		and bare_jid ~= jid_bare(dest_occupant.bare_jid) then
 		-- new nick or has different bare real jid
 		log("debug", "%s couldn't join due to nick conflict: %s", real_jid, dest_occupant.nick);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, self.jid):up();
 		reply.tags[1].attr.code = "409";
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -613,7 +642,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 				x:tag("status", {code = "303";}):up();
 				x:tag("status", {code = "110";}):up();
 				self:route_stanza(generated_unavail:add_child(x));
-				dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant
+				dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant
 			end
 		end
 
@@ -696,7 +725,7 @@ function room_mt:handle_presence_to_occupant(origin, stanza)
 		return self:handle_normal_presence(origin, stanza);
 	elseif type ~= 'result' then -- bad type
 		if type ~= 'visible' and type ~= 'invisible' then -- COMPAT ejabberd can broadcast or forward XEP-0018 presences
-			origin.send(st.error_reply(stanza, "modify", "bad-request")); -- FIXME correct error?
+			origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid)); -- FIXME correct error?
 		end
 	end
 	return true;
@@ -731,11 +760,11 @@ function room_mt:handle_iq_to_occupant(origin, stanza)
 	else -- Type is "get" or "set"
 		local current_nick = self:get_occupant_jid(from);
 		if not current_nick then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 			return true;
 		end
 		if not occupant then -- recipient not in room
-			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 			return true;
 		end
 		-- XEP-0410 MUC Self-Ping #1220
@@ -764,12 +793,12 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 	local type = stanza.attr.type;
 	if not current_nick then -- not in room
 		if type ~= "error" then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 		end
 		return true;
 	end
 	if type == "groupchat" then -- groupchat messages not allowed in PM
-		origin.send(st.error_reply(stanza, "modify", "bad-request"));
+		origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid));
 		return true;
 	elseif type == "error" and is_kickable_error(stanza) then
 		log("debug", "%s kicked from %s for sending an error message", current_nick, self.jid);
@@ -778,7 +807,7 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 
 	local o_data = self:get_occupant_by_nick(to);
 	if not o_data then
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 		return true;
 	end
 	log("debug", "%s sent private message stanza to %s (%s)", from, to, o_data.jid);
@@ -815,10 +844,12 @@ function room_mt:process_form(origin, stanza)
 	if form.attr.type == "cancel" then
 		origin.send(st.reply(stanza));
 	elseif form.attr.type == "submit" then
+		-- luacheck: ignore 231/errors
 		local fields, errors, present;
 		if form.tags[1] == nil then -- Instant room
 			fields, present = {}, {};
 		else
+			-- FIXME handle form errors
 			fields, errors, present = self:get_form_layout(stanza.attr.from):data(form);
 			if fields.FORM_TYPE ~= "http://jabber.org/protocol/muc#roomconfig" then
 				origin.send(st.error_reply(stanza, "cancel", "bad-request", "Form is not of type room configuration"));
@@ -879,7 +910,11 @@ function room_mt:clear(x)
 	end
 	for occupant in pairs(occupants_updated) do
 		self:publicise_occupant_status(occupant, x);
-		module:fire_event("muc-occupant-left", { room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 end
 
@@ -972,7 +1007,7 @@ function room_mt:handle_admin_query_get_command(origin, stanza)
 	local _aff_rank = valid_affiliations[_aff or "none"];
 	local _rol = item.attr.role;
 	if _aff and _aff_rank and not _rol then
-		-- You need to be at least an admin, and be requesting info about your affifiliation or lower
+		-- You need to be at least an admin, and be requesting info about your affiliation or lower
 		-- e.g. an admin can't ask for a list of owners
 		local affiliation_rank = valid_affiliations[affiliation or "none"];
 		if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank)
@@ -1049,6 +1084,9 @@ end
 function room_mt:handle_groupchat_to_room(origin, stanza)
 	local from = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(from);
+	if not stanza.attr.id then
+		stanza.attr.id = new_id()
+	end
 	if module:fire_event("muc-occupant-groupchat", {
 		room = self; origin = origin; stanza = stanza; from = from; occupant = occupant;
 	}) then return true; end
@@ -1218,7 +1256,7 @@ function room_mt:route_stanza(stanza) -- luacheck: ignore 212
 end
 
 function room_mt:get_affiliation(jid)
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	-- Affiliations are granted, revoked, and maintained based on the user's bare JID.
 	local bare = node and node.."@"..host or host;
 	local result = self._affiliations[bare];
@@ -1241,7 +1279,7 @@ end
 function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 	if not actor then return nil, "modify", "not-acceptable"; end;
 
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	if not host then return nil, "modify", "not-acceptable"; end
 	jid = jid_join(node, host); -- Bare
 	local is_host_only = node == nil;
@@ -1297,7 +1335,7 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 			-- Outcast can be by host.
 			is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host
 		) then
-			-- need to publcize in all cases; as affiliation in <item/> has changed.
+			-- need to publicize in all cases; as affiliation in <item/> has changed.
 			occupants_updated[occupant] = occupant.role;
 			if occupant.role ~= role and (
 				is_downgrade or
@@ -1324,7 +1362,11 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 		for occupant, old_role in pairs(occupants_updated) do
 			self:publicise_occupant_status(occupant, x, nil, actor, reason);
 			if occupant.role == nil then
-				module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+				module:fire_event("muc-occupant-left", {
+						room = self;
+						nick = occupant.nick;
+						occupant = occupant;
+					});
 			elseif is_semi_anonymous and
 				(old_role == "moderator" and occupant.role ~= "moderator") or
 				(old_role ~= "moderator" and occupant.role == "moderator") then -- Has gained or lost moderator status
@@ -1376,6 +1418,42 @@ function room_mt:get_role(nick)
 	return occupant and occupant.role or nil;
 end
 
+function room_mt:may_set_role(actor, occupant, role)
+	local event = {
+		room = self,
+		actor = actor,
+		occupant = occupant,
+		role = role,
+	};
+
+	module:fire_event("muc-pre-set-role", event);
+	if event.allowed ~= nil then
+		return event.allowed, event.error, event.condition;
+	end
+
+	-- Can't do anything to other owners or admins
+	local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
+	if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	-- If you are trying to give or take moderator role you need to be an owner or admin
+	if occupant.role == "moderator" or role == "moderator" then
+		local actor_affiliation = self:get_affiliation(actor);
+		if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
+			return nil, "cancel", "not-allowed";
+		end
+	end
+
+	-- Need to be in the room and a moderator
+	local actor_occupant = self:get_occupant_by_real_jid(actor);
+	if not actor_occupant or actor_occupant.role ~= "moderator" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	return true;
+end
+
 function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not actor then return nil, "modify", "not-acceptable"; end
 
@@ -1390,24 +1468,9 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if actor == true then
 		actor = nil -- So we can pass it safely to 'publicise_occupant_status' below
 	else
-		-- Can't do anything to other owners or admins
-		local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
-		if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
-			return nil, "cancel", "not-allowed";
-		end
-
-		-- If you are trying to give or take moderator role you need to be an owner or admin
-		if occupant.role == "moderator" or role == "moderator" then
-			local actor_affiliation = self:get_affiliation(actor);
-			if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
-				return nil, "cancel", "not-allowed";
-			end
-		end
-
-		-- Need to be in the room and a moderator
-		local actor_occupant = self:get_occupant_by_real_jid(actor);
-		if not actor_occupant or actor_occupant.role ~= "moderator" then
-			return nil, "cancel", "not-allowed";
+		local allowed, err, condition = self:may_set_role(actor, occupant, role)
+		if not allowed then
+			return allowed, err, condition;
 		end
 	end
 
@@ -1415,11 +1478,17 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not role then
 		x:tag("status", {code = "307"}):up();
 	end
+
+	local prev_role = occupant.role;
 	occupant.role = role;
 	self:save_occupant(occupant);
-	self:publicise_occupant_status(occupant, x, nil, actor, reason);
+	self:publicise_occupant_status(occupant, x, nil, actor, reason, prev_role);
 	if role == nil then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -1504,7 +1573,7 @@ function _M.restore_room(frozen, state)
 	else
 		-- New storage format
 		for jid, data in pairs(frozen) do
-			local node, host, resource = jid_split(jid);
+			local _, host, resource = jid_split(jid);
 			if host:sub(1,1) ~= "_" and not resource and type(data) == "string" then
 				-- bare jid: affiliation
 				room._affiliations[jid] = data;
diff --git a/plugins/muc/password.lib.lua b/plugins/muc/password.lib.lua
index 1f4b2add..6695c0cf 100644
--- a/plugins/muc/password.lib.lua
+++ b/plugins/muc/password.lib.lua
@@ -50,7 +50,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	if get_password(room) ~= password then
 		local from, to = stanza.attr.from, stanza.attr.to;
 		module:log("debug", "%s couldn't join due to invalid password: %s", from, to);
-		local reply = st.error_reply(stanza, "auth", "not-authorized"):up();
+		local reply = st.error_reply(stanza, "auth", "not-authorized", nil, room.jid):up();
 		reply.tags[1].attr.code = "401";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
diff --git a/plugins/muc/presence_broadcast.lib.lua b/plugins/muc/presence_broadcast.lib.lua
new file mode 100644
index 00000000..613e6403
--- /dev/null
+++ b/plugins/muc/presence_broadcast.lib.lua
@@ -0,0 +1,87 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+-- Copyright (C) 2014 Daurnimator
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local valid_roles = { "visitor", "participant", "moderator" };
+local default_broadcast = {
+	none = true;
+	visitor = true;
+	participant = true;
+	moderator = true;
+};
+
+local function get_presence_broadcast(room)
+	return room._data.presence_broadcast or default_broadcast;
+end
+
+local function set_presence_broadcast(room, broadcast_roles)
+	broadcast_roles = broadcast_roles or default_broadcast;
+
+	-- Ensure that unavailable presence is always sent when role changes to none
+	broadcast_roles.none = true;
+
+	local changed = false;
+	local old_broadcast_roles = get_presence_broadcast(room);
+	for _, role in ipairs(valid_roles) do
+		if old_broadcast_roles[role] ~= broadcast_roles[role] then
+			changed = true;
+		end
+	end
+
+	if not changed then return false; end
+
+	room._data.presence_broadcast = broadcast_roles;
+
+	for _, occupant in room:each_occupant() do
+		local x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local role = occupant.role or "none";
+		if broadcast_roles[role] and not old_broadcast_roles[role] then
+			-- Presence broadcast is now enabled, so announce existing user
+			room:publicise_occupant_status(occupant, x);
+		elseif old_broadcast_roles[role] and not broadcast_roles[role] then
+			-- Presence broadcast is now disabled, so mark existing user as unavailable
+			room:publicise_occupant_status(occupant, x, nil, nil, nil, nil, true);
+		end
+	end
+
+	return true;
+end
+
+module:hook("muc-config-form", function(event)
+	local values = {};
+	for role, value in pairs(get_presence_broadcast(event.room)) do
+		if value then
+			values[#values + 1] = role;
+		end
+	end
+
+	table.insert(event.form, {
+		name = "muc#roomconfig_presencebroadcast";
+		type = "list-multi";
+		label = "Only show participants with roles:";
+		value = values;
+		options = valid_roles;
+	});
+end, 70-7);
+
+module:hook("muc-config-submitted/muc#roomconfig_presencebroadcast", function(event)
+	local broadcast_roles = {};
+	for _, role in ipairs(event.value) do
+		broadcast_roles[role] = true;
+	end
+	if set_presence_broadcast(event.room, broadcast_roles) then
+		event.status_codes["104"] = true;
+	end
+end);
+
+return {
+	get = get_presence_broadcast;
+	set = set_presence_broadcast;
+};
diff --git a/plugins/muc/register.lib.lua b/plugins/muc/register.lib.lua
index 95ed1a84..f0a15dd4 100644
--- a/plugins/muc/register.lib.lua
+++ b/plugins/muc/register.lib.lua
@@ -15,8 +15,7 @@ local function get_reserved_nicks(room)
 	end
 	module:log("debug", "Refreshing reserved nicks...");
 	local reserved_nicks = {};
-	for jid in room:each_affiliation() do
-		local data = room._affiliation_data[jid];
+	for jid, _, data in room:each_affiliation() do
 		local nick = data and data.reserved_nickname;
 		module:log("debug", "Refreshed for %s: %s", jid, nick);
 		if nick then
@@ -54,7 +53,7 @@ end);
 
 local registration_form = dataforms.new {
 	{ name = "FORM_TYPE", type = "hidden", value = "http://jabber.org/protocol/muc#register" },
-	{ name = "muc#register_roomnick", type = "text-single", label = "Nickname"},
+	{ name = "muc#register_roomnick", type = "text-single", required = true, label = "Nickname"},
 };
 
 local function enforce_nick_policy(event)
@@ -67,7 +66,7 @@ local function enforce_nick_policy(event)
 	local reserved_by = get_registered_jid(room, requested_nick);
 	if reserved_by and reserved_by ~= jid_bare(stanza.attr.from) then
 		module:log("debug", "%s attempted to use nick %s reserved by %s", stanza.attr.from, requested_nick, reserved_by);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, room.jid):up();
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
 	end
@@ -80,7 +79,7 @@ local function enforce_nick_policy(event)
 				event.occupant.nick = jid_bare(event.occupant.nick) .. "/" .. nick;
 			elseif event.dest_occupant.nick ~= jid_bare(event.dest_occupant.nick) .. "/" .. nick then
 				module:log("debug", "Attempt by %s to join as %s, but their reserved nick is %s", stanza.attr.from, requested_nick, nick);
-				local reply = st.error_reply(stanza, "cancel", "not-acceptable"):up();
+				local reply = st.error_reply(stanza, "cancel", "not-acceptable", nil, room.jid):up();
 				origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 				return true;
 			end
@@ -104,7 +103,7 @@ local function handle_register_iq(room, origin, stanza)
 	local user_jid = jid_bare(stanza.attr.from)
 	local affiliation = room:get_affiliation(user_jid);
 	if affiliation == "outcast" then
-		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		origin.send(st.error_reply(stanza, "auth", "forbidden", room.jid));
 		return true;
 	elseif not (affiliation or allow_unaffiliated) then
 		origin.send(st.error_reply(stanza, "auth", "registration-required"));
@@ -135,7 +134,19 @@ local function handle_register_iq(room, origin, stanza)
 			return true;
 		end
 		local form_tag = query:get_child("x", "jabber:x:data");
-		local reg_data = form_tag and registration_form:data(form_tag);
+		if not form_tag then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing dataform"));
+			return true;
+		end
+		local form_type, err = dataforms.get_type(form_tag);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error with form: "..err));
+			return true;
+		elseif form_type ~= "http://jabber.org/protocol/muc#register" then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
+			return true;
+		end
+		local reg_data = registration_form:data(form_tag);
 		if not reg_data then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
 			return true;
diff --git a/plugins/muc/subject.lib.lua b/plugins/muc/subject.lib.lua
index 938abf61..c8b99cc7 100644
--- a/plugins/muc/subject.lib.lua
+++ b/plugins/muc/subject.lib.lua
@@ -94,6 +94,12 @@ module:hook("muc-occupant-groupchat", function(event)
 	local stanza = event.stanza;
 	local subject = stanza:get_child("subject");
 	if subject then
+		if stanza:get_child("body") or stanza:get_child("thread") then
+			-- Note: A message with a <subject/> and a <body/> or a <subject/> and
+			-- a <thread/> is a legitimate message, but it SHALL NOT be interpreted
+			-- as a subject change.
+			return;
+		end
 		local room = event.room;
 		local occupant = event.occupant;
 		-- Role check for subject changes
diff --git a/prosody b/prosody
index 204fb36d..e82318d1 100755
--- a/prosody
+++ b/prosody
@@ -94,4 +94,6 @@ cleanup();
 prosody.events.fire_event("server-stopped");
 prosody.log("info", "Shutdown complete");
 
+prosody.log("debug", "Shutdown reason was: %s", prosody.shutdown_reason or "not specified");
+prosody.log("debug", "Exiting with status code: %d", prosody.shutdown_code or 0);
 os.exit(prosody.shutdown_code);
diff --git a/prosody.cfg.lua.dist b/prosody.cfg.lua.dist
index f7f7b731..c6a210e5 100644
--- a/prosody.cfg.lua.dist
+++ b/prosody.cfg.lua.dist
@@ -32,6 +32,10 @@ admins = { }
 -- will look for modules first. For community modules, see https://modules.prosody.im/
 --plugin_paths = {}
 
+-- Single directory for custom prosody plugins and/or Lua libraries installation
+-- This path takes priority over plugin_paths, when prosody is searching for modules
+--installer_plugin_path = ""
+
 -- This is the list of modules Prosody will load on startup.
 -- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too.
 -- Documentation for bundled modules can be found at: https://prosody.im/doc/modules
@@ -88,7 +92,7 @@ modules_disabled = {
 	-- "offline"; -- Store offline messages
 	-- "c2s"; -- Handle client connections
 	-- "s2s"; -- Handle server-to-server connections
-	-- "posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
+	-- "posix"; -- POSIX functionality, sends server to background, etc.
 }
 
 -- Disable account creation by default, for security
diff --git a/prosodyctl b/prosodyctl
index a732a73e..5243c366 100755
--- a/prosodyctl
+++ b/prosodyctl
@@ -10,7 +10,6 @@
 -- prosodyctl - command-line controller for Prosody XMPP server
 
 -- Will be modified by configure script if run --
-
 CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
 CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
 CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
@@ -77,13 +76,38 @@ local show_usage = prosodyctl.show_usage;
 local show_yesno = prosodyctl.show_yesno;
 local show_prompt = prosodyctl.show_prompt;
 local read_password = prosodyctl.read_password;
+local call_luarocks = prosodyctl.call_luarocks;
 
 local jid_split = require "util.jid".prepped_split;
 
 local prosodyctl_timeout = (configmanager.get("*", "prosodyctl_timeout") or 5) * 2;
 -----------------------
 local commands = {};
-local command = arg[1];
+local command = table.remove(arg, 1);
+
+function commands.install(arg)
+	if arg[1] == "--help" then
+		show_usage([[install]], [[Installs a prosody/luarocks plugin]]);
+		return 1;
+	end
+	call_luarocks(arg[1], "install")
+end
+
+function commands.remove(arg)
+	if arg[1] == "--help" then
+		show_usage([[remove]], [[Removes a module installed in the working directory's plugins folder]]);
+		return 1;
+	end
+	call_luarocks(arg[1], "remove")
+end
+
+function commands.list(arg)
+	if arg[1] == "--help" then
+		show_usage([[list]], [[Shows installed rocks]]);
+		return 1;
+	end
+	call_luarocks(arg[1], "list")
+end
 
 function commands.adduser(arg)
 	if not arg[1] or arg[1] == "--help" then
@@ -120,7 +144,7 @@ function commands.adduser(arg)
 
 	if ok then return 0; end
 
-	show_message(msg)
+	show_message(error_messages[msg])
 	return 1;
 end
 
@@ -222,7 +246,15 @@ function commands.start(arg)
 	end
 
 	--luacheck: ignore 411/ret
-	local ok, ret = prosodyctl.start(prosody.paths.source);
+	local lua;
+	do
+		local i = 0;
+		repeat
+			i = i - 1;
+		until arg[i-1] == nil
+		lua = arg[i];
+	end
+	local ok, ret = prosodyctl.start(prosody.paths.source, lua);
 	if ok then
 		local daemonize = configmanager.get("*", "daemonize");
 		if daemonize == nil then
@@ -363,6 +395,13 @@ function commands.about(arg)
 				.."\n  ";
 		end)));
 	print("");
+	local have_pposix, pposix = pcall(require, "util.pposix");
+	if have_pposix and pposix.uname then
+		print("# Operating system");
+		local uname, err = pposix.uname();
+		print(uname and uname.sysname .. " " .. uname.release or "Unknown POSIX", err or "");
+		print("");
+	end
 	print("# Lua environment");
 	print("Lua version:             ", _G._VERSION);
 	print("");
@@ -811,7 +850,7 @@ function commands.check(arg)
 		print("Checking config...");
 		local deprecated = set.new({
 			"bosh_ports", "disallow_s2s", "no_daemonize", "anonymous_login", "require_encryption",
-			"vcard_compatibility",
+			"vcard_compatibility", "cross_domain_bosh", "cross_domain_websocket"
 		});
 		local known_global_options = set.new({
 			"pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize",
@@ -1313,8 +1352,6 @@ local command_runner = async.runner(function ()
 			end
 		end
 
-		table.remove(arg, 1);
-
 		local module = modulemanager.get_module("*", module_name);
 		if not module then
 			show_message("Failed to load module '"..module_name.."': Unknown error");
@@ -1353,7 +1390,8 @@ local command_runner = async.runner(function ()
 		print("Where COMMAND may be one of:\n");
 
 		local hidden_commands = require "util.set".new{ "register", "unregister", "addplugin" };
-		local commands_order = { "adduser", "passwd", "deluser", "start", "stop", "restart", "reload", "about" };
+		local commands_order = { "install", "remove", "list", "adduser", "passwd", "deluser", "start", "stop", "restart", "reload",
+			"about" };
 
 		local done = {};
 
@@ -1378,7 +1416,7 @@ local command_runner = async.runner(function ()
 		os.exit(0);
 	end
 
-	os.exit(commands[command]({ select(2, unpack(arg)) }));
+	os.exit(commands[command](arg));
 end, watchers);
 
 command_runner:run(true);
diff --git a/spec/core_configmanager_spec.lua b/spec/core_configmanager_spec.lua
index afb7d492..7958ec6b 100644
--- a/spec/core_configmanager_spec.lua
+++ b/spec/core_configmanager_spec.lua
@@ -9,7 +9,9 @@ describe("core.configmanager", function()
 
 			configmanager.set("*", "testkey1", 321);
 			assert.are.equal(321, configmanager.get("*", "testkey1"), "Retrieving a set global key");
-			assert.are.equal(321, configmanager.get("example.com", "testkey1"), "Retrieving a set key of undefined host, of which only a globally set one exists");
+			assert.are.equal(321, configmanager.get("example.com", "testkey1"),
+				"Retrieving a set key of undefined host, of which only a globally set one exists"
+			);
 
 			configmanager.set("example.com", ""); -- Creates example.com host in config
 			assert.are.equal(321, configmanager.get("example.com", "testkey1"), "Retrieving a set key, of which only a globally set one exists");
diff --git a/spec/core_storagemanager_spec.lua b/spec/core_storagemanager_spec.lua
index a0a8b5ef..c3534cd5 100644
--- a/spec/core_storagemanager_spec.lua
+++ b/spec/core_storagemanager_spec.lua
@@ -1,4 +1,4 @@
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local server = require "net.server_select";
 package.loaded["net.server"] = server;
 
@@ -119,6 +119,7 @@ describe("storagemanager", function ()
 
 				describe("can be queried", function ()
 					it("for all items", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {});
 						assert.truthy(data);
 						local count = 0;
@@ -135,6 +136,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by JID", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							with = "contact@example.com";
 						});
@@ -153,6 +155,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by time (end)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["end"] = test_time;
 						});
@@ -171,6 +174,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by time (start)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["start"] = test_time;
 						});
@@ -189,6 +193,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by time (start+end)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["start"] = test_time;
 							["end"] = test_time+1;
@@ -239,6 +244,7 @@ describe("storagemanager", function ()
 				end);
 
 				it("can be purged", function ()
+					-- luacheck: ignore 211/err
 					local ok, err = archive:delete("user");
 					assert.truthy(ok);
 					local data, err = archive:find("user", {
diff --git a/spec/net_http_parser_spec.lua b/spec/net_http_parser_spec.lua
index 6bba087c..8310a451 100644
--- a/spec/net_http_parser_spec.lua
+++ b/spec/net_http_parser_spec.lua
@@ -1,16 +1,68 @@
-local httpstreams = { [[
+local http_parser = require "net.http.parser";
+
+local function test_stream(stream, expect)
+	local success_cb = spy.new(function (packet)
+		assert.is_table(packet);
+		assert.is_equal(expect.body, packet.body);
+	end);
+
+	stream = stream:gsub("\n", "\r\n");
+	local parser = http_parser.new(success_cb, error, stream:sub(1,4) == "HTTP" and "client" or "server")
+	for chunk in stream:gmatch("..?.?") do
+		parser:feed(chunk);
+	end
+
+	assert.spy(success_cb).was_called(expect.count or 1);
+end
+
+
+describe("net.http.parser", function()
+	describe("parser", function()
+		it("should handle requests with no content-length or body", function ()
+			test_stream(
+[[
 GET / HTTP/1.1
 Host: example.com
 
-]], [[
+]],
+				{
+					body = "";
+				}
+			);
+		end);
+
+		it("should handle responses with empty body", function ()
+			test_stream(
+[[
 HTTP/1.1 200 OK
 Content-Length: 0
 
-]], [[
+]],
+				{
+					body = "";
+				}
+			);
+		end);
+
+		it("should handle simple responses", function ()
+			test_stream(
+
+[[
 HTTP/1.1 200 OK
 Content-Length: 7
 
 Hello
+]],
+				{
+					body = "Hello\r\n", count = 1;
+				}
+			);
+		end);
+
+		it("should handle chunked encoding in responses", function ()
+			test_stream(
+
+[[
 HTTP/1.1 200 OK
 Transfer-Encoding: chunked
 
@@ -25,28 +77,40 @@ o
 0
 
 
-]]
-}
+]],
+				{
+					body = "Hello", count = 1;
+				}
+			);
+		end);
 
+		it("should handle a stream of responses", function ()
+			test_stream(
 
-local http_parser = require "net.http.parser";
+[[
+HTTP/1.1 200 OK
+Content-Length: 5
 
-describe("net.http.parser", function()
-	describe("#new()", function()
-		it("should work", function()
-			for _, stream in ipairs(httpstreams) do
-				local success;
-				local function success_cb(packet)
-					success = true;
-				end
-				stream = stream:gsub("\n", "\r\n");
-				local parser = http_parser.new(success_cb, error, stream:sub(1,4) == "HTTP" and "client" or "server")
-				for chunk in stream:gmatch("..?.?") do
-					parser:feed(chunk);
-				end
-
-				assert.is_true(success);
-			end
+Hello
+HTTP/1.1 200 OK
+Transfer-Encoding: chunked
+
+1
+H
+1
+e
+2
+ll
+1
+o
+0
+
+
+]],
+				{
+					body = "Hello", count = 2;
+				}
+			);
 		end);
 	end);
 end);
diff --git a/spec/scansion/basic_message.scs b/spec/scansion/basic_message.scs
index fb21c465..1258dbf5 100644
--- a/spec/scansion/basic_message.scs
+++ b/spec/scansion/basic_message.scs
@@ -79,7 +79,7 @@ Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>Hello Juliet, are you there?</body>
 		<delay xmlns='urn:xmpp:delay' from='localhost' stamp='{scansion:any}' />
-	</message>	
+	</message>
 
 # Romeo sends another bare-JID message, it should be delivered
 # instantly to Juliet's phone
@@ -92,7 +92,7 @@ Romeo sends:
 Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>Oh, hi!</body>
-	</message>	
+	</message>
 
 # Juliet's laptop goes online, but with a negative priority
 
@@ -122,7 +122,7 @@ Romeo sends:
 Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>How are you?</body>
-	</message>	
+	</message>
 
 # Romeo sends direct to Juliet's full JID, and she should receive it
 
diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs
new file mode 100644
index 00000000..41ffec0d
--- /dev/null
+++ b/spec/scansion/keep_full_sub_req.scs
@@ -0,0 +1,58 @@
+# server MUST keep a record of the complete presence stanza comprising the subscription request (#689)
+
+[Client] Alice
+	jid: pars-a@localhost
+	password: password
+
+[Client] Bob
+	jid: pars-b@localhost
+	password: password
+
+[Client] Bob's phone
+	jid: pars-b@localhost/phone
+	password: password
+
+---------
+
+Alice connects
+
+Alice sends:
+	<presence to="${Bob's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Alice disconnects
+
+Bob connects
+
+Bob sends:
+	<presence/>
+
+Bob receives:
+	<presence from="${Bob's full JID}"/>
+
+Bob receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob disconnects
+
+# Works if they reconnect too
+
+Bob's phone connects
+
+Bob's phone sends:
+	<presence/>
+
+Bob's phone receives:
+	<presence from="${Bob's phone's full JID}"/>
+
+
+Bob's phone receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob's phone disconnects
+
diff --git a/spec/scansion/muc_create_destroy.scs b/spec/scansion/muc_create_destroy.scs
new file mode 100644
index 00000000..fea759a3
--- /dev/null
+++ b/spec/scansion/muc_create_destroy.scs
@@ -0,0 +1,250 @@
+# MUC creation, basic messages and destruction
+
+[Client] Romeo
+	jid: romeo@localhost/mK0dD6Ha
+	password: password
+
+[Client] Juliet
+	jid: juliet@localhost/lVwkim_k
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence to="garden@conference.localhost/romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/romeo">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="moderator"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message from="garden@conference.localhost" type="groupchat">
+		<subject/>
+	</message>
+
+Romeo sends:
+	<iq to="garden@conference.localhost" id="lx3" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#owner">
+			<x type="submit" xmlns="jabber:x:data"/>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="lx3" type="result" from="garden@conference.localhost"/>
+
+Juliet connects
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm1">
+		<body>Where are thou my Juliet?</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="garden@conference.localhost/romeo" id="rm1">
+		<body>Where are thou my Juliet?</body>
+	</message>
+
+Juliet sends:
+	<presence to="garden@conference.localhost/juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/romeo">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator"/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/juliet">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="none" jid="${Juliet's full JID}" role="participant"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<message from="garden@conference.localhost/romeo" id="rm1" type="groupchat">
+		<body>Where are thou my Juliet?</body>
+		<delay stamp="{scansion:any}" xmlns="urn:xmpp:delay" from="garden@conference.localhost"/>
+		<x stamp="{scansion:any}" xmlns="jabber:x:delay" from="garden@conference.localhost"/>
+	</message>
+
+Juliet receives:
+	<message from="garden@conference.localhost" type="groupchat">
+		<subject/>
+	</message>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/juliet">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="none" jid="${Juliet's full JID}" role="participant"/>
+		</x>
+	</presence>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm1">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm1" from="garden@conference.localhost/juliet">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm1" from="garden@conference.localhost/juliet">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm2">
+		<body>Here I am!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm2" from="garden@conference.localhost/juliet">
+		<body>Here I am!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm2" from="garden@conference.localhost/juliet">
+		<body>Here I am!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm2">
+		<body>What is this place?</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm2" from="garden@conference.localhost/romeo">
+		<body>What is this place?</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm2" from="garden@conference.localhost/romeo">
+		<body>What is this place?</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm3">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm3" from="garden@conference.localhost/juliet">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm3" from="garden@conference.localhost/juliet">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm3">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm3" from="garden@conference.localhost/romeo">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm3" from="garden@conference.localhost/romeo">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm4">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm4" from="garden@conference.localhost/juliet">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm4" from="garden@conference.localhost/juliet">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm4">
+		<body>Yes!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm4" from="garden@conference.localhost/romeo">
+		<body>Yes!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm4" from="garden@conference.localhost/romeo">
+		<body>Yes!</body>
+	</message>
+
+Romeo sends:
+	<iq to="garden@conference.localhost" id="lx4" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#owner">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+		</query>
+	</iq>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/juliet" type="unavailable">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+			<item affiliation="none" jid="${Juliet's full JID}" role="none"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/romeo" type="unavailable">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="none"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<iq id="lx4" type="result" from="garden@conference.localhost"/>
+
+Juliet disconnects
+
+Romeo disconnects
+
+# recording ended on 2019-08-31T13:45:32Z
diff --git a/spec/scansion/muc_members_only_change.scs b/spec/scansion/muc_members_only_change.scs
index dc40b5a0..a708dbfb 100644
--- a/spec/scansion/muc_members_only_change.scs
+++ b/spec/scansion/muc_members_only_change.scs
@@ -94,7 +94,7 @@ Romeo sends:
 			<item affiliation='none' jid="${Juliet's JID}" />
 		</query>
 	</iq>
-	
+
 # As a non-member, Juliet must now be removed from the room
 Romeo receives:
 	<presence type='unavailable' from='room@conference.localhost/Juliet'>
diff --git a/spec/scansion/muc_nickname_change.scs b/spec/scansion/muc_nickname_change.scs
new file mode 100644
index 00000000..73f81203
--- /dev/null
+++ b/spec/scansion/muc_nickname_change.scs
@@ -0,0 +1,127 @@
+# MUC: Change nickname
+# Make sure a role is not reset, see #1466
+
+[Client] Romeo
+	jid: user@localhost
+	password: password
+
+[Client] Juliet
+	jid: user2@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence to="room@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='201'/>
+			<item jid="${Romeo's full JID}" affiliation='owner' role='moderator'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo sends:
+	<iq id='config1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#owner'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#roomconfig</value>
+				</field>
+				<field var='muc#roomconfig_moderatedroom'>
+					<value>1</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="config1" from="room@conference.localhost" type="result"/>
+
+Juliet connects
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item role='moderator' xmlns='http://jabber.org/protocol/muc#user' affiliation='owner'/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation="none" role="visitor" jid="${Juliet's full JID}"/>
+		</x>
+	</presence>
+
+Romeo sends:
+	<iq id='config1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#owner'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#roomconfig</value>
+				</field>
+				<field var='muc#roomconfig_moderatedroom'>
+					<value>0</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="config1" from="room@conference.localhost" type="result"/>
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status xmlns='http://jabber.org/protocol/muc#user' code='104'/>
+		</x>
+	</message>
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet2">
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet' type='unavailable'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='303'/>
+			<item nick='Juliet2' jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet2'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
diff --git a/spec/scansion/muc_password.scs b/spec/scansion/muc_password.scs
index 82611183..63c821e0 100644
--- a/spec/scansion/muc_password.scs
+++ b/spec/scansion/muc_password.scs
@@ -58,7 +58,7 @@ Juliet sends:
 
 Juliet receives:
 	<presence from="room@conference.localhost/Juliet" type="error">
-		<error type="auth" code="401">
+		<error type="auth" code="401" by="room@conference.localhost">
 			<not-authorized xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 	</presence>
diff --git a/spec/scansion/muc_register.scs b/spec/scansion/muc_register.scs
index e1eaf4e0..81612829 100644
--- a/spec/scansion/muc_register.scs
+++ b/spec/scansion/muc_register.scs
@@ -100,7 +100,9 @@ Juliet receives:
 				<field type='hidden' var='FORM_TYPE'>
 					<value>http://jabber.org/protocol/muc#register</value>
 				</field>
-				<field type='text-single' label='Nickname' var='muc#register_roomnick'/>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
 			</x>
 		</query>
 	</iq>
@@ -175,7 +177,7 @@ Rosaline sends:
 
 Rosaline receives:
 	<presence type='error' from='room@conference.localhost/Juliet'>
-		<error type='cancel'>
+		<error type='cancel' by='room@conference.localhost'>
 			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 		<x xmlns='http://jabber.org/protocol/muc'/>
@@ -286,7 +288,7 @@ Rosaline sends:
 
 Rosaline receives:
 	<presence type='error' from='room@conference.localhost/Juliet'>
-		<error type='cancel'>
+		<error type='cancel' by='room@conference.localhost'>
 			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 		<x xmlns='http://jabber.org/protocol/muc'/>
@@ -326,7 +328,7 @@ Romeo receives:
 	</iq>
 
 # Romeo updates his own registration
-	
+
 Romeo sends:
 	<iq id='jw81b36f' to='room@conference.localhost' type='get'>
 		<query xmlns='jabber:iq:register'/>
@@ -339,7 +341,9 @@ Romeo receives:
 				<field type='hidden' var='FORM_TYPE'>
 					<value>http://jabber.org/protocol/muc#register</value>
 				</field>
-				<field type='text-single' label='Nickname' var='muc#register_roomnick'/>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
 			</x>
 		</query>
 	</iq>
diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs
new file mode 100644
index 00000000..74980073
--- /dev/null
+++ b/spec/scansion/muc_subject_issue_667.scs
@@ -0,0 +1,129 @@
+# #667 MUC message with subject and body SHALL NOT be interpreted as a subject change
+
+[Client] Romeo
+	password: password
+	jid: romeo@localhost
+
+-----
+
+Romeo connects
+
+# and creates a room
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# the default (empty) subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this should be treated as a normal message
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# the still empty subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this is a subject change
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+# a message without <subject>
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# History
+# These have delay tags but we ignore those for now
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Finally, the topic
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo disconnects
+
diff --git a/spec/scansion/presence_preapproval.scs b/spec/scansion/presence_preapproval.scs
new file mode 100644
index 00000000..e34ac7cf
--- /dev/null
+++ b/spec/scansion/presence_preapproval.scs
@@ -0,0 +1,74 @@
+# server supports contact subscription pre-approval (RFC 6121 3.4)
+
+[Client] Alice
+	jid: preappove-a@localhost
+	password: password
+
+[Client] Bob
+	jid: preapprove-b@localhost
+	password: password
+
+---------
+
+Alice connects
+
+Alice sends:
+	<presence/>
+
+Alice receives:
+	<presence/>
+
+Alice sends:
+	<presence to="${Bob's JID}" type="subscribed"/>
+
+Bob connects
+
+Bob sends:
+	<iq type="get" id="roster1">
+		<query xmlns="jabber:iq:roster"/>
+	</iq>
+
+Bob receives:
+	<iq type="result" id="roster1">
+		<query xmlns="jabber:iq:roster" ver="{scansion:any}">
+		</query>
+	</iq>
+
+Bob sends:
+	<presence/>
+
+Bob receives:
+	<presence from="${Bob's full JID}"/>
+
+Bob sends:
+	<presence to="${Alice's JID}" type="subscribe" />
+
+Bob receives:
+	<iq type='set' id='{scansion:any}'>
+		<query ver='1' xmlns='jabber:iq:roster'>
+			<item jid="${Alice's JID}" subscription='none' ask='subscribe' />
+		</query>
+	</iq>
+
+
+
+Bob receives:
+	<presence from="${Alice's JID}" type="subscribed" />
+
+Bob disconnects
+
+Alice sends:
+	<iq type="get" id="roster1">
+		<query xmlns="jabber:iq:roster"/>
+	</iq>
+
+Alice receives:
+	<iq type="result" id="roster1">
+		<query xmlns="jabber:iq:roster" ver="{scansion:any}">
+			<item jid="${Bob's JID}" subscription="from" />
+		</query>
+	</iq>
+
+Alice disconnects
+
+Bob disconnects
diff --git a/spec/scansion/prosody.cfg.lua b/spec/scansion/prosody.cfg.lua
index f95ea31b..3f804c4e 100644
--- a/spec/scansion/prosody.cfg.lua
+++ b/spec/scansion/prosody.cfg.lua
@@ -8,16 +8,17 @@ modules_enabled = {
 	-- Generally required
 		"roster"; -- Allow users to have a roster. Recommended ;)
 		"saslauth"; -- Authentication for clients and servers. Recommended if you want to log in.
-		"tls"; -- Add support for secure TLS on c2s/s2s connections
-		"dialback"; -- s2s dialback support
+		--"tls"; -- Add support for secure TLS on c2s/s2s connections
+		--"dialback"; -- s2s dialback support
 		"disco"; -- Service discovery
 
 	-- Not essential, but recommended
 		"carbons"; -- Keep multiple clients in sync
-		"pep"; -- Enables users to publish their mood, activity, playing music and more
+		"pep"; -- Enables users to publish their avatar, mood, activity, playing music and more
 		"private"; -- Private XML storage (for room bookmarks, etc.)
 		"blocklist"; -- Allow users to block communications with other users
-		"vcard"; -- Allow users to set vCards
+		"vcard4"; -- User profiles (stored in PEP)
+		"vcard_legacy"; -- Conversion between legacy vCard and PEP Avatar, vcard
 
 	-- Nice to have
 		"version"; -- Replies to server version requests
@@ -26,6 +27,11 @@ modules_enabled = {
 		"ping"; -- Replies to XMPP pings with pongs
 		"register"; -- Allow users to register on this server using a client and change passwords
 		"mam"; -- Store messages in an archive and allow users to access it
+		--"csi_simple"; -- Simple Mobile optimizations
+
+	-- Admin interfaces
+		--"admin_adhoc"; -- Allows administration via an XMPP client that supports ad-hoc commands
+		--"admin_telnet"; -- Opens telnet console interface on localhost port 5582
 
 	-- HTTP modules
 		--"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
@@ -47,6 +53,9 @@ modules_enabled = {
 		--"scansion_record"; -- Records things that happen in scansion test case format
 }
 
+modules_disabled = {
+	"s2s";
+}
 certificate = "certs"
 
 allow_registration = false
diff --git a/spec/scansion/pubsub_basic.scs b/spec/scansion/pubsub_basic.scs
index d983ff66..0adf6857 100644
--- a/spec/scansion/pubsub_basic.scs
+++ b/spec/scansion/pubsub_basic.scs
@@ -32,7 +32,7 @@ Juliet connects
 -- 			<subscribe node="princely_musings" jid="${Romeo's full JID}"/>
 -- 		</pubsub>
 -- 	</iq>
--- 
+--
 -- Juliet receives:
 -- 	<iq type="error"/>
 
diff --git a/spec/scansion/pubsub_preconditions.scs b/spec/scansion/pubsub_preconditions.scs
new file mode 100644
index 00000000..25afaa8d
--- /dev/null
+++ b/spec/scansion/pubsub_preconditions.scs
@@ -0,0 +1,234 @@
+# Pubsub preconditions are enforced
+
+[Client] Romeo
+	password: password
+	jid: jqpcrbq2@localhost
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current">
+					<tune xmlns="http://jabber.org/protocol/tune"/>
+				</item>
+			</publish>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="result">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current"/>
+			</publish>
+		</pubsub>
+	</iq>
+
+Romeo sends:
+	<iq id="52d74a36-afb0-4028-87ed-b25b988b049e" type="get">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune"/>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="52d74a36-afb0-4028-87ed-b25b988b049e" type="result">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune">
+				<x xmlns="jabber:x:data" type="form">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#node_config</value>
+					</field>
+					<field var="pubsub#title" label="Title" type="text-single"/>
+					<field var="pubsub#description" label="Description" type="text-single"/>
+					<field var="pubsub#type" label="The type of node data, usually specified by the namespace of the payload (if any)" type="text-single"/>
+					<field var="pubsub#max_items" label="Max # of items to persist" type="text-single">
+						<validate xmlns="http://jabber.org/protocol/xdata-validate" datatype="xs:integer"/>
+						<value>1</value>
+					</field>
+					<field var="pubsub#persist_items" label="Persist items to storage" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#access_model" label="Specify the subscriber model" type="list-single">
+						<option label="authorize">
+							<value>authorize</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<option label="presence">
+							<value>presence</value>
+						</option>
+						<option label="roster">
+							<value>roster</value>
+						</option>
+						<option label="whitelist">
+							<value>whitelist</value>
+						</option>
+						<value>presence</value>
+					</field>
+					<field var="pubsub#publish_model" label="Specify the publisher model" type="list-single">
+						<option label="publishers">
+							<value>publishers</value>
+						</option>
+						<option label="subscribers">
+							<value>subscribers</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<value>publishers</value>
+					</field>
+					<field var="pubsub#deliver_notifications" label="Whether to deliver event notifications" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#deliver_payloads" label="Whether to deliver payloads with event notifications" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notification_type" label="Specify the delivery style for notifications" type="list-single">
+						<option label="Messages of type normal">
+							<value>normal</value>
+						</option>
+						<option label="Messages of type headline">
+							<value>headline</value>
+						</option>
+						<value>headline</value>
+					</field>
+					<field var="pubsub#notify_delete" label="Whether to notify subscribers when the node is deleted" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notify_retract" label="Whether to notify subscribers when items are removed from the node" type="boolean">
+						<value>1</value>
+					</field>
+				</x>
+			</configure>
+		</pubsub>
+	</iq>
+
+Romeo sends:
+	<iq id="a73aac09-74be-4ee2-97e5-571bbdbcd956" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune">
+				<x xmlns="jabber:x:data" type="submit">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#node_config</value>
+					</field>
+					<field var="pubsub#title" type="text-single" label="Title">
+						<value>Nice tunes</value>
+					</field>
+					<field var="pubsub#description" type="text-single" label="Description"/>
+					<field var="pubsub#type" type="text-single" label="The type of node data, usually specified by the namespace of the payload (if any)"/>
+					<field var="pubsub#max_items" type="text-single" label="Max # of items to persist">
+						<validate xmlns="http://jabber.org/protocol/xdata-validate" datatype="xs:integer"/>
+						<value>1</value>
+					</field>
+					<field var="pubsub#persist_items" type="boolean" label="Persist items to storage">
+						<value>1</value>
+					</field>
+					<field var="pubsub#access_model" type="list-single" label="Specify the subscriber model">
+						<option label="authorize">
+							<value>authorize</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<option label="presence">
+							<value>presence</value>
+						</option>
+						<option label="roster">
+							<value>roster</value>
+						</option>
+						<option label="whitelist">
+							<value>whitelist</value>
+						</option>
+						<value>presence</value>
+					</field>
+					<field var="pubsub#publish_model" type="list-single" label="Specify the publisher model">
+						<option label="publishers">
+							<value>publishers</value>
+						</option>
+						<option label="subscribers">
+							<value>subscribers</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<value>publishers</value>
+					</field>
+					<field var="pubsub#deliver_notifications" type="boolean" label="Whether to deliver event notifications">
+						<value>1</value>
+					</field>
+					<field var="pubsub#deliver_payloads" type="boolean" label="Whether to deliver payloads with event notifications">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notification_type" type="list-single" label="Specify the delivery style for notifications">
+						<option label="Messages of type normal">
+							<value>normal</value>
+						</option>
+						<option label="Messages of type headline">
+							<value>headline</value>
+						</option>
+						<value>headline</value>
+					</field>
+					<field var="pubsub#notify_delete" type="boolean" label="Whether to notify subscribers when the node is deleted">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notify_retract" type="boolean" label="Whether to notify subscribers when items are removed from the node">
+						<value>1</value>
+					</field>
+				</x>
+			</configure>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="a73aac09-74be-4ee2-97e5-571bbdbcd956" type="result"/>
+
+Romeo sends:
+	<iq id="ab0e92d2-c06b-4987-9d45-f9f9e7721709" type="get">
+		<query xmlns="http://jabber.org/protocol/disco#items"/>
+	</iq>
+
+Romeo receives:
+	<iq id="ab0e92d2-c06b-4987-9d45-f9f9e7721709" type="result">
+		<query xmlns="http://jabber.org/protocol/disco#items">
+			<item name="Nice tunes" node="http://jabber.org/protocol/tune" jid="${Romeo's JID}"/>
+		</query>
+	</iq>
+
+Romeo sends:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current">
+					<tune xmlns="http://jabber.org/protocol/tune"/>
+				</item>
+			</publish>
+			<publish-options>
+				<x xmlns="jabber:x:data">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#publish-options</value>
+					</field>
+					<field var="pubsub#access_model">
+						<value>whitelist</value>
+					</field>
+				</x>
+			</publish-options>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq type='error' id='67eb1f47-1e69-4cb3-91e2-4d5943e72d4c'>
+		<error type='cancel'>
+			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+			<text xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'>Field does not match: access_model</text>
+			<precondition-not-met xmlns='http://jabber.org/protocol/pubsub#errors'/>
+		</error>
+	</iq>
+
+Romeo disconnects
+
diff --git a/spec/util_array_spec.lua b/spec/util_array_spec.lua
new file mode 100644
index 00000000..04d0cc28
--- /dev/null
+++ b/spec/util_array_spec.lua
@@ -0,0 +1,154 @@
+local array = require "util.array";
+describe("util.array", function ()
+	describe("creation", function ()
+		describe("from table", function ()
+			it("works", function ()
+				local a = array({"a", "b", "c"});
+				assert.same({"a", "b", "c"}, a);
+			end);
+		end);
+
+		describe("from iterator", function ()
+			it("works", function ()
+				-- collects the first value, ie the keys
+				local a = array(ipairs({true, true, true}));
+				assert.same({1, 2, 3}, a);
+			end);
+		end);
+
+		describe("collect", function ()
+			it("works", function ()
+				-- collects the first value, ie the keys
+				local a = array.collect(ipairs({true, true, true}));
+				assert.same({1, 2, 3}, a);
+			end);
+		end);
+
+	end);
+
+	describe("metatable", function ()
+		describe("operator", function ()
+			describe("addition", function ()
+				it("works", function ()
+					local a = array({ "a", "b" });
+					local b = array({ "c", "d" });
+					assert.same({"a", "b", "c", "d"}, a + b);
+				end);
+			end);
+
+			describe("equality", function ()
+				it("works", function ()
+					local a1 = array({ "a", "b" });
+					local a2 = array({ "a", "b" });
+					local b = array({ "c", "d" });
+					assert.truthy(a1 == a2);
+					assert.falsy(a1 == b);
+				end);
+			end);
+
+			describe("division", function ()
+				it("works", function ()
+					local a = array({ "a", "b", "c" });
+					local b = a / function (i) if i ~= "b" then return i .. "x" end end;
+					assert.same({ "ax", "cx" }, b);
+				end);
+			end);
+
+		end);
+	end);
+
+	describe("methods", function ()
+		describe("map", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				local b = a:map(string.upper);
+				assert.same({ "A", "B", "C" }, b);
+			end);
+		end);
+
+		describe("filter", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:filter(function (i) return i ~= "b" end);
+				assert.same({ "a", "c" }, a);
+			end);
+		end);
+
+		describe("sort", function ()
+			it("works", function ()
+				local a = array({ 5, 4, 3, 1, 2, });
+				a:sort();
+				assert.same({ 1, 2, 3, 4, 5, }, a);
+			end);
+		end);
+
+		describe("unique", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c", "c", "a", "b" });
+				a:unique();
+				assert.same({ "a", "b", "c" }, a);
+			end);
+		end);
+
+		describe("pluck", function ()
+			it("works", function ()
+				local a = array({ { a = 1, b = -1 }, { a = 2, b = -2 }, });
+				a:pluck("a");
+				assert.same({ 1, 2 }, a);
+			end);
+		end);
+
+
+		describe("reverse", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:reverse();
+				assert.same({ "c", "b", "a" }, a);
+			end);
+		end);
+
+		-- TODO :shuffle
+
+		describe("append", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:append(array({ "d", "e", }));
+				assert.same({ "a", "b", "c", "d", "e" }, a);
+			end);
+		end);
+
+		describe("push", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:push("d"):push("e");
+				assert.same({ "a", "b", "c", "d", "e" }, a);
+			end);
+		end);
+
+		describe("pop", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal("c", a:pop());
+				assert.same({ "a", "b", }, a);
+			end);
+		end);
+
+		describe("concat", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal("a,b,c", a:concat(","));
+			end);
+		end);
+
+		describe("length", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal(3, a:length());
+			end);
+		end);
+
+	end);
+
+	-- TODO The various array.foo(array ina, array outa) functions
+end);
+
diff --git a/spec/util_async_spec.lua b/spec/util_async_spec.lua
index d2de8c94..8123503b 100644
--- a/spec/util_async_spec.lua
+++ b/spec/util_async_spec.lua
@@ -544,6 +544,8 @@ describe("util.async", function()
 			assert.equal(r1.state, "ready");
 		end);
 
+		-- luacheck: ignore 211/rf
+		-- FIXME what's rf?
 		it("should support multiple done() calls", function ()
 			local processed_item;
 			local wait, done;
diff --git a/spec/util_dataforms_spec.lua b/spec/util_dataforms_spec.lua
index 89759035..085128d1 100644
--- a/spec/util_dataforms_spec.lua
+++ b/spec/util_dataforms_spec.lua
@@ -110,7 +110,7 @@ describe("util.dataforms", function ()
 		xform = some_form:form();
 	end);
 
-	it("works", function ()
+	it("XML serialization looks like it should", function ()
 		assert.truthy(xform);
 		assert.truthy(st.is_stanza(xform));
 		assert.equal("x", xform.name);
@@ -316,7 +316,7 @@ describe("util.dataforms", function ()
 	end);
 
 	describe(":data", function ()
-		it("works", function ()
+		it("returns something", function ()
 			assert.truthy(some_form:data(xform));
 		end);
 	end);
@@ -402,7 +402,7 @@ describe("util.dataforms", function ()
 		end);
 	end);
 
-	describe("validation", function ()
+	describe("datatype validation", function ()
 		local f = dataforms.new {
 			{
 				name = "number",
@@ -411,12 +411,12 @@ describe("util.dataforms", function ()
 			},
 		};
 
-		it("works", function ()
+		it("integer roundtrip works", function ()
 			local d = f:data(f:form({number = 1}));
 			assert.equal(1, d.number);
 		end);
 
-		it("works", function ()
+		it("integer error handling works", function ()
 			local d,e = f:data(f:form({number = "nan"}));
 			assert.not_equal(1, d.number);
 			assert.table(e);
diff --git a/spec/util_error_spec.lua b/spec/util_error_spec.lua
new file mode 100644
index 00000000..ca053285
--- /dev/null
+++ b/spec/util_error_spec.lua
@@ -0,0 +1,70 @@
+local errors = require "util.error"
+
+describe("util.error", function ()
+	describe("new()", function ()
+		it("works", function ()
+			local err = errors.new("bork", "bork bork");
+			assert.not_nil(err);
+			assert.equal("cancel", err.type);
+			assert.equal("undefined-condition", err.condition);
+			assert.same("bork bork", err.context);
+		end);
+
+		describe("templates", function ()
+			it("works", function ()
+				local templates = {
+					["fail"] = {
+						type = "wait",
+						condition = "internal-server-error",
+						code = 555;
+					};
+				};
+				local err = errors.new("fail", { traceback = "in some file, somewhere" }, templates);
+				assert.equal("wait", err.type);
+				assert.equal("internal-server-error", err.condition);
+				assert.equal(555, err.code);
+				assert.same({ traceback = "in some file, somewhere" }, err.context);
+			end);
+		end);
+
+	end);
+
+	describe("is_err()", function ()
+		it("works", function ()
+			assert.truthy(errors.is_err(errors.new()));
+			assert.falsy(errors.is_err("not an error"));
+		end);
+	end);
+
+	describe("coerce", function ()
+		it("works", function ()
+			local ok, err = errors.coerce(nil, "it dun goofed");
+			assert.is_nil(ok);
+			assert.truthy(errors.is_err(err))
+		end);
+	end);
+
+	describe("from_stanza", function ()
+		it("works", function ()
+			local st = require "util.stanza";
+			local m = st.message({ type = "chat" });
+			local e = st.error_reply(m, "modify", "bad-request");
+			local err = errors.from_stanza(e);
+			assert.truthy(errors.is_err(err));
+			assert.equal("modify", err.type);
+			assert.equal("bad-request", err.condition);
+			assert.equal(e, err.context.stanza);
+		end);
+	end);
+
+	describe("__tostring", function ()
+		it("doesn't throw", function ()
+			assert.has_no.errors(function ()
+				-- See 6f317e51544d
+				tostring(errors.new());
+			end);
+		end);
+	end);
+
+end);
+
diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua
index 7e6a0c6e..50509630 100644
--- a/spec/util_format_spec.lua
+++ b/spec/util_format_spec.lua
@@ -5,10 +5,15 @@ describe("util.format", function()
 		it("should work", function()
 			assert.equal("hello", format("%s", "hello"));
 			assert.equal("<nil>", format("%s"));
+			assert.equal("<nil>", format("%d"));
+			assert.equal("<nil>", format("%q"));
 			assert.equal(" [<nil>]", format("", nil));
 			assert.equal("true", format("%s", true));
 			assert.equal("[true]", format("%d", true));
 			assert.equal("% [true]", format("%%", true));
+			assert.equal("{ }", format("%q", { }));
+			assert.equal("[1.5]", format("%d", 1.5));
+			assert.equal("[7.3786976294838e+19]", format("%d", 73786976294838206464));
 		end);
 	end);
 end);
diff --git a/spec/util_hashes_spec.lua b/spec/util_hashes_spec.lua
new file mode 100644
index 00000000..1e6187bb
--- /dev/null
+++ b/spec/util_hashes_spec.lua
@@ -0,0 +1,37 @@
+-- Test vectors from RFC 6070
+local hashes = require "util.hashes";
+local hex = require "util.hex";
+
+-- Also see spec for util.hmac where HMAC test cases reside
+
+describe("PBKDF2-SHA1", function ()
+	it("test vector 1", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 1
+		local DK = "0c60c80f961f0e71f3a9b524af6012062fe037a6";
+		assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c)));
+	end);
+	it("test vector 2", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 2
+		local DK = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957";
+		assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c)));
+	end);
+	it("test vector 3", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 4096
+		local DK = "4b007901b765489abead49d926f721d065a429c1";
+		assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c)));
+	end);
+	it("test vector 4 #SLOW", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 16777216
+		local DK = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984";
+		assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c)));
+	end);
+end);
+
diff --git a/spec/util_hashring_spec.lua b/spec/util_hashring_spec.lua
new file mode 100644
index 00000000..d8801774
--- /dev/null
+++ b/spec/util_hashring_spec.lua
@@ -0,0 +1,85 @@
+local hashring = require "util.hashring";
+
+describe("util.hashring", function ()
+
+	local sha256 = require "util.hashes".sha256;
+
+	local ring = hashring.new(128, sha256);
+
+	it("should fail to get a node that does not exist", function ()
+		assert.is_nil(ring:get_node("foo"))
+	end);
+
+	it("should support adding nodes", function ()
+		ring:add_node("node1");
+	end);
+
+	it("should return a single node for all keys if only one node exists", function ()
+		for i = 1, 100 do
+			assert.is_equal("node1", ring:get_node(tostring(i)))
+		end
+	end);
+
+	it("should support adding a second node", function ()
+		ring:add_node("node2");
+	end);
+
+	it("should fail to remove a non-existent node", function ()
+		assert.is_falsy(ring:remove_node("node3"));
+	end);
+
+	it("should succeed to remove a node", function ()
+		assert.is_truthy(ring:remove_node("node1"));
+	end);
+
+	it("should return the only node for all keys", function ()
+		for i = 1, 100 do
+			assert.is_equal("node2", ring:get_node(tostring(i)))
+		end
+	end);
+
+	it("should support adding multiple nodes", function ()
+		ring:add_nodes({ "node1", "node3", "node4", "node5" });
+	end);
+
+	it("should disrupt a minimal number of keys on node removal", function ()
+		local orig_ring = ring:clone();
+		local node_tallies = {};
+
+		local n = 1000;
+
+		for i = 1, n do
+			local key = tostring(i);
+			local node = ring:get_node(key);
+			node_tallies[node] = (node_tallies[node] or 0) + 1;
+		end
+
+		--[[
+		for node, key_count in pairs(node_tallies) do
+			print(node, key_count, ("%.2f%%"):format((key_count/n)*100));
+		end
+		]]
+
+		ring:remove_node("node5");
+
+		local disrupted_keys = 0;
+		for i = 1, n do
+			local key = tostring(i);
+			if orig_ring:get_node(key) ~= ring:get_node(key) then
+				disrupted_keys = disrupted_keys + 1;
+			end
+		end
+		assert.is_equal(node_tallies["node5"], disrupted_keys);
+	end);
+
+	it("should support removing multiple nodes", function ()
+		ring:remove_nodes({"node2", "node3", "node4", "node5"});
+	end);
+
+	it("should return a single node for all keys if only one node remains", function ()
+		for i = 1, 100 do
+			assert.is_equal("node1", ring:get_node(tostring(i)))
+		end
+	end);
+
+end);
diff --git a/spec/util_hmac_spec.lua b/spec/util_hmac_spec.lua
new file mode 100644
index 00000000..a2125c3a
--- /dev/null
+++ b/spec/util_hmac_spec.lua
@@ -0,0 +1,106 @@
+-- Test cases from RFC 4231
+
+-- Yes, the lines are long, it's annoying to split the long hex things.
+-- luacheck: ignore 631
+
+local hmac = require "util.hmac";
+local hex = require "util.hex";
+
+describe("Test case 1", function ()
+	local Key  = hex.from("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b");
+	local Data = hex.from("4869205468657265");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 2", function ()
+	local Key  = hex.from("4a656665");
+	local Data = hex.from("7768617420646f2079612077616e7420666f72206e6f7468696e673f");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 3", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("fa73b0089d56a284efb0f0756c890be9b1b5dbdd8ee81a3655f83e33b2279d39bf3e848279a722c806b485a47e67c807b946a337bee8942674278859e13292fb", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 4", function ()
+	local Key  = hex.from("0102030405060708090a0b0c0d0e0f10111213141516171819");
+	local Data = hex.from("cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("b0ba465637458c6990e5a8c5f61d4af7e576d97ff94b872de76f8050361ee3dba91ca5c11aa25eb4d679275cc5788063a5f19741120c4f2de2adebeb10a298dd", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 5", function ()
+	local Key  = hex.from("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c");
+	local Data = hex.from("546573742057697468205472756e636174696f6e");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("a3b6167473100ee06e0c796c2955552b", hmac.sha256(Key, Data, true):sub(1,128/4))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("415fad6271580a531d4179bc891d87a6", hmac.sha512(Key, Data, true):sub(1,128/4))
+		end);
+	end);
+end);
+describe("Test case 6", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("54657374205573696e67204c6172676572205468616e20426c6f636b2d53697a65204b6579202d2048617368204b6579204669727374");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("80b24263c7c1a3ebb71493c1dd7be8b49b46d1f41b4aeec1121b013783f8f3526b56d037e05f2598bd0fd2215d6a1e5295e64f73f63f0aec8b915a985d786598", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 7", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("5468697320697320612074657374207573696e672061206c6172676572207468616e20626c6f636b2d73697a65206b657920616e642061206c6172676572207468616e20626c6f636b2d73697a6520646174612e20546865206b6579206e6565647320746f20626520686173686564206265666f7265206265696e6720757365642062792074686520484d414320616c676f726974686d2e");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
diff --git a/spec/util_http_spec.lua b/spec/util_http_spec.lua
index 0f51a86c..d38645f7 100644
--- a/spec/util_http_spec.lua
+++ b/spec/util_http_spec.lua
@@ -28,6 +28,11 @@ describe("util.http", function()
 		it("should decode important URL characters", function()
 			assert.are.equal("This & that = something", http.urldecode("This%20%26%20that%20%3d%20something"), "Important URL chars escaped");
 		end);
+
+		it("should decode both lower and uppercase", function ()
+			assert.are.equal("This & that = {something}.", http.urldecode("This%20%26%20that%20%3D%20%7Bsomething%7D%2E"), "Important URL chars escaped");
+		end);
+
 	end);
 
 	describe("#formencode()", function()
diff --git a/spec/util_interpolation_spec.lua b/spec/util_interpolation_spec.lua
new file mode 100644
index 00000000..76000d94
--- /dev/null
+++ b/spec/util_interpolation_spec.lua
@@ -0,0 +1,47 @@
+local template = [[
+{greet!?Hi}, {name?world}!
+]];
+local expect1 = [[
+Hello, WORLD!
+]];
+local expect2 = [[
+Hello, world!
+]];
+local expect3 = [[
+Hi, YOU!
+]];
+local template_array = [[
+{foo#{idx}. {item}
+}]]
+local expect_array = [[
+1. HELLO
+2. WORLD
+]]
+local template_func_pipe = [[
+{foo|sort#{idx}. {item}
+}]]
+local expect_func_pipe = [[
+1. A
+2. B
+3. C
+4. D
+]]
+local template_map = [[
+{foo%{idx}: {item!}
+}]]
+local expect_map = [[
+FOO: bar
+]]
+
+describe("util.interpolation", function ()
+	it("renders", function ()
+		local render = require "util.interpolation".new("%b{}", string.upper, { sort = function (t) table.sort(t) return t end });
+		assert.equal(expect1, render(template, { greet = "Hello", name = "world" }));
+		assert.equal(expect2, render(template, { greet = "Hello" }));
+		assert.equal(expect3, render(template, { name = "you" }));
+		assert.equal(expect_array, render(template_array, { foo = { "Hello", "World" } }));
+		assert.equal(expect_func_pipe, render(template_func_pipe, { foo = { "c", "a", "d", "b", } }));
+		-- assert.equal("", render(template_func_pipe, { foo = nil })); -- FIXME
+		assert.equal(expect_map, render(template_map, { foo = { foo = "bar" } }));
+	end);
+end);
diff --git a/spec/util_promise_spec.lua b/spec/util_promise_spec.lua
index 65d252f6..0008c6a2 100644
--- a/spec/util_promise_spec.lua
+++ b/spec/util_promise_spec.lua
@@ -248,6 +248,30 @@ describe("util.promise", function ()
 			assert.spy(cb3).was_called(1);
 			assert.spy(cb3).was_called_with("goodbye");
 		end);
+
+		it("ordinary values", function ()
+			local p = promise.resolve()
+			local cb = spy.new(function ()
+				return "hello"
+			end);
+			local cb2 = spy.new(function () end);
+			p:next(cb):next(cb2);
+			assert.spy(cb).was_called(1);
+			assert.spy(cb2).was_called(1);
+			assert.spy(cb2).was_called_with("hello");
+		end);
+
+		it("nil", function ()
+			local p = promise.resolve()
+			local cb = spy.new(function ()
+				return
+			end);
+			local cb2 = spy.new(function () end);
+			p:next(cb):next(cb2);
+			assert.spy(cb).was_called(1);
+			assert.spy(cb2).was_called(1);
+			assert.spy(cb2).was_called_with(nil);
+		end);
 	end);
 
 	describe("race()", function ()
diff --git a/spec/util_pubsub_spec.lua b/spec/util_pubsub_spec.lua
index c982fb36..75f893e3 100644
--- a/spec/util_pubsub_spec.lua
+++ b/spec/util_pubsub_spec.lua
@@ -101,13 +101,14 @@ describe("util.pubsub", function ()
 			assert(service:publish("node", true, "1", "item 1", { myoption = true }));
 
 			local ok, config = assert(service:get_node_config("node", true));
+			assert.truthy(ok);
 			assert.equals(true, config.myoption);
 		end);
 
 		it("fails to publish to a node with differing config", function ()
 			local ok, err = service:publish("node", true, "1", "item 2", { myoption = false });
 			assert.falsy(ok);
-			assert.equals("precondition-not-met", err);
+			assert.equals("precondition-not-met", err.pubsub_condition);
 		end);
 
 		it("allows to publish to a node with differing config when only defaults are suggested", function ()
@@ -229,6 +230,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be the default", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("open", config.access_model);
 			end);
 			it("should allow anyone to subscribe", function ()
@@ -250,6 +252,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be present in the configuration", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("whitelist", config.access_model);
 			end);
 			it("should not allow anyone to subscribe", function ()
@@ -294,6 +297,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be the default", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("publishers", config.publish_model);
 			end);
 			it("should not allow anyone to publish", function ()
@@ -304,6 +308,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should allow publishers to publish", function ()
 				assert(service:set_affiliation("test", true, "mypublisher", "publisher"));
+				-- luacheck: ignore 211/err
 				local ok, err = service:publish("test", "mypublisher", "item1", "foo");
 				assert.is_true(ok);
 			end);
@@ -342,6 +347,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should allow publishers to publish without a subscription", function ()
 				assert(service:set_affiliation("test", true, "mypublisher", "publisher"));
+				-- luacheck: ignore 211/err
 				local ok, err = service:publish("test", "mypublisher", "item1", "foo");
 				assert.is_true(ok);
 			end);
diff --git a/spec/util_queue_spec.lua b/spec/util_queue_spec.lua
index 7cd3d695..d73f523d 100644
--- a/spec/util_queue_spec.lua
+++ b/spec/util_queue_spec.lua
@@ -100,4 +100,41 @@ describe("util.queue", function()
 
 		end);
 	end);
+	describe("consume()", function ()
+		it("should work", function ()
+			local q = queue.new(10);
+			for i = 1, 5 do
+				q:push(i);
+			end
+			local c = 0;
+			for i in q:consume() do
+				assert(i == c + 1);
+				assert(q:count() == (5-i));
+				c = i;
+			end
+		end);
+
+		it("should work even if items are pushed in the loop", function ()
+			local q = queue.new(10);
+			for i = 1, 5 do
+				q:push(i);
+			end
+			local c = 0;
+			for i in q:consume() do
+				assert(i == c + 1);
+				if c < 3 then
+					assert(q:count() == (5-i));
+				else
+					assert(q:count() == (6-i));
+				end
+
+				c = i;
+
+				if c == 3 then
+					q:push(6);
+				end
+			end
+			assert.equal(c, 6);
+		end);
+	end);
 end);
diff --git a/spec/util_sasl_spec.lua b/spec/util_sasl_spec.lua
new file mode 100644
index 00000000..368291b3
--- /dev/null
+++ b/spec/util_sasl_spec.lua
@@ -0,0 +1,43 @@
+local sasl = require "util.sasl";
+
+-- profile * mechanism
+-- callbacks could use spies instead
+
+describe("util.sasl", function ()
+	describe("plain_test profile", function ()
+		local profile = {
+			plain_test = function (_, username, password, realm)
+				assert.equals("user", username)
+				assert.equals("pencil", password)
+				assert.equals("sasl.test", realm)
+				return true, true;
+			end;
+		};
+		it("works with PLAIN", function ()
+			local plain = sasl.new("sasl.test", profile);
+			assert.truthy(plain:select("PLAIN"));
+			assert.truthy(plain:process("\000user\000pencil"));
+			assert.equals("user", plain.username);
+		end);
+	end);
+
+	describe("plain profile", function ()
+		local profile = {
+			plain = function (_, username, realm)
+				assert.equals("user", username)
+				assert.equals("sasl.test", realm)
+				return "pencil", true;
+			end;
+		};
+
+		it("works with PLAIN", function ()
+			local plain = sasl.new("sasl.test", profile);
+			assert.truthy(plain:select("PLAIN"));
+			assert.truthy(plain:process("\000user\000pencil"));
+			assert.equals("user", plain.username);
+		end);
+
+		-- TODO SCRAM
+	end);
+end);
+
diff --git a/spec/util_stanza_spec.lua b/spec/util_stanza_spec.lua
index 6fbae41a..d38a609f 100644
--- a/spec/util_stanza_spec.lua
+++ b/spec/util_stanza_spec.lua
@@ -1,5 +1,6 @@
 
 local st = require "util.stanza";
+local errors = require "util.error";
 
 describe("util.stanza", function()
 	describe("#preserialize()", function()
@@ -95,20 +96,31 @@ describe("util.stanza", function()
 
 	describe("#iq()", function()
 		it("should create an iq stanza", function()
-			local i = st.iq({ id = "foo" });
+			local i = st.iq({ type = "get", id = "foo" });
 			assert.are.equal("iq", i.name);
 			assert.are.equal("foo", i.attr.id);
+			assert.are.equal("get", i.attr.type);
 		end);
 
-		it("should reject stanzas with no id", function ()
+		it("should reject stanzas with no attributes", function ()
 			assert.has.error_match(function ()
 				st.iq();
-			end, "id attribute");
+			end, "attributes");
+		end);
 
+
+		it("should reject stanzas with no id", function ()
 			assert.has.error_match(function ()
-				st.iq({ foo = "bar" });
+				st.iq({ type = "get" });
 			end, "id attribute");
 		end);
+
+		it("should reject stanzas with no type", function ()
+			assert.has.error_match(function ()
+				st.iq({ id = "foo" });
+			end, "type attribute");
+
+		end);
 	end);
 
 	describe("#presence()", function ()
@@ -159,6 +171,19 @@ describe("util.stanza", function()
 			assert.are.equal(r.attr.type, "result");
 			assert.are.equal(#r.tags, 0, "A reply should not include children of the original stanza");
 		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.reply(not "a stanza");
+			end, "expected stanza");
+		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.reply({name="x"});
+			end, "expected stanza");
+		end);
+
 	end);
 
 	describe("#error_reply()", function()
@@ -167,13 +192,14 @@ describe("util.stanza", function()
 			local s = st.stanza("s", { to = "touser", from = "fromuser", id = "123" })
 				:tag("child1");
 			-- Make reply stanza
-			local r = st.error_reply(s, "cancel", "service-unavailable");
+			local r = st.error_reply(s, "cancel", "service-unavailable", nil, "host");
 			assert.are.equal(r.name, s.name);
 			assert.are.equal(r.id, s.id);
 			assert.are.equal(r.attr.to, s.attr.from);
 			assert.are.equal(r.attr.from, s.attr.to);
 			assert.are.equal(#r.tags, 1);
 			assert.are.equal(r.tags[1].tags[1].name, "service-unavailable");
+			assert.are.equal(r.tags[1].attr.by, "host");
 		end);
 
 		it("should work for <iq get>", function()
@@ -190,6 +216,38 @@ describe("util.stanza", function()
 			assert.are.equal(#r.tags, 1);
 			assert.are.equal(r.tags[1].tags[1].name, "service-unavailable");
 		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.error_reply(not "a stanza", "modify", "bad-request");
+			end, "expected stanza");
+		end);
+
+		it("should reject stanzas of type error", function ()
+			assert.has.error_match(function ()
+				st.error_reply(st.message({type="error"}), "cancel", "conflict");
+			end, "got stanza of type error");
+			assert.has.error_match(function ()
+				st.error_reply(st.error_reply(st.message({type="chat"}), "modify", "forbidden"), "cancel", "service-unavailable");
+			end, "got stanza of type error");
+		end);
+
+		it("should accept util.error objects", function ()
+			local s = st.message({ to = "touser", from = "fromuser", id = "123", type = "chat" }, "Hello");
+			local e = errors.new({ type = "modify", condition = "not-acceptable", text = "Bork bork bork" });
+			local r = st.error_reply(s, e);
+
+			assert.are.equal(r.name, s.name);
+			assert.are.equal(r.id, s.id);
+			assert.are.equal(r.attr.to, s.attr.from);
+			assert.are.equal(r.attr.from, s.attr.to);
+			assert.are.equal(r.attr.type, "error");
+			assert.are.equal(r.tags[1].name, "error");
+			assert.are.equal(r.tags[1].attr.type, e.type);
+			assert.are.equal(r.tags[1].tags[1].name, e.condition);
+			assert.are.equal(r.tags[1].tags[2]:get_text(), e.text);
+		end);
+
 	end);
 
 	describe("should reject #invalid", function ()
@@ -370,4 +428,35 @@ describe("util.stanza", function()
 			end);
 		end);
 	end);
+
+	describe("top_tag", function ()
+		local xml_parse = require "util.xml".parse;
+		it("works", function ()
+			local s = st.message({type="chat"}, "Hello");
+			local top_tag = s:top_tag();
+			assert.is_string(top_tag);
+			assert.not_equal("/>", top_tag:sub(-2, -1));
+			assert.equal(">", top_tag:sub(-1, -1));
+			local s2 = xml_parse(top_tag.."</message>");
+			assert(st.is_stanza(s2));
+			assert.equal("message", s2.name);
+			assert.equal(0, #s2);
+			assert.equal(0, #s2.tags);
+			assert.equal("chat", s2.attr.type);
+		end);
+
+		it("works with namespaced attributes", function ()
+			local s = xml_parse[[<message foo:bar='true' xmlns:foo='my-awesome-ns'/>]];
+			local top_tag = s:top_tag();
+			assert.is_string(top_tag);
+			assert.not_equal("/>", top_tag:sub(-2, -1));
+			assert.equal(">", top_tag:sub(-1, -1));
+			local s2 = xml_parse(top_tag.."</message>");
+			assert(st.is_stanza(s2));
+			assert.equal("message", s2.name);
+			assert.equal(0, #s2);
+			assert.equal(0, #s2.tags);
+			assert.equal("true", s2.attr["my-awesome-ns\1bar"]);
+		end);
+	end);
 end);
diff --git a/spec/util_table_spec.lua b/spec/util_table_spec.lua
new file mode 100644
index 00000000..76f54b69
--- /dev/null
+++ b/spec/util_table_spec.lua
@@ -0,0 +1,17 @@
+local u_table = require "util.table";
+describe("util.table", function ()
+	describe("create()", function ()
+		it("works", function ()
+			-- Can't test the allocated sizes of the table, so what you gonna do?
+			assert.is.table(u_table.create(1,1));
+		end);
+	end);
+
+	describe("pack()", function ()
+		it("works", function ()
+			assert.same({ "lorem", "ipsum", "dolor", "sit", "amet", n = 5 }, u_table.pack("lorem", "ipsum", "dolor", "sit", "amet"));
+		end);
+	end);
+end);
+
+
diff --git a/spec/util_throttle_spec.lua b/spec/util_throttle_spec.lua
index 75daf1b9..985afae8 100644
--- a/spec/util_throttle_spec.lua
+++ b/spec/util_throttle_spec.lua
@@ -88,7 +88,7 @@ describe("util.throttle", function()
 				later(0.1);
 				a:update();
 			end
-			assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rouding errors
+			assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rounding errors
 		end);
 	end);
 
diff --git a/tools/make_repo.lua b/tools/make_repo.lua
new file mode 100644
index 00000000..5b5e6234
--- /dev/null
+++ b/tools/make_repo.lua
@@ -0,0 +1,44 @@
+print("Getting all the available modules")
+if os.execute '[ -e "./downloaded_modules" ]' then
+	os.execute("rm -rf downloaded_modules")
+end
+os.execute("hg clone https://hg.prosody.im/prosody-modules/ downloaded_modules")
+local i, popen = 0, io.popen
+local flag = "mod_"
+if os.execute '[ -e "./repository" ]' then
+	os.execute("mkdir repository")
+end
+local pfile = popen('ls -a "downloaded_modules"')
+for filename in pfile:lines() do
+	i = i + 1
+	if filename:sub(1, #flag) == flag then
+		local file = io.open("repository/"..filename.."-scm-1.rockspec", "w")
+		file:write('package = "'..filename..'"', '\n')
+		file:write('version = "scm-1"', '\n')
+		file:write('source = {', '\n')
+		file:write('\turl = "hg+https://hg.prosody.im/prosody-modules",', '\n')
+		file:write('\tdir = "prosody-modules"', '\n')
+		file:write('}', '\n')
+		file:write('description = {', '\n')
+		file:write('\thomepage = "https://prosody.im/",', '\n')
+		file:write('\tlicense = "MIT"', '\n')
+		file:write('}', '\n')
+		file:write('dependencies = {', '\n')
+		file:write('\t"lua >= 5.1"', '\n')
+		file:write('}', '\n')
+		file:write('build = {', '\n')
+		file:write('\ttype = "builtin",', '\n')
+		file:write('\tmodules = {', '\n')
+		file:write('\t\t["'..filename..'.'..filename..'"] = "'..filename..'/'..filename..'.lua"', '\n')
+		file:write('\t}', '\n')
+		file:write('}', '\n')
+		file:close()
+	end
+end
+pfile:close()
+os.execute("cd repository/ && luarocks-admin make_manifest ./ && chmod -R 644 ./*")
+print("")
+print("Done!. Modules' sources are locally available at ./downloaded_modules")
+print("Repository is available at ./repository")
+print("The repository contains all of prosody modules' respective rockspecs, as well as manifest files and an html Index")
+print("You can now either point your server to this folder, or copy its contents to another configured folder.")
diff --git a/tools/migration/Makefile b/tools/migration/Makefile
index 713831d2..dc14e0da 100644
--- a/tools/migration/Makefile
+++ b/tools/migration/Makefile
@@ -12,16 +12,13 @@ INSTALLEDCONFIG = $(SYSCONFDIR)
 INSTALLEDMODULES = $(LIBDIR)/prosody/modules
 INSTALLEDDATA = $(DATADIR)
 
-SOURCE_FILES = migrator/*.lua
-
-all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES)
+all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua
 
 install: prosody-migrator.install migrator.cfg.lua.install
-	install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator
+	install -d $(BIN) $(CONFIG) $(SOURCE)
 	install -d $(MAN)/man1
 	install -d $(SOURCE)/migrator
 	install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator
-	install -m644 $(SOURCE_FILES) $(SOURCE)/migrator
 	test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua
 
 clean:
diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua
index fa37f2a3..c26fa869 100644
--- a/tools/migration/migrator.cfg.lua
+++ b/tools/migration/migrator.cfg.lua
@@ -1,12 +1,38 @@
 local data_path = "../../data";
 
+local vhost = {
+	"accounts",
+	"account_details",
+	"roster",
+	"vcard",
+	"private",
+	"blocklist",
+	"privacy",
+	"archive-archive",
+	"offline-archive",
+	"pubsub_nodes",
+	-- "pubsub_*-archive",
+	"pep",
+	-- "pep_*-archive",
+}
+local muc = {
+	"persistent",
+	"config",
+	"state",
+	"muc_log-archive",
+};
+
 input {
-	type = "prosody_files";
+	hosts = {
+		["example.com"] = vhost;
+		["conference.example.com"] = muc;
+	};
+	type = "internal";
 	path = data_path;
 }
 
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
@@ -14,11 +40,11 @@ output {
 --[[
 
 input {
-	type = "prosody_files";
+	type = "internal";
 	path = data_path;
 }
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua
deleted file mode 100644
index cfbfcce8..00000000
--- a/tools/migration/migrator/mtools.lua
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-local print = print;
-local t_insert = table.insert;
-local t_sort = table.sort;
-
-
-local function sorted(params)
-
-	local reader = params.reader; -- iterator to get items from
-	local sorter = params.sorter; -- sorting function
-	local filter = params.filter; -- filter function
-
-	local cache = {};
-	for item in reader do
-		if filter then item = filter(item); end
-		if item then t_insert(cache, item); end
-	end
-	if sorter then
-		t_sort(cache, sorter);
-	end
-	local i = 0;
-	return function()
-		i = i + 1;
-		return cache[i];
-	end;
-
-end
-
-local function merged(reader, merger)
-
-	local item1 = reader();
-	local merged = { item1 };
-	return function()
-		while true do
-			if not item1 then return nil; end
-			local item2 = reader();
-			if not item2 then item1 = nil; return merged; end
-			if merger(item1, item2) then
-			--print("merged")
-				item1 = item2;
-				t_insert(merged, item1);
-			else
-			--print("unmerged", merged)
-				item1 = item2;
-				local tmp = merged;
-				merged = { item1 };
-				return tmp;
-			end
-		end
-	end;
-
-end
-
-return {
-	sorted = sorted;
-	merged = merged;
-}
diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua
deleted file mode 100644
index 4de09273..00000000
--- a/tools/migration/migrator/prosody_files.lua
+++ /dev/null
@@ -1,144 +0,0 @@
-
-local print = print;
-local assert = assert;
-local setmetatable = setmetatable;
-local tonumber = tonumber;
-local char = string.char;
-local coroutine = coroutine;
-local lfs = require "lfs";
-local loadfile = loadfile;
-local pcall = pcall;
-local mtools = require "migrator.mtools";
-local next = next;
-local pairs = pairs;
-local json = require "util.json";
-local os_getenv = os.getenv;
-local error = error;
-
-prosody = {};
-local dm = require "util.datamanager"
-
-
-local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end
-local function is_file(path) return lfs.attributes(path, "mode") == "file"; end
-local function clean_path(path)
-	return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~");
-end
-local encode, decode; do
-	local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });
-	decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end
-	encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end
-end
-local function decode_dir(x)
-	if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x);
-	end
-end
-local function decode_file(x)
-	if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x:gsub("%.dat$", ""));
-	end
-end
-local function prosody_dir(path, ondir, onfile, ...)
-	for x in lfs.dir(path) do
-		local xpath = path.."/"..x;
-		if decode_dir(x) and is_dir(xpath) then
-			ondir(xpath, x, ...);
-		elseif decode_file(x) and is_file(xpath) then
-			onfile(xpath, x, ...);
-		end
-	end
-end
-
-local function handle_root_file(path, name)
-	--print("root file: ", decode_file(name))
-	coroutine.yield { user = nil, host = nil, store = decode_file(name) };
-end
-local function handle_host_file(path, name, host)
-	--print("host file: ", decode_dir(host).."/"..decode_file(name))
-	coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };
-end
-local function handle_store_file(path, name, store, host)
-	--print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))
-	coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };
-end
-local function handle_host_store(path, name, host)
-	prosody_dir(path, function() end, handle_store_file, name, host);
-end
-local function handle_host_dir(path, name)
-	prosody_dir(path, handle_host_store, handle_host_file, name);
-end
-local function handle_root_dir(path)
-	prosody_dir(path, handle_host_dir, handle_root_file);
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1].user;
-		host = item[1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		userdata.stores[store.store] = store.data;
-		store.user = nil; store.host = nil; store.store = nil;
-	end
-	return userdata;
-end
-
-local function reader(input)
-	local path = clean_path(assert(input.path, "no input.path specified"));
-	assert(is_dir(path), "input.path is not a directory");
-	local iter = coroutine.wrap(function()handle_root_dir(path);end);
-	-- get per-user stores, sorted
-	local iter = mtools.sorted {
-		reader = function()
-			local x = iter();
-			while x do
-				dm.set_data_path(path);
-				local err;
-				x.data, err = dm.load(x.user, x.host, x.store);
-				if x.data == nil and err then
-					local p = dm.getpath(x.user, x.host, x.store);
-					print(("Error loading data at path %s for %s@%s (%s store): %s")
-						:format(p, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>", err or "<nil>"));
-				else
-					return x;
-				end
-				x = iter();
-			end
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user);
-	end);
-
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end
-end
-
-local function writer(output)
-	local path = clean_path(assert(output.path, "no output.path specified"));
-	assert(is_dir(path), "output.path is not a directory");
-	return function(item)
-		if not item then return; end -- end of input
-		dm.set_data_path(path);
-		for store, data in pairs(item.stores) do
-			assert(dm.store(item.user, item.host, store, data));
-		end
-	end
-end
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua
deleted file mode 100644
index 6df2b025..00000000
--- a/tools/migration/migrator/prosody_sql.lua
+++ /dev/null
@@ -1,190 +0,0 @@
-
-local assert = assert;
-local have_DBI = pcall(require,"DBI");
-local print = print;
-local type = type;
-local next = next;
-local pairs = pairs;
-local t_sort = table.sort;
-local json = require "util.json";
-local mtools = require "migrator.mtools";
-local tostring = tostring;
-local tonumber = tonumber;
-
-if not have_DBI then
-	error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0);
-end
-
-local sql = require "util.sql";
-
-local function create_table(engine, name) -- luacheck: ignore 431/engine
-	local Table, Column, Index = sql.Table, sql.Column, sql.Index;
-
-	local ProsodyTable = Table {
-		name= name or "prosody";
-		Column { name="host", type="TEXT", nullable=false };
-		Column { name="user", type="TEXT", nullable=false };
-		Column { name="store", type="TEXT", nullable=false };
-		Column { name="key", type="TEXT", nullable=false };
-		Column { name="type", type="TEXT", nullable=false };
-		Column { name="value", type="MEDIUMTEXT", nullable=false };
-		Index { name="prosody_index", "host", "user", "store", "key" };
-	};
-	engine:transaction(function()
-		ProsodyTable:create(engine);
-	end);
-
-end
-
-local function serialize(value)
-	local t = type(value);
-	if t == "string" or t == "boolean" or t == "number" then
-		return t, tostring(value);
-	elseif t == "table" then
-		local value,err = json.encode(value);
-		if value then return "json", value; end
-		return nil, err;
-	end
-	return nil, "Unhandled value type: "..t;
-end
-local function deserialize(t, value)
-	if t == "string" then return value;
-	elseif t == "boolean" then
-		if value == "true" then return true;
-		elseif value == "false" then return false; end
-	elseif t == "number" then return tonumber(value);
-	elseif t == "json" then
-		return json.decode(value);
-	end
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1][1].user;
-		host = item[1][1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		for i=1,#store do -- loop over store data
-			local row = store[i];
-			local k = row.key;
-			local v = deserialize(row.type, row.value);
-			if k and v then
-				if k ~= "" then result[k] = v; elseif type(v) == "table" then
-					for a,b in pairs(v) do
-						result[a] = b;
-					end
-				end
-			end
-			userdata.stores[store[1].store] = result;
-		end
-	end
-	return userdata;
-end
-
-local function needs_upgrade(engine, params)
-	if params.driver == "MySQL" then
-		local success = engine:transaction(function()
-			local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'");
-			assert(result:rowcount() == 0);
-
-			-- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already
-			local check_encoding_query = [[
-			SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME"
-			FROM "information_schema"."columns"
-			WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' );
-			]];
-			check_encoding_query = check_encoding_query:format(engine.charset, engine.charset);
-			local result = engine:execute(check_encoding_query);
-			assert(result:rowcount() == 0)
-		end);
-		if not success then
-			-- Upgrade required
-			return true;
-		end
-	end
-	return false;
-end
-
-local function reader(input)
-	local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, input) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-	end));
-	local keys = {"host", "user", "store", "key", "type", "value"};
-	assert(engine:connect());
-	local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";"));
-	-- get SQL rows, sorted
-	local iter = mtools.sorted {
-		reader = function() val = f(s, val); return val; end;
-		filter = function(x)
-			for i=1,#keys do
-				x[ keys[i] ] = x[i];
-			end
-			if x.host  == "" then x.host  = nil; end
-			if x.user  == "" then x.user  = nil; end
-			if x.store == "" then x.store = nil; end
-			return x;
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge rows to get stores
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user and a.store == b.store);
-	end);
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a[1].host == b[1].host and a[1].user == b[1].user);
-	end);
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end;
-end
-
-local function writer(output, iter)
-	local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, output) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-		create_table(engine);
-	end));
-	assert(engine:connect());
-	assert(engine:delete("DELETE FROM \"prosody\""));
-	local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)";
-
-	return function(item)
-		if not item then assert(engine.conn:commit()) return end -- end of input
-		local host = item.host or "";
-		local user = item.user or "";
-		for store, data in pairs(item.stores) do
-			-- TODO transactions
-			local extradata = {};
-			for key, value in pairs(data) do
-				if type(key) == "string" and key ~= "" then
-					local t, value = assert(serialize(value));
-					local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value));
-				else
-					extradata[key] = value;
-				end
-			end
-			if next(extradata) ~= nil then
-				local t, extradata = assert(serialize(extradata));
-				local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata));
-			end
-		end
-	end;
-end
-
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua
index 1219d891..5c4800ad 100644
--- a/tools/migration/prosody-migrator.lua
+++ b/tools/migration/prosody-migrator.lua
@@ -1,19 +1,43 @@
 #!/usr/bin/env lua
 
-CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR");
-CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR");
+CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
+CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
+CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
+CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
 
--- Substitute ~ with path to home directory in paths
-if CFG_CONFIGDIR then
-	CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME"));
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+local function is_relative(path)
+	local path_sep = package.config:sub(1,1);
+        return ((path_sep == "/" and path:sub(1,1) ~= "/")
+	or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
 end
 
+-- Tell Lua where to find our libraries
 if CFG_SOURCEDIR then
-	CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME"));
+	local function filter_relative_paths(path)
+		if is_relative(path) then return ""; end
+	end
+	local function sanitise_paths(paths)
+		return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
+	end
+	package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
+	package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
+end
+
+-- Substitute ~ with path to home directory in data path
+if CFG_DATADIR then
+	if os.getenv("HOME") then
+		CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
+	end
 end
 
 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
 
+local startup = require "util.startup";
+startup.prosodyctl();
+-- TODO startup.migrator ?
+
 -- Command-line parsing
 local options = {};
 local i = 1;
@@ -29,13 +53,6 @@ while arg[i] do
 	end
 end
 
-if CFG_SOURCEDIR then
-	package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
-	package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
-else
-	package.path = "../../?.lua;"..package.path
-	package.cpath = "../../?.so;"..package.cpath
-end
 
 local envloadfile = require "util.envload".envloadfile;
 
@@ -69,24 +86,14 @@ if not config[to_store] then
 	print("Error: Output store '"..to_store.."' not found in the config file.");
 end
 
-function load_store_handler(name)
-	local store_type = config[name].type;
-	if not store_type then
-		print("Error: "..name.." store type not specified in the config file");
-		return false;
-	else
-		local ok, err = pcall(require, "migrator."..store_type);
-		if not ok then
-			print(("Error: Failed to initialize '%s' store:\n\t%s")
-				:format(name, err));
-			return false;
-		end
+for store, conf in pairs(config) do -- COMPAT
+	if conf.type == "prosody_files" then
+		conf.type = "internal";
+	elseif conf.type == "prosody_sql" then
+		conf.type = "sql";
 	end
-	return true;
 end
 
-have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
-
 if have_err then
 	print("");
 	print("Usage: "..arg[0].." FROM_STORE TO_STORE");
@@ -101,17 +108,88 @@ if have_err then
 	os.exit(1);
 end
 
-local itype = config[from_store].type;
-local otype = config[to_store].type;
-local reader = require("migrator."..itype).reader(config[from_store]);
-local writer = require("migrator."..otype).writer(config[to_store]);
+local async = require "util.async";
+local server = require "net.server";
+local watchers = {
+	error = function (_, err)
+		error(err);
+	end;
+	waiting = function ()
+		server.loop();
+	end;
+};
+
+local cm = require "core.configmanager";
+local hm = require "core.hostmanager";
+local sm = require "core.storagemanager";
+local um = require "core.usermanager";
+
+local function users(store, host)
+	if store.users then
+		return store:users();
+	else
+		return um.users(host);
+	end
+end
 
-local json = require "util.json";
+local function prepare_config(host, conf)
+	if conf.type == "internal" then
+		sm.olddm.set_data_path(conf.path or prosody.paths.data);
+	elseif conf.type == "sql" then
+		cm.set(host, "sql", conf);
+	end
+end
 
-io.stderr:write("Migrating...\n");
-for x in reader do
-	--print(json.encode(x))
-	writer(x);
+local function get_driver(host, conf)
+	prepare_config(host, conf);
+	return assert(sm.load_driver(host, conf.type));
 end
-writer(nil); -- close
+
+local migration_runner = async.runner(function (job)
+	for host, stores in pairs(job.input.hosts) do
+		prosody.hosts[host] = startup.make_host(host);
+		sm.initialize_host(host);
+		um.initialize_host(host);
+
+		local input_driver = get_driver(host, job.input);
+
+		local output_driver = get_driver(host, job.output);
+
+		for _, store in ipairs(stores) do
+			local p, typ = store:match("()%-(%w+)$");
+			if typ then store = store:sub(1, p-1); else typ = "keyval"; end
+			log("info", "Migrating host %s store %s (%s)", host, store, typ);
+
+			local origin = assert(input_driver:open(store, typ));
+			local destination = assert(output_driver:open(store, typ));
+
+			if typ == "keyval" then -- host data
+				local data, err = origin:get(nil);
+				assert(not err, err);
+				assert(destination:set(nil, data));
+			end
+
+			for user in users(origin, host) do
+				if typ == "keyval" then
+					local data, err = origin:get(user);
+					assert(not err, err);
+					assert(destination:set(user, data));
+				elseif typ == "archive" then
+					local iter, err = origin:find(user);
+					assert(iter, err);
+					for id, item, when, with in iter do
+						assert(destination:append(user, id, item, when, with));
+					end
+				else
+					error("Don't know how to migrate data of type '"..typ.."'.");
+				end
+			end
+		end
+	end
+end, watchers);
+
+io.stderr:write("Migrating...\n");
+
+migration_runner:run({ input = config[from_store], output = config[to_store] });
+
 io.stderr:write("Done!\n");
diff --git a/util-src/encodings.c b/util-src/encodings.c
index e55a3f44..996a6d4c 100644
--- a/util-src/encodings.c
+++ b/util-src/encodings.c
@@ -216,7 +216,7 @@ static const char *utf8_decode(const char *o, int *val) {
  * Check that a string is valid UTF-8
  * Returns NULL if not
  */
-const char *check_utf8(lua_State *L, int idx, size_t *l) {
+static const char *check_utf8(lua_State *L, int idx, size_t *l) {
 	size_t pos, len;
 	const char *s = luaL_checklstring(L, idx, &len);
 	pos = 0;
@@ -268,30 +268,36 @@ static const luaL_Reg Reg_utf8[] = {
 #include <unicode/usprep.h>
 #include <unicode/ustring.h>
 #include <unicode/utrace.h>
+#include <unicode/uspoof.h>
+#include <unicode/uidna.h>
 
 static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile) {
 	size_t input_len;
 	int32_t unprepped_len, prepped_len, output_len;
 	const char *input;
 	char output[1024];
+	int flags = USPREP_ALLOW_UNASSIGNED;
 
 	UChar unprepped[1024]; /* Temporary unicode buffer (1024 characters) */
 	UChar prepped[1024];
 
 	UErrorCode err = U_ZERO_ERROR;
 
-	if(!lua_isstring(L, 1)) {
-		lua_pushnil(L);
-		return 1;
-	}
-
-	input = lua_tolstring(L, 1, &input_len);
+	input = luaL_checklstring(L, 1, &input_len);
 
 	if(input_len >= 1024) {
 		lua_pushnil(L);
 		return 1;
 	}
 
+	/* strict */
+	if(!lua_isnoneornil(L, 2)) {
+		luaL_checktype(L, 2, LUA_TBOOLEAN);
+		if(lua_toboolean(L, 2)) {
+			flags = 0;
+		}
+	}
+
 	u_strFromUTF8(unprepped, 1024, &unprepped_len, input, input_len, &err);
 
 	if(U_FAILURE(err)) {
@@ -299,7 +305,7 @@ static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile)
 		return 1;
 	}
 
-	prepped_len = usprep_prepare(profile, unprepped, unprepped_len, prepped, 1024, USPREP_ALLOW_UNASSIGNED, NULL, &err);
+	prepped_len = usprep_prepare(profile, unprepped, unprepped_len, prepped, 1024, flags, NULL, &err);
 
 	if(U_FAILURE(err)) {
 		lua_pushnil(L);
@@ -317,22 +323,55 @@ static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile)
 	}
 }
 
-UStringPrepProfile *icu_nameprep;
-UStringPrepProfile *icu_nodeprep;
-UStringPrepProfile *icu_resourceprep;
-UStringPrepProfile *icu_saslprep;
+static UStringPrepProfile *icu_nameprep;
+static UStringPrepProfile *icu_nodeprep;
+static UStringPrepProfile *icu_resourceprep;
+static UStringPrepProfile *icu_saslprep;
+static USpoofChecker *icu_spoofcheck;
+static UIDNA *icu_idna2008;
+
+#if (U_ICU_VERSION_MAJOR_NUM < 58)
+/* COMPAT */
+#define USPOOF_CONFUSABLE (USPOOF_SINGLE_SCRIPT_CONFUSABLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_WHOLE_SCRIPT_CONFUSABLE)
+#endif
 
 /* initialize global ICU stringprep profiles */
-void init_icu() {
+static void init_icu(void) {
 	UErrorCode err = U_ZERO_ERROR;
 	utrace_setLevel(UTRACE_VERBOSE);
 	icu_nameprep = usprep_openByType(USPREP_RFC3491_NAMEPREP, &err);
 	icu_nodeprep = usprep_openByType(USPREP_RFC3920_NODEPREP, &err);
 	icu_resourceprep = usprep_openByType(USPREP_RFC3920_RESOURCEPREP, &err);
 	icu_saslprep = usprep_openByType(USPREP_RFC4013_SASLPREP, &err);
+	icu_spoofcheck = uspoof_open(&err);
+	uspoof_setChecks(icu_spoofcheck, USPOOF_CONFUSABLE, &err);
+	int options = UIDNA_DEFAULT;
+#if 0
+	/* COMPAT with future Unicode versions */
+	options |= UIDNA_ALLOW_UNASSIGNED;
+#endif
+#if 1
+	/* Forbid eg labels starting with _ */
+	options |= UIDNA_USE_STD3_RULES;
+#endif
+#if 0
+	/* TODO determine if we need this */
+	options |= UIDNA_CHECK_BIDI;
+#endif
+#if 0
+	/* UTS46 makes it sound like these are the responsibility of registrars */
+	options |= UIDNA_CHECK_CONTEXTJ;
+	options |= UIDNA_CHECK_CONTEXTO;
+#endif
+#if 0
+	/* This disables COMPAT with IDNA 2003 */
+	options |= UIDNA_NONTRANSITIONAL_TO_ASCII;
+	options |= UIDNA_NONTRANSITIONAL_TO_UNICODE;
+#endif
+	icu_idna2008 = uidna_openUTS46(options, &err);
 
 	if(U_FAILURE(err)) {
-		fprintf(stderr, "[c] util.encodings: error: %s\n", u_errorName((UErrorCode)err));
+		fprintf(stderr, "[c] util.encodings: error: %s\n", u_errorName(err));
 	}
 }
 
@@ -362,21 +401,25 @@ static int stringprep_prep(lua_State *L, const Stringprep_profile *profile) {
 	const char *s;
 	char string[1024];
 	int ret;
-
-	if(!lua_isstring(L, 1)) {
-		lua_pushnil(L);
-		return 1;
-	}
+	Stringprep_profile_flags flags = 0;
 
 	s = check_utf8(L, 1, &len);
 
+	/* strict */
+	if(!lua_isnoneornil(L, 2)) {
+		luaL_checktype(L, 2, LUA_TBOOLEAN);
+		if(lua_toboolean(L, 2)) {
+			flags = STRINGPREP_NO_UNASSIGNED;
+		}
+	}
+
 	if(s == NULL || len >= 1024 || len != strlen(s)) {
 		lua_pushnil(L);
 		return 1; /* TODO return error message */
 	}
 
 	strcpy(string, s);
-	ret = stringprep(string, 1024, (Stringprep_profile_flags)0, profile);
+	ret = stringprep(string, 1024, flags, profile);
 
 	if(ret == STRINGPREP_OK) {
 		lua_pushstring(L, string);
@@ -425,9 +468,10 @@ static int Lidna_to_ascii(lua_State *L) {	/** idna.to_ascii(s) */
 		return 1;
 	}
 
-	dest_len = uidna_IDNToASCII(ustr, ulen, dest, 1024, UIDNA_USE_STD3_RULES, NULL, &err);
+	UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+	dest_len = uidna_nameToASCII(icu_idna2008, ustr, ulen, dest, 256, &info, &err);
 
-	if(U_FAILURE(err)) {
+	if(U_FAILURE(err) || info.errors) {
 		lua_pushnil(L);
 		return 1;
 	} else {
@@ -459,9 +503,10 @@ static int Lidna_to_unicode(lua_State *L) {	/** idna.to_unicode(s) */
 		return 1;
 	}
 
-	dest_len = uidna_IDNToUnicode(ustr, ulen, dest, 1024, UIDNA_USE_STD3_RULES, NULL, &err);
+	UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+	dest_len = uidna_nameToUnicode(icu_idna2008, ustr, ulen, dest, 1024, &info, &err);
 
-	if(U_FAILURE(err)) {
+	if(U_FAILURE(err) || info.errors) {
 		lua_pushnil(L);
 		return 1;
 	} else {
@@ -477,6 +522,40 @@ static int Lidna_to_unicode(lua_State *L) {	/** idna.to_unicode(s) */
 	}
 }
 
+static int Lskeleton(lua_State *L) {
+	size_t len;
+	int32_t ulen, dest_len, output_len;
+	const char *s = luaL_checklstring(L, 1, &len);
+	UErrorCode err = U_ZERO_ERROR;
+	UChar ustr[1024];
+	UChar dest[1024];
+	char output[1024];
+
+	u_strFromUTF8(ustr, 1024, &ulen, s, len, &err);
+
+	if(U_FAILURE(err)) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	dest_len = uspoof_getSkeleton(icu_spoofcheck, 0, ustr, ulen, dest, 1024, &err);
+
+	if(U_FAILURE(err)) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	u_strToUTF8(output, 1024, &output_len, dest, dest_len, &err);
+
+	if(U_SUCCESS(err)) {
+		lua_pushlstring(L, output, output_len);
+		return 1;
+	}
+
+	lua_pushnil(L);
+	return 1;
+}
+
 #else /* USE_STRINGPREP_ICU */
 /****************** libidn ********************/
 
@@ -558,6 +637,13 @@ LUALIB_API int luaopen_util_encodings(lua_State *L) {
 	luaL_setfuncs(L, Reg_utf8, 0);
 	lua_setfield(L, -2, "utf8");
 
+#ifdef USE_STRINGPREP_ICU
+	lua_newtable(L);
+	lua_pushcfunction(L, Lskeleton);
+	lua_setfield(L, -2, "skeleton");
+	lua_setfield(L, -2, "confusable");
+#endif
+
 	lua_pushliteral(L, "-3.14");
 	lua_setfield(L, -2, "version");
 	return 1;
diff --git a/util-src/hashes.c b/util-src/hashes.c
index 903ecb6e..51c7611c 100644
--- a/util-src/hashes.c
+++ b/util-src/hashes.c
@@ -26,6 +26,7 @@ typedef unsigned __int32 uint32_t;
 #include <openssl/sha.h>
 #include <openssl/md5.h>
 #include <openssl/hmac.h>
+#include <openssl/evp.h>
 
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
@@ -34,8 +35,8 @@ typedef unsigned __int32 uint32_t;
 #define HMAC_IPAD 0x36363636
 #define HMAC_OPAD 0x5c5c5c5c
 
-const char *hex_tab = "0123456789abcdef";
-void toHex(const unsigned char *in, int length, unsigned char *out) {
+static const char *hex_tab = "0123456789abcdef";
+static void toHex(const unsigned char *in, int length, unsigned char *out) {
 	int i;
 
 	for(i = 0; i < length; i++) {
@@ -75,44 +76,6 @@ struct hash_desc {
 	void *ctx, *ctxo;
 };
 
-static void hmac(struct hash_desc *desc, const char *key, size_t key_len,
-                 const char *msg, size_t msg_len, unsigned char *result) {
-	union xory {
-		unsigned char bytes[64];
-		uint32_t quadbytes[16];
-	};
-
-	int i;
-	unsigned char hashedKey[64]; /* Maximum used digest length */
-	union xory k_ipad, k_opad;
-
-	if(key_len > 64) {
-		desc->Init(desc->ctx);
-		desc->Update(desc->ctx, key, key_len);
-		desc->Final(hashedKey, desc->ctx);
-		key = (const char *)hashedKey;
-		key_len = desc->digestLength;
-	}
-
-	memcpy(k_ipad.bytes, key, key_len);
-	memset(k_ipad.bytes + key_len, 0, 64 - key_len);
-	memcpy(k_opad.bytes, k_ipad.bytes, 64);
-
-	for(i = 0; i < 16; i++) {
-		k_ipad.quadbytes[i] ^= HMAC_IPAD;
-		k_opad.quadbytes[i] ^= HMAC_OPAD;
-	}
-
-	desc->Init(desc->ctx);
-	desc->Update(desc->ctx, k_ipad.bytes, 64);
-	desc->Init(desc->ctxo);
-	desc->Update(desc->ctxo, k_opad.bytes, 64);
-	desc->Update(desc->ctx, msg, msg_len);
-	desc->Final(result, desc->ctx);
-	desc->Update(desc->ctxo, result, desc->digestLength);
-	desc->Final(result, desc->ctxo);
-}
-
 #define MAKE_HMAC_FUNCTION(myFunc, evp, size, type) \
 static int myFunc(lua_State *L) { \
 	unsigned char hash[size], result[2*size]; \
@@ -136,56 +99,37 @@ MAKE_HMAC_FUNCTION(Lhmac_sha256, EVP_sha256, SHA256_DIGEST_LENGTH, SHA256_CTX)
 MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX)
 MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX)
 
-static int LscramHi(lua_State *L) {
-	union xory {
-		unsigned char bytes[SHA_DIGEST_LENGTH];
-		uint32_t quadbytes[SHA_DIGEST_LENGTH / 4];
-	};
-	int i;
-	SHA_CTX ctx, ctxo;
-	unsigned char Ust[SHA_DIGEST_LENGTH];
-	union xory Und;
-	union xory res;
-	size_t str_len, salt_len;
-	struct hash_desc desc;
-	const char *str = luaL_checklstring(L, 1, &str_len);
-	const char *salt = luaL_checklstring(L, 2, &salt_len);
-	char *salt2;
-	const int iter = luaL_checkinteger(L, 3);
-
-	desc.Init = (int (*)(void *))SHA1_Init;
-	desc.Update = (int (*)(void *, const void *, size_t))SHA1_Update;
-	desc.Final = (int (*)(unsigned char *, void *))SHA1_Final;
-	desc.digestLength = SHA_DIGEST_LENGTH;
-	desc.ctx = &ctx;
-	desc.ctxo = &ctxo;
+static int Lpbkdf2_sha1(lua_State *L) {
+	unsigned char out[SHA_DIGEST_LENGTH];
 
-	salt2 = malloc(salt_len + 4);
+	size_t pass_len, salt_len;
+	const char *pass = luaL_checklstring(L, 1, &pass_len);
+	const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len);
+	const int iter = luaL_checkinteger(L, 3);
 
-	if(salt2 == NULL) {
-		return luaL_error(L, "Out of memory in scramHi");
+	if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), SHA_DIGEST_LENGTH, out) == 0) {
+		return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed");
 	}
 
-	memcpy(salt2, salt, salt_len);
-	memcpy(salt2 + salt_len, "\0\0\0\1", 4);
-	hmac(&desc, str, str_len, salt2, salt_len + 4, Ust);
-	free(salt2);
+	lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH);
 
-	memcpy(res.bytes, Ust, sizeof(res));
+	return 1;
+}
 
-	for(i = 1; i < iter; i++) {
-		int j;
-		hmac(&desc, str, str_len, (char *)Ust, sizeof(Ust), Und.bytes);
 
-		for(j = 0; j < SHA_DIGEST_LENGTH / 4; j++) {
-			res.quadbytes[j] ^= Und.quadbytes[j];
-		}
+static int Lpbkdf2_sha256(lua_State *L) {
+	unsigned char out[SHA256_DIGEST_LENGTH];
 
-		memcpy(Ust, Und.bytes, sizeof(Ust));
-	}
+	size_t pass_len, salt_len;
+	const char *pass = luaL_checklstring(L, 1, &pass_len);
+	const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len);
+	const int iter = luaL_checkinteger(L, 3);
 
-	lua_pushlstring(L, (char *)res.bytes, SHA_DIGEST_LENGTH);
+	if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), SHA256_DIGEST_LENGTH, out) == 0) {
+		return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed");
+	}
 
+	lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH);
 	return 1;
 }
 
@@ -200,7 +144,9 @@ static const luaL_Reg Reg[] = {
 	{ "hmac_sha256",	Lhmac_sha256	},
 	{ "hmac_sha512",	Lhmac_sha512	},
 	{ "hmac_md5",		Lhmac_md5	},
-	{ "scram_Hi_sha1",	LscramHi	},
+	{ "scram_Hi_sha1",	Lpbkdf2_sha1	}, /* COMPAT */
+	{ "pbkdf2_hmac_sha1",	Lpbkdf2_sha1	},
+	{ "pbkdf2_hmac_sha256",	Lpbkdf2_sha256	},
 	{ NULL,			NULL		}
 };
 
@@ -209,7 +155,7 @@ LUALIB_API int luaopen_util_hashes(lua_State *L) {
 	luaL_checkversion(L);
 #endif
 	lua_newtable(L);
-	luaL_setfuncs(L, Reg, 0);;
+	luaL_setfuncs(L, Reg, 0);
 	lua_pushliteral(L, "-3.14");
 	lua_setfield(L, -2, "version");
 	return 1;
diff --git a/util-src/net.c b/util-src/net.c
index 5f706d81..d9c6b914 100644
--- a/util-src/net.c
+++ b/util-src/net.c
@@ -36,7 +36,7 @@
 
 /* Enumerate all locally configured IP addresses */
 
-const char *const type_strings[] = {
+static const char *const type_strings[] = {
 	"both",
 	"ipv4",
 	"ipv6",
diff --git a/util-src/poll.c b/util-src/poll.c
index 0ca0cf28..21cb9581 100644
--- a/util-src/poll.c
+++ b/util-src/poll.c
@@ -59,7 +59,7 @@ typedef struct Lpoll_state {
 /*
  * Add an FD to be watched
  */
-int Ladd(lua_State *L) {
+static int Ladd(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -137,7 +137,7 @@ int Ladd(lua_State *L) {
 /*
  * Set events to watch for, readable and/or writable
  */
-int Lset(lua_State *L) {
+static int Lset(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -172,6 +172,7 @@ int Lset(lua_State *L) {
 		lua_pushnil(L);
 		lua_pushstring(L, strerror(ENOENT));
 		lua_pushinteger(L, ENOENT);
+		return 3;
 	}
 
 	if(!lua_isnoneornil(L, 3)) {
@@ -200,7 +201,7 @@ int Lset(lua_State *L) {
 /*
  * Remove FDs
  */
-int Ldel(lua_State *L) {
+static int Ldel(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -229,6 +230,7 @@ int Ldel(lua_State *L) {
 		lua_pushnil(L);
 		lua_pushstring(L, strerror(ENOENT));
 		lua_pushinteger(L, ENOENT);
+		return 3;
 	}
 
 	FD_CLR(fd, &state->wantread);
@@ -247,7 +249,7 @@ int Ldel(lua_State *L) {
 /*
  * Check previously manipulated event state for FDs ready for reading or writing
  */
-int Lpushevent(lua_State *L, struct Lpoll_state *state) {
+static int Lpushevent(lua_State *L, struct Lpoll_state *state) {
 #ifdef USE_EPOLL
 
 	if(state->processed > 0) {
@@ -281,7 +283,7 @@ int Lpushevent(lua_State *L, struct Lpoll_state *state) {
 /*
  * Wait for event
  */
-int Lwait(lua_State *L) {
+static int Lwait(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 
 	int ret = Lpushevent(L, state);
@@ -344,7 +346,7 @@ int Lwait(lua_State *L) {
 /*
  * Return Epoll FD
  */
-int Lgetfd(lua_State *L) {
+static int Lgetfd(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	lua_pushinteger(L, state->epoll_fd);
 	return 1;
@@ -353,7 +355,7 @@ int Lgetfd(lua_State *L) {
 /*
  * Close epoll FD
  */
-int Lgc(lua_State *L) {
+static int Lgc(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 
 	if(state->epoll_fd == -1) {
@@ -375,7 +377,7 @@ int Lgc(lua_State *L) {
 /*
  * String representation
  */
-int Ltos(lua_State *L) {
+static int Ltos(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	lua_pushfstring(L, "%s: %p", STATE_MT, state);
 	return 1;
@@ -384,7 +386,7 @@ int Ltos(lua_State *L) {
 /*
  * Create a new context
  */
-int Lnew(lua_State *L) {
+static int Lnew(lua_State *L) {
 	/* Allocate state */
 	Lpoll_state *state = lua_newuserdata(L, sizeof(Lpoll_state));
 	luaL_setmetatable(L, STATE_MT);
diff --git a/util-src/pposix.c b/util-src/pposix.c
index 004f61a6..2ebf0444 100644
--- a/util-src/pposix.c
+++ b/util-src/pposix.c
@@ -25,14 +25,18 @@
 #define _DEFAULT_SOURCE
 #endif
 #endif
+
 #if defined(__APPLE__)
 #ifndef _DARWIN_C_SOURCE
 #define _DARWIN_C_SOURCE
 #endif
 #endif
+
+#if ! defined(__FreeBSD__)
 #ifndef _POSIX_C_SOURCE
 #define _POSIX_C_SOURCE 200809L
 #endif
+#endif
 
 #include <stdlib.h>
 #include <math.h>
@@ -133,7 +137,7 @@ static int lc_daemonize(lua_State *L) {
 
 /* Syslog support */
 
-const char *const facility_strings[] = {
+static const char *const facility_strings[] = {
 	"auth",
 #if !(defined(sun) || defined(__sun))
 	"authpriv",
@@ -159,7 +163,7 @@ const char *const facility_strings[] = {
 	"uucp",
 	NULL
 };
-int facility_constants[] =	{
+static int facility_constants[] =	{
 	LOG_AUTH,
 #if !(defined(sun) || defined(__sun))
 	LOG_AUTHPRIV,
@@ -195,9 +199,9 @@ int facility_constants[] =	{
        constant.
    " -- syslog manpage
 */
-char *syslog_ident = NULL;
+static char *syslog_ident = NULL;
 
-int lc_syslog_open(lua_State *L) {
+static int lc_syslog_open(lua_State *L) {
 	int facility = luaL_checkoption(L, 2, "daemon", facility_strings);
 	facility = facility_constants[facility];
 
@@ -213,7 +217,7 @@ int lc_syslog_open(lua_State *L) {
 	return 0;
 }
 
-const char *const level_strings[] = {
+static const char *const level_strings[] = {
 	"debug",
 	"info",
 	"notice",
@@ -221,7 +225,7 @@ const char *const level_strings[] = {
 	"error",
 	NULL
 };
-int level_constants[] = 	{
+static int level_constants[] = 	{
 	LOG_DEBUG,
 	LOG_INFO,
 	LOG_NOTICE,
@@ -229,7 +233,7 @@ int level_constants[] = 	{
 	LOG_CRIT,
 	-1
 };
-int lc_syslog_log(lua_State *L) {
+static int lc_syslog_log(lua_State *L) {
 	int level = level_constants[luaL_checkoption(L, 1, "notice", level_strings)];
 
 	if(lua_gettop(L) == 3) {
@@ -241,7 +245,7 @@ int lc_syslog_log(lua_State *L) {
 	return 0;
 }
 
-int lc_syslog_close(lua_State *L) {
+static int lc_syslog_close(lua_State *L) {
 	(void)L;
 	closelog();
 
@@ -253,7 +257,7 @@ int lc_syslog_close(lua_State *L) {
 	return 0;
 }
 
-int lc_syslog_setmask(lua_State *L) {
+static int lc_syslog_setmask(lua_State *L) {
 	int level_idx = luaL_checkoption(L, 1, "notice", level_strings);
 	int mask = 0;
 
@@ -267,24 +271,24 @@ int lc_syslog_setmask(lua_State *L) {
 
 /* getpid */
 
-int lc_getpid(lua_State *L) {
+static int lc_getpid(lua_State *L) {
 	lua_pushinteger(L, getpid());
 	return 1;
 }
 
 /* UID/GID functions */
 
-int lc_getuid(lua_State *L) {
+static int lc_getuid(lua_State *L) {
 	lua_pushinteger(L, getuid());
 	return 1;
 }
 
-int lc_getgid(lua_State *L) {
+static int lc_getgid(lua_State *L) {
 	lua_pushinteger(L, getgid());
 	return 1;
 }
 
-int lc_setuid(lua_State *L) {
+static int lc_setuid(lua_State *L) {
 	int uid = -1;
 
 	if(lua_gettop(L) < 1) {
@@ -342,7 +346,7 @@ int lc_setuid(lua_State *L) {
 	return 2;
 }
 
-int lc_setgid(lua_State *L) {
+static int lc_setgid(lua_State *L) {
 	int gid = -1;
 
 	if(lua_gettop(L) < 1) {
@@ -400,7 +404,7 @@ int lc_setgid(lua_State *L) {
 	return 2;
 }
 
-int lc_initgroups(lua_State *L) {
+static int lc_initgroups(lua_State *L) {
 	int ret;
 	gid_t gid;
 	struct passwd *p;
@@ -464,7 +468,7 @@ int lc_initgroups(lua_State *L) {
 	return 2;
 }
 
-int lc_umask(lua_State *L) {
+static int lc_umask(lua_State *L) {
 	char old_mode_string[7];
 	mode_t old_mode = umask(strtoul(luaL_checkstring(L, 1), NULL, 8));
 
@@ -475,7 +479,7 @@ int lc_umask(lua_State *L) {
 	return 1;
 }
 
-int lc_mkdir(lua_State *L) {
+static int lc_mkdir(lua_State *L) {
 	int ret = mkdir(luaL_checkstring(L, 1), S_IRUSR | S_IWUSR | S_IXUSR
 	                | S_IRGRP | S_IWGRP | S_IXGRP
 	                | S_IROTH | S_IXOTH); /* mode 775 */
@@ -500,7 +504,7 @@ int lc_mkdir(lua_State *L) {
  *	Example usage:
  *	pposix.setrlimit("NOFILE", 1000, 2000)
  */
-int string2resource(const char *s) {
+static int string2resource(const char *s) {
 	if(!strcmp(s, "CORE")) {
 		return RLIMIT_CORE;
 	}
@@ -550,7 +554,7 @@ int string2resource(const char *s) {
 	return -1;
 }
 
-rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
+static rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
 	switch(lua_type(L, idx)) {
 		case LUA_TSTRING:
 
@@ -571,7 +575,7 @@ rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
 	}
 }
 
-int lc_setrlimit(lua_State *L) {
+static int lc_setrlimit(lua_State *L) {
 	struct rlimit lim;
 	int arguments = lua_gettop(L);
 	int rid = -1;
@@ -610,7 +614,7 @@ int lc_setrlimit(lua_State *L) {
 	return 1;
 }
 
-int lc_getrlimit(lua_State *L) {
+static int lc_getrlimit(lua_State *L) {
 	int arguments = lua_gettop(L);
 	const char *resource = NULL;
 	int rid = -1;
@@ -655,13 +659,13 @@ int lc_getrlimit(lua_State *L) {
 	return 3;
 }
 
-int lc_abort(lua_State *L) {
+static int lc_abort(lua_State *L) {
 	(void)L;
 	abort();
 	return 0;
 }
 
-int lc_uname(lua_State *L) {
+static int lc_uname(lua_State *L) {
 	struct utsname uname_info;
 
 	if(uname(&uname_info) != 0) {
@@ -688,7 +692,7 @@ int lc_uname(lua_State *L) {
 	return 1;
 }
 
-int lc_setenv(lua_State *L) {
+static int lc_setenv(lua_State *L) {
 	const char *var = luaL_checkstring(L, 1);
 	const char *value;
 
@@ -717,7 +721,7 @@ int lc_setenv(lua_State *L) {
 }
 
 #ifdef WITH_MALLINFO
-int lc_meminfo(lua_State *L) {
+static int lc_meminfo(lua_State *L) {
 	struct mallinfo info = mallinfo();
 	lua_createtable(L, 0, 5);
 	/* This is the total size of memory allocated with sbrk by malloc, in bytes. */
@@ -745,7 +749,7 @@ int lc_meminfo(lua_State *L) {
  * Attempt to allocate space first
  * Truncate to original size on failure
  */
-int lc_atomic_append(lua_State *L) {
+static int lc_atomic_append(lua_State *L) {
 	int err;
 	size_t len;
 
diff --git a/util-src/ringbuffer.c b/util-src/ringbuffer.c
index 8f9013f7..3e17cdf5 100644
--- a/util-src/ringbuffer.c
+++ b/util-src/ringbuffer.c
@@ -15,23 +15,18 @@ typedef struct {
 	char buffer[];
 } ringbuffer;
 
-char readchar(ringbuffer *b) {
-	b->blen--;
-	return b->buffer[(b->rpos++) % b->alen];
-}
-
-void writechar(ringbuffer *b, char c) {
+static void writechar(ringbuffer *b, char c) {
 	b->blen++;
 	b->buffer[(b->wpos++) % b->alen] = c;
 }
 
 /* make sure position counters stay within the allocation */
-void modpos(ringbuffer *b) {
+static void modpos(ringbuffer *b) {
 	b->rpos = b->rpos % b->alen;
 	b->wpos = b->wpos % b->alen;
 }
 
-int find(ringbuffer *b, const char *s, size_t l) {
+static int find(ringbuffer *b, const char *s, size_t l) {
 	size_t i, j;
 	int m;
 
@@ -64,7 +59,7 @@ int find(ringbuffer *b, const char *s, size_t l) {
  * Find first position of a substring in buffer
  * (buffer, string) -> number
  */
-int rb_find(lua_State *L) {
+static int rb_find(lua_State *L) {
 	size_t l, m;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
@@ -82,7 +77,7 @@ int rb_find(lua_State *L) {
  * Move read position forward without returning the data
  * (buffer, number) -> boolean
  */
-int rb_discard(lua_State *L) {
+static int rb_discard(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	size_t r = luaL_checkinteger(L, 2);
 
@@ -103,7 +98,7 @@ int rb_discard(lua_State *L) {
  * Read bytes from buffer
  * (buffer, number, boolean?) -> string
  */
-int rb_read(lua_State *L) {
+static int rb_read(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	size_t r = luaL_checkinteger(L, 2);
 	int peek = lua_toboolean(L, 3);
@@ -135,7 +130,7 @@ int rb_read(lua_State *L) {
  * Read buffer until first occurrence of a substring
  * (buffer, string) -> string
  */
-int rb_readuntil(lua_State *L) {
+static int rb_readuntil(lua_State *L) {
 	size_t l, m;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
@@ -154,7 +149,7 @@ int rb_readuntil(lua_State *L) {
  * Write bytes into the buffer
  * (buffer, string) -> integer
  */
-int rb_write(lua_State *L) {
+static int rb_write(lua_State *L) {
 	size_t l, w = 0;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
@@ -177,31 +172,31 @@ int rb_write(lua_State *L) {
 	return 1;
 }
 
-int rb_tostring(lua_State *L) {
+static int rb_tostring(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushfstring(L, "ringbuffer: %p %d/%d", b, b->blen, b->alen);
 	return 1;
 }
 
-int rb_length(lua_State *L) {
+static int rb_length(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->blen);
 	return 1;
 }
 
-int rb_size(lua_State *L) {
+static int rb_size(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->alen);
 	return 1;
 }
 
-int rb_free(lua_State *L) {
+static int rb_free(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->alen - b->blen);
 	return 1;
 }
 
-int rb_new(lua_State *L) {
+static int rb_new(lua_State *L) {
 	size_t size = luaL_optinteger(L, 1, sysconf(_SC_PAGESIZE));
 	ringbuffer *b = lua_newuserdata(L, sizeof(ringbuffer) + size);
 
diff --git a/util-src/signal.c b/util-src/signal.c
index c696a3a2..835a601a 100644
--- a/util-src/signal.c
+++ b/util-src/signal.c
@@ -164,8 +164,8 @@ static lua_Hook Hsig = NULL;
 static int Hmask = 0;
 static int Hcount = 0;
 
-int signals[MAX_PENDING_SIGNALS];
-int nsig = 0;
+static int signals[MAX_PENDING_SIGNALS];
+static int nsig = 0;
 
 static void sighook(lua_State *L, lua_Debug *ar) {
 	(void)ar;
diff --git a/util-src/time.c b/util-src/time.c
index bfad52ee..afef3df5 100644
--- a/util-src/time.c
+++ b/util-src/time.c
@@ -1,22 +1,22 @@
 #ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 199309L
+#define _POSIX_C_SOURCE 200809L
 #endif
 
 #include <time.h>
 #include <lua.h>
 
-lua_Number tv2number(struct timespec *tv) {
+static lua_Number tv2number(struct timespec *tv) {
 	return tv->tv_sec + tv->tv_nsec * 1e-9;
 }
 
-int lc_time_realtime(lua_State *L) {
+static int lc_time_realtime(lua_State *L) {
 	struct timespec t;
 	clock_gettime(CLOCK_REALTIME, &t);
 	lua_pushnumber(L, tv2number(&t));
 	return 1;
 }
 
-int lc_time_monotonic(lua_State *L) {
+static int lc_time_monotonic(lua_State *L) {
 	struct timespec t;
 	clock_gettime(CLOCK_MONOTONIC, &t);
 	lua_pushnumber(L, tv2number(&t));
diff --git a/util/async.lua b/util/async.lua
index 20397785..d338071f 100644
--- a/util/async.lua
+++ b/util/async.lua
@@ -246,9 +246,25 @@ local function ready()
 	return pcall(checkthread);
 end
 
+local function wait(promise)
+	local async_wait, async_done = waiter();
+	local ret, err = nil, nil;
+	promise:next(
+		function (r) ret = r; end,
+		function (e) err = e; end)
+		:finally(async_done);
+	async_wait();
+	if ret then
+		return ret;
+	else
+		return nil, err;
+	end
+end
+
 return {
 	ready = ready;
 	waiter = waiter;
 	guarder = guarder;
 	runner = runner;
+	wait = wait;
 };
diff --git a/util/bit53.lua b/util/bit53.lua
new file mode 100644
index 00000000..4b5c2e9c
--- /dev/null
+++ b/util/bit53.lua
@@ -0,0 +1,7 @@
+-- Only the operators needed by net.websocket.frames are provided at this point
+return {
+	band   = function (a, b) return a & b end;
+	bor    = function (a, b) return a | b end;
+	bxor   = function (a, b) return a ~ b end;
+};
+
diff --git a/util/bitcompat.lua b/util/bitcompat.lua
new file mode 100644
index 00000000..454181af
--- /dev/null
+++ b/util/bitcompat.lua
@@ -0,0 +1,32 @@
+-- Compatibility layer for bitwise operations
+
+-- First try the bit32 lib
+-- Lua 5.3 has it with compat enabled
+-- Lua 5.2 has it by default
+if _G.bit32 then
+	return _G.bit32;
+else
+	-- Lua 5.1 may have it as a standalone module that can be installed
+	local ok, bitop = pcall(require, "bit32")
+	if ok then
+		return bitop;
+	end
+end
+
+do
+	-- Lua 5.3 and 5.4 would be able to use native infix operators
+	local ok, bitop = pcall(require, "util.bit53")
+	if ok then
+		return bitop;
+	end
+end
+
+do
+	-- Lastly, try the LuaJIT bitop library
+	local ok, bitop = pcall(require, "bit")
+	if ok then
+		return bitop;
+	end
+end
+
+error "No bit module found. See https://prosody.im/doc/depends#bitop";
diff --git a/util/datamanager.lua b/util/datamanager.lua
index cf96887b..1c578fee 100644
--- a/util/datamanager.lua
+++ b/util/datamanager.lua
@@ -24,7 +24,7 @@ local t_concat = table.concat;
 local envloadfile = require"util.envload".envloadfile;
 local serialize = require "util.serialization".serialize;
 local lfs = require "lfs";
--- Extract directory seperator from package.config (an undocumented string that comes with lua)
+-- Extract directory separator from package.config (an undocumented string that comes with lua)
 local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" )
 
 local prosody = prosody;
@@ -157,7 +157,8 @@ end
 
 local function atomic_store(filename, data)
 	local scratch = filename.."~";
-	local f, ok, msg, errno;
+	local f, ok, msg, errno; -- luacheck: ignore errno
+	-- TODO return util.error with code=errno?
 
 	f, msg, errno = io_open(scratch, "w");
 	if not f then
diff --git a/util/dependencies.lua b/util/dependencies.lua
index 7c7b938e..ede8c6ac 100644
--- a/util/dependencies.lua
+++ b/util/dependencies.lua
@@ -13,7 +13,8 @@ if not softreq "luarocks.loader" then -- LuaRocks 2.x
 	softreq "luarocks.require"; -- LuaRocks <1.x
 end
 
-local function missingdep(name, sources, msg)
+local function missingdep(name, sources, msg, err) -- luacheck: ignore err
+	-- TODO print something about the underlying error, useful for debugging
 	print("");
 	print("**************************");
 	print("Prosody was unable to find "..tostring(name));
@@ -44,25 +45,25 @@ local function check_dependencies()
 
 	local fatal;
 
-	local lxp = softreq "lxp"
+	local lxp, err = softreq "lxp"
 
 	if not lxp then
 		missingdep("luaexpat", {
 				["Debian/Ubuntu"] = "sudo apt-get install lua-expat";
 				["luarocks"] = "luarocks install luaexpat";
 				["Source"] = "http://matthewwild.co.uk/projects/luaexpat/";
-			});
+			}, nil, err);
 		fatal = true;
 	end
 
-	local socket = softreq "socket"
+	local socket, err = softreq "socket"
 
 	if not socket then
 		missingdep("luasocket", {
 				["Debian/Ubuntu"] = "sudo apt-get install lua-socket";
 				["luarocks"] = "luarocks install luasocket";
 				["Source"] = "http://www.tecgraf.puc-rio.br/~diego/professional/luasocket/";
-			});
+			}, nil, err);
 		fatal = true;
 	elseif not socket.tcp4 then
 		-- COMPAT LuaSocket before being IP-version agnostic
@@ -76,28 +77,28 @@ local function check_dependencies()
 			["luarocks"] = "luarocks install luafilesystem";
 			["Debian/Ubuntu"] = "sudo apt-get install lua-filesystem";
 			["Source"] = "http://www.keplerproject.org/luafilesystem/";
-		});
+		}, nil, err);
 		fatal = true;
 	end
 
-	local ssl = softreq "ssl"
+	local ssl, err = softreq "ssl"
 
 	if not ssl then
 		missingdep("LuaSec", {
 				["Debian/Ubuntu"] = "sudo apt-get install lua-sec";
 				["luarocks"] = "luarocks install luasec";
 				["Source"] = "https://github.com/brunoos/luasec";
-			}, "SSL/TLS support will not be available");
+			}, "SSL/TLS support will not be available", err);
 	end
 
-	local bit = _G.bit32 or softreq"bit";
+	local bit, err = softreq"util.bitcompat";
 
 	if not bit then
 		missingdep("lua-bitops", {
 			["Debian/Ubuntu"] = "sudo apt-get install lua-bitop";
 			["luarocks"] = "luarocks install luabitop";
 			["Source"] = "http://bitop.luajit.org/";
-		}, "WebSocket support will not be available");
+		}, "WebSocket support will not be available", err);
 	end
 
 	local encodings, err = softreq "util.encodings"
@@ -140,7 +141,7 @@ local function check_dependencies()
 end
 
 local function log_warnings()
-	if _VERSION > "Lua 5.2" then
+	if _VERSION > "Lua 5.3" then
 		prosody.log("warn", "Support for %s is experimental, please report any issues", _VERSION);
 	end
 	local ssl = softreq"ssl";
diff --git a/util/error.lua b/util/error.lua
new file mode 100644
index 00000000..ca960dd9
--- /dev/null
+++ b/util/error.lua
@@ -0,0 +1,60 @@
+local error_mt = { __name = "error" };
+
+function error_mt:__tostring()
+	return ("error<%s:%s:%s>"):format(self.type, self.condition, self.text or "");
+end
+
+local function is_err(e)
+	return getmetatable(e) == error_mt;
+end
+
+-- Do we want any more well-known fields?
+-- Or could we just copy all fields from `e`?
+-- Sometimes you want variable details in the `text`, how to handle that?
+-- Translations?
+-- Should the `type` be restricted to the stanza error types or free-form?
+-- What to set `type` to for stream errors or SASL errors? Those don't have a 'type' attr.
+
+local function new(e, context, registry)
+	local template = (registry and registry[e]) or e or {};
+	return setmetatable({
+		type = template.type or "cancel";
+		condition = template.condition or "undefined-condition";
+		text = template.text;
+		code = template.code;
+
+		context = context or template.context or { _error_id = e };
+	}, error_mt);
+end
+
+local function coerce(ok, err, ...)
+	if ok or is_err(err) then
+		return ok, err, ...;
+	end
+
+	local new_err = setmetatable({
+		native = err;
+
+		type = "cancel";
+		condition = "undefined-condition";
+	}, error_mt);
+	return ok, new_err, ...;
+end
+
+local function from_stanza(stanza, context)
+	local error_type, condition, text = stanza:get_error();
+	return setmetatable({
+		type = error_type or "cancel";
+		condition = condition or "undefined-condition";
+		text = text;
+
+		context = context or { stanza = stanza };
+	}, error_mt);
+end
+
+return {
+	new = new;
+	coerce = coerce;
+	is_err = is_err;
+	from_stanza = from_stanza;
+}
diff --git a/util/format.lua b/util/format.lua
index c5e513fa..1ce670f3 100644
--- a/util/format.lua
+++ b/util/format.lua
@@ -3,12 +3,20 @@
 --
 
 local tostring = tostring;
-local select = select;
 local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack
+local pack = require "util.table".pack; -- TODO table.pack in 5.2+
 local type = type;
+local dump = require "util.serialization".new("debug");
+local num_type = math.type or function (n)
+	return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float";
+end
+
+-- In Lua 5.3+ these formats throw an error if given a float
+local expects_integer = { c = true, d = true, i = true, o = true, u = true, X = true, x = true, };
 
 local function format(formatstring, ...)
-	local args, args_length = { ... }, select('#', ...);
+	local args = pack(...);
+	local args_length = args.n;
 
 	-- format specifier spec:
 	-- 1. Start: '%%'
@@ -28,17 +36,22 @@ local function format(formatstring, ...)
 		if spec ~= "%%" then
 			i = i + 1;
 			local arg = args[i];
-			if arg == nil then -- special handling for nil
-				arg = "<nil>"
-				args[i] = "<nil>";
-			end
 
 			local option = spec:sub(-1);
-			if option == "q" or option == "s" then -- arg should be string
+			if arg == nil then
+				args[i] = "nil";
+				spec = "<%s>";
+			elseif option == "q" then
+				args[i] = dump(arg);
+				spec = "%s";
+			elseif option == "s" then
 				args[i] = tostring(arg);
 			elseif type(arg) ~= "number" then -- arg isn't number as expected?
 				args[i] = tostring(arg);
 				spec = "[%s]";
+			elseif expects_integer[option] and num_type(arg) ~= "integer" then
+				args[i] = tostring(arg);
+				spec = "[%s]";
 			end
 		end
 		return spec;
diff --git a/util/hashring.lua b/util/hashring.lua
new file mode 100644
index 00000000..322bc005
--- /dev/null
+++ b/util/hashring.lua
@@ -0,0 +1,88 @@
+local function generate_ring(nodes, num_replicas, hash)
+	local new_ring = {};
+        for _, node_name in ipairs(nodes) do
+                for replica = 1, num_replicas do
+                        local replica_hash = hash(node_name..":"..replica);
+                        new_ring[replica_hash] = node_name;
+                        table.insert(new_ring, replica_hash);
+                end
+        end
+        table.sort(new_ring);
+	return new_ring;
+end
+
+local hashring_methods = {};
+local hashring_mt = {
+	__index = function (self, k)
+		-- Automatically build self.ring if it's missing
+		if k == "ring" then
+			local ring = generate_ring(self.nodes, self.num_replicas, self.hash);
+			rawset(self, "ring", ring);
+			return ring;
+		end
+		return rawget(hashring_methods, k);
+	end
+};
+
+local function new(num_replicas, hash_function)
+	return setmetatable({ nodes = {}, num_replicas = num_replicas, hash = hash_function }, hashring_mt);
+end;
+
+function hashring_methods:add_node(name)
+	self.ring = nil;
+	self.nodes[name] = true;
+	table.insert(self.nodes, name);
+	return true;
+end
+
+function hashring_methods:add_nodes(nodes)
+	self.ring = nil;
+	for _, node_name in ipairs(nodes) do
+		if not self.nodes[node_name] then
+			self.nodes[node_name] = true;
+			table.insert(self.nodes, node_name);
+		end
+	end
+	return true;
+end
+
+function hashring_methods:remove_node(node_name)
+	self.ring = nil;
+	if self.nodes[node_name] then
+		for i, stored_node_name in ipairs(self.nodes) do
+			if node_name == stored_node_name then
+				self.nodes[node_name] = nil;
+				table.remove(self.nodes, i);
+				return true;
+			end
+		end
+	end
+	return false;
+end
+
+function hashring_methods:remove_nodes(nodes)
+	self.ring = nil;
+	for _, node_name in ipairs(nodes) do
+		self:remove_node(node_name);
+	end
+end
+
+function hashring_methods:clone()
+	local clone_hashring = new(self.num_replicas, self.hash);
+	clone_hashring:add_nodes(self.nodes);
+	return clone_hashring;
+end
+
+function hashring_methods:get_node(key)
+	local key_hash = self.hash(key);
+	for _, replica_hash in ipairs(self.ring) do
+		if key_hash < replica_hash then
+			return self.ring[replica_hash];
+		end
+	end
+	return self.ring[self.ring[1]];
+end
+
+return {
+	new = new;
+}
diff --git a/util/hmac.lua b/util/hmac.lua
index 2c4cc6ef..4cad17cc 100644
--- a/util/hmac.lua
+++ b/util/hmac.lua
@@ -10,6 +10,9 @@
 
 local hashes = require "util.hashes"
 
-return { md5 = hashes.hmac_md5,
-	 sha1 = hashes.hmac_sha1,
-	 sha256 = hashes.hmac_sha256 };
+return {
+	md5 = hashes.hmac_md5,
+	sha1 = hashes.hmac_sha1,
+	sha256 = hashes.hmac_sha256,
+	sha512 = hashes.hmac_sha512,
+};
diff --git a/util/http.lua b/util/http.lua
index cfb89193..3852f91c 100644
--- a/util/http.lua
+++ b/util/http.lua
@@ -6,24 +6,26 @@
 --
 
 local format, char = string.format, string.char;
-local pairs, ipairs, tonumber = pairs, ipairs, tonumber;
+local pairs, ipairs = pairs, ipairs;
 local t_insert, t_concat = table.insert, table.concat;
 
+local url_codes = {};
+for i = 0, 255 do
+	local c = char(i);
+	local u = format("%%%02x", i);
+	url_codes[c] = u;
+	url_codes[u] = c;
+	url_codes[u:upper()] = c;
+end
 local function urlencode(s)
-	return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end));
+	return s and (s:gsub("[^a-zA-Z0-9.~_-]", url_codes));
 end
 local function urldecode(s)
-	return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end));
+	return s and (s:gsub("%%%x%x", url_codes));
 end
 
 local function _formencodepart(s)
-	return s and (s:gsub("%W", function (c)
-		if c ~= " " then
-			return format("%%%02x", c:byte());
-		else
-			return "+";
-		end
-	end));
+	return s and (urlencode(s):gsub("%%20", "+"));
 end
 
 local function formencode(form)
diff --git a/util/import.lua b/util/import.lua
index 8ecfe43c..1007bc0a 100644
--- a/util/import.lua
+++ b/util/import.lua
@@ -8,7 +8,7 @@
 
 
 
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 local t_insert = table.insert;
 function _G.import(module, ...)
 	local m = package.loaded[module] or require(module);
diff --git a/util/iterators.lua b/util/iterators.lua
index 302cca36..c03c2fd6 100644
--- a/util/iterators.lua
+++ b/util/iterators.lua
@@ -11,9 +11,9 @@
 local it = {};
 
 local t_insert = table.insert;
-local select, next = select, next;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
-local pack = table.pack or function (...) return { n = select("#", ...), ... }; end -- luacheck: ignore 143
+local next = next;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+local pack = table.pack or require "util.table".pack;
 local type = type;
 local table, setmetatable = table, setmetatable;
 
diff --git a/util/jid.lua b/util/jid.lua
index ec31f180..1ddf33d4 100644
--- a/util/jid.lua
+++ b/util/jid.lua
@@ -45,20 +45,20 @@ local function bare(jid)
 	return host;
 end
 
-local function prepped_split(jid)
+local function prepped_split(jid, strict)
 	local node, host, resource = split(jid);
 	if host and host ~= "." then
 		if sub(host, -1, -1) == "." then -- Strip empty root label
 			host = sub(host, 1, -2);
 		end
-		host = nameprep(host);
+		host = nameprep(host, strict);
 		if not host then return; end
 		if node then
-			node = nodeprep(node);
+			node = nodeprep(node, strict);
 			if not node then return; end
 		end
 		if resource then
-			resource = resourceprep(resource);
+			resource = resourceprep(resource, strict);
 			if not resource then return; end
 		end
 		return node, host, resource;
@@ -77,8 +77,8 @@ local function join(node, host, resource)
 	return host;
 end
 
-local function prep(jid)
-	local node, host, resource = prepped_split(jid);
+local function prep(jid, strict)
+	local node, host, resource = prepped_split(jid, strict);
 	return join(node, host, resource);
 end
 
diff --git a/util/mercurial.lua b/util/mercurial.lua
index 3f75c4c1..0f2b1d04 100644
--- a/util/mercurial.lua
+++ b/util/mercurial.lua
@@ -19,7 +19,7 @@ function hg.check_id(path)
 			hg_changelog:close();
 		end
 	else
-		local hg_archival,e = io.open(path.."/.hg_archival.txt");
+		local hg_archival,e = io.open(path.."/.hg_archival.txt"); -- luacheck: ignore 211/e
 		if hg_archival then
 			local repo = hg_archival:read("*l");
 			local node = hg_archival:read("*l");
diff --git a/util/multitable.lua b/util/multitable.lua
index 8d32ed8a..4f2cd972 100644
--- a/util/multitable.lua
+++ b/util/multitable.lua
@@ -9,7 +9,7 @@
 local select = select;
 local t_insert = table.insert;
 local pairs, next, type = pairs, next, type;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 
 local _ENV = nil;
 -- luacheck: std none
diff --git a/util/paths.lua b/util/paths.lua
index 89f4cad9..036f315a 100644
--- a/util/paths.lua
+++ b/util/paths.lua
@@ -41,4 +41,20 @@ function path_util.join(...)
 	return t_concat({...}, path_sep);
 end
 
+function path_util.complement_lua_path(installer_plugin_path)
+	-- Checking for duplicates
+	-- The commands using luarocks need the path to the directory that has the /share and /lib folders.
+	local lua_version = _VERSION:match(" (.+)$");
+	local lua_path_sep = package.config:sub(3,3);
+	local dir_sep = package.config:sub(1,1);
+	local sub_path = dir_sep.."lua"..dir_sep..lua_version..dir_sep;
+	if not string.find(package.path, installer_plugin_path, 1, true) then
+		package.path = package.path..lua_path_sep..installer_plugin_path..dir_sep.."share"..sub_path.."?.lua";
+		package.path = package.path..lua_path_sep..installer_plugin_path..dir_sep.."share"..sub_path.."?"..dir_sep.."init.lua";
+	end
+	if not string.find(package.path, installer_plugin_path, 1, true) then
+		package.cpath = package.cpath..lua_path_sep..installer_plugin_path..dir_sep.."lib"..sub_path.."?.so";
+	end
+end
+
 return path_util;
diff --git a/util/pluginloader.lua b/util/pluginloader.lua
index 9ab8f245..af0428c4 100644
--- a/util/pluginloader.lua
+++ b/util/pluginloader.lua
@@ -36,12 +36,13 @@ end
 
 local function load_resource(plugin, resource)
 	resource = resource or "mod_"..plugin..".lua";
-
+	local lua_version = _VERSION:match(" (.+)$");
 	local names = {
 		"mod_"..plugin..dir_sep..plugin..dir_sep..resource; -- mod_hello/hello/mod_hello.lua
 		"mod_"..plugin..dir_sep..resource;                  -- mod_hello/mod_hello.lua
 		plugin..dir_sep..resource;                          -- hello/mod_hello.lua
 		resource;                                           -- mod_hello.lua
+		"share"..dir_sep.."lua"..dir_sep..lua_version..dir_sep.."mod_"..plugin..dir_sep..resource;
 	};
 
 	return load_file(names);
diff --git a/util/promise.lua b/util/promise.lua
index 07c9c4dc..0b182b54 100644
--- a/util/promise.lua
+++ b/util/promise.lua
@@ -49,6 +49,9 @@ local function promise_settle(promise, new_state, new_next, cbs, value)
 	for _, cb in ipairs(cbs) do
 		cb(value);
 	end
+	-- No need to keep references to callbacks
+	promise._pending_on_fulfilled = nil;
+	promise._pending_on_rejected = nil;
 	return true;
 end
 
diff --git a/util/prosodyctl.lua b/util/prosodyctl.lua
index 5f0c4d12..0787b8c3 100644
--- a/util/prosodyctl.lua
+++ b/util/prosodyctl.lua
@@ -39,6 +39,16 @@ local function show_usage(usage, desc)
 	end
 end
 
+local function show_module_configuration_help(mod_name)
+	print("Done.")
+	print("If you installed a prosody plugin, don't forget to add its name under the 'modules_enabled' section inside your configuration file.")
+	print("Depending on the module, there might be further configuration steps required.")
+	print("")
+	print("More info about: ")
+	print("	modules_enabled: https://prosody.im/doc/modules_enabled")
+	print("	"..mod_name..": https://modules.prosody.im/"..mod_name..".html")
+end
+
 local function getchar(n)
 	local stty_ret = os.execute("stty raw -echo 2>/dev/null");
 	local ok, char;
@@ -124,7 +134,7 @@ end
 
 -- Server control
 local function adduser(params)
-	local user, host, password = nodeprep(params.user), nameprep(params.host), params.password;
+	local user, host, password = nodeprep(params.user, true), nameprep(params.host), params.password;
 	if not user then
 		return false, "invalid-username";
 	elseif not host then
@@ -200,7 +210,7 @@ local function getpid()
 		return false, "pidfile-read-failed", err;
 	end
 
-	local locked, err = lfs.lock(file, "w");
+	local locked, err = lfs.lock(file, "w"); -- luacheck: ignore 211/err
 	if locked then
 		file:close();
 		return false, "pidfile-not-locked";
@@ -217,7 +227,7 @@ local function getpid()
 end
 
 local function isrunning()
-	local ok, pid, err = getpid();
+	local ok, pid, err = getpid(); -- luacheck: ignore 211/err
 	if not ok then
 		if pid == "pidfile-read-failed" or pid == "pidfile-not-locked" then
 			-- Report as not running, since we can't open the pidfile
@@ -229,7 +239,8 @@ local function isrunning()
 	return true, signal.kill(pid, 0) == 0;
 end
 
-local function start(source_dir)
+local function start(source_dir, lua)
+	lua = lua and lua .. " " or "";
 	local ok, ret = isrunning();
 	if not ok then
 		return ok, ret;
@@ -238,9 +249,9 @@ local function start(source_dir)
 		return false, "already-running";
 	end
 	if not source_dir then
-		os.execute("./prosody");
+		os.execute(lua .. "./prosody");
 	else
-		os.execute(source_dir.."/../../bin/prosody");
+		os.execute(lua .. source_dir.."/../../bin/prosody");
 	end
 	return true;
 end
@@ -277,10 +288,36 @@ local function reload()
 	return true;
 end
 
+local function get_path_custom_plugins(plugin_paths)
+		-- I'm considering that the custom plugins' path is the first one at prosody.paths.plugins
+	-- luacheck: ignore 512
+	for path in plugin_paths:gmatch("[^;]+") do
+		return path;
+	end
+end
+
+local function call_luarocks(mod, operation)
+	local dir = get_path_custom_plugins(prosody.paths.plugins);
+	if operation == "install" then
+		show_message("Installing %s at %s", mod, dir);
+	elseif operation == "remove" then
+		show_message("Removing %s from %s", mod, dir);
+	end
+	if operation == "list" then
+		os.execute("luarocks list --tree='"..dir.."'")
+	else
+		os.execute("luarocks --tree='"..dir.."' --server='http://localhost/' "..operation.." "..mod);
+	end
+	if operation == "install" then
+		show_module_configuration_help(mod);
+	end
+end
+
 return {
 	show_message = show_message;
 	show_warning = show_message;
 	show_usage = show_usage;
+	show_module_configuration_help = show_module_configuration_help;
 	getchar = getchar;
 	getline = getline;
 	getpass = getpass;
@@ -296,4 +333,6 @@ return {
 	start = start;
 	stop = stop;
 	reload = reload;
+	get_path_custom_plugins = get_path_custom_plugins;
+	call_luarocks = call_luarocks;
 };
diff --git a/util/pubsub.lua b/util/pubsub.lua
index 1674b9a7..cfac7a68 100644
--- a/util/pubsub.lua
+++ b/util/pubsub.lua
@@ -1,5 +1,6 @@
 local events = require "util.events";
 local cache = require "util.cache";
+local errors = require "util.error";
 
 local service_mt = {};
 
@@ -280,7 +281,8 @@ function service:set_affiliation(node, actor, jid, affiliation) --> ok, err
 	node_obj.affiliations[jid] = affiliation;
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.affiliations[jid] = old_affiliation;
 			return ok, "internal-server-error";
@@ -344,7 +346,8 @@ function service:add_subscription(node, actor, jid, options) --> ok, err
 	end
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.subscribers[jid] = old_subscription;
 			self.subscriptions[normal_jid][jid][node] = old_subscription and true or nil;
@@ -396,7 +399,8 @@ function service:remove_subscription(node, actor, jid) --> ok, err
 	end
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.subscribers[jid] = old_subscription;
 			self.subscriptions[normal_jid][jid][node] = old_subscription and true or nil;
@@ -454,7 +458,8 @@ function service:create(node, actor, options) --> ok, err
 	};
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, self.nodes[node]);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, self.nodes[node]); -- luacheck: ignore 211/err
 		if not ok then
 			self.nodes[node] = nil;
 			return ok, "internal-server-error";
@@ -511,7 +516,7 @@ local function check_preconditions(node_config, required_config)
 	end
 	for config_field, value in pairs(required_config) do
 		if node_config[config_field] ~= value then
-			return false;
+			return false, config_field;
 		end
 	end
 	return true;
@@ -547,8 +552,13 @@ function service:publish(node, actor, id, item, requested_config) --> ok, err
 		node_obj = self.nodes[node];
 	elseif requested_config and not requested_config._defaults_only then
 		-- Check that node has the requested config before we publish
-		if not check_preconditions(node_obj.config, requested_config) then
-			return false, "precondition-not-met";
+		local ok, field = check_preconditions(node_obj.config, requested_config);
+		if not ok then
+			local err = errors.new({
+				type = "cancel", condition = "conflict", text = "Field does not match: "..field;
+			});
+			err.pubsub_condition = "precondition-not-met";
+			return false, err;
 		end
 	end
 	if not self.config.itemcheck(item) then
@@ -768,7 +778,8 @@ function service:set_node_config(node, actor, new_config) --> ok, err
 	node_obj.config = new_config;
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.config = old_config;
 			return ok, "internal-server-error";
diff --git a/util/queue.lua b/util/queue.lua
index 728e905f..66ed098b 100644
--- a/util/queue.lua
+++ b/util/queue.lua
@@ -52,18 +52,20 @@ local function new(size, allow_wrapping)
 			return t[tail];
 		end;
 		items = function (self)
-			--luacheck: ignore 431/t
-			return function (t, pos)
-				if pos >= t:count() then
+			return function (_, pos)
+				if pos >= items then
 					return nil;
 				end
 				local read_pos = tail + pos;
-				if read_pos > t.size then
+				if read_pos > self.size then
 					read_pos = (read_pos%size);
 				end
-				return pos+1, t._items[read_pos];
+				return pos+1, t[read_pos];
 			end, self, 0;
 		end;
+		consume = function (self)
+			return self.pop, self;
+		end;
 	};
 end
 
diff --git a/util/sasl/scram.lua b/util/sasl/scram.lua
index 043f328b..e2ce00f5 100644
--- a/util/sasl/scram.lua
+++ b/util/sasl/scram.lua
@@ -14,9 +14,7 @@
 local s_match = string.match;
 local type = type
 local base64 = require "util.encodings".base64;
-local hmac_sha1 = require "util.hashes".hmac_sha1;
-local sha1 = require "util.hashes".sha1;
-local Hi = require "util.hashes".scram_Hi_sha1;
+local hashes = require "util.hashes";
 local generate_uuid = require "util.uuid".generate;
 local saslprep = require "util.encodings".stringprep.saslprep;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
@@ -99,20 +97,22 @@ local function hashprep(hashname)
 	return hashname:lower():gsub("-", "_");
 end
 
-local function getAuthenticationDatabaseSHA1(password, salt, iteration_count)
-	if type(password) ~= "string" or type(salt) ~= "string" or type(iteration_count) ~= "number" then
-		return false, "inappropriate argument types"
-	end
-	if iteration_count < 4096 then
-		log("warn", "Iteration count < 4096 which is the suggested minimum according to RFC 5802.")
+local function get_scram_hasher(H, HMAC, Hi)
+	return function (password, salt, iteration_count)
+		if type(password) ~= "string" or type(salt) ~= "string" or type(iteration_count) ~= "number" then
+			return false, "inappropriate argument types"
+		end
+		if iteration_count < 4096 then
+			log("warn", "Iteration count < 4096 which is the suggested minimum according to RFC 5802.")
+		end
+		local salted_password = Hi(password, salt, iteration_count);
+		local stored_key = H(HMAC(salted_password, "Client Key"))
+		local server_key = HMAC(salted_password, "Server Key");
+		return true, stored_key, server_key
 	end
-	local salted_password = Hi(password, salt, iteration_count);
-	local stored_key = sha1(hmac_sha1(salted_password, "Client Key"))
-	local server_key = hmac_sha1(salted_password, "Server Key");
-	return true, stored_key, server_key
 end
 
-local function scram_gen(hash_name, H_f, HMAC_f)
+local function scram_gen(hash_name, H_f, HMAC_f, get_auth_db)
 	local profile_name = "scram_" .. hashprep(hash_name);
 	local function scram_hash(self, message)
 		local support_channel_binding = false;
@@ -125,6 +125,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 			local client_first_message = message;
 
 			-- TODO: fail if authzid is provided, since we don't support them yet
+			-- luacheck: ignore 211/authzid
 			local gs2_header, gs2_cbind_flag, gs2_cbind_name, authzid, client_first_message_bare, username, clientnonce
 				= s_match(client_first_message, "^(([pny])=?([^,]*),([^,]*),)(m?=?[^,]*,?n=([^,]*),r=([^,]*),?.*)$");
 
@@ -177,7 +178,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 				iteration_count = default_i;
 
 				local succ;
-				succ, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, iteration_count);
+				succ, stored_key, server_key = get_auth_db(password, salt, iteration_count);
 				if not succ then
 					log("error", "Generating authentication database failed. Reason: %s", stored_key);
 					return "failure", "temporary-auth-failure";
@@ -190,7 +191,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 			end
 
 			local nonce = clientnonce .. generate_uuid();
-			local server_first_message = "r="..nonce..",s="..base64.encode(salt)..",i="..iteration_count;
+			local server_first_message = ("r=%s,s=%s,i=%d"):format(nonce, base64.encode(salt), iteration_count);
 			self.state = {
 				gs2_header = gs2_header;
 				gs2_cbind_name = gs2_cbind_name;
@@ -247,22 +248,28 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 	return scram_hash;
 end
 
+local auth_db_getters = {}
 local function init(registerMechanism)
-	local function registerSCRAMMechanism(hash_name, hash, hmac_hash)
+	local function registerSCRAMMechanism(hash_name, hash, hmac_hash, pbkdf2)
+		local get_auth_db = get_scram_hasher(hash, hmac_hash, pbkdf2);
+		auth_db_getters[hash_name] = get_auth_db;
 		registerMechanism("SCRAM-"..hash_name,
 			{"plain", "scram_"..(hashprep(hash_name))},
-			scram_gen(hash_name:lower(), hash, hmac_hash));
+			scram_gen(hash_name:lower(), hash, hmac_hash, get_auth_db));
 
 		-- register channel binding equivalent
 		registerMechanism("SCRAM-"..hash_name.."-PLUS",
 			{"plain", "scram_"..(hashprep(hash_name))},
-			scram_gen(hash_name:lower(), hash, hmac_hash), {"tls-unique"});
+			scram_gen(hash_name:lower(), hash, hmac_hash, get_auth_db), {"tls-unique"});
 	end
 
-	registerSCRAMMechanism("SHA-1", sha1, hmac_sha1);
+	registerSCRAMMechanism("SHA-1", hashes.sha1, hashes.hmac_sha1, hashes.pbkdf2_hmac_sha1);
+	registerSCRAMMechanism("SHA-256", hashes.sha256, hashes.hmac_sha256, hashes.pbkdf2_hmac_sha256);
 end
 
 return {
-	getAuthenticationDatabaseSHA1 = getAuthenticationDatabaseSHA1;
+	get_hash = get_scram_hasher;
+	hashers = auth_db_getters;
+	getAuthenticationDatabaseSHA1 = get_scram_hasher(hashes.sha1, hashes.hmac_sha1, hashes.pbkdf2_hmac_sha1); -- COMPAT
 	init = init;
 }
diff --git a/util/serialization.lua b/util/serialization.lua
index 5121a9f9..d70e92ba 100644
--- a/util/serialization.lua
+++ b/util/serialization.lua
@@ -16,22 +16,18 @@ local s_char = string.char;
 local s_match = string.match;
 local t_concat = table.concat;
 
+local to_hex = require "util.hex".to;
+
 local pcall = pcall;
 local envload = require"util.envload".envload;
 
 local pos_inf, neg_inf = math.huge, -math.huge;
--- luacheck: ignore 143/math
 local m_type = math.type or function (n)
 	return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float";
 end;
 
-local char_to_hex = {};
-for i = 0,255 do
-	char_to_hex[s_char(i)] = s_format("%02x", i);
-end
-
-local function to_hex(s)
-	return (s_gsub(s, ".", char_to_hex));
+local function rawpairs(t)
+	return next, t, nil;
 end
 
 local function fatal_error(obj, why)
@@ -123,6 +119,7 @@ local function new(opt)
 	local freeze = opt.freeze;
 	local maxdepth = opt.maxdepth or 127;
 	local multirefs = opt.multiref;
+	local table_pairs = opt.table_iterator or rawpairs;
 
 	-- serialize one table, recursively
 	-- t - table being serialized
@@ -164,7 +161,9 @@ local function new(opt)
 		local indent = s_rep(indentwith, d);
 		local numkey = 1;
 		local ktyp, vtyp;
-		for k,v in next,t do
+		local had_items = false;
+		for k,v in table_pairs(t) do
+			had_items = true;
 			o[l], l = itemstart, l + 1;
 			o[l], l = indent, l + 1;
 			ktyp, vtyp = type(k), type(v);
@@ -195,14 +194,10 @@ local function new(opt)
 			else
 				o[l], l = ser(v), l + 1;
 			end
-			-- last item?
-			if next(t, k) ~= nil then
-				o[l], l = itemsep, l + 1;
-			else
-				o[l], l = itemlast, l + 1;
-			end
+			o[l], l = itemsep, l + 1;
 		end
-		if next(t) ~= nil then
+		if had_items then
+			o[l - 1] = itemlast;
 			o[l], l = s_rep(indentwith, d-1), l + 1;
 		end
 		o[l], l = tend, l +1;
diff --git a/util/session.lua b/util/session.lua
index b2a726ce..25b22faf 100644
--- a/util/session.lua
+++ b/util/session.lua
@@ -4,12 +4,13 @@ local logger = require "util.logger";
 local function new_session(typ)
 	local session = {
 		type = typ .. "_unauthed";
+		base_type = typ;
 	};
 	return session;
 end
 
 local function set_id(session)
-	local id = session.type .. tostring(session):match("%x+$"):lower();
+	local id = session.base_type .. tostring(session):match("%x+$"):lower();
 	session.id = id;
 	return session;
 end
@@ -30,7 +31,7 @@ local function set_send(session)
 	local conn = session.conn;
 	if not conn then
 		function session.send(data)
-			session.log("debug", "Discarding data sent to unconnected session: %s", tostring(data));
+			session.log("debug", "Discarding data sent to unconnected session: %s", data);
 			return false;
 		end
 		return session;
@@ -46,7 +47,7 @@ local function set_send(session)
 			if t then
 				local ret, err = w(conn, t);
 				if not ret then
-					session.log("debug", "Error writing to connection: %s", tostring(err));
+					session.log("debug", "Error writing to connection: %s", err);
 					return false, err;
 				end
 			end
diff --git a/util/sql.lua b/util/sql.lua
index 00c7b57f..9d1c86ca 100644
--- a/util/sql.lua
+++ b/util/sql.lua
@@ -201,31 +201,31 @@ function engine:_transaction(func, ...)
 		if not ok then return ok, err; end
 	end
 	--assert(not self.__transaction, "Recursive transactions not allowed");
-	log("debug", "SQL transaction begin [%s]", tostring(func));
+	log("debug", "SQL transaction begin [%s]", func);
 	self.__transaction = true;
 	local success, a, b, c = xpcall(func, handleerr, ...);
 	self.__transaction = nil;
 	if success then
-		log("debug", "SQL transaction success [%s]", tostring(func));
+		log("debug", "SQL transaction success [%s]", func);
 		local ok, err = self.conn:commit();
 		-- LuaDBI doesn't actually return an error message here, just a boolean
 		if not ok then return ok, err or "commit failed"; end
 		return success, a, b, c;
 	else
-		log("debug", "SQL transaction failure [%s]: %s", tostring(func), a.err);
+		log("debug", "SQL transaction failure [%s]: %s", func, a.err);
 		if self.conn then self.conn:rollback(); end
 		return success, a.err;
 	end
 end
 function engine:transaction(...)
-	local ok, ret = self:_transaction(...);
+	local ok, ret, b, c = self:_transaction(...);
 	if not ok then
 		local conn = self.conn;
 		if not conn or not conn:ping() then
 			log("debug", "Database connection was closed. Will reconnect and retry.");
 			self.conn = nil;
-			log("debug", "Retrying SQL transaction [%s]", tostring((...)));
-			ok, ret = self:_transaction(...);
+			log("debug", "Retrying SQL transaction [%s]", (...));
+			ok, ret, b, c = self:_transaction(...);
 			log("debug", "SQL transaction retry %s", ok and "succeeded" or "failed");
 		else
 			log("debug", "SQL connection is up, so not retrying");
@@ -234,7 +234,7 @@ function engine:transaction(...)
 			log("error", "Error in SQL transaction: %s", ret);
 		end
 	end
-	return ok, ret;
+	return ok, ret, b, c;
 end
 function engine:_create_index(index)
 	local sql = "CREATE INDEX \""..index.name.."\" ON \""..index.table.."\" (";
@@ -335,6 +335,9 @@ function engine:set_encoding() -- to UTF-8
 		local ok, actual_charset = self:transaction(function ()
 			return self:select"SHOW SESSION VARIABLES LIKE 'character_set_client'";
 		end);
+		if not ok then
+			return false, "Failed to detect connection encoding";
+		end
 		local charset_ok = true;
 		for row in actual_charset do
 			if row[2] ~= charset then
diff --git a/util/stanza.lua b/util/stanza.lua
index a90d56b3..f5cd5668 100644
--- a/util/stanza.lua
+++ b/util/stanza.lua
@@ -98,7 +98,7 @@ function stanza_mt:query(xmlns)
 end
 
 function stanza_mt:body(text, attr)
-	return self:tag("body", attr):text(text);
+	return self:text_tag("body", text, attr);
 end
 
 function stanza_mt:text_tag(name, text, attr, namespaces)
@@ -270,6 +270,34 @@ function stanza_mt:find(path)
 	until not self
 end
 
+local function _clone(stanza, only_top)
+	local attr, tags = {}, {};
+	for k,v in pairs(stanza.attr) do attr[k] = v; end
+	local old_namespaces, namespaces = stanza.namespaces;
+	if old_namespaces then
+		namespaces = {};
+		for k,v in pairs(old_namespaces) do namespaces[k] = v; end
+	end
+	local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags };
+	if not only_top then
+		for i=1,#stanza do
+			local child = stanza[i];
+			if child.name then
+				child = _clone(child);
+				t_insert(tags, child);
+			end
+			t_insert(new, child);
+		end
+	end
+	return setmetatable(new, stanza_mt);
+end
+
+local function clone(stanza, only_top)
+	if not is_stanza(stanza) then
+		error("bad argument to clone: expected stanza, got "..type(stanza));
+	end
+	return _clone(stanza, only_top);
+end
 
 local escape_table = { ["'"] = "&apos;", ["\""] = "&quot;", ["<"] = "&lt;", [">"] = "&gt;", ["&"] = "&amp;" };
 local function xml_escape(str) return (s_gsub(str, "['&<>\"]", escape_table)); end
@@ -310,11 +338,8 @@ function stanza_mt.__tostring(t)
 end
 
 function stanza_mt.top_tag(t)
-	local attr_string = "";
-	if t.attr then
-		for k, v in pairs(t.attr) do if type(k) == "string" then attr_string = attr_string .. s_format(" %s='%s'", k, xml_escape(tostring(v))); end end
-	end
-	return s_format("<%s%s>", t.name, attr_string);
+	local top_tag_clone = clone(t, true);
+	return tostring(top_tag_clone):sub(1,-3)..">";
 end
 
 function stanza_mt.get_text(t)
@@ -388,50 +413,32 @@ local function deserialize(serialized)
 	end
 end
 
-local function _clone(stanza)
-	local attr, tags = {}, {};
-	for k,v in pairs(stanza.attr) do attr[k] = v; end
-	local old_namespaces, namespaces = stanza.namespaces;
-	if old_namespaces then
-		namespaces = {};
-		for k,v in pairs(old_namespaces) do namespaces[k] = v; end
-	end
-	local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags };
-	for i=1,#stanza do
-		local child = stanza[i];
-		if child.name then
-			child = _clone(child);
-			t_insert(tags, child);
-		end
-		t_insert(new, child);
-	end
-	return setmetatable(new, stanza_mt);
-end
-
-local function clone(stanza)
-	if not is_stanza(stanza) then
-		error("bad argument to clone: expected stanza, got "..type(stanza));
-	end
-	return _clone(stanza);
-end
-
 local function message(attr, body)
 	if not body then
 		return new_stanza("message", attr);
 	else
-		return new_stanza("message", attr):tag("body"):text(body):up();
+		return new_stanza("message", attr):text_tag("body", body);
 	end
 end
 local function iq(attr)
-	if not (attr and attr.id) then
+	if not attr then
+		error("iq stanzas require id and type attributes");
+	end
+	if not attr.id then
 		error("iq stanzas require an id attribute");
 	end
+	if not attr.type then
+		error("iq stanzas require a type attribute");
+	end
 	return new_stanza("iq", attr);
 end
 
 local function reply(orig)
+	if not is_stanza(orig) then
+		error("bad argument to reply: expected stanza, got "..type(orig));
+	end
 	return new_stanza(orig.name,
-		orig.attr and {
+		{
 			to = orig.attr.from,
 			from = orig.attr.to,
 			id = orig.attr.id,
@@ -440,12 +447,23 @@ local function reply(orig)
 end
 
 local xmpp_stanzas_attr = { xmlns = xmlns_stanzas };
-local function error_reply(orig, error_type, condition, error_message)
+local function error_reply(orig, error_type, condition, error_message, error_by)
+	if not is_stanza(orig) then
+		error("bad argument to error_reply: expected stanza, got "..type(orig));
+	elseif orig.attr.type == "error" then
+		error("bad argument to error_reply: got stanza of type error which must not be replied to");
+	end
 	local t = reply(orig);
 	t.attr.type = "error";
-	t:tag("error", {type = error_type}) --COMPAT: Some day xmlns:stanzas goes here
+	if t.attr.from == error_by then
+		error_by = nil;
+	end
+	if type(error_type) == "table" then -- an util.error or similar object
+		error_type, condition, error_message = error_type.type, error_type.condition, error_type.text;
+	end
+	t:tag("error", {type = error_type, by = error_by}) --COMPAT: Some day xmlns:stanzas goes here
 	:tag(condition, xmpp_stanzas_attr):up();
-	if error_message then t:tag("text", xmpp_stanzas_attr):text(error_message):up(); end
+	if error_message then t:text_tag("text", error_message, xmpp_stanzas_attr); end
 	return t; -- stanza ready for adding app-specific errors
 end
 
diff --git a/util/startup.lua b/util/startup.lua
index e88ed709..dd757dd1 100644
--- a/util/startup.lua
+++ b/util/startup.lua
@@ -7,6 +7,7 @@ local logger = require "util.logger";
 local log = logger.init("startup");
 
 local config = require "core.configmanager";
+local config_warnings;
 
 local dependencies = require "util.dependencies";
 
@@ -65,6 +66,8 @@ function startup.read_config()
 		print("**************************");
 		print("");
 		os.exit(1);
+	elseif err and #err > 0 then
+		config_warnings = err;
 	end
 	prosody.config_loaded = true;
 end
@@ -97,8 +100,13 @@ function startup.init_logging()
 	end);
 end
 
-function startup.log_dependency_warnings()
+function startup.log_startup_warnings()
 	dependencies.log_warnings();
+	if config_warnings then
+		for _, warning in ipairs(config_warnings) do
+			log("warn", "Configuration warning: %s", warning);
+		end
+	end
 end
 
 function startup.sanity_check()
@@ -220,8 +228,8 @@ end
 
 function startup.setup_plugindir()
 	local custom_plugin_paths = config.get("*", "plugin_paths");
+	local path_sep = package.config:sub(3,3);
 	if custom_plugin_paths then
-		local path_sep = package.config:sub(3,3);
 		-- path1;path2;path3;defaultpath...
 		-- luacheck: ignore 111
 		CFG_PLUGINDIR = table.concat(custom_plugin_paths, path_sep)..path_sep..(CFG_PLUGINDIR or "plugins");
@@ -229,6 +237,19 @@ function startup.setup_plugindir()
 	end
 end
 
+function startup.setup_plugin_install_path()
+	local installer_plugin_path = config.get("*", "installer_plugin_path") or "custom_plugins";
+	local path_sep = package.config:sub(3,3);
+	-- TODO Figure out what this should be relative to, because CWD could be anywhere
+	installer_plugin_path = config.resolve_relative_path(require "lfs".currentdir(), installer_plugin_path);
+	-- TODO Can probably move directory creation to the install command
+	require "lfs".mkdir(installer_plugin_path);
+	require"util.paths".complement_lua_path(installer_plugin_path);
+	-- luacheck: ignore 111
+	CFG_PLUGINDIR = installer_plugin_path..path_sep..(CFG_PLUGINDIR or "plugins");
+	prosody.paths.plugins = CFG_PLUGINDIR;
+end
+
 function startup.chdir()
 	if prosody.installed then
 		local lfs = require "lfs";
@@ -250,9 +271,9 @@ function startup.add_global_prosody_functions()
 		local ok, level, err = config.load(prosody.config_file);
 		if not ok then
 			if level == "parser" then
-				log("error", "There was an error parsing the configuration file: %s", tostring(err));
+				log("error", "There was an error parsing the configuration file: %s", err);
 			elseif level == "file" then
-				log("error", "Couldn't read the config file when trying to reload: %s", tostring(err));
+				log("error", "Couldn't read the config file when trying to reload: %s", err);
 			end
 		else
 			prosody.events.fire_event("config-reloaded", {
@@ -427,7 +448,7 @@ function startup.switch_user()
 				print("Warning: Couldn't switch to Prosody user/group '"..tostring(desired_user).."'/'"..tostring(desired_group).."': "..tostring(err));
 			else
 				-- Make sure the Prosody user can read the config
-				local conf, err, errno = io.open(prosody.config_file);
+				local conf, err, errno = io.open(prosody.config_file); --luacheck: ignore 211/errno
 				if conf then
 					conf:close();
 				else
@@ -520,12 +541,13 @@ function startup.prosodyctl()
 	startup.force_console_logging();
 	startup.init_logging();
 	startup.setup_plugindir();
+	-- startup.setup_plugin_install_path();
 	startup.setup_datadir();
 	startup.chdir();
 	startup.read_version();
 	startup.switch_user();
 	startup.check_dependencies();
-	startup.log_dependency_warnings();
+	startup.log_startup_warnings();
 	startup.check_unwriteable();
 	startup.load_libraries();
 	startup.init_http_client();
@@ -545,12 +567,13 @@ function startup.prosody()
 	startup.init_logging();
 	startup.load_libraries();
 	startup.setup_plugindir();
+	-- startup.setup_plugin_install_path();
 	startup.setup_datadir();
 	startup.chdir();
 	startup.add_global_prosody_functions();
 	startup.read_version();
 	startup.log_greeting();
-	startup.log_dependency_warnings();
+	startup.log_startup_warnings();
 	startup.load_secondary_libraries();
 	startup.init_http_client();
 	startup.init_data_store();
diff --git a/util/statistics.lua b/util/statistics.lua
index 39954652..0ec88e21 100644
--- a/util/statistics.lua
+++ b/util/statistics.lua
@@ -57,12 +57,14 @@ local function new_registry(config)
 			end;
 		end;
 		rate = function (name)
-			local since, n = time(), 0;
+			local since, n, total = time(), 0, 0;
 			registry[name..":rate"] = function ()
+				total = total + n;
 				local t = time();
 				local stats = {
 					rate = n/(t-since);
 					count = n;
+					total = total;
 				};
 				since, n = t, 0;
 				return "rate", stats.rate, stats;
diff --git a/util/termcolours.lua b/util/termcolours.lua
index 829d84af..2c13d0ff 100644
--- a/util/termcolours.lua
+++ b/util/termcolours.lua
@@ -83,7 +83,7 @@ end
 setmetatable(stylemap, { __index = function(_, style)
 	if type(style) == "string" and style:find("%x%x%x%x%x%x") == 1 then
 		local g = style:sub(7) == " background" and "48;5;" or "38;5;";
-		return g .. color(hex2rgb(style));
+		return format("%s%d", g, color(hex2rgb(style)));
 	end
 end } );
 
diff --git a/util/x509.lua b/util/x509.lua
index 15cc4d3c..342dafde 100644
--- a/util/x509.lua
+++ b/util/x509.lua
@@ -20,9 +20,12 @@
 
 local nameprep = require "util.encodings".stringprep.nameprep;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local idna_to_unicode = require "util.encodings".idna.to_unicode;
 local base64 = require "util.encodings".base64;
 local log = require "util.logger".init("x509");
+local mt = require "util.multitable";
 local s_format = string.format;
+local ipairs = ipairs;
 
 local _ENV = nil;
 -- luacheck: std none
@@ -216,6 +219,60 @@ local function verify_identity(host, service, cert)
 	return false
 end
 
+-- TODO Support other SANs
+local function get_identities(cert) --> map of names to sets of services
+	if cert.setencode then
+		cert:setencode("utf8");
+	end
+
+	local names = mt.new();
+
+	local ext = cert:extensions();
+	local sans = ext[oid_subjectaltname];
+	if sans then
+		if sans["dNSName"] then -- Valid for any service
+			for _, name in ipairs(sans["dNSName"]) do
+				name = idna_to_unicode(nameprep(name));
+				if name then
+					names:set(name, "*", true);
+				end
+			end
+		end
+		if sans[oid_xmppaddr] then
+			for _, name in ipairs(sans[oid_xmppaddr]) do
+				name = nameprep(name);
+				if name then
+					names:set(name, "xmpp-client", true);
+					names:set(name, "xmpp-server", true);
+				end
+			end
+		end
+		if sans[oid_dnssrv] then
+			for _, srvname in ipairs(sans[oid_dnssrv]) do
+				local srv, name = srvname:match("^_([^.]+)%.(.*)");
+				if srv then
+					name = nameprep(name);
+					if name then
+						names:set(name, srv, true);
+					end
+				end
+			end
+		end
+	end
+
+	local subject = cert:subject();
+	for i = 1, #subject do
+		local dn = subject[i];
+		if dn.oid == oid_commonname then
+			local name = nameprep(dn.value);
+			if name and idna_to_ascii(name) then
+				names:set(name, "*", true);
+			end
+		end
+	end
+	return names.data;
+end
+
 local pat = "%-%-%-%-%-BEGIN ([A-Z ]+)%-%-%-%-%-\r?\n"..
 "([0-9A-Za-z+/=\r\n]*)\r?\n%-%-%-%-%-END %1%-%-%-%-%-";
 
@@ -237,6 +294,7 @@ end
 
 return {
 	verify_identity = verify_identity;
+	get_identities = get_identities;
 	pem2der = pem2der;
 	der2pem = der2pem;
 };
diff --git a/util/xmppstream.lua b/util/xmppstream.lua
index 58cbd18e..6aa1def3 100644
--- a/util/xmppstream.lua
+++ b/util/xmppstream.lua
@@ -64,6 +64,8 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 
 	local stream_default_ns = stream_callbacks.default_ns;
 
+	local stream_lang = "en";
+
 	local stack = {};
 	local chardata, stanza = {};
 	local stanza_size = 0;
@@ -101,6 +103,7 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 			if session.notopen then
 				if tagname == stream_tag then
 					non_streamns_depth = 0;
+					stream_lang = attr["xml:lang"] or stream_lang;
 					if cb_streamopened then
 						if lxp_supports_bytecount then
 							cb_handleprogress(stanza_size);
@@ -178,6 +181,9 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 					cb_handleprogress(stanza_size);
 				end
 				stanza_size = 0;
+				if stanza.attr["xml:lang"] == nil then
+					stanza.attr["xml:lang"] = stream_lang;
+				end
 				if tagname ~= stream_error_tag then
 					cb_handlestanza(session, stanza);
 				else