Merge 0.11->trunk

256 files changed, 15356 insertions, 5631 deletions

diff --git a/.busted b/.busted
index f5f9472e..0206c093 100644
--- a/.busted
+++ b/.busted
@@ -2,7 +2,7 @@ return {
   _all = {
   },
   default = {
-    ["exclude-tags"] = "mod_bosh,storage";
+    ["exclude-tags"] = "mod_bosh,storage,SLOW";
   };
   bosh = {
     tags = "mod_bosh";
diff --git a/.luacheckrc b/.luacheckrc
index ce3d377b..cbfcef8d 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -1,7 +1,8 @@
 cache = true
 codes = true
-ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", "113/unpack" }
+ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", }
 
+std = "lua53c"
 max_line_length = 150
 
 read_globals = {
@@ -33,7 +34,6 @@ files["plugins/"] = {
 		"module.name",
 		"module.host",
 		"module._log",
-		"module.log",
 		"module.event_handlers",
 		"module.reloading",
 		"module.saved_state",
@@ -64,12 +64,15 @@ files["plugins/"] = {
 		"module.get_option_scalar",
 		"module.get_option_set",
 		"module.get_option_string",
+		"module.get_status",
 		"module.handle_items",
 		"module.hook",
 		"module.hook_global",
 		"module.hook_object_event",
 		"module.hook_tag",
 		"module.load_resource",
+		"module.log",
+		"module.log_status",
 		"module.measure",
 		"module.measure_event",
 		"module.measure_global_event",
@@ -79,7 +82,9 @@ files["plugins/"] = {
 		"module.remove_item",
 		"module.require",
 		"module.send",
+		"module.send_iq",
 		"module.set_global",
+		"module.set_status",
 		"module.shared",
 		"module.unhook",
 		"module.unhook_object_event",
@@ -123,46 +128,37 @@ files["prosody.cfg.lua"] = {
 if os.getenv("PROSODY_STRICT_LINT") ~= "1" then
 	-- These files have not yet been brought up to standard
 	-- Do not add more files here, but do help us fix these!
-	unused_secondaries = false
 
 	local exclude_files = {
-	"doc/net.server.lua";
-
-	"fallbacks/bit.lua";
-	"fallbacks/lxp.lua";
+		"doc/net.server.lua";
 
-	"net/adns.lua";
-	"net/cqueues.lua";
-	"net/dns.lua";
-	"net/server_select.lua";
+		"fallbacks/bit.lua";
+		"fallbacks/lxp.lua";
 
-	"plugins/mod_storage_sql1.lua";
+		"net/dns.lua";
+		"net/server_select.lua";
 
-	"spec/core_configmanager_spec.lua";
-	"spec/core_moduleapi_spec.lua";
-	"spec/net_http_parser_spec.lua";
-	"spec/util_events_spec.lua";
-	"spec/util_http_spec.lua";
-	"spec/util_ip_spec.lua";
-	"spec/util_multitable_spec.lua";
-	"spec/util_rfc6724_spec.lua";
-	"spec/util_throttle_spec.lua";
-	"spec/util_xmppstream_spec.lua";
+		"plugins/mod_storage_sql1.lua";
 
-	"tools/ejabberd2prosody.lua";
-	"tools/ejabberdsql2prosody.lua";
-	"tools/erlparse.lua";
-	"tools/jabberd14sql2prosody.lua";
-	"tools/migration/migrator.cfg.lua";
-	"tools/migration/migrator/jabberd14.lua";
-	"tools/migration/migrator/mtools.lua";
-	"tools/migration/migrator/prosody_files.lua";
-	"tools/migration/migrator/prosody_sql.lua";
-	"tools/migration/prosody-migrator.lua";
-	"tools/openfire2prosody.lua";
-	"tools/xep227toprosody.lua";
+		"spec/core_moduleapi_spec.lua";
+		"spec/util_http_spec.lua";
+		"spec/util_ip_spec.lua";
+		"spec/util_multitable_spec.lua";
+		"spec/util_rfc6724_spec.lua";
+		"spec/util_throttle_spec.lua";
 
-	"util/sasl/digest-md5.lua";
+		"tools/ejabberd2prosody.lua";
+		"tools/ejabberdsql2prosody.lua";
+		"tools/erlparse.lua";
+		"tools/jabberd14sql2prosody.lua";
+		"tools/migration/migrator.cfg.lua";
+		"tools/migration/migrator/jabberd14.lua";
+		"tools/migration/migrator/mtools.lua";
+		"tools/migration/migrator/prosody_files.lua";
+		"tools/migration/migrator/prosody_sql.lua";
+		"tools/migration/prosody-migrator.lua";
+		"tools/openfire2prosody.lua";
+		"tools/xep227toprosody.lua";
 	}
 	for _, file in ipairs(exclude_files) do
 		files[file] = { only = {} }
diff --git a/CHANGES b/CHANGES
index 136b7d2b..619dec03 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,25 @@
+TRUNK
+=====
+
+-   Module statuses
+-   SNI support (not completely finished)
+-   CORS handling now provided by mod\_http
+-   CSI improvements
+-   mod\_limits: Exempted JIDs
+-   Archive quotas
+-   mod\_mimicking
+-   Rewritten migrator
+-   SCRAM-SHA-256
+-   Bi-directional server-to-server (XEP-0288)
+-   Built-in HTTP server now handles HEAD requests
+-   MUC presence broadcast controls
+-   ALPN support in mod\_net\_multiplex
+-   `daemonize` option deprecated
+-   SASL DIGEST-MD5 removed
+-   Switch to libunbound for DNS queries
+-   mod_external_services (XEP-0215)
+-   util.error for encapsulating errors
+
 0.11.0
 ======
 
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 00000000..2e732c73
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,9 @@
+Thanks for your interest in contributing to the project!
+
+There are many ways to contribute, such as helping improve the
+documentation, reporting bugs, spreading the word or testing the latest
+development version.
+
+You can find more information on how to contribute at <https://prosody.im/doc/contributing>
+
+See also the HACKERS and README files.
diff --git a/GNUmakefile b/GNUmakefile
index 6c134679..1846d72d 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -21,6 +21,7 @@ MKDIR_PRIVATE=$(MKDIR) -m750
 
 LUACHECK=luacheck
 BUSTED=busted
+SCANSION=scansion
 
 .PHONY: all test coverage clean install
 
@@ -48,9 +49,12 @@ install: prosody.install prosodyctl.install prosody.cfg.lua.install util/encodin
 	$(INSTALL_DATA) util/*.so $(SOURCE)/util
 	$(MKDIR) $(SOURCE)/util/sasl
 	$(INSTALL_DATA) util/sasl/*.lua $(SOURCE)/util/sasl
-	$(MKDIR) $(MODULES)/mod_s2s $(MODULES)/mod_pubsub $(MODULES)/adhoc $(MODULES)/muc $(MODULES)/mod_mam
+	$(MKDIR) $(SOURCE)/util/human
+	$(INSTALL_DATA) util/human/*.lua $(SOURCE)/util/human
+	$(MKDIR) $(SOURCE)/util/prosodyctl
+	$(INSTALL_DATA) util/prosodyctl/*.lua $(SOURCE)/util/prosodyctl
+	$(MKDIR) $(MODULES)/mod_pubsub $(MODULES)/adhoc $(MODULES)/muc $(MODULES)/mod_mam
 	$(INSTALL_DATA) plugins/*.lua $(MODULES)
-	$(INSTALL_DATA) plugins/mod_s2s/*.lua $(MODULES)/mod_s2s
 	$(INSTALL_DATA) plugins/mod_pubsub/*.lua $(MODULES)/mod_pubsub
 	$(INSTALL_DATA) plugins/adhoc/*.lua $(MODULES)/adhoc
 	$(INSTALL_DATA) plugins/muc/*.lua $(MODULES)/muc
@@ -71,6 +75,23 @@ clean:
 test:
 	$(BUSTED) --lua=$(RUNWITH)
 
+test-%:
+	$(BUSTED) --lua=$(RUNWITH) -r $*
+
+integration-test: all
+	$(MKDIR) data
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua start
+	$(SCANSION) -d ./spec/scansion; R=$$? \
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop \
+	exit $$R
+
+integration-test-%: all
+	$(MKDIR) data
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua start
+	$(SCANSION) ./spec/scansion/$*.scs; R=$$? \
+	$(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop \
+	exit $$R
+
 coverage:
 	-rm -- luacov.*
 	$(BUSTED) --lua=$(RUNWITH) -c
diff --git a/HACKERS b/HACKERS
index cae98091..240de63d 100644
--- a/HACKERS
+++ b/HACKERS
@@ -5,7 +5,7 @@ involved you can join us on our mailing list and discussion rooms. More
 information on these at https://prosody.im/discuss
 
 Patches are welcome, though before sending we would appreciate if you read 
-docs/coding_style.txt for guidelines on how to format your code, and other tips.
+docs/coding_style.md for guidelines on how to format your code, and other tips.
 
 Documentation for developers can be found at https://prosody.im/doc/developers
 
diff --git a/README b/README
index 797a2175..647327ac 100644
--- a/README
+++ b/README
@@ -12,12 +12,13 @@ rapidly prototype new protocols.
 Homepage:        https://prosody.im/
 Download:        https://prosody.im/download
 Documentation:   https://prosody.im/doc/
+Issue tracker:   https://issues.prosody.im/
 
 Jabber/XMPP Chat:
                Address:
                  prosody@conference.prosody.im
                Web interface:
-                 https://prosody.im/webchat
+                 https://chat.prosody.im/
                
 Mailing lists:
                User support and discussion:
@@ -26,9 +27,6 @@ Mailing lists:
                Development discussion:
                  https://groups.google.com/group/prosody-dev
                
-               Issue tracker changes:
-                 https://groups.google.com/group/prosody-issues
-
 ## Installation
 
 See the accompanying INSTALL file for help on building Prosody from source. Alternatively 
diff --git a/TODO b/TODO
index d22d20f4..9ec820b2 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,4 @@
 == 1.0 ==
 - Roster providers
-- Statistics
 - Clustering
 - World domination
diff --git a/configure b/configure
index 0fe186b5..abb5ffef 100755
--- a/configure
+++ b/configure
@@ -23,7 +23,8 @@ EXCERTS="yes"
 PRNG=
 PRNGLIBS=
 
-CFLAGS="-fPIC -Wall -pedantic -std=c99"
+CFLAGS="-fPIC -std=c99"
+CFLAGS="$CFLAGS -Wall -pedantic -Wextra -Wshadow -Wformat=2"
 LDFLAGS="-shared"
 
 IDN_LIBRARY="idn"
@@ -152,74 +153,8 @@ do
       SYSCONFDIR_SET=yes
       ;;
    --ostype)
-	# TODO make this a switch?
       OSPRESET="$value"
-      if [ "$OSPRESET" = "debian" ]; then
-         if [ "$LUA_SUFFIX_SET" != "yes" ]; then
-            LUA_SUFFIX="5.1";
-            LUA_SUFFIX_SET=yes
-         fi
-         if [ "$RUNWITH_SET" != "yes" ]; then
-            RUNWITH="lua$LUA_SUFFIX";
-            RUNWITH_SET=yes
-         fi
-         LUA_INCDIR="/usr/include/lua$LUA_SUFFIX"
-         LUA_INCDIR_SET=yes
-         CFLAGS="$CFLAGS -ggdb"
-      fi
-      if [ "$OSPRESET" = "macosx" ]; then
-         LUA_INCDIR=/usr/local/include;
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR=/usr/local/lib
-         LUA_LIBDIR_SET=yes
-         CFLAGS="$CFLAGS -mmacosx-version-min=10.3"
-         LDFLAGS="-bundle -undefined dynamic_lookup"
-      fi
-      if [ "$OSPRESET" = "linux" ]; then
-         LUA_INCDIR=/usr/local/include;
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR=/usr/local/lib
-         LUA_LIBDIR_SET=yes
-         CFLAGS="$CFLAGS -ggdb"
-      fi
-      if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then
-         LUA_INCDIR="/usr/local/include/lua51"
-         LUA_INCDIR_SET=yes
-         CFLAGS="-Wall -fPIC -I/usr/local/include"
-         LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared"
-         LUA_SUFFIX="51"
-         LUA_SUFFIX_SET=yes
-         LUA_DIR=/usr/local
-         LUA_DIR_SET=yes
-         CC=cc
-         LD=ld
-      fi
-      if [ "$OSPRESET" = "openbsd" ]; then
-         LUA_INCDIR="/usr/local/include";
-         LUA_INCDIR_SET="yes"
-      fi
-      if [ "$OSPRESET" = "netbsd" ]; then
-         LUA_INCDIR="/usr/pkg/include/lua-5.1"
-         LUA_INCDIR_SET=yes
-         LUA_LIBDIR="/usr/pkg/lib/lua/5.1"
-         LUA_LIBDIR_SET=yes
-         CFLAGS="-Wall -fPIC -I/usr/pkg/include"
-         LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared"
-      fi
-      if [ "$OSPRESET" = "pkg-config" ]; then
-         if [ "$LUA_SUFFIX_SET" != "yes" ]; then
-            LUA_SUFFIX="5.1";
-            LUA_SUFFIX_SET=yes
-         fi
-         LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)"
-         LUA_CF="${LUA_CF#*-I}"
-         LUA_CF="${LUA_CF%% *}"
-         if [ "$LUA_CF" != "" ]; then
-            LUA_INCDIR="$LUA_CF"
-            LUA_INCDIR_SET=yes
-         fi
-         CFLAGS="$CFLAGS"
-      fi
+      OSPRESET_SET="yes"
       ;;
    --libdir)
       LIBDIR="$value"
@@ -237,7 +172,7 @@ do
    --lua-version|--with-lua-version)
       [ -n "$value" ] || die "Missing value in flag $key."
       LUA_VERSION="$value"
-      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || die "Invalid Lua version in flag $key."
+      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || [ "$LUA_VERSION" = "5.4" ] || die "Invalid Lua version in flag $key."
       LUA_VERSION_SET=yes
       ;;
    --with-lua)
@@ -318,6 +253,66 @@ do
    shift
 done
 
+if [ "$OSPRESET_SET" = "yes" ]; then
+	# TODO make this a switch?
+   if [ "$OSPRESET" = "debian" ]; then
+      CFLAGS="$CFLAGS -ggdb"
+   fi
+   if [ "$OSPRESET" = "macosx" ]; then
+      if [ "$LUA_INCDIR_SET" != "yes" ]; then
+         LUA_INCDIR=/usr/local/include;
+         LUA_INCDIR_SET=yes
+      fi
+      if [ "$LUA_LIBDIR_SET" != "yes" ]; then
+         LUA_LIBDIR=/usr/local/lib
+         LUA_LIBDIR_SET=yes
+      fi
+      CFLAGS="$CFLAGS -mmacosx-version-min=10.3"
+      LDFLAGS="-bundle -undefined dynamic_lookup"
+   fi
+   if [ "$OSPRESET" = "linux" ]; then
+      CFLAGS="$CFLAGS -ggdb"
+   fi
+   if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then
+      LUA_INCDIR="/usr/local/include/lua51"
+      LUA_INCDIR_SET=yes
+      CFLAGS="-Wall -fPIC -I/usr/local/include"
+      LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared"
+      LUA_SUFFIX="51"
+      LUA_SUFFIX_SET=yes
+      LUA_DIR=/usr/local
+      LUA_DIR_SET=yes
+      CC=cc
+      LD=ld
+   fi
+   if [ "$OSPRESET" = "openbsd" ]; then
+      LUA_INCDIR="/usr/local/include";
+      LUA_INCDIR_SET="yes"
+   fi
+   if [ "$OSPRESET" = "netbsd" ]; then
+      LUA_INCDIR="/usr/pkg/include/lua-5.1"
+      LUA_INCDIR_SET=yes
+      LUA_LIBDIR="/usr/pkg/lib/lua/5.1"
+      LUA_LIBDIR_SET=yes
+      CFLAGS="-Wall -fPIC -I/usr/pkg/include"
+      LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared"
+   fi
+   if [ "$OSPRESET" = "pkg-config" ]; then
+      if [ "$LUA_SUFFIX_SET" != "yes" ]; then
+         LUA_SUFFIX="5.1";
+         LUA_SUFFIX_SET=yes
+      fi
+      LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)"
+      LUA_CF="${LUA_CF#*-I}"
+      LUA_CF="${LUA_CF%% *}"
+      if [ "$LUA_CF" != "" ]; then
+         LUA_INCDIR="$LUA_CF"
+         LUA_INCDIR_SET=yes
+      fi
+      CFLAGS="$CFLAGS"
+   fi
+fi
+
 if [ "$PREFIX_SET" = "yes" ] && [ ! "$SYSCONFDIR_SET" = "yes" ]
 then
    if [ "$PREFIX" = "/usr" ]
@@ -340,7 +335,7 @@ then
 fi
 
 detect_lua_version() {
-   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[123])$"))' 2> /dev/null)
+   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[1234])$"))' 2> /dev/null)
    if [ "$detected_lua" != "nil" ]
    then
       if [ "$LUA_VERSION_SET" != "yes" ]
@@ -403,8 +398,14 @@ then
    elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.3" ]
    then
       suffixes="5.3 53 -5.3 -53"
+   elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.4" ]
+   then
+      suffixes="5.4 54 -5.4 -54"
    else
-      suffixes="5.1 51 -5.1 -51 5.2 52 -5.2 -52 5.3 53 -5.3 -53"
+      suffixes="5.1 51 -5.1 -51"
+      suffixes="$suffixes 5.2 52 -5.2 -52"
+      suffixes="$suffixes 5.3 53 -5.3 -53"
+      suffixes="$suffixes 5.4 54 -5.4 -54"
    fi
    for suffix in "" $suffixes
    do
@@ -464,30 +465,46 @@ then
    LUA_LIBDIR="$LUA_DIR/lib"
 fi
 
-echo_n "Checking Lua includes... "
 lua_h="$LUA_INCDIR/lua.h"
+echo_n "Looking for lua.h at $lua_h..."
 if [ -f "$lua_h" ]
 then
-   echo "lua.h found in $lua_h"
+   echo found
 else
-   v_dir="$LUA_INCDIR/lua/$LUA_VERSION"
-   lua_h="$v_dir/lua.h"
-   if [ -f "$lua_h" ]
-   then
-      echo "lua.h found in $lua_h"
+  echo "not found"
+  for postfix in "$LUA_VERSION" "$LUA_SUFFIX"; do
+    if ! [ "$postfix" = "" ]; then
+      v_dir="$LUA_INCDIR/lua/$postfix";
+    else
+      v_dir="$LUA_INCDIR/lua";
+    fi
+    lua_h="$v_dir/lua.h"
+    echo_n "Looking for lua.h at $lua_h..."
+    if [ -f "$lua_h" ]
+    then
       LUA_INCDIR="$v_dir"
-   else
-      d_dir="$LUA_INCDIR/lua$LUA_VERSION"
+      echo found
+      break;
+    else
+      echo "not found"
+      d_dir="$LUA_INCDIR/lua$postfix"
       lua_h="$d_dir/lua.h"
+      echo_n "Looking for lua.h at $lua_h..."
       if [ -f "$lua_h" ]
       then
-         echo "lua.h found in $lua_h (Debian/Ubuntu)"
-         LUA_INCDIR="$d_dir"
+        echo found
+        LUA_INCDIR="$d_dir"
+        break;
       else
-         echo "lua.h not found (looked in $LUA_INCDIR, $v_dir, $d_dir)"
-         die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+        echo "not found"
       fi
-   fi
+    fi
+  done
+  if [ ! -f "$lua_h" ]; then
+    echo "lua.h not found."
+    echo
+    die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+  fi
 fi
 
 if [ "$lua_interp_found" = "yes" ]
@@ -507,7 +524,7 @@ fi
 if [ "$IDN_LIBRARY" = "icu" ]
 then
    IDNA_LIBS="$ICU_FLAGS"
-   CFLAGS="$CFLAGS -DUSE_STRINGPREP_ICU"
+   IDNA_FLAGS="-DUSE_STRINGPREP_ICU"
 fi
 if [ "$IDN_LIBRARY" = "idn" ]
 then
@@ -552,6 +569,7 @@ LUA_INCDIR=$LUA_INCDIR
 LUA_LIBDIR=$LUA_LIBDIR
 LUA_BINDIR=$LUA_BINDIR
 IDN_LIB=$IDN_LIB
+IDNA_FLAGS=$IDNA_FLAGS
 IDNA_LIBS=$IDNA_LIBS
 OPENSSL_LIBS=$OPENSSL_LIBS
 CFLAGS=$CFLAGS
diff --git a/core/certmanager.lua b/core/certmanager.lua
index 20b91318..023218fa 100644
--- a/core/certmanager.lua
+++ b/core/certmanager.lua
@@ -20,7 +20,6 @@ end
 local configmanager = require "core.configmanager";
 local log = require "util.logger".init("certmanager");
 local ssl_context = ssl.context or softreq"ssl.context";
-local ssl_x509 = ssl.x509 or softreq"ssl.x509";
 local ssl_newcontext = ssl.newcontext;
 local new_config = require"util.sslconfig".new;
 local stat = require "lfs".attributes;
@@ -38,6 +37,9 @@ local config_path = prosody.paths.config or ".";
 
 local luasec_major, luasec_minor = ssl._VERSION:match("^(%d+)%.(%d+)");
 local luasec_version = tonumber(luasec_major) * 100 + tonumber(luasec_minor);
+-- TODO Use ssl.config instead of require here once we are sure that the fix
+-- in LuaSec has been widely distributed
+-- https://github.com/brunoos/luasec/issues/149
 local luasec_has = softreq"ssl.config" or {
 	algorithms = {
 		ec = luasec_version >= 5;
@@ -108,7 +110,7 @@ local core_defaults = {
 	capath = "/etc/ssl/certs";
 	depth = 9;
 	protocol = "tlsv1+";
-	verify = (ssl_x509 and { "peer", "client_once", }) or "none";
+	verify = "none";
 	options = {
 		cipher_server_preference = luasec_has.options.cipher_server_preference;
 		no_ticket = luasec_has.options.no_ticket;
@@ -150,13 +152,6 @@ local path_options = { -- These we pass through resolve_path()
 	key = true, certificate = true, cafile = true, capath = true, dhparam = true
 }
 
-if luasec_version < 5 and ssl_x509 then
-	-- COMPAT mw/luasec-hg
-	for i=1,#core_defaults.verifyext do -- Remove lsec_ prefix
-		core_defaults.verify[#core_defaults.verify+1] = core_defaults.verifyext[i]:sub(6);
-	end
-end
-
 local function create_context(host, mode, ...)
 	local cfg = new_config();
 	cfg:apply(core_defaults);
@@ -179,8 +174,10 @@ local function create_context(host, mode, ...)
 	local user_ssl_config = cfg:final();
 
 	if mode == "server" then
-		if not user_ssl_config.certificate then return nil, "No certificate present in SSL/TLS configuration for "..host; end
-		if not user_ssl_config.key then return nil, "No key present in SSL/TLS configuration for "..host; end
+		if not user_ssl_config.certificate then
+			log("info", "No certificate present in SSL/TLS configuration for %s. SNI will be required.", host);
+		end
+		if user_ssl_config.certificate and not user_ssl_config.key then return nil, "No key present in SSL/TLS configuration for "..host; end
 	end
 
 	for option in pairs(path_options) do
@@ -260,4 +257,5 @@ return {
 	create_context = create_context;
 	reload_ssl_config = reload_ssl_config;
 	find_cert = find_cert;
+	find_host_cert = find_host_cert;
 };
diff --git a/core/configmanager.lua b/core/configmanager.lua
index 1e67da9b..ae0a274a 100644
--- a/core/configmanager.lua
+++ b/core/configmanager.lua
@@ -7,15 +7,16 @@
 --
 
 local _G = _G;
-local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs =
-      setmetatable, rawget, rawset, io, os, error, dofile, type, pairs;
-local format, math_max = string.format, math.max;
+local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs =
+      setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs;
+local format, math_max, t_insert = string.format, math.max, table.insert;
 
 local envload = require"util.envload".envload;
 local deps = require"util.dependencies";
 local resolve_relative_path = require"util.paths".resolve_relative_path;
 local glob_to_pattern = require"util.paths".glob_to_pattern;
 local path_sep = package.config:sub(1,1);
+local get_traceback_table = require "util.debug".get_traceback_table;
 
 local encodings = deps.softreq"util.encodings";
 local nameprep = encodings and encodings.stringprep.nameprep or function (host) return host:lower(); end
@@ -100,8 +101,18 @@ end
 -- Built-in Lua parser
 do
 	local pcall = _G.pcall;
+	local function get_line_number(config_file)
+		local tb = get_traceback_table(nil, 2);
+		for i = 1, #tb do
+			if tb[i].info.short_src == config_file then
+				return tb[i].info.currentline;
+			end
+		end
+	end
 	parser = {};
 	function parser.load(data, config_file, config_table)
+		local set_options = {}; -- set_options[host.."/"..option_name] = true (when the option has been set already in this file)
+		local warnings = {};
 		local env;
 		-- The ' = true' are needed so as not to set off __newindex when we assign the functions below
 		env = setmetatable({
@@ -115,13 +126,26 @@ do
 					return rawget(_G, k);
 				end,
 				__newindex = function (_, k, v)
+					local host = env.__currenthost or "*";
+					local option_path = host.."/"..k;
+					if set_options[option_path] then
+						t_insert(warnings, ("%s:%d: Duplicate option '%s'"):format(config_file, get_line_number(config_file), k));
+					end
+					set_options[option_path] = true;
 					set(config_table, env.__currenthost or "*", k, v);
 				end
 		});
 
 		rawset(env, "__currenthost", "*") -- Default is global
 		function env.VirtualHost(name)
-			name = nameprep(name);
+			if not name then
+				error("Host must have a name", 2);
+			end
+			local prepped_name = nameprep(name);
+			if not prepped_name then
+				error(format("Name of Host %q contains forbidden characters", name), 0);
+			end
+			name = prepped_name;
 			if rawget(config_table, name) and rawget(config_table[name], "component_module") then
 				error(format("Host %q clashes with previously defined %s Component %q, for services use a sub-domain like conference.%s",
 					name, config_table[name].component_module:gsub("^%a+$", { component = "external", muc = "MUC"}), name, name), 0);
@@ -139,7 +163,14 @@ do
 		env.Host, env.host = env.VirtualHost, env.VirtualHost;
 
 		function env.Component(name)
-			name = nameprep(name);
+			if not name then
+				error("Component must have a name", 2);
+			end
+			local prepped_name = nameprep(name);
+			if not prepped_name then
+				error(format("Name of Component %q contains forbidden characters", name), 0);
+			end
+			name = prepped_name;
 			if rawget(config_table, name) and rawget(config_table[name], "defined")
 				and not rawget(config_table[name], "component_module") then
 				error(format("Component %q clashes with previously defined Host %q, for services use a sub-domain like conference.%s",
@@ -195,6 +226,11 @@ do
 			if f then
 				local ret, err = parser.load(f:read("*a"), file, config_table);
 				if not ret then error(err:gsub("%[string.-%]", file), 0); end
+				if err then
+					for _, warning in ipairs(err) do
+						t_insert(warnings, warning);
+					end
+				end
 			end
 			if not f then error("Error loading included "..file..": "..err, 0); end
 			return f, err;
@@ -217,7 +253,7 @@ do
 			return nil, err;
 		end
 
-		return true;
+		return true, warnings;
 	end
 
 end
diff --git a/core/loggingmanager.lua b/core/loggingmanager.lua
index cfa8246a..85a6380b 100644
--- a/core/loggingmanager.lua
+++ b/core/loggingmanager.lua
@@ -18,6 +18,9 @@ local getstyle, getstring = require "util.termcolours".getstyle, require "util.t
 local config = require "core.configmanager";
 local logger = require "util.logger";
 
+local have_pposix, pposix = pcall(require, "util.pposix");
+have_pposix = have_pposix and pposix._VERSION == "0.4.0";
+
 local _ENV = nil;
 -- luacheck: std none
 
@@ -232,6 +235,22 @@ local function log_to_console(sink_config)
 end
 log_sink_types.console = log_to_console;
 
+if have_pposix then
+	local syslog_opened;
+	local function log_to_syslog(sink_config) -- luacheck: ignore 212/sink_config
+		if not syslog_opened then
+			local facility = sink_config.syslog_facility or config.get("*", "syslog_facility");
+			pposix.syslog_open(sink_config.syslog_name or "prosody", facility);
+			syslog_opened = true;
+		end
+		local syslog = pposix.syslog_log;
+		return function (name, level, message, ...)
+			syslog(level, name, format(message, ...));
+		end;
+	end
+	log_sink_types.syslog = log_to_syslog;
+end
+
 local function register_sink_type(name, sink_maker)
 	local old_sink_maker = log_sink_types[name];
 	log_sink_types[name] = sink_maker;
diff --git a/core/moduleapi.lua b/core/moduleapi.lua
index 10f9f04d..82002737 100644
--- a/core/moduleapi.lua
+++ b/core/moduleapi.lua
@@ -14,13 +14,19 @@ local pluginloader = require "util.pluginloader";
 local timer = require "util.timer";
 local resolve_relative_path = require"util.paths".resolve_relative_path;
 local st = require "util.stanza";
+local cache = require "util.cache";
+local errors = require "util.error";
+local promise = require "util.promise";
+local time_now = require "util.time".now;
+local format = require "util.format".format;
+local jid_node = require "util.jid".node;
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 local error, setmetatable, type = error, setmetatable, type;
 local ipairs, pairs, select = ipairs, pairs, select;
 local tonumber, tostring = tonumber, tostring;
 local require = require;
-local pack = table.pack or function(...) return {n=select("#",...), ...}; end -- table.pack is only in 5.2
+local pack = table.pack or require "util.table".pack; -- table.pack is only in 5.2
 local unpack = table.unpack or unpack; --luacheck: ignore 113 -- renamed in 5.2
 
 local prosody = prosody;
@@ -361,6 +367,100 @@ function api:send(stanza, origin)
 	return core_post_stanza(origin or hosts[self.host], stanza);
 end
 
+function api:send_iq(stanza, origin, timeout)
+	local iq_cache = self._iq_cache;
+	if not iq_cache then
+		iq_cache = cache.new(256, function (_, iq)
+			iq.reject(errors.new({
+				type = "wait", condition = "resource-constraint",
+				text = "evicted from iq tracking cache"
+			}));
+		end);
+		self._iq_cache = iq_cache;
+	end
+
+	local event_type;
+	if not jid_node(stanza.attr.from) then
+		event_type = "host";
+	else -- assume bare since we can't hook full jids
+		event_type = "bare";
+	end
+	local result_event = "iq-result/"..event_type.."/"..stanza.attr.id;
+	local error_event = "iq-error/"..event_type.."/"..stanza.attr.id;
+	local cache_key = event_type.."/"..stanza.attr.id;
+
+	local p = promise.new(function (resolve, reject)
+		local function result_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				resolve(event);
+				return true;
+			end
+		end
+
+		local function error_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				reject(errors.from_stanza(event.stanza, event));
+				return true;
+			end
+		end
+
+		if iq_cache:get(cache_key) then
+			reject(errors.new({
+				type = "modify", condition = "conflict",
+				text = "IQ stanza id attribute already used",
+			}));
+			return;
+		end
+
+		self:hook(result_event, result_handler);
+		self:hook(error_event, error_handler);
+
+		local timeout_handle = self:add_timer(timeout or 120, function ()
+			reject(errors.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "IQ stanza timed out",
+			}));
+		end);
+
+		local ok = iq_cache:set(cache_key, {
+			reject = reject, resolve = resolve,
+			timeout_handle = timeout_handle,
+			result_handler = result_handler, error_handler = error_handler;
+		});
+
+		if not ok then
+			reject(errors.new({
+				type = "wait", condition = "internal-server-error",
+				text = "Could not store IQ tracking data"
+			}));
+			return;
+		end
+
+		local wrapped_origin = setmetatable({
+				-- XXX Needed in some cases for replies to work correctly when sending queries internally.
+				send = function (reply)
+					resolve({ stanza = reply });
+				end;
+			}, {
+				__index = origin or hosts[self.host];
+			});
+
+		self:send(stanza, wrapped_origin);
+	end);
+
+	p:finally(function ()
+		local iq = iq_cache:get(cache_key);
+		if iq then
+			self:unhook(result_event, iq.result_handler);
+			self:unhook(error_event, iq.error_handler);
+			iq.timeout_handle:stop();
+			iq_cache:set(cache_key, nil);
+		end
+	end);
+
+	return p;
+end
+
 function api:broadcast(jids, stanza, iter)
 	for jid in (iter or it.values)(jids) do
 		local new_stanza = st.clone(stanza);
@@ -396,7 +496,7 @@ end
 
 local path_sep = package.config:sub(1,1);
 function api:get_directory()
-	return self.path and (self.path:gsub("%"..path_sep.."[^"..path_sep.."]*$", "")) or nil;
+	return self.resource_path or self.path and (self.path:gsub("%"..path_sep.."[^"..path_sep.."]*$", "")) or nil;
 end
 
 function api:load_resource(path, mode)
@@ -408,9 +508,9 @@ function api:open_store(name, store_type)
 	return require"core.storagemanager".open(self.host, name or self.name, store_type);
 end
 
-function api:measure(name, stat_type)
+function api:measure(name, stat_type, conf)
 	local measure = require "core.statsmanager".measure;
-	return measure(stat_type, "/"..self.host.."/mod_"..self.name.."/"..name);
+	return measure(stat_type, "/"..self.host.."/mod_"..self.name.."/"..name, conf);
 end
 
 function api:measure_object_event(events_object, event_name, stat_name)
@@ -432,4 +532,32 @@ function api:measure_global_event(event_name, stat_name)
 	return self:measure_object_event(prosody.events.wrappers, event_name, stat_name);
 end
 
+local status_priorities = { error = 3, warn = 2, info = 1, core = 0 };
+
+function api:set_status(status_type, status_message, override)
+	local priority = status_priorities[status_type];
+	if not priority then
+		self:log("error", "set_status: Invalid status type '%s', assuming 'info'");
+		status_type, priority = "info", status_priorities.info;
+	end
+	local current_priority = status_priorities[self.status_type] or 0;
+	-- By default an 'error' status can only be overwritten by another 'error' status
+	if (current_priority >= status_priorities.error and priority < current_priority and override ~= true)
+	or (override == false and current_priority > priority) then
+		self:log("debug", "moduleapi: ignoring status [prio %d override %s]: %s", priority, override, status_message);
+		return;
+	end
+	self.status_type, self.status_message, self.status_time = status_type, status_message, time_now();
+	self:fire_event("module-status/updated", { name = self.name });
+end
+
+function api:log_status(level, msg, ...)
+	self:set_status(level, format(msg, ...));
+	return self:log(level, msg, ...);
+end
+
+function api:get_status()
+	return self.status_type, self.status_message, self.status_time;
+end
+
 return api;
diff --git a/core/modulemanager.lua b/core/modulemanager.lua
index a824d36a..3f4233e6 100644
--- a/core/modulemanager.lua
+++ b/core/modulemanager.lua
@@ -10,6 +10,7 @@ local logger = require "util.logger";
 local log = logger.init("modulemanager");
 local config = require "core.configmanager";
 local pluginloader = require "util.pluginloader";
+local envload = require "util.envload";
 local set = require "util.set";
 
 local new_multitable = require "util.multitable".new;
@@ -22,9 +23,26 @@ local xpcall = require "util.xpcall".xpcall;
 local debug_traceback = debug.traceback;
 local setmetatable, rawget = setmetatable, rawget;
 local ipairs, pairs, type, t_insert = ipairs, pairs, type, table.insert;
-
-local autoload_modules = {prosody.platform, "presence", "message", "iq", "offline", "c2s", "s2s", "s2s_auth_certs"};
-local component_inheritable_modules = {"tls", "saslauth", "dialback", "iq", "s2s"};
+local lua_version = _VERSION:match("5%.%d$");
+
+local autoload_modules = {
+	prosody.platform,
+	"presence",
+	"message",
+	"iq",
+	"offline",
+	"c2s",
+	"s2s",
+	"s2s_auth_certs",
+};
+local component_inheritable_modules = {
+	"tls",
+	"saslauth",
+	"dialback",
+	"iq",
+	"s2s",
+	"s2s_bidi",
+};
 
 -- We need this to let modules access the real global namespace
 local _G = _G;
@@ -174,11 +192,48 @@ local function do_load_module(host, module_name, state)
 	local mod, err = pluginloader.load_code(module_name, nil, pluginenv);
 	if not mod then
 		log("error", "Unable to load module '%s': %s", module_name or "nil", err or "nil");
+		api_instance:set_status("error", "Failed to load (see log)");
 		return nil, err;
 	end
 
 	api_instance.path = err;
 
+	local custom_plugins = prosody.paths.installer;
+	if custom_plugins and err:sub(1, #custom_plugins+1) == custom_plugins.."/" then
+		-- Stage 1: Make it work (you are here)
+		-- Stage 2: Make it less hacky (TODO)
+		local manifest = {};
+		local luarocks_path = custom_plugins.."/lib/luarocks/rocks-"..lua_version;
+		local manifest_filename = luarocks_path.."/manifest";
+		local load_manifest, err = envload.envloadfile(manifest_filename, manifest);
+		if not load_manifest then
+			-- COMPAT Luarocks 2.x
+			log("debug", "Could not load LuaRocks 3.x manifest, trying 2.x", err);
+			luarocks_path = custom_plugins.."/lib/luarocks/rocks-"..lua_version;
+			manifest_filename = luarocks_path.."/manifest";
+			load_manifest, err = envload.envloadfile(manifest_filename, manifest);
+		end
+		if not load_manifest then
+			log("error", "Could not load manifest of installed plugins: %s", err, load_manifest);
+		else
+			local ok, err = xpcall(load_manifest, debug_traceback);
+			if not ok then
+				log("error", "Could not load manifest of installed plugins: %s", err);
+			elseif type(manifest.modules) ~= "table" then
+				log("debug", "Expected 'table' but manifest.modules = %q", manifest.modules);
+				log("error", "Can't look up resource path for mod_%s because '%s' does not appear to be a LuaRocks manifest", module_name, manifest_filename);
+			else
+				local versions = manifest.modules["mod_"..module_name];
+				if type(versions) == "table" and versions[1] then
+					-- Not going to deal with multiple installed versions
+					api_instance.resource_path = luarocks_path.."/"..versions[1];
+				else
+					log("debug", "mod_%s does not appear in the installation manifest", module_name);
+				end
+			end
+		end
+	end
+
 	modulemap[host][module_name] = pluginenv;
 	local ok, err = xpcall(mod, debug_traceback);
 	if ok then
@@ -187,6 +242,7 @@ local function do_load_module(host, module_name, state)
 			ok, err = call_module_method(pluginenv, "load");
 			if not ok then
 				log("warn", "Error loading module '%s' on '%s': %s", module_name, host, err or "nil");
+				api_instance:set_status("warn", "Error during load (see log)");
 			end
 		end
 		api_instance.reloading, api_instance.saved_state = nil, nil;
@@ -209,6 +265,9 @@ local function do_load_module(host, module_name, state)
 	if not ok then
 		modulemap[api_instance.host][module_name] = nil;
 		log("error", "Error initializing module '%s' on '%s': %s", module_name, host, err or "nil");
+		api_instance:set_status("warn", "Error during load (see log)");
+	else
+		api_instance:set_status("core", "Loaded", false);
 	end
 	return ok and pluginenv, err;
 end
@@ -225,7 +284,8 @@ local function do_reload_module(host, name)
 
 	local saved;
 	if module_has_method(mod, "save") then
-		local ok, ret, err = call_module_method(mod, "save");
+		-- FIXME What goes in 'err' here?
+		local ok, ret, err = call_module_method(mod, "save"); -- luacheck: ignore 211/err
 		if ok then
 			saved = ret;
 		else
diff --git a/core/portmanager.lua b/core/portmanager.lua
index bed5eca5..2f9aa8b2 100644
--- a/core/portmanager.lua
+++ b/core/portmanager.lua
@@ -9,7 +9,8 @@ local set = require "util.set";
 
 local table = table;
 local setmetatable, rawset, rawget = setmetatable, rawset, rawget;
-local type, tonumber, tostring, ipairs = type, tonumber, tostring, ipairs;
+local type, tonumber, ipairs = type, tonumber, ipairs;
+local pairs = pairs;
 
 local prosody = prosody;
 local fire_event = prosody.events.fire_event;
@@ -95,25 +96,25 @@ local function activate(service_name)
 		   }
 	bind_ports = set.new(type(bind_ports) ~= "table" and { bind_ports } or bind_ports );
 
-	local mode, ssl = listener.default_mode or default_mode;
+	local mode = listener.default_mode or default_mode;
 	local hooked_ports = {};
 
 	for interface in bind_interfaces do
 		for port in bind_ports do
 			local port_number = tonumber(port);
 			if not port_number then
-				log("error", "Invalid port number specified for service '%s': %s", service_info.name, tostring(port));
+				log("error", "Invalid port number specified for service '%s': %s", service_info.name, port);
 			elseif #active_services:search(nil, interface, port_number) > 0 then
 				log("error", "Multiple services configured to listen on the same port ([%s]:%d): %s, %s", interface, port,
 					active_services:search(nil, interface, port)[1][1].service.name or "<unnamed>", service_name or "<unnamed>");
 			else
-				local err;
+				local ssl, cfg, err;
 				-- Create SSL context for this service/port
 				if service_info.encryption == "ssl" then
 					local global_ssl_config = config.get("*", "ssl") or {};
 					local prefix_ssl_config = config.get("*", config_prefix.."ssl") or global_ssl_config;
 					log("debug", "Creating context for direct TLS service %s on port %d", service_info.name, port);
-					ssl, err = certmanager.create_context(service_info.name.." port "..port, "server",
+					ssl, err, cfg = certmanager.create_context(service_info.name.." port "..port, "server",
 						prefix_ssl_config[interface],
 						prefix_ssl_config[port],
 						prefix_ssl_config,
@@ -127,7 +128,12 @@ local function activate(service_name)
 				end
 				if not err then
 					-- Start listening on interface+port
-					local handler, err = server.addserver(interface, port_number, listener, mode, ssl);
+					local handler, err = server.listen(interface, port_number, listener, {
+						read_size = mode,
+						tls_ctx = ssl,
+						tls_direct = service_info.encryption == "ssl";
+						sni_hosts = {},
+					});
 					if not handler then
 						log("error", "Failed to open server port %d on %s, %s", port_number, interface,
 							error_to_friendly_message(service_name, port_number, err));
@@ -137,6 +143,7 @@ local function activate(service_name)
 						active_services:add(service_name, interface, port_number, {
 							server = handler;
 							service = service_info;
+							tls_cfg = cfg;
 						});
 					end
 				end
@@ -163,7 +170,7 @@ end
 local function register_service(service_name, service_info)
 	table.insert(services[service_name], service_info);
 
-	if not active_services:get(service_name) then
+	if not active_services:get(service_name) and prosody.process_type == "prosody" then
 		log("debug", "No active service for %s, activating...", service_name);
 		local ok, err = activate(service_name);
 		if not ok then
@@ -222,15 +229,46 @@ end
 
 -- Event handlers
 
+local function add_sni_host(host, service)
+	for name, interface, port, n, active_service --luacheck: ignore 213
+		in active_services:iter(service, nil, nil, nil) do
+		if active_service.server.hosts and active_service.tls_cfg then
+			local config_prefix = (active_service.config_prefix or name).."_";
+			if config_prefix == "_" then config_prefix = ""; end
+			local prefix_ssl_config = config.get(host, config_prefix.."ssl");
+			local autocert = certmanager.find_host_cert(host);
+			-- luacheck: ignore 211/cfg
+			local ssl, err, cfg = certmanager.create_context(host, "server", prefix_ssl_config, autocert, active_service.tls_cfg);
+			if ssl then
+				active_service.server.hosts[host] = ssl;
+			else
+				log("error", "Error creating TLS context for SNI host %s: %s", host, err);
+			end
+		end
+	end
+end
 prosody.events.add_handler("item-added/net-provider", function (event)
 	local item = event.item;
 	register_service(item.name, item);
+	for host in pairs(prosody.hosts) do
+		add_sni_host(host, item.name);
+	end
 end);
 prosody.events.add_handler("item-removed/net-provider", function (event)
 	local item = event.item;
 	unregister_service(item.name, item);
 end);
 
+prosody.events.add_handler("host-activated", add_sni_host);
+prosody.events.add_handler("host-deactivated", function (host)
+	for name, interface, port, n, active_service --luacheck: ignore 213
+		in active_services:iter(nil, nil, nil, nil) do
+		if active_service.tls_cfg then
+			active_service.server.hosts[host] = nil;
+		end
+	end
+end);
+
 return {
 	activate = activate;
 	deactivate = deactivate;
diff --git a/core/rostermanager.lua b/core/rostermanager.lua
index 7bfad0a0..7b104339 100644
--- a/core/rostermanager.lua
+++ b/core/rostermanager.lua
@@ -285,15 +285,15 @@ end
 
 function is_contact_pending_in(username, host, jid)
 	local roster = load_roster(username, host);
-	return roster[false].pending[jid];
+	return roster[false].pending[jid] ~= nil;
 end
-local function set_contact_pending_in(username, host, jid)
+local function set_contact_pending_in(username, host, jid, stanza)
 	local roster = load_roster(username, host);
 	local item = roster[jid];
 	if item and (item.subscription == "from" or item.subscription == "both") then
 		return; -- false
 	end
-	roster[false].pending[jid] = true;
+	roster[false].pending[jid] = st.is_stanza(stanza) and st.preserialize(stanza) or true;
 	return save_roster(username, host, roster, jid);
 end
 function is_contact_pending_out(username, host, jid)
@@ -301,6 +301,11 @@ function is_contact_pending_out(username, host, jid)
 	local item = roster[jid];
 	return item and item.ask;
 end
+local function is_contact_preapproved(username, host, jid)
+	local roster = load_roster(username, host);
+	local item = roster[jid];
+	return item and (item.approved == "true");
+end
 local function set_contact_pending_out(username, host, jid) -- subscribe
 	local roster = load_roster(username, host);
 	local item = roster[jid];
@@ -331,9 +336,10 @@ local function unsubscribe(username, host, jid)
 	return save_roster(username, host, roster, jid);
 end
 local function subscribed(username, host, jid)
+	local roster = load_roster(username, host);
+	local item = roster[jid];
+
 	if is_contact_pending_in(username, host, jid) then
-		local roster = load_roster(username, host);
-		local item = roster[jid];
 		if not item then -- FIXME should roster item be auto-created?
 			item = {subscription = "none", groups = {}};
 			roster[jid] = item;
@@ -345,7 +351,17 @@ local function subscribed(username, host, jid)
 		end
 		roster[false].pending[jid] = nil;
 		return save_roster(username, host, roster, jid);
-	end -- TODO else implement optional feature pre-approval (ask = subscribed)
+	elseif not item or item.subscription == "none" or item.subscription == "to" then
+		-- Contact is not subscribed and has not sent a subscription request.
+		-- We store a pre-approval as per RFC6121 3.4
+		if not item then
+			item = {subscription = "none", groups = {}};
+			roster[jid] = item;
+		end
+		item.approved = "true";
+		log("debug", "Storing preapproval for %s", jid);
+		return save_roster(username, host, roster, jid);
+	end
 end
 local function unsubscribed(username, host, jid)
 	local roster = load_roster(username, host);
@@ -403,6 +419,7 @@ return {
 	set_contact_pending_in = set_contact_pending_in;
 	is_contact_pending_out = is_contact_pending_out;
 	set_contact_pending_out = set_contact_pending_out;
+	is_contact_preapproved = is_contact_preapproved;
 	unsubscribe = unsubscribe;
 	subscribed = subscribed;
 	unsubscribed = unsubscribed;
diff --git a/core/s2smanager.lua b/core/s2smanager.lua
index 58269c49..7f156f12 100644
--- a/core/s2smanager.lua
+++ b/core/s2smanager.lua
@@ -9,10 +9,10 @@
 
 
 local hosts = prosody.hosts;
-local tostring, pairs, setmetatable
-    = tostring, pairs, setmetatable;
+local pairs, setmetatable = pairs, setmetatable;
 
 local logger_init = require "util.logger".init;
+local sessionlib = require "util.session";
 
 local log = logger_init("s2smanager");
 
@@ -26,18 +26,29 @@ local _ENV = nil;
 -- luacheck: std none
 
 local function new_incoming(conn)
-	local session = { conn = conn, type = "s2sin_unauthed", direction = "incoming", hosts = {} };
-	session.log = logger_init("s2sin"..tostring(session):match("[a-f0-9]+$"));
-	incoming_s2s[session] = true;
-	return session;
+	local host_session = sessionlib.new("s2sin");
+	sessionlib.set_id(host_session);
+	sessionlib.set_logger(host_session);
+	sessionlib.set_conn(host_session, conn);
+	host_session.direction = "incoming";
+	host_session.incoming = true;
+	host_session.hosts = {};
+	incoming_s2s[host_session] = true;
+	return host_session;
 end
 
 local function new_outgoing(from_host, to_host)
-	local host_session = { to_host = to_host, from_host = from_host, host = from_host,
-		               notopen = true, type = "s2sout_unauthed", direction = "outgoing" };
+	local host_session = sessionlib.new("s2sout");
+	sessionlib.set_id(host_session);
+	sessionlib.set_logger(host_session);
+	host_session.to_host = to_host;
+	host_session.from_host = from_host;
+	host_session.host = from_host;
+	host_session.notopen = true;
+	host_session.direction = "outgoing";
+	host_session.outgoing = true;
+	host_session.hosts = {};
 	hosts[from_host].s2sout[to_host] = host_session;
-	local conn_name = "s2sout"..tostring(host_session):match("[a-f0-9]*$");
-	host_session.log = logger_init(conn_name);
 	return host_session;
 end
 
@@ -50,6 +61,9 @@ local resting_session = { -- Resting, not dead
 		close = function (session)
 			session.log("debug", "Attempt to close already-closed session");
 		end;
+		reset_stream = function (session)
+			session.log("debug", "Attempt to reset stream of already-closed session");
+		end;
 		filter = function (type, data) return data; end; --luacheck: ignore 212/type
 	}; resting_session.__index = resting_session;
 
@@ -63,23 +77,25 @@ local function retire_session(session, reason)
 
 	session.destruction_reason = reason;
 
-	function session.send(data) log("debug", "Discarding data sent to resting session: %s", tostring(data)); end
-	function session.data(data) log("debug", "Discarding data received from resting session: %s", tostring(data)); end
+	function session.send(data) log("debug", "Discarding data sent to resting session: %s", data); end
+	function session.data(data) log("debug", "Discarding data received from resting session: %s", data); end
 	session.thread = { run = function (_, data) return session.data(data) end };
 	session.sends2s = session.send;
 	return setmetatable(session, resting_session);
 end
 
-local function destroy_session(session, reason)
+local function destroy_session(session, reason, bounce_reason)
 	if session.destroyed then return; end
-	(session.log or log)("debug", "Destroying "..tostring(session.direction)
-		.." session "..tostring(session.from_host).."->"..tostring(session.to_host)
-		..(reason and (": "..reason) or ""));
+	local log = session.log or log;
+	log("debug", "Destroying %s session %s->%s%s%s", session.direction, session.from_host, session.to_host, reason and ": " or "", reason or "");
 
 	if session.direction == "outgoing" then
 		hosts[session.from_host].s2sout[session.to_host] = nil;
-		session:bounce_sendq(reason);
+		session:bounce_sendq(bounce_reason or reason);
 	elseif session.direction == "incoming" then
+		if session.outgoing then
+			hosts[session.to_host].s2sout[session.from_host] = nil;
+		end
 		incoming_s2s[session] = nil;
 	end
 
diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua
index 2843001a..eb515819 100644
--- a/core/sessionmanager.lua
+++ b/core/sessionmanager.lua
@@ -21,6 +21,7 @@ local config_get = require "core.configmanager".get;
 local resourceprep = require "util.encodings".stringprep.resourceprep;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
 local generate_identifier = require "util.id".short;
+local sessionlib = require "util.session";
 
 local initialize_filters = require "util.filters".initialize;
 local gettime = require "socket".gettime;
@@ -29,23 +30,34 @@ local _ENV = nil;
 -- luacheck: std none
 
 local function new_session(conn)
-	local session = { conn = conn, type = "c2s_unauthed", conntime = gettime() };
+	local session = sessionlib.new("c2s");
+	sessionlib.set_id(session);
+	sessionlib.set_logger(session);
+	sessionlib.set_conn(session, conn);
+
+	session.conntime = gettime();
 	local filter = initialize_filters(session);
 	local w = conn.write;
+
+	function session.rawsend(t)
+		t = filter("bytes/out", tostring(t));
+		if t then
+			local ret, err = w(conn, t);
+			if not ret then
+				session.log("debug", "Error writing to connection: %s", err);
+				return false, err;
+			end
+		end
+		return true;
+	end
+
 	session.send = function (t)
 		session.log("debug", "Sending[%s]: %s", session.type, t.top_tag and t:top_tag() or t:match("^[^>]*>?"));
 		if t.name then
 			t = filter("stanzas/out", t);
 		end
 		if t then
-			t = filter("bytes/out", tostring(t));
-			if t then
-				local ret, err = w(conn, t);
-				if not ret then
-					session.log("debug", "Error writing to connection: %s", tostring(err));
-					return false, err;
-				end
-			end
+			return session.rawsend(t);
 		end
 		return true;
 	end
@@ -73,8 +85,8 @@ local function retire_session(session)
 		end
 	end
 
-	function session.send(data) log("debug", "Discarding data sent to resting session: %s", tostring(data)); return false; end
-	function session.data(data) log("debug", "Discarding data received from resting session: %s", tostring(data)); end
+	function session.send(data) log("debug", "Discarding data sent to resting session: %s", data); return false; end
+	function session.data(data) log("debug", "Discarding data received from resting session: %s", data); end
 	session.thread = { run = function (_, data) return session.data(data) end };
 	return setmetatable(session, resting_session);
 end
@@ -110,14 +122,15 @@ local function destroy_session(session, err)
 	retire_session(session);
 end
 
-local function make_authenticated(session, username)
+local function make_authenticated(session, username, scope)
 	username = nodeprep(username);
 	if not username or #username == 0 then return nil, "Invalid username"; end
 	session.username = username;
 	if session.type == "c2s_unauthed" then
 		session.type = "c2s_unbound";
 	end
-	session.log("info", "Authenticated as %s@%s", username or "(unknown)", session.host or "(unknown)");
+	session.auth_scope = scope;
+	session.log("info", "Authenticated as %s@%s", username, session.host or "(unknown)");
 	return true;
 end
 
@@ -138,7 +151,7 @@ local function bind_resource(session, resource)
 		resource = event_payload.resource;
 	end
 
-	resource = resourceprep(resource);
+	resource = resourceprep(resource or "", true);
 	resource = resource ~= "" and resource or generate_identifier();
 	--FIXME: Randomly-generated resources must be unique per-user, and never conflict with existing
 
diff --git a/core/stanza_router.lua b/core/stanza_router.lua
index f5a34f59..774c077e 100644
--- a/core/stanza_router.lua
+++ b/core/stanza_router.lua
@@ -12,6 +12,7 @@ local hosts = _G.prosody.hosts;
 local tostring = tostring;
 local st = require "util.stanza";
 local jid_split = require "util.jid".split;
+local jid_host = require "util.jid".host;
 local jid_prepped_split = require "util.jid".prepped_split;
 
 local full_sessions = _G.prosody.full_sessions;
@@ -27,7 +28,7 @@ local function handle_unhandled_stanza(host, origin, stanza) --luacheck: ignore
 		local st_type = stanza.attr.type;
 		if st_type == "error" or (name == "iq" and st_type == "result") then
 			if st_type == "error" then
-				local err_type, err_condition, err_message = stanza:get_error();
+				local err_type, err_condition, err_message = stanza:get_error(); -- luacheck: ignore 211/err_message
 				log("debug", "Discarding unhandled error %s (%s, %s) from %s: %s",
 					name, err_type, err_condition or "unknown condition", origin_type, stanza:top_tag());
 			else
@@ -81,7 +82,7 @@ function core_process_stanza(origin, stanza)
 	local to_bare, from_bare;
 	if to then
 		if full_sessions[to] or bare_sessions[to] or hosts[to] then
-			node, host = jid_split(to); -- TODO only the host is needed, optimize
+			host = jid_host(to);
 		else
 			node, host, resource = jid_prepped_split(to);
 			if not host then
@@ -111,8 +112,8 @@ function core_process_stanza(origin, stanza)
 		stanza.attr.from = from;
 	end
 
-	if (origin.type == "s2sin" or origin.type == "c2s" or origin.type == "component") and xmlns == nil then
-		if origin.type == "s2sin" and not origin.dummy then
+	if (origin.type == "s2sin" or origin.type == "s2sout" or origin.type == "c2s" or origin.type == "component") and xmlns == nil then
+		if (origin.type == "s2sin" or origin.type == "s2sout") and not origin.dummy then
 			local host_status = origin.hosts[from_host];
 			if not host_status or not host_status.authed then -- remote server trying to impersonate some other server?
 				log("warn", "Received a stanza claiming to be from %s, over a stream authed for %s!", from_host, origin.from_host);
@@ -171,8 +172,15 @@ function core_post_stanza(origin, stanza, preevents)
 		end
 	end
 
-	local event_data = {origin=origin, stanza=stanza};
+	local event_data = {origin=origin, stanza=stanza, to_self=to_self};
+
 	if preevents then -- c2s connection
+		local result = hosts[origin.host].events.fire_event("pre-stanza", event_data);
+		if result ~= nil then
+			log("debug", "Stanza rejected by pre-stanza handler: %s", event_data.reason or "unknown reason");
+			return;
+		end
+
 		if hosts[origin.host].events.fire_event('pre-'..stanza.name..to_type, event_data) then return; end -- do preprocessing
 	end
 	local h = hosts[to_bare] or hosts[host or origin.host];
@@ -186,8 +194,8 @@ function core_post_stanza(origin, stanza, preevents)
 end
 
 function core_route_stanza(origin, stanza)
-	local node, host, resource = jid_split(stanza.attr.to);
-	local from_node, from_host, from_resource = jid_split(stanza.attr.from);
+	local host = jid_host(stanza.attr.to);
+	local from_host = jid_host(stanza.attr.from);
 
 	-- Auto-detect origin if not specified
 	origin = origin or hosts[from_host];
@@ -199,7 +207,7 @@ function core_route_stanza(origin, stanza)
 	else
 		local host_session = hosts[from_host];
 		if not host_session then
-			log("error", "No hosts[from_host] (please report): %s", tostring(stanza));
+			log("error", "No hosts[from_host] (please report): %s", stanza);
 		else
 			local xmlns = stanza.attr.xmlns;
 			stanza.attr.xmlns = nil;
diff --git a/core/statsmanager.lua b/core/statsmanager.lua
index 237b1dd5..2297c959 100644
--- a/core/statsmanager.lua
+++ b/core/statsmanager.lua
@@ -60,9 +60,9 @@ local changed_stats = {};
 local stats_extra = {};
 
 if stats then
-	function measure(type, name)
+	function measure(type, name, conf)
 		local f = assert(stats[type], "unknown stat type: "..type);
-		return f(name);
+		return f(name, conf);
 	end
 
 	if stats_interval then
@@ -79,6 +79,7 @@ if stats then
 			if stats.get_stats then
 				changed_stats, stats_extra = {}, {};
 				for stat_name, getter in pairs(stats.get_stats()) do
+					-- luacheck: ignore 211/type
 					local type, value, extra = getter();
 					local old_value = latest_stats[stat_name];
 					latest_stats[stat_name] = value;
@@ -97,6 +98,7 @@ if stats then
 		end
 		timer.add_task(stats_interval, collect);
 		prosody.events.add_handler("server-started", function () collect() end, -1);
+		prosody.events.add_handler("server-stopped", function () collect() end, -1);
 	else
 		log("debug", "Statistics enabled using %s provider, collection is disabled", stats_provider_name);
 	end
diff --git a/core/storagemanager.lua b/core/storagemanager.lua
index dea71733..856acad3 100644
--- a/core/storagemanager.lua
+++ b/core/storagemanager.lua
@@ -167,6 +167,39 @@ local map_shim_mt = {
 			return self.keyval_store:set(username, current);
 		end;
 		remove = {};
+		get_all = function (self, key)
+			if type(key) ~= "string" or key == "" then
+				return nil, "get_all only supports non-empty string keys";
+			end
+			local ret;
+			for username in self.keyval_store:users() do
+				local key_data = self:get(username, key);
+				if key_data then
+					if not ret then
+						ret = {};
+					end
+					ret[username] = key_data;
+				end
+			end
+			return ret;
+		end;
+		delete_all = function (self, key)
+			if type(key) ~= "string" or key == "" then
+				return nil, "delete_all only supports non-empty string keys";
+			end
+			local data = { [key] = self.remove };
+			local last_err;
+			for username in self.keyval_store:users() do
+				local ok, err = self:set_keys(username, data);
+				if not ok then
+					last_err = err;
+				end
+			end
+			if last_err then
+				return nil, last_err;
+			end
+			return true;
+		end;
 	};
 }
 
diff --git a/core/usermanager.lua b/core/usermanager.lua
index bb5669cf..11707450 100644
--- a/core/usermanager.lua
+++ b/core/usermanager.lua
@@ -9,12 +9,13 @@
 local modulemanager = require "core.modulemanager";
 local log = require "util.logger".init("usermanager");
 local type = type;
-local ipairs = ipairs;
 local jid_bare = require "util.jid".bare;
+local jid_split = require "util.jid".split;
 local jid_prep = require "util.jid".prep;
 local config = require "core.configmanager";
 local sasl_new = require "util.sasl".new;
 local storagemanager = require "core.storagemanager";
+local set = require "util.set";
 
 local prosody = _G.prosody;
 local hosts = prosody.hosts;
@@ -34,10 +35,32 @@ local function new_null_provider()
 	});
 end
 
+local global_admins_config = config.get("*", "admins");
+if type(global_admins_config) ~= "table" then
+	global_admins_config = nil; -- TODO: factor out moduleapi magic config handling and use it here
+end
+local global_admins = set.new(global_admins_config) / jid_prep;
+
+local admin_role = { ["prosody:admin"] = true };
+local global_authz_provider = {
+	get_user_roles = function (user) end; --luacheck: ignore 212/user
+	get_jid_roles = function (jid)
+		if global_admins:contains(jid) then
+			return admin_role;
+		end
+	end;
+};
+
 local provider_mt = { __index = new_null_provider() };
 
 local function initialize_host(host)
 	local host_session = hosts[host];
+
+	local authz_provider_name = config.get(host, "authorization") or "internal";
+
+	local authz_mod = modulemanager.load(host, "authz_"..authz_provider_name);
+	host_session.authz = authz_mod or global_authz_provider;
+
 	if host_session.type ~= "local" then return; end
 
 	host_session.events.add_handler("item-added/auth-provider", function (event)
@@ -66,6 +89,7 @@ local function initialize_host(host)
 	if auth_provider ~= "null" then
 		modulemanager.load(host, "auth_"..auth_provider);
 	end
+
 end;
 prosody.events.add_handler("host-activated", initialize_host, 100);
 
@@ -113,45 +137,30 @@ local function get_provider(host)
 	return hosts[host].users;
 end
 
-local function is_admin(jid, host)
+local function get_roles(jid, host)
 	if host and not hosts[host] then return false; end
 	if type(jid) ~= "string" then return false; end
 
 	jid = jid_bare(jid);
 	host = host or "*";
 
-	local host_admins = config.get(host, "admins");
-	local global_admins = config.get("*", "admins");
-
-	if host_admins and host_admins ~= global_admins then
-		if type(host_admins) == "table" then
-			for _,admin in ipairs(host_admins) do
-				if jid_prep(admin) == jid then
-					return true;
-				end
-			end
-		elseif host_admins then
-			log("error", "Option 'admins' for host '%s' is not a list", host);
-		end
-	end
+	local actor_user, actor_host = jid_split(jid);
+	local roles;
 
-	if global_admins then
-		if type(global_admins) == "table" then
-			for _,admin in ipairs(global_admins) do
-				if jid_prep(admin) == jid then
-					return true;
-				end
-			end
-		elseif global_admins then
-			log("error", "Global option 'admins' is not a list");
-		end
-	end
+	local authz_provider = (host ~= "*" and hosts[host].authz) or global_authz_provider;
 
-	-- Still not an admin, check with auth provider
-	if host ~= "*" and hosts[host].users and hosts[host].users.is_admin then
-		return hosts[host].users.is_admin(jid);
+	if actor_user and actor_host == host then -- Local user
+		roles = authz_provider.get_user_roles(actor_user);
+	else -- Remote user/JID
+		roles = authz_provider.get_jid_roles(jid);
 	end
-	return false;
+
+	return roles;
+end
+
+local function is_admin(jid, host)
+	local roles = get_roles(jid, host);
+	return roles and roles["prosody:admin"];
 end
 
 return {
@@ -166,5 +175,6 @@ return {
 	users = users;
 	get_sasl_handler = get_sasl_handler;
 	get_provider = get_provider;
+	get_roles = get_roles;
 	is_admin = is_admin;
 };
diff --git a/doc/coding_style.md b/doc/coding_style.md
new file mode 100644
index 00000000..2635fa2f
--- /dev/null
+++ b/doc/coding_style.md
@@ -0,0 +1,804 @@
+
+# Prosody Coding Style Guide
+
+This style guides lists the coding conventions used in the
+[Prosody](https://prosody.im/) project. It is based heavily on the [style guide used by the LuaRocks project](https://github.com/luarocks/lua-style-guide).
+
+## Indentation and formatting
+
+* Prosody code is indented with tabs at the start of the line, a single
+  tab per logical indent level:
+
+```lua
+for i, pkg in ipairs(packages) do
+    for name, version in pairs(pkg) do
+        if name == searched then
+            print(version);
+        end
+    end
+end
+```
+
+Tab width is configurable in editors, so never assume a particular width.
+Specically this means you should not mix tabs and spaces, or use tabs for
+alignment of items at different indentation levels.
+
+* Use LF (Unix) line endings.
+
+## Comments
+
+* Comments are encouraged where necessary to explain non-obvious code.
+
+* In general comments should be used to explain 'why', not 'how'
+
+### Comment tags
+
+A comment may be prefixed with one of the following tags:
+
+* **FIXME**: Indicates a serious problem with the code that should be addressed
+* **TODO**: Indicates an open task, feature request or code restructuring that
+  is primarily of interest to developers (otherwise it should be in the
+  issue tracker).
+* **COMPAT**: Must be used on all code that is present only for backwards-compatibility,
+  and may be removed one day. For example code that is added to support old
+  or buggy third-party software or dependencies.
+
+**Example:**
+
+```lua
+-- TODO: implement method
+local function something()
+   -- FIXME: check conditions
+end
+
+```
+
+## Variable names
+
+* Variable names with larger scope should be more descriptive than those with
+smaller scope. One-letter variable names should be avoided except for very
+small scopes (less than ten lines) or for iterators.
+
+* `i` should be used only as a counter variable in for loops (either numeric for
+or `ipairs`).
+
+* Prefer more descriptive names than `k` and `v` when iterating with `pairs`,
+unless you are writing a function that operates on generic tables.
+
+* Use `_` for ignored variables (e.g. in for loops:)
+
+```lua
+for _, item in ipairs(items) do
+   do_something_with_item(item);
+end
+```
+
+* Generally all identifiers (variables and function names) should use `snake_case`,
+  i.e. lowercase words joined by `_`.
+
+```lua
+-- bad
+local OBJEcttsssss = {}
+local thisIsMyObject = {}
+local c = function()
+   -- ...stuff...
+end
+
+-- good
+local this_is_my_object = {};
+
+local function do_that_thing()
+   -- ...stuff...
+end
+```
+
+> **Rationale:** The standard library uses lowercase APIs, with `joinedlowercase`
+names, but this does not scale too well for more complex APIs. `snake_case`
+tends to look good enough and not too out-of-place along side the standard
+APIs.
+
+```lua
+for _, name in pairs(names) do
+   -- ...stuff...
+end
+```
+
+* Prefer using `is_` when naming boolean functions:
+
+```lua
+-- bad
+local function evil(alignment)
+   return alignment < 100
+end
+
+-- good
+local function is_evil(alignment)
+   return alignment < 100;
+end
+```
+
+* `UPPER_CASE` is to be used sparingly, with "constants" only.
+
+> **Rationale:** "Sparingly", since Lua does not have real constants. This
+notation is most useful in libraries that bind C libraries, when bringing over
+constants from C.
+
+* Do not use uppercase names starting with `_`, they are reserved by Lua.
+
+## Tables
+
+* When creating a table, prefer populating its fields all at once, if possible:
+
+```lua
+local player = { name = "Jack", class = "Rogue" };
+```
+
+* Items should be separated by commas. If there are many items, put each
+  key/value on a separate line and use a semi-colon after each item (including
+  the last one):
+
+```lua
+local player = {
+   name = "Jack";
+   class = "Rogue";
+}
+```
+
+> **Rationale:** This makes the structure of your tables more evident at a glance.
+Trailing semi-colons make it quicker to add new fields and produces shorter diffs.
+
+* Use plain `key` syntax whenever possible, use `["key"]` syntax when using names
+that can't be represented as identifiers and avoid mixing representations in
+a declaration:
+
+```lua
+local mytable = {
+   ["1394-E"] = val1;
+   ["UTF-8"] = val2;
+   ["and"] = val2;
+}
+```
+
+## Strings
+
+* Use `"double quotes"` for strings; use `'single quotes'` when writing strings
+that contain double quotes.
+
+```lua
+local name = "Prosody";
+local sentence = 'The name of the program is "Prosody"';
+```
+
+> **Rationale:** Double quotes are used as string delimiters in a larger number of
+programming languages. Single quotes are useful for avoiding escaping when
+using double quotes in literals.
+
+## Line lengths
+
+* There are no hard or soft limits on line lengths. Line lengths are naturally
+limited by using one statement per line. If that still produces lines that are
+too long (e.g. an expression that produces a line over 256-characters long,
+for example), this means the expression is too complex and would do better
+split into subexpressions with reasonable names.
+
+> **Rationale:** No one works on VT100 terminals anymore. If line lengths are a proxy
+for code complexity, we should address code complexity instead of using line
+breaks to fit mind-bending statements over multiple lines.
+
+## Function declaration syntax
+
+* Prefer function syntax over variable syntax. This helps differentiate between
+named and anonymous functions.
+
+```lua
+-- bad
+local nope = function(name, options)
+   -- ...stuff...
+end
+
+-- good
+local function yup(name, options)
+   -- ...stuff...
+end
+```
+
+* Perform validation early and return as early as possible.
+
+```lua
+-- bad
+local function is_good_name(name, options, arg)
+   local is_good = #name > 3
+   is_good = is_good and #name < 30
+
+   -- ...stuff...
+
+   return is_good
+end
+
+-- good
+local function is_good_name(name, options, args)
+   if #name < 3 or #name > 30 then
+      return false;
+   end
+
+   -- ...stuff...
+
+   return true;
+end
+```
+
+## Function calls
+
+* Even though Lua allows it, generally you should not omit parentheses
+  for functions that take a unique string literal argument.
+
+```lua
+-- bad
+local data = get_data"KRP"..tostring(area_number)
+-- good
+local data = get_data("KRP"..tostring(area_number));
+local data = get_data("KRP")..tostring(area_number);
+```
+
+> **Rationale:** It is not obvious at a glace what the precedence rules are
+when omitting the parentheses in a function call. Can you quickly tell which
+of the two "good" examples in equivalent to the "bad" one? (It's the second
+one).
+
+* You should not omit parenthesis for functions that take a unique table
+argument on a single line. You may do so for table arguments that span several
+lines.
+
+```lua
+local an_instance = a_module.new {
+   a_parameter = 42;
+   another_parameter = "yay";
+}
+```
+
+> **Rationale:** The use as in `a_module.new` above occurs alone in a statement,
+so there are no precedence issues.
+
+## Table attributes
+
+* Use dot notation when accessing known properties.
+
+```lua
+local luke = {
+   jedi = true;
+   age = 28;
+}
+
+-- bad
+local is_jedi = luke["jedi"]
+
+-- good
+local is_jedi = luke.jedi;
+```
+
+* Use subscript notation `[]` when accessing properties with a variable or if using a table as a list.
+
+```lua
+local vehicles = load_vehicles_from_disk("vehicles.dat")
+
+if vehicles["Porsche"] then
+   porsche_handler(vehicles["Porsche"]);
+   vehicles["Porsche"] = nil;
+end
+for name, cars in pairs(vehicles) do
+   regular_handler(cars);
+end
+```
+
+> **Rationale:** Using dot notation makes it clearer that the given key is meant
+to be used as a record/object field.
+
+## Functions in tables
+
+* When declaring modules and classes, declare functions external to the table definition:
+
+```lua
+local my_module = {};
+
+function my_module.a_function(x)
+   -- code
+end
+```
+
+* When declaring metatables, declare function internal to the table definition.
+
+```lua
+local version_mt = {
+   __eq = function(a, b)
+      -- code
+   end;
+   __lt = function(a, b)
+      -- code
+   end;
+}
+```
+
+> **Rationale:** Metatables contain special behavior that affect the tables
+they're assigned (and are used implicitly at the call site), so it's good to
+be able to get a view of the complete behavior of the metatable at a glance.
+
+This is not as important for objects and modules, which usually have way more
+code, and which don't fit in a single screen anyway, so nesting them inside
+the table does not gain much: when scrolling a longer file, it is more evident
+that `check_version` is a method of `Api` if it says `function Api:check_version()`
+than if it says `check_version = function()` under some indentation level.
+
+## Variable declaration
+
+* Always use `local` to declare variables.
+
+```lua
+-- bad
+superpower = get_superpower()
+
+-- good
+local superpower = get_superpower();
+```
+
+> **Rationale:** Not doing so will result in global variables to avoid polluting
+the global namespace.
+
+## Variable scope
+
+* Assign variables with the smallest possible scope.
+
+```lua
+-- bad
+local function good()
+   local name = get_name()
+
+   test()
+   print("doing stuff..")
+
+   --...other stuff...
+
+   if name == "test" then
+      return false
+   end
+
+   return name
+end
+
+-- good
+local bad = function()
+   test();
+   print("doing stuff..");
+
+   --...other stuff...
+
+   local name = get_name();
+
+   if name == "test" then
+      return false;
+   end
+
+   return name;
+end
+```
+
+> **Rationale:** Lua has proper lexical scoping. Declaring the function later means that its
+scope is smaller, so this makes it easier to check for the effects of a variable.
+
+## Conditional expressions
+
+* False and nil are falsy in conditional expressions. Use shortcuts when you
+can, unless you need to know the difference between false and nil.
+
+```lua
+-- bad
+if name ~= nil then
+   -- ...stuff...
+end
+
+-- good
+if name then
+   -- ...stuff...
+end
+```
+
+* Avoid designing APIs which depend on the difference between `nil` and `false`.
+
+* Use the `and`/`or` idiom for the pseudo-ternary operator when it results in
+more straightforward code. When nesting expressions, use parentheses to make it
+easier to scan visually:
+
+```lua
+local function default_name(name)
+   -- return the default "Waldo" if name is nil
+   return name or "Waldo";
+end
+
+local function brew_coffee(machine)
+   return (machine and machine.is_loaded) and "coffee brewing" or "fill your water";
+end
+```
+
+Note that the `x and y or z` as a substitute for `x ? y : z` does not work if
+`y` may be `nil` or `false` so avoid it altogether for returning booleans or
+values which may be nil.
+
+## Blocks
+
+* Use single-line blocks only for `then return`, `then break` and `function return` (a.k.a "lambda") constructs:
+
+```lua
+-- good
+if test then break end
+
+-- good
+if not ok then return nil, "this failed for this reason: " .. reason end
+
+-- good
+use_callback(x, function(k) return k.last end);
+
+-- good
+if test then
+  return false
+end
+
+-- bad
+if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then do_other_complicated_function() end
+
+-- good
+if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then
+   do_other_complicated_function();
+   return false;
+end
+```
+
+* Separate statements onto multiple lines. Use semicolons as statement terminators.
+
+```lua
+-- bad
+local whatever = "sure"
+a = 1 b = 2
+
+-- good
+local whatever = "sure";
+a = 1;
+b = 2;
+```
+
+## Spacing
+
+* Use a space after `--`.
+
+```lua
+--bad
+-- good
+```
+
+* Always put a space after commas and between operators and assignment signs:
+
+```lua
+-- bad
+local x = y*9
+local numbers={1,2,3}
+numbers={1 , 2 , 3}
+numbers={1 ,2 ,3}
+local strings = { "hello"
+                , "Lua"
+                , "world"
+                }
+dog.set( "attr",{
+  age="1 year",
+  breed="Bernese Mountain Dog"
+})
+
+-- good
+local x = y * 9;
+local numbers = {1, 2, 3};
+local strings = {
+    "hello";
+    "Lua";
+    "world";
+}
+dog.set("attr", {
+   age = "1 year";
+   breed = "Bernese Mountain Dog";
+});
+```
+
+* Indent tables and functions according to the start of the line, not the construct:
+
+```lua
+-- bad
+local my_table = {
+                    "hello",
+                    "world",
+                 }
+using_a_callback(x, function(...)
+                       print("hello")
+                    end)
+
+-- good
+local my_table = {
+    "hello";
+    "world";
+}
+using_a_callback(x, function(...)
+   print("hello");
+end)
+```
+
+> **Rationale:** This keep indentation levels aligned at predictable places. You don't
+need to realign the entire block if something in the first line changes (such as
+replacing `x` with `xy` in the `using_a_callback` example above).
+
+* The concatenation operator gets a pass for avoiding spaces:
+
+```lua
+-- okay
+local message = "Hello, "..user.."! This is your day # "..day.." in our platform!";
+```
+
+> **Rationale:** Being at the baseline, the dots already provide some visual spacing.
+
+* No spaces after the name of a function in a declaration or in its arguments:
+
+```lua
+-- bad
+local function hello ( name, language )
+   -- code
+end
+
+-- good
+local function hello(name, language)
+   -- code
+end
+```
+
+* Add blank lines between functions:
+
+```lua
+-- bad
+local function foo()
+   -- code
+end
+local function bar()
+   -- code
+end
+
+-- good
+local function foo()
+   -- code
+end
+
+local function bar()
+   -- code
+end
+```
+
+* Avoid aligning variable declarations:
+
+```lua
+-- bad
+local a               = 1
+local long_identifier = 2
+
+-- good
+local a = 1;
+local long_identifier = 2;
+```
+
+> **Rationale:** This produces extra diffs which add noise to `hg annotate`.
+
+* Alignment is occasionally useful when logical correspondence is to be highlighted:
+
+```lua
+-- okay
+sys_command(form, UI_FORM_UPDATE_NODE, "a",      FORM_NODE_HIDDEN,  false);
+sys_command(form, UI_FORM_UPDATE_NODE, "sample", FORM_NODE_VISIBLE, false);
+```
+
+## Typing
+
+* In non-performance critical code, it can be useful to add type-checking assertions
+for function arguments:
+
+```lua
+function manif.load_manifest(repo_url, lua_version)
+   assert(type(repo_url) == "string");
+   assert(type(lua_version) == "string" or not lua_version);
+
+   -- ...
+end
+```
+
+* Use the standard functions for type conversion, avoid relying on coercion:
+
+```lua
+-- bad
+local total_score = review_score .. ""
+
+-- good
+local total_score = tostring(review_score);
+```
+
+## Errors
+
+* Functions that can fail for reasons that are expected (e.g. I/O) should
+return `nil` and a (string) error message on error, possibly followed by other
+return values such as an error code.
+
+* On errors such as API misuse, an error should be thrown, either with `error()`
+or `assert()`.
+
+## Modules
+
+Follow [these guidelines](http://hisham.hm/2014/01/02/how-to-write-lua-modules-in-a-post-module-world/) for writing modules. In short:
+
+* Always require a module into a local variable named after the last component of the module’s full name.
+
+```lua
+local bar = require("foo.bar"); -- requiring the module
+
+bar.say("hello"); -- using the module
+```
+
+* Don’t rename modules arbitrarily:
+
+```lua
+-- bad
+local skt = require("socket")
+```
+
+> **Rationale:** Code is much harder to read if we have to keep going back to the top
+to check how you chose to call a module.
+
+* Start a module by declaring its table using the same all-lowercase local
+name that will be used to require it. You may use an LDoc comment to identify
+the whole module path.
+
+```lua
+--- @module foo.bar
+local bar = {};
+```
+
+* Try to use names that won't clash with your local variables. For instance, don't
+name your module something like “size”.
+
+* Use `local function` to declare _local_ functions only: that is, functions
+that won’t be accessible from outside the module.
+
+That is, `local function helper_foo()` means that `helper_foo` is really local.
+
+* Public functions are declared in the module table, with dot syntax:
+
+```lua
+function bar.say(greeting)
+   print(greeting);
+end
+```
+
+> **Rationale:** Visibility rules are made explicit through syntax.
+
+* Do not set any globals in your module and always return a table in the end.
+
+* If you would like your module to be used as a function, you may set the
+`__call` metamethod on the module table instead.
+
+> **Rationale:** Modules should return tables in order to be amenable to have their
+contents inspected via the Lua interactive interpreter or other tools.
+
+* Requiring a module should cause no side-effect other than loading other
+modules and returning the module table.
+
+* A module should not have state. If a module needs configuration, turn
+  it into a factory. For example, do not make something like this:
+
+```lua
+-- bad
+local mp = require "MessagePack"
+mp.set_integer("unsigned")
+```
+
+and do something like this instead:
+
+```lua
+-- good
+local messagepack = require("messagepack");
+local mpack = messagepack.new({integer = "unsigned"});
+```
+
+* The invocation of require may omit parentheses around the module name:
+
+```lua
+local bla = require "bla";
+```
+
+## Metatables, classes and objects
+
+If creating a new type of object that has a metatable and methods, the
+metatable and methods table should be separate, and the metatable name
+should end with `_mt`.
+
+```lua
+local mytype_methods = {};
+local mytype_mt = { __index = mytype_methods };
+
+function mytype_methods:add_new_thing(thing)
+end
+
+local function new()
+    return setmetatable({}, mytype_mt);
+end
+
+return { new = new };
+```
+
+* Use the method notation when invoking methods:
+
+```
+-- bad
+my_object.my_method(my_object)
+
+-- good
+my_object:my_method();
+```
+
+> **Rationale:** This makes it explicit that the intent is to use the function as a method.
+
+* Do not rely on the `__gc` metamethod to release resources other than memory.
+If your object manage resources such as files, add a `close` method to their
+APIs and do not auto-close via `__gc`. Auto-closing via `__gc` would entice
+users of your module to not close resources as soon as possible. (Note that
+the standard `io` library does not follow this recommendation, and users often
+forget that not closing files immediately can lead to "too many open files"
+errors when the program runs for a while.)
+
+> **Rationale:** The garbage collector performs automatic *memory* management,
+dealing with memory only. There is no guarantees as to when the garbage
+collector will be invoked, and memory pressure does not correlate to pressure
+on other resources.
+
+## File structure
+
+* Lua files should be named in all lowercase.
+
+* Tests should be in a top-level `spec` directory. Prosody uses
+[Busted](http://olivinelabs.com/busted/) for testing.
+
+## Static checking
+
+All code should pass [luacheck](https://github.com/mpeterv/luacheck) using
+the `.luacheckrc` provided in the Prosody repository, and using miminal
+inline exceptions.
+
+* luacheck warnings of class 211, 212, 213 (unused variable, argument or loop
+variable) may be ignored, if the unused variable was added explicitly: for
+example, sometimes it is useful, for code understandability, to spell out what
+the keys and values in a table are, even if you're only using one of them.
+Another example is a function that needs to follow a given signature for API
+reasons (e.g. a callback that follows a given format) but doesn't use some of
+its arguments; it's better to spell out in the argument what the API the
+function implements is, instead of adding `_` variables.
+
+```
+local foo, bar = some_function(); --luacheck: ignore 212/foo
+print(bar);
+```
+
+* luacheck warning 542 (empty if branch) can also be ignored, when a sequence
+of `if`/`elseif`/`else` blocks implements a "switch/case"-style list of cases,
+and one of the cases is meant to mean "pass". For example:
+
+```lua
+if warning >= 600 and warning <= 699 then
+   print("no whitespace warnings");
+elseif warning == 542 then --luacheck: ignore 542
+   -- pass
+else
+   print("got a warning: "..warning);
+end
+```
+
+> **Rationale:** This avoids writing negated conditions in the final fallback
+case, and it's easy to add another case to the construct without having to
+edit the fallback.
+
diff --git a/doc/coding_style.txt b/doc/coding_style.txt
deleted file mode 100644
index af44da1a..00000000
--- a/doc/coding_style.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-This file describes some coding styles to try and adhere to when contributing to this project.
-Please try to follow, and feel free to fix code you see not following this standard.
-
-== Indentation ==
-
-	1 tab indentation for all blocks
-
-== Spacing ==
-
-No space between function names and parenthesis and parenthesis and parameters:
-
-		function foo(bar, baz)
-
-Single space between braces and key/value pairs in table constructors:
-
-		{ foo = "bar", bar = "foo" }
-
-== Local variable naming ==
-
-In this project there are many places where use of globals is restricted, and locals used for faster access.
-
-Local versions of standard functions should follow the below form:
-
-	math.random -> m_random
-	string.char -> s_char	
-
-== Miscellaneous ==
-
-Single-statement blocks may be written on one line when short
-	
-	if foo then bar(); end
-
-'do' and 'then' keywords should be placed at the end of the line, and never on a line by themself.
diff --git a/doc/doap.xml b/doc/doap.xml
new file mode 100644
index 00000000..0b507236
--- /dev/null
+++ b/doc/doap.xml
@@ -0,0 +1,596 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:xmpp="https://linkmauve.fr/ns/xmpp-doap#" xml:lang="en">
+  <Project xmlns="http://usefulinc.com/ns/doap#">
+    <name>Prosody IM</name>
+    <shortdesc>Lightweight XMPP server</shortdesc>
+    <description>Prosody is a server for Jabber/XMPP written in Lua. It aims to be easy to use and light on resources. For developers, it aims to give a flexible system on which to rapidly develop added functionality or rapidly prototype new protocols.</description>
+    <created>2008-08-22</created>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-xmpp"/>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-jabber"/>
+    <category rdf:resource="https://linkmauve.fr/ns/xmpp-doap#category-server"/>
+    <homepage rdf:resource="https://prosody.im/"/>
+    <download-page rdf:resource="https://prosody.im/download/"/>
+    <license rdf:resource="https://hg.prosody.im/trunk/file/tip/COPYING"/>
+    <bug-database rdf:resource="https://issues.prosody.im/"/>
+    <support-forum rdf:resource="xmpp:prosody@conference.prosody.im?join"/>
+    <repository>
+      <HgRepository>
+        <location rdf:resource="https://hg.prosody.im/trunk/"/>
+        <browse rdf:location="https://hg.prosody.im/trunk/"/>
+      </HgRepository>
+    </repository>
+    <programming-langauge>Lua</programming-langauge>
+    <programming-langauge>C</programming-langauge>
+    <os>Linux</os>
+    <os>macOS</os>
+    <os>FreeBSD</os>
+    <os>OpenBSD</os>
+    <os>NetBSD</os>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Matthew Wild</foaf:name>
+        <foaf:nick>MattJ</foaf:nick>
+        <foaf:homepage>https://matthewwild.co.uk/</foaf:homepage>
+      </foaf:Person>
+    </maintainer>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Waqas Hussain</foaf:name>
+        <foaf:nick>waqas</foaf:nick>
+      </foaf:Person>
+    </maintainer>
+    <maintainer>
+      <foaf:Person>
+        <foaf:name>Kim Alvefur</foaf:name>
+        <foaf:nick>Zash</foaf:nick>
+        <foaf:homepage>https://www.zash.se/</foaf:homepage>
+      </foaf:Person>
+    </maintainer>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc5802"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6120"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6121"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6122"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc6455"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc7301"/>
+    <implements rdf:resource="https://www.rfc-editor.org/info/rfc7395"/>
+    <!-- Added in hg:0bbbc9042361 released in 0.6.0 -->
+    <implements rdf:resource="https://datatracker.ietf.org/doc/draft-cridland-xmpp-session/"/>
+    <implements rdf:resource="http://www.unicode.org/reports/tr39/"/>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0004.html"/>
+        <xmpp:version>2.9</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0009.html"/>
+        <xmpp:since>0.4</xmpp:since>
+        <xmpp:until>0.7</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0012.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:note>mod_lastactivity and mod_uptime</xmpp:note>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0016.html"/>
+        <xmpp:since>0.7</xmpp:since>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0030.html"/>
+        <xmpp:since>0.10</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0045.html"/>
+        <xmpp:version>1.32.0</xmpp:version>
+        <xmpp:since>0.3</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0049.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0050.html"/>
+        <xmpp:since>0.8</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0054.html"/>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0060.html"/>
+        <xmpp:version>1.15.8</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0065.html"/>
+        <xmpp:version>1.8.1</xmpp:version>
+        <xmpp:since>0.7</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0068.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0077.html"/>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:version>2.4</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0078.html"/>
+        <xmpp:version>2.5</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0080.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0082.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0084.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0090.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:note>mod_time</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0091.html"/>
+        <xmpp:version>1.4</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:until>0.12</xmpp:until>
+        <xmpp:note>Gone from offline messages in 0.10.0, gone from MUC in 0.12</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0092.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0106.html"/>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0107.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0108.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0114.html"/>
+        <xmpp:version>1.6</xmpp:version>
+        <xmpp:since>0.4</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0115.html"/>
+        <xmpp:since>0.8</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0118.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0122.html"/>
+        <xmpp:version>1.0.2</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>partial</xmpp:status>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0124.html"/>
+        <xmpp:since>0.2</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0126.html"/>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0128.html"/>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0133.html"/>
+        <xmpp:since>0.7</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0138.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:since>0.6</xmpp:since>
+        <xmpp:until>0.10</xmpp:until>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0153.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>via XEP-0398</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0157.html"/>
+        <xmpp:version>1.1.0</xmpp:version>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0160.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0163.html"/>
+        <xmpp:version>1.2.1</xmpp:version>
+        <xmpp:since>0.5</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0170.html"/>
+        <xmpp:version>1.0</xmpp:version>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0172.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0175.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.4</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0178.html"/>
+        <xmpp:version>1.1</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0182.html"/>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0185.html"/>
+        <xmpp:version>1.0</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.9.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0189.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0191.html"/>
+        <xmpp:version>1.3</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0194.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0195.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0196.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0197.html"/>
+        <xmpp:note>via XEP-0163</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0199.html"/>
+        <xmpp:version>2.0.1</xmpp:version>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0202.html"/>
+        <xmpp:version>2.0</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.1</xmpp:since>
+        <xmpp:note>mod_time</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0203.html"/>
+        <xmpp:version>2.0</xmpp:version>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0206.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.2</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0212.html"/>
+        <xmpp:note>required level</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0215.html"/>
+        <xmpp:version>0.7</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.12</xmpp:since>
+        <xmpp:note>mod_external_services</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0220.html"/>
+        <xmpp:since>0.1</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0227.html"/>
+        <xmpp:since>0.7</xmpp:since>
+        <xmpp:note>Used in migrator tools</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0237.html"/>
+        <xmpp:since>0.4</xmpp:since>
+        <xmpp:note>implied by rfc6121</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0249.html"/>
+        <xmpp:version>1.2</xmpp:version>
+        <xmpp:since>0.12</xmpp:since>
+        <xmpp:note>mod_csi_simple</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0280.html"/>
+        <xmpp:version>0.12.1</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.10</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0286.html"/>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>mod_csi_simple</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0288.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.12</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0292.html"/>
+        <xmpp:version>0.10</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>mod_vcard4, mod_vcard_legacy</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0302.html"/>
+        <xmpp:note>Core Server</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0307.html"/>
+        <xmpp:version>0.1</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.6</xmpp:since>
+        <xmpp:note>Moved into mod_muc_unique in 0.11</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0313.html"/>
+        <xmpp:version>0.6.3</xmpp:version>
+        <xmpp:since>0.10</xmpp:since>
+        <xmpp:note>mod_mam, mod_muc_mam</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0317.html"/>
+        <xmpp:version>0.1</xmpp:version>
+        <xmpp:status>planned</xmpp:status>
+        <xmpp:since>0.12</xmpp:since>
+        <xmpp:note>muc/hats</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0318.html"/>
+        <xmpp:version>0.2</xmpp:version>
+        <xmpp:since>0.9</xmpp:since>
+        <xmpp:note>refers to inclusion of delay stamp in presence</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0352.html"/>
+        <xmpp:version>0.3.0</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0353.html"/>
+        <xmpp:version>0.3</xmpp:version>
+        <xmpp:since>0.11.6</xmpp:since>
+        <xmpp:note>triggers buffer flush in mod_csi_simple since 0.11.6; recognised by mod_carbons and mod_mam since 0.12</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0359.html"/>
+        <xmpp:version>0.6.0</xmpp:version>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:since>0.10.0</xmpp:since>
+        <xmpp:note>Used in context of XEP-0313 by mod_mam and mod_muc_mam</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0368.html"/>
+        <xmpp:version>1.1.0</xmpp:version>
+        <xmpp:status>partial</xmpp:status>
+        <xmpp:since>0.2</xmpp:since>
+        <xmpp:note>legacy_ssl_ports</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0380.html"/>
+        <xmpp:version>0.3.0</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:note>Used in context of XEP-0352</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0384.html"/>
+        <xmpp:note>via XEP-0163, XEP-0222</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0398.html"/>
+        <xmpp:version>0.2.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:note>mod_vcard_legacy</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+    <implements>
+      <xmpp:SupportedXep>
+        <xmpp:xep rdf:resource="https://xmpp.org/extensions/xep-0410.html"/>
+        <xmpp:version>1.0.1</xmpp:version>
+        <xmpp:since>0.11</xmpp:since>
+        <xmpp:status>complete</xmpp:status>
+        <xmpp:note>Server Optimization</xmpp:note>
+      </xmpp:SupportedXep>
+    </implements>
+  </Project>
+</rdf:RDF>
diff --git a/doc/net.server.lua b/doc/net.server.lua
index 7342c549..aa9a4a9d 100644
--- a/doc/net.server.lua
+++ b/doc/net.server.lua
@@ -160,6 +160,26 @@ Returns:
 local function addserver(address, port, listeners, pattern, sslctx)
 end
 
+--[[ Binds and listens on the given address and port
+Mostly the same as addserver but with all optional arguments in a table
+
+Arguments:
+  - address: address to bind to, may be "*" to bind all addresses. will be resolved if it is a string.
+  - port: port to bind (as number)
+  - listeners: a table of listeners
+	- config: table of extra settings
+		- read_size: the amount of bytes to read or a read pattern
+		- tls_ctx: is a valid luasec constructor
+		- tls_direct: boolean true for direct TLS, false (or nil) for starttls
+
+Returns:
+  - handle
+  - nil, "an error message": on failure (e.g. out of file descriptors)
+]]
+local function listen(address, port, listeners, config)
+end
+
+
 --[[ Wraps a lua-socket socket client socket in a handle.
 The socket must be already connected to the remote end.
 If `sslctx` is given, a SSL session will be negotiated before listeners are called.
@@ -255,4 +275,5 @@ return {
 	closeall = closeall;
 	hook_signal = hook_signal;
 	watchfd = watchfd;
+	listen = listen;
 }
diff --git a/doc/storage.tld b/doc/storage.tld
index f1d33e58..0fc8ff54 100644
--- a/doc/storage.tld
+++ b/doc/storage.tld
@@ -47,6 +47,13 @@ interface archive_store
 
 	-- Array of dates which do have messages (Optional?)
 	dates  : ( self, string? ) -> ({ string }) | (nil, string)
+
+	-- Map of counts per "with" field
+	summary : ( self, string?, archive_query? ) -> ( { string : integer } ) | (nil, string)
+
+	-- Map-store API
+	get    : ( self, string, string ) -> (stanza, number?, string?) | (nil, string)
+	set    : ( self, string, string, stanza, number?, string? ) -> (boolean) | (nil, string)
 end
 
 -- This represents moduleapi
diff --git a/makefile b/makefile
index d19ec24d..c0aabb81 100644
--- a/makefile
+++ b/makefile
@@ -19,6 +19,9 @@ INSTALL_EXEC=$(INSTALL) -m755
 MKDIR=install -d
 MKDIR_PRIVATE=$(MKDIR) -m750
 
+LUACHECK=luacheck
+BUSTED=busted
+
 .PHONY: all test clean install
 
 all: prosody.install prosodyctl.install prosody.cfg.lua.install prosody.version
@@ -45,9 +48,12 @@ install: prosody.install prosodyctl.install prosody.cfg.lua.install util/encodin
 	$(INSTALL_DATA) util/*.so $(SOURCE)/util
 	$(MKDIR) $(SOURCE)/util/sasl
 	$(INSTALL_DATA) util/sasl/*.lua $(SOURCE)/util/sasl
-	$(MKDIR) $(MODULES)/mod_s2s $(MODULES)/mod_pubsub $(MODULES)/adhoc $(MODULES)/muc $(MODULES)/mod_mam
+	$(MKDIR) $(SOURCE)/util/human
+	$(INSTALL_DATA) util/human/*.lua $(SOURCE)/util/human
+	$(MKDIR) $(SOURCE)/util/prosodyctl
+	$(INSTALL_DATA) util/prosodyctl/*.lua $(SOURCE)/util/prosodyctl
+	$(MKDIR) $(MODULES)/mod_pubsub $(MODULES)/adhoc $(MODULES)/muc $(MODULES)/mod_mam
 	$(INSTALL_DATA) plugins/*.lua $(MODULES)
-	$(INSTALL_DATA) plugins/mod_s2s/*.lua $(MODULES)/mod_s2s
 	$(INSTALL_DATA) plugins/mod_pubsub/*.lua $(MODULES)/mod_pubsub
 	$(INSTALL_DATA) plugins/adhoc/*.lua $(MODULES)/adhoc
 	$(INSTALL_DATA) plugins/muc/*.lua $(MODULES)/muc
@@ -68,8 +74,13 @@ clean:
 	rm -f prosody.version
 	$(MAKE) clean -C util-src
 
+lint:
+	$(LUACHECK) -q $$(HGPLAIN= hg files -I '**.lua') prosody prosodyctl
+	@echo $$(sed -n '/^\tlocal exclude_files/,/^}/p;' .luacheckrc | sed '1d;$d' | wc -l) files ignored
+	shellcheck configure
+
 test:
-	busted --lua=$(RUNWITH)
+	$(BUSTED) --lua=$(RUNWITH)
 
 
 prosody.install: prosody
diff --git a/net/adns.lua b/net/adns.lua
index 560e4b53..8fe05653 100644
--- a/net/adns.lua
+++ b/net/adns.lua
@@ -8,13 +8,17 @@
 
 local server = require "net.server";
 local new_resolver = require "net.dns".resolver;
+local promise = require "util.promise";
 
 local log = require "util.logger".init("adns");
 
-local coroutine, tostring, pcall = coroutine, tostring, pcall;
+log("debug", "Using legacy DNS API (missing lua-unbound?)"); -- TODO write docs about luaunbound
+-- TODO Raise log level once packages are available
+
+local coroutine, pcall = coroutine, pcall;
 local setmetatable = setmetatable;
 
-local function dummy_send(sock, data, i, j) return (j-i)+1; end
+local function dummy_send(sock, data, i, j) return (j-i)+1; end -- luacheck: ignore 212
 
 local _ENV = nil;
 -- luacheck: std none
@@ -29,8 +33,7 @@ local function new_async_socket(sock, resolver)
 	local peername = "<unknown>";
 	local listener = {};
 	local handler = {};
-	local err;
-	function listener.onincoming(conn, data)
+	function listener.onincoming(conn, data) -- luacheck: ignore 212/conn
 		if data then
 			resolver:feed(handler, data);
 		end
@@ -46,9 +49,12 @@ local function new_async_socket(sock, resolver)
 			resolver:servfail(conn); -- Let the magic commence
 		end
 	end
-	handler, err = server.wrapclient(sock, "dns", 53, listener);
-	if not handler then
-		return nil, err;
+	do
+		local err;
+		handler, err = server.wrapclient(sock, "dns", 53, listener);
+		if not handler then
+			return nil, err;
+		end
 	end
 
 	handler.settimeout = function () end
@@ -71,11 +77,11 @@ function async_resolver_methods:lookup(handler, qname, qtype, qclass)
 					handler(peek);
 					return;
 				end
-				log("debug", "Records for %s not in cache, sending query (%s)...", qname, tostring(coroutine.running()));
+				log("debug", "Records for %s not in cache, sending query (%s)...", qname, coroutine.running());
 				local ok, err = resolver:query(qname, qtype, qclass);
 				if ok then
 					coroutine.yield(setmetatable({ resolver, qclass or "IN", qtype or "A", qname, coroutine.running()}, query_mt)); -- Wait for reply
-					log("debug", "Reply for %s (%s)", qname, tostring(coroutine.running()));
+					log("debug", "Reply for %s (%s)", qname, coroutine.running());
 				end
 				if ok then
 					ok, err = pcall(handler, resolver:peek(qname, qtype, qclass));
@@ -84,13 +90,25 @@ function async_resolver_methods:lookup(handler, qname, qtype, qclass)
 					ok, err = pcall(handler, nil, err);
 				end
 				if not ok then
-					log("error", "Error in DNS response handler: %s", tostring(err));
+					log("error", "Error in DNS response handler: %s", err);
 				end
 			end)(resolver:peek(qname, qtype, qclass));
 end
 
-function query_methods:cancel(call_handler, reason)
-	log("warn", "Cancelling DNS lookup for %s", tostring(self[4]));
+function async_resolver_methods:lookup_promise(qname, qtype, qclass)
+	return promise.new(function (resolve, reject)
+		local function handler(answer)
+			if not answer then
+				return reject();
+			end
+			resolve(answer);
+		end
+		self:lookup(handler, qname, qtype, qclass);
+	end);
+end
+
+function query_methods:cancel(call_handler, reason) -- luacheck: ignore 212/reason
+	log("warn", "Cancelling DNS lookup for %s", self[4]);
 	self[1].cancel(self[2], self[3], self[4], self[5], call_handler);
 end
 
diff --git a/net/connect.lua b/net/connect.lua
index b812ffcd..d52d3901 100644
--- a/net/connect.lua
+++ b/net/connect.lua
@@ -2,6 +2,12 @@ local server = require "net.server";
 local log = require "util.logger".init("net.connect");
 local new_id = require "util.id".short;
 
+-- TODO #1246 Happy Eyeballs
+-- FIXME RFC 6724
+-- FIXME Error propagation from resolvers doesn't work
+-- FIXME #1428 Reuse DNS resolver object between service and basic resolver
+-- FIXME #1429 Close DNS resolver object when done
+
 local pending_connection_methods = {};
 local pending_connection_mt = {
 	__name = "pending_connection";
@@ -38,7 +44,7 @@ local function attempt_connection(p)
 		p:log("debug", "Next target to try is %s:%d", ip, port);
 		local conn, err = server.addclient(ip, port, pending_connection_listeners, p.options.pattern or "*a", p.options.sslctx, conn_type, extra);
 		if not conn then
-			log("debug", "Connection attempt failed immediately: %s", tostring(err));
+			log("debug", "Connection attempt failed immediately: %s", err);
 			p.last_error = err or "unknown reason";
 			return attempt_connection(p);
 		end
diff --git a/net/connlisteners.lua b/net/connlisteners.lua
deleted file mode 100644
index 9b8f88c3..00000000
--- a/net/connlisteners.lua
+++ /dev/null
@@ -1,18 +0,0 @@
--- COMPAT w/pre-0.9
-local log = require "util.logger".init("net.connlisteners");
-local traceback = debug.traceback;
-
-local _ENV = nil;
--- luacheck: std none
-
-local function fail()
-	log("error", "Attempt to use legacy connlisteners API. For more info see https://prosody.im/doc/developers/network");
-	log("error", "Legacy connlisteners API usage, %s", traceback("", 2));
-end
-
-return {
-	register = fail;
-	get = fail;
-	start = fail;
-	-- epic fail
-};
diff --git a/net/cqueues.lua b/net/cqueues.lua
index 8c4c756f..65d2a019 100644
--- a/net/cqueues.lua
+++ b/net/cqueues.lua
@@ -9,6 +9,7 @@
 
 local server = require "net.server";
 local cqueues = require "cqueues";
+local timer = require "util.timer";
 assert(cqueues.VERSION >= 20150113, "cqueues newer than 20150113 required")
 
 -- Create a single top level cqueue
@@ -16,55 +17,24 @@ local cq;
 
 if server.cq then -- server provides cqueues object
 	cq = server.cq;
-elseif server.get_backend() == "select" and server._addtimer then -- server_select
+elseif server.watchfd then
 	cq = cqueues.new();
-	local function step()
+	local timeout = timer.add_task(cq:timeout() or 0, function ()
+		-- FIXME It should be enough to reschedule this timeout instead of replacing it, but this does not work.  See https://issues.prosody.im/1572
 		assert(cq:loop(0));
-	end
-
-	-- Use wrapclient (as wrapconnection isn't exported) to get server_select to watch cq fd
-	local handler = server.wrapclient({
-		getfd = function() return cq:pollfd(); end;
-		settimeout = function() end; -- Method just needs to exist
-		close = function() end; -- Need close method for 'closeall'
-	}, nil, nil, {});
-
-	-- Only need to listen for readable; cqueues handles everything under the hood
-	-- readbuffer is called when `select` notes an fd as readable
-	handler.readbuffer = step;
-
-	-- Use server_select low lever timer facility,
-	-- this callback gets called *every* time there is a timeout in the main loop
-	server._addtimer(function(current_time)
-		-- This may end up in extra step()'s, but cqueues handles it for us.
-		step();
 		return cq:timeout();
 	end);
-elseif server.event and server.base then -- server_event
-	cq = cqueues.new();
-	-- Only need to listen for readable; cqueues handles everything under the hood
-	local EV_READ = server.event.EV_READ;
-	-- Convert a cqueues timeout to an acceptable timeout for luaevent
-	local function luaevent_safe_timeout(cq)
+	server.watchfd(cq:pollfd(), function ()
+		assert(cq:loop(0));
 		local t = cq:timeout();
-		-- if you give luaevent 0 or nil, it re-uses the previous timeout.
-		if t == 0 then
-			t = 0.000001; -- 1 microsecond is the smallest that works (goes into a `struct timeval`)
-		elseif t == nil then -- pick something big if we don't have one
-			t = 0x7FFFFFFF; -- largest 32bit int
+		if t then
+			timer.stop(timeout);
+			timeout = timer.add_task(cq:timeout(), function ()
+				assert(cq:loop(0));
+				return cq:timeout();
+			end);
 		end
-		return t
-	end
-	local event_handle;
-	event_handle = server.base:addevent(cq:pollfd(), EV_READ, function(e)
-			-- Need to reference event_handle or this callback will get collected
-			-- This creates a circular reference that can only be broken if event_handle is manually :close()'d
-			local _ = event_handle;
-			-- Run as many cqueues things as possible (with a timeout of 0)
-			-- If an error is thrown, it will break the libevent loop; but prosody resumes after logging a top level error
-			assert(cq:loop(0));
-			return EV_READ, luaevent_safe_timeout(cq);
-		end, luaevent_safe_timeout(cq));
+	end);
 else
 	error "NYI"
 end
diff --git a/net/dns.lua b/net/dns.lua
index 3902f95c..17119152 100644
--- a/net/dns.lua
+++ b/net/dns.lua
@@ -13,10 +13,12 @@
 
 
 local socket = require "socket";
-local timer = require "util.timer";
+local have_timer, timer = pcall(require, "util.timer");
 local new_ip = require "util.ip".new_ip;
 local have_util_net, util_net = pcall(require, "util.net");
 
+local log = require "util.logger".init("dns");
+
 local _, windows = pcall(require, "util.windows");
 local is_windows = (_ and windows) or os.getenv("WINDIR");
 
@@ -69,7 +71,9 @@ local ztact = { -- public domain 20080404 lua@ztact.com
 };
 local get, set = ztact.get, ztact.set;
 
-local default_timeout = 15;
+local default_timeout = 5;
+local default_jitter = 1;
+local default_retry_jitter = 2;
 
 -------------------------------------------------- module dns
 local _ENV = nil;
@@ -664,8 +668,10 @@ end
 -- socket layer -------------------------------------------------- socket layer
 
 
-resolver.delays = { 1, 3 };
+resolver.delays = { 1, 2, 3, 5 };
 
+resolver.jitter = have_timer and default_jitter or nil;
+resolver.retry_jitter = have_timer and default_retry_jitter or nil;
 
 function resolver:addnameserver(address)    -- - - - - - - - - - addnameserver
 	self.server = self.server or {};
@@ -853,7 +859,10 @@ function resolver:query(qname, qtype, qclass)    -- - - - - - - - - - -- query
 		packet = header..question,
 		server = self.best_server,
 		delay  = 1,
-		retry  = socket.gettime() + self.delays[1]
+		retry  = socket.gettime() + self.delays[1];
+		qclass = qclass;
+		qtype  = qtype;
+		qname  = qname;
 	};
 
 	-- remember the query
@@ -864,30 +873,32 @@ function resolver:query(qname, qtype, qclass)    -- - - - - - - - - - -- query
 	if not conn then
 		return nil, err;
 	end
-	conn:send (o.packet)
+	if self.jitter then
+		timer.add_task(math.random()*self.jitter, function ()
+			conn:send(o.packet);
+		end);
+	else
+		conn:send(o.packet);
+	end
 
 	-- remember which coroutine wants the answer
 	if co then
 		set(self.wanted, qclass, qtype, qname, co, true);
 	end
 	
-	if timer and self.timeout then
+	if have_timer and self.timeout then
 		local num_servers = #self.server;
 		local i = 1;
 		timer.add_task(self.timeout, function ()
 			if get(self.wanted, qclass, qtype, qname, co) then
-				if i < num_servers then
+				log("debug", "DNS request timeout %d/%d", i, num_servers)
 					i = i + 1;
-					self:servfail(conn);
-					o.server = self.best_server;
-					conn, err = self:getsocket(o.server);
-					if conn then
-						conn:send(o.packet);
-						return self.timeout;
-					end
-				end
-				-- Tried everything, failed
-				self:cancel(qclass, qtype, qname);
+					self:servfail(self.socket[o.server]);
+--				end
+			end
+			-- Still outstanding? (i.e. retried)
+			if get(self.wanted, qclass, qtype, qname, co) then
+				return self.timeout; -- Then wait
 			end
 		end)
 	end
@@ -904,6 +915,7 @@ function resolver:servfail(sock, err)
 
 	-- Find all requests to the down server, and retry on the next server
 	self.time = socket.gettime();
+	log("debug", "servfail %d (of %d)", num, #self.server);
 	for id,queries in pairs(self.active) do
 		for question,o in pairs(queries) do
 			if o.server == num then -- This request was to the broken server
@@ -913,12 +925,27 @@ function resolver:servfail(sock, err)
 				end
 
 				o.retries = (o.retries or 0) + 1;
-				if o.retries >= #self.server then
-					--print('timeout');
-					queries[question] = nil;
-				else
+				local retried;
+				if o.retries < #self.server then
 					sock, err = self:getsocket(o.server);
-					if sock then sock:send(o.packet); end
+					if sock then
+						retried = true;
+						if self.retry_jitter then
+							local delay = self.delays[((o.retries-1)%#self.delays)+1] + (math.random()*self.retry_jitter);
+							log("debug", "retry %d in %0.2fs", o.retries, delay);
+							timer.add_task(delay, function ()
+								sock:send(o.packet);
+							end);
+						else
+							log("debug", "retry %d (immediate)", o.retries);
+							sock:send(o.packet);
+						end
+					end
+				end	
+				if not retried then
+					log("debug", 'tried all servers, giving up');
+					self:cancel(o.qclass, o.qtype, o.qname);
+					queries[question] = nil;
 				end
 			end
 		end
@@ -1164,6 +1191,7 @@ end
 
 local _resolver = dns.resolver();
 dns._resolver = _resolver;
+_resolver.jitter, _resolver.retry_jitter = false, false;
 
 function dns.lookup(...)    -- - - - - - - - - - - - - - - - - - - - -  lookup
 	return _resolver:lookup(...);
diff --git a/net/http.lua b/net/http.lua
index 7335d210..bb247d18 100644
--- a/net/http.lua
+++ b/net/http.lua
@@ -12,6 +12,8 @@ local httpstream_new = require "net.http.parser".new;
 local util_http = require "util.http";
 local events = require "util.events";
 local verify_identity = require"util.x509".verify_identity;
+local promise = require "util.promise";
+local http_errors = require "net.http.errors";
 
 local basic_resolver = require "net.resolvers.basic";
 local connect = require "net.connect".connect;
@@ -40,7 +42,7 @@ local listener = { default_port = 80, default_mode = "*a" };
 local function handleerr(err) log("error", "Traceback[http]: %s", traceback(tostring(err), 2)); return err; end
 local function log_if_failed(req, ret, ...)
 	if not ret then
-		log("error", "Request '%s': error in callback: %s", req.id, tostring((...)));
+		log("error", "Request '%s': error in callback: %s", req.id, (...));
 		if not req.suppress_errors then
 			error(...);
 		end
@@ -81,7 +83,24 @@ local function request_reader(request, data, err)
 			return;
 		end
 
+		local finalize_sink;
 		local function success_cb(r)
+			if r.partial then
+				-- Request should be streamed
+				log("debug", "Request '%s': partial response (%s%s)",
+					request.id,
+					r.chunked and "chunked, " or "",
+					r.body_length and ("%d bytes"):format(r.body_length) or "unknown length"
+				);
+				if request.streaming_handler then
+					log("debug", "Request '%s': Streaming via handler");
+					r.body_sink, finalize_sink = request.streaming_handler(r);
+				end
+				return;
+			elseif finalize_sink then
+				log("debug", "Request '%s': Finalizing response stream");
+				finalize_sink(r);
+			end
 			if request.callback then
 				request.callback(r.body, r.code, r, request);
 				request.callback = nil;
@@ -161,7 +180,7 @@ function listener.onincoming(conn, data)
 	local request = requests[conn];
 
 	if not request then
-		log("warn", "Received response from connection %s with no request attached!", tostring(conn));
+		log("warn", "Received response from connection %s with no request attached!", conn);
 		return;
 	end
 
@@ -254,6 +273,7 @@ local function request(self, u, ex, callback)
 		end
 		req.insecure = ex.insecure;
 		req.suppress_errors = ex.suppress_errors;
+		req.streaming_handler = ex.streaming_handler;
 	end
 
 	log("debug", "Making %s %s request '%s' to %s", req.scheme:upper(), method or "GET", req.id, (ex and ex.suppress_url and host_header) or u);
@@ -282,7 +302,22 @@ end
 local function new(options)
 	local http = {
 		options = options;
-		request = request;
+		request = function (self, u, ex, callback)
+			if callback ~= nil then
+				return request(self, u, ex, callback);
+			else
+				return promise.new(function (resolve, reject)
+					request(self, u, ex, function (body, code, a, b)
+						if code == 0 then
+							reject(http_errors.new(body, { request = a }));
+						else
+							a.request = b;
+							resolve(a);
+						end
+					end);
+				end);
+			end
+		end;
 		new = options and function (new_options)
 			local final_options = {};
 			for k, v in pairs(options) do final_options[k] = v; end
@@ -297,7 +332,7 @@ local function new(options)
 end
 
 local default_http = new({
-	sslctx = { mode = "client", protocol = "sslv23", options = { "no_sslv2", "no_sslv3" } };
+	sslctx = { mode = "client", protocol = "sslv23", options = { "no_sslv2", "no_sslv3" }, alpn = "http/1.1" };
 	suppress_errors = true;
 });
 
diff --git a/net/http/codes.lua b/net/http/codes.lua
index 8098b5c3..4327f151 100644
--- a/net/http/codes.lua
+++ b/net/http/codes.lua
@@ -82,5 +82,5 @@ local response_codes = {
 	-- [512-599] = "Unassigned";
 };
 
-for k,v in pairs(response_codes) do response_codes[k] = k.." "..v; end
+for k,v in pairs(response_codes) do response_codes[k] = ("%03d %s"):format(k, v); end
 return setmetatable(response_codes, { __index = function(_, k) return k.." Unassigned"; end })
diff --git a/net/http/errors.lua b/net/http/errors.lua
new file mode 100644
index 00000000..ae45d5a8
--- /dev/null
+++ b/net/http/errors.lua
@@ -0,0 +1,115 @@
+-- This module returns a table that is suitable for use as a util.error registry,
+-- and a function to return a util.error object given callback 'code' and 'body'
+-- parameters.
+
+local codes = require "net.http.codes";
+local util_error = require "util.error";
+
+local error_templates = {
+	-- This code is used by us to report a client-side or connection error.
+	-- Instead of using the code, use the supplied body text to get one of
+	-- the more detailed errors below.
+	[0] = {
+		code = 0, type = "cancel", condition = "internal-server-error";
+		text = "Connection or internal error";
+	};
+
+	-- These are net.http built-in errors, they are returned in
+	-- the body parameter when code == 0
+	["cancelled"] = {
+		code = 0, type = "cancel", condition = "remote-server-timeout";
+		text = "Request cancelled";
+	};
+	["connection-closed"] = {
+		code = 0, type = "wait", condition = "remote-server-timeout";
+		text = "Connection closed";
+	};
+	["certificate-chain-invalid"] = {
+		code = 0, type = "cancel", condition = "remote-server-timeout";
+		text = "Server certificate not trusted";
+	};
+	["certificate-verify-failed"] = {
+		code = 0, type = "cancel", condition = "remote-server-timeout";
+		text = "Server certificate invalid";
+	};
+	["connection failed"] = {
+		code = 0, type = "cancel", condition = "remote-server-not-found";
+		text = "Connection failed";
+	};
+	["invalid-url"] = {
+		code = 0, type = "modify", condition = "bad-request";
+		text = "Invalid URL";
+	};
+
+	-- This doesn't attempt to map every single HTTP code (not all have sane mappings),
+	-- but all the common ones should be covered. XEP-0086 was used as reference for
+	-- most of these.
+	[400] = { type = "modify", condition = "bad-request" };
+	[401] = { type = "auth", condition = "not-authorized" };
+	[402] = { type = "auth", condition = "payment-required" };
+	[403] = { type = "auth", condition = "forbidden" };
+	[404] = { type = "cancel", condition = "item-not-found" };
+	[405] = { type = "cancel", condition = "not-allowed" };
+	[406] = { type = "modify", condition = "not-acceptable" };
+	[407] = { type = "auth", condition = "registration-required" };
+	[408] = { type = "wait", condition = "remote-server-timeout" };
+	[409] = { type = "cancel", condition = "conflict" };
+	[410] = { type = "cancel", condition = "gone" };
+	[411] = { type = "modify", condition = "bad-request" };
+	[412] = { type = "cancel", condition = "conflict" };
+	[413] = { type = "modify", condition = "resource-constraint" };
+	[414] = { type = "modify", condition = "resource-constraint" };
+	[415] = { type = "cancel", condition = "feature-not-implemented" };
+	[416] = { type = "modify", condition = "bad-request" };
+
+	[422] = { type = "modify", condition = "bad-request" };
+	[423] = { type = "wait", condition = "resource-constraint" };
+
+	[429] = { type = "wait", condition = "resource-constraint" };
+	[431] = { type = "modify", condition = "resource-constraint" };
+	[451] = { type = "auth", condition = "forbidden" };
+
+	[500] = { type = "wait", condition = "internal-server-error" };
+	[501] = { type = "cancel", condition = "feature-not-implemented" };
+	[502] = { type = "wait", condition = "remote-server-timeout" };
+	[503] = { type = "cancel", condition = "service-unavailable" };
+	[504] = { type = "wait", condition = "remote-server-timeout" };
+	[507] = { type = "wait", condition = "resource-constraint" };
+	[511] = { type = "auth", condition = "not-authorized" };
+};
+
+for k, v in pairs(codes) do
+	if error_templates[k] then
+		error_templates[k].code = k;
+		error_templates[k].text = v;
+	else
+		error_templates[k] = { type = "cancel", condition = "undefined-condition", text = v, code = k };
+	end
+end
+
+setmetatable(error_templates, {
+	__index = function(_, k)
+		if type(k) ~= "number" then
+			return nil;
+		end
+		return {
+			type = "cancel";
+			condition = "undefined-condition";
+			text = codes[k] or (k.." Unassigned");
+			code = k;
+		};
+	end
+});
+
+local function new(code, body, context)
+	if code == 0 then
+		return util_error.new(body, context, error_templates);
+	else
+		return util_error.new(code, context, error_templates);
+	end
+end
+
+return {
+	registry = error_templates;
+	new = new;
+};
diff --git a/net/http/files.lua b/net/http/files.lua
new file mode 100644
index 00000000..583f7514
--- /dev/null
+++ b/net/http/files.lua
@@ -0,0 +1,149 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local server = require"net.http.server";
+local lfs = require "lfs";
+local new_cache = require "util.cache".new;
+local log = require "util.logger".init("net.http.files");
+
+local os_date = os.date;
+local open = io.open;
+local stat = lfs.attributes;
+local build_path = require"socket.url".build_path;
+local path_sep = package.config:sub(1,1);
+
+
+local forbidden_chars_pattern = "[/%z]";
+if package.config:sub(1,1) == "\\" then
+	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+end
+
+local urldecode = require "util.http".urldecode;
+local function sanitize_path(path) --> util.paths or util.http?
+	if not path then return end
+	local out = {};
+
+	local c = 0;
+	for component in path:gmatch("([^/]+)") do
+		component = urldecode(component);
+		if component:find(forbidden_chars_pattern) then
+			return nil;
+		elseif component == ".." then
+			if c <= 0 then
+				return nil;
+			end
+			out[c] = nil;
+			c = c - 1;
+		elseif component ~= "." then
+			c = c + 1;
+			out[c] = component;
+		end
+	end
+	if path:sub(-1,-1) == "/" then
+		out[c+1] = "";
+	end
+	return "/"..table.concat(out, "/");
+end
+
+local function serve(opts)
+	if type(opts) ~= "table" then -- assume path string
+		opts = { path = opts };
+	end
+	local mime_map = opts.mime_map or { html = "text/html" };
+	local cache = new_cache(opts.cache_size or 256);
+	local cache_max_file_size = tonumber(opts.cache_max_file_size) or 1024
+	-- luacheck: ignore 431
+	local base_path = opts.path;
+	local dir_indices = opts.index_files or { "index.html", "index.htm" };
+	local directory_index = opts.directory_index;
+	local function serve_file(event, path)
+		local request, response = event.request, event.response;
+		local sanitized_path = sanitize_path(path);
+		if path and not sanitized_path then
+			return 400;
+		end
+		path = sanitized_path;
+		local orig_path = sanitize_path(request.path);
+		local full_path = base_path .. (path or ""):gsub("/", path_sep);
+		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
+		if not attr then
+			return 404;
+		end
+
+		local request_headers, response_headers = request.headers, response.headers;
+
+		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
+		response_headers.last_modified = last_modified;
+
+		local etag = ('"%x-%x-%x"'):format(attr.change or 0, attr.size or 0, attr.modification or 0);
+		response_headers.etag = etag;
+
+		local if_none_match = request_headers.if_none_match
+		local if_modified_since = request_headers.if_modified_since;
+		if etag == if_none_match
+		or (not if_none_match and last_modified == if_modified_since) then
+			return 304;
+		end
+
+		local data = cache:get(orig_path);
+		if data and data.etag == etag then
+			response_headers.content_type = data.content_type;
+			data = data.data;
+			cache:set(orig_path, data);
+		elseif attr.mode == "directory" and path then
+			if full_path:sub(-1) ~= "/" then
+				local dir_path = { is_absolute = true, is_directory = true };
+				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
+				response_headers.location = build_path(dir_path);
+				return 301;
+			end
+			for i=1,#dir_indices do
+				if stat(full_path..dir_indices[i], "mode") == "file" then
+					return serve_file(event, path..dir_indices[i]);
+				end
+			end
+
+			if directory_index then
+				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
+			end
+			if not data then
+				return 403;
+			end
+			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
+			response_headers.content_type = mime_map.html;
+
+		else
+			local f, err = open(full_path, "rb");
+			if not f then
+				log("debug", "Could not open %s. Error was %s", full_path, err);
+				return 403;
+			end
+			local ext = full_path:match("%.([^./]+)$");
+			local content_type = ext and mime_map[ext];
+			response_headers.content_type = content_type;
+			if attr.size > cache_max_file_size then
+				response_headers.content_length = ("%d"):format(attr.size);
+				log("debug", "%d > cache_max_file_size", attr.size);
+				return response:send_file(f);
+			else
+				data = f:read("*a");
+				f:close();
+			end
+			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
+		end
+
+		return response:send(data);
+	end
+
+	return serve_file;
+end
+
+return {
+	serve = serve;
+}
+
diff --git a/net/http/parser.lua b/net/http/parser.lua
index 4e4ae9fb..96f17fdb 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -1,8 +1,8 @@
 local tonumber = tonumber;
 local assert = assert;
-local t_insert, t_concat = table.insert, table.concat;
 local url_parse = require "socket.url".parse;
 local urldecode = require "util.http".urldecode;
+local dbuffer = require "util.dbuffer";
 
 local function preprocess_path(path)
 	path = urldecode((path:gsub("//+", "/")));
@@ -28,10 +28,13 @@ local httpstream = {};
 function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 	local client = true;
 	if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
-	local buf, buflen, buftable = {}, 0, true;
 	local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024;
+	-- https://stackoverflow.com/a/686243
+	-- Indiviual headers can be up to 16k? What madness?
+	local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024;
 	local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2;
-	local chunked, chunk_size, chunk_start;
+	local buffer = dbuffer.new(buflimit);
+	local chunked;
 	local state = nil;
 	local packet;
 	local len;
@@ -41,32 +44,27 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 		feed = function(_, data)
 			if error then return nil, "parse has failed"; end
 			if not data then -- EOF
-				if buftable then buf, buftable = t_concat(buf), false; end
 				if state and client and not len then -- reading client body until EOF
-					packet.body = buf;
+					buffer:collapse();
+					packet.body = buffer:read_chunk() or "";
+					packet.partial = nil;
 					success_cb(packet);
-				elseif buf ~= "" then -- unexpected EOF
+					state = nil;
+				elseif buffer:length() ~= 0 then -- unexpected EOF
 					error = true; return error_cb("unexpected-eof");
 				end
 				return;
 			end
-			if buftable then
-				t_insert(buf, data);
-			else
-				buf = { buf, data };
-				buftable = true;
-			end
-			buflen = buflen + #data;
-			if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end
-			while buflen > 0 do
+			if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end
+			while buffer:length() > 0 do
 				if state == nil then -- read request
-					if buftable then buf, buftable = t_concat(buf), false; end
-					local index = buf:find("\r\n\r\n", nil, true);
+					local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true);
 					if not index then return; end -- not enough data
-					local method, path, httpversion, status_code, reason_phrase;
+					-- FIXME was reason_phrase meant to be passed on somewhere?
+					local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase
 					local first_line;
 					local headers = {};
-					for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request
+					for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request
 						if first_line then
 							local key, val = line:match("^([^%s:]+): *(.*)$");
 							if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers
@@ -91,7 +89,6 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 					if not first_line then error = true; return error_cb("invalid-status-line"); end
 					chunked = have_body and headers["transfer-encoding"] == "chunked";
 					len = tonumber(headers["content-length"]); -- TODO check for invalid len
-					if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end
 					if client then
 						-- FIXME handle '100 Continue' response (by skipping it)
 						if not have_body then len = 0; end
@@ -99,7 +96,10 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 							code = status_code;
 							httpversion = httpversion;
 							headers = headers;
-							body = have_body and "" or nil;
+							body = false;
+							body_length = len;
+							chunked = chunked;
+							partial = true;
 							-- COMPAT the properties below are deprecated
 							responseversion = httpversion;
 							responseheaders = headers;
@@ -124,60 +124,81 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
 							path = path;
 							httpversion = httpversion;
 							headers = headers;
-							body = nil;
+							body = false;
+							body_sink = nil;
+							chunked = chunked;
+							partial = true;
 						};
 					end
-					buf = buf:sub(index + 4);
-					buflen = #buf;
+					if len and len > bodylimit then
+						-- Early notification, for redirection
+						success_cb(packet);
+						if not packet.body_sink then error = true; return error_cb("content-length-limit-exceeded"); end
+					end
+					if chunked and not packet.body_sink then
+						success_cb(packet);
+						if not packet.body_sink then
+							packet.body_buffer = dbuffer.new(buflimit);
+						end
+					end
 					state = true;
 				end
 				if state then -- read body
-					if client then
-						if chunked then
-							if chunk_start and buflen - chunk_start - 2 < chunk_size then
-								return;
-							end -- not enough data
-							if buftable then buf, buftable = t_concat(buf), false; end
-							if not buf:find("\r\n", nil, true) then
-								return;
-							end -- not enough data
-							if not chunk_size then
-								chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()");
-								chunk_size = chunk_size and tonumber(chunk_size, 16);
-								if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end
-							end
-							if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then
-								state, chunk_size = nil, nil;
-								buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped
-								success_cb(packet);
-							elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk
-								packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1));
-								buf = buf:sub(chunk_start + chunk_size + 2);
-								buflen = buflen - (chunk_start + chunk_size + 2 - 1);
-								chunk_size, chunk_start = nil, nil;
-							else -- Partial chunk remaining
-								break;
+					if chunked then
+						local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow?
+						local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()");
+						if not chunk_size then return; end
+						chunk_size = chunk_size and tonumber(chunk_size, 16);
+						if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end
+						if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then
+							local body_buffer = packet.body_buffer;
+							if body_buffer then
+								packet.body_buffer = nil;
+								body_buffer:collapse();
+								packet.body = body_buffer:read_chunk() or "";
 							end
-						elseif len and buflen >= len then
-							if buftable then buf, buftable = t_concat(buf), false; end
-							if packet.code == 101 then
-								packet.body, buf, buflen, buftable = buf, {}, 0, true;
+
+							buffer:collapse();
+							local buf = buffer:read_chunk();
+							buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped
+							buffer:write(buf);
+							state, chunked = nil, nil;
+							packet.partial = nil;
+							success_cb(packet);
+						elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk
+							buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one
+							(packet.body_sink or packet.body_buffer):write(buffer:read(chunk_size));
+							buffer:discard(2); -- CRLF
+						else -- Partial chunk remaining
+							break;
+						end
+					elseif packet.body_sink then
+						local chunk = buffer:read_chunk(len);
+						while chunk and len > 0 do
+							if packet.body_sink:write(chunk) then
+								len = len - #chunk;
+								chunk = buffer:read_chunk(len);
 							else
-								packet.body, buf = buf:sub(1, len), buf:sub(len + 1);
-								buflen = #buf;
+								error = true;
+								return error_cb("body-sink-write-failure");
 							end
-							state = nil; success_cb(packet);
-						else
-							break;
 						end
-					elseif buflen >= len then
-						if buftable then buf, buftable = t_concat(buf), false; end
-						packet.body, buf = buf:sub(1, len), buf:sub(len + 1);
-						buflen = #buf;
-						state = nil; success_cb(packet);
+						if len == 0 then
+							state = nil;
+							packet.partial = nil;
+							success_cb(packet);
+						end
+					elseif buffer:length() >= len then
+						assert(not chunked)
+						packet.body = buffer:read(len) or "";
+						state = nil;
+						packet.partial = nil;
+						success_cb(packet);
 					else
 						break;
 					end
+				else
+					break;
 				end
 			end
 		end;
diff --git a/net/http/server.lua b/net/http/server.lua
index 3873bbe0..75859719 100644
--- a/net/http/server.lua
+++ b/net/http/server.lua
@@ -13,6 +13,8 @@ local traceback = debug.traceback;
 local tostring = tostring;
 local cache = require "util.cache";
 local codes = require "net.http.codes";
+local promise = require "util.promise";
+local errors = require "util.error";
 local blocksize = 2^16;
 
 local _M = {};
@@ -170,6 +172,49 @@ local headerfix = setmetatable({}, {
 	end
 });
 
+local function handle_result(request, response, result)
+	if result == nil then
+		result = 404;
+	end
+
+	if result == true then
+		return;
+	end
+
+	local body;
+	local result_type = type(result);
+	if result_type == "number" then
+		response.status_code = result;
+		if result >= 400 then
+			body = events.fire_event("http-error", { request = request, response = response, code = result });
+		end
+	elseif result_type == "string" then
+		body = result;
+	elseif errors.is_err(result) then
+		response.status_code = result.code or 500;
+		body = events.fire_event("http-error", { request = request, response = response, code = result.code or 500, error = result });
+	elseif promise.is_promise(result) then
+		result:next(function (ret)
+			handle_result(request, response, ret);
+		end, function (err)
+			response.status_code = 500;
+			handle_result(request, response, err or 500);
+		end);
+		return true;
+	elseif result_type == "table" then
+		for k, v in pairs(result) do
+			if k ~= "headers" then
+				response[k] = v;
+			else
+				for header_name, header_value in pairs(v) do
+					response.headers[header_name] = header_value;
+				end
+			end
+		end
+	end
+	return response:send(body);
+end
+
 function _M.hijack_response(response, listener) -- luacheck: ignore
 	error("TODO");
 end
@@ -194,8 +239,11 @@ function handle_request(conn, request, finish_cb)
 		response_conn_header = httpversion == "1.1" and "close" or nil
 	end
 
+	local is_head_request = request.method == "HEAD";
+
 	local response = {
 		request = request;
+		is_head_request = is_head_request;
 		status_code = 200;
 		headers = { date = date_header, connection = response_conn_header };
 		persistent = persistent;
@@ -227,6 +275,11 @@ function handle_request(conn, request, finish_cb)
 	local payload = { request = request, response = response };
 	log("debug", "Firing event: %s", global_event);
 	local result = events.fire_event(global_event, payload);
+	if result == nil and is_head_request then
+		local global_head_event = "GET "..request.path:match("[^?]*");
+		log("debug", "Firing event: %s", global_head_event);
+		result = events.fire_event(global_head_event, payload);
+	end
 	if result == nil then
 		if not hosts[host] then
 			if hosts[default_host] then
@@ -247,40 +300,17 @@ function handle_request(conn, request, finish_cb)
 		local host_event = request.method.." "..host..request.path:match("[^?]*");
 		log("debug", "Firing event: %s", host_event);
 		result = events.fire_event(host_event, payload);
-	end
-	if result ~= nil then
-		if result ~= true then
-			local body;
-			local result_type = type(result);
-			if result_type == "number" then
-				response.status_code = result;
-				if result >= 400 then
-					payload.code = result;
-					body = events.fire_event("http-error", payload);
-				end
-			elseif result_type == "string" then
-				body = result;
-			elseif result_type == "table" then
-				for k, v in pairs(result) do
-					if k ~= "headers" then
-						response[k] = v;
-					else
-						for header_name, header_value in pairs(v) do
-							response.headers[header_name] = header_value;
-						end
-					end
-				end
-			end
-			response:send(body);
+
+		if result == nil and is_head_request then
+			local host_head_event = "GET "..host..request.path:match("[^?]*");
+			log("debug", "Firing event: %s", host_head_event);
+			result = events.fire_event(host_head_event, payload);
 		end
-		return;
 	end
 
-	-- if handler not called, return 404
-	response.status_code = 404;
-	payload.code = 404;
-	response:send(events.fire_event("http-error", payload));
+	return handle_result(request, response, result);
 end
+
 local function prepare_header(response)
 	local status_line = "HTTP/"..response.request.httpversion.." "..(response.status or codes[response.status_code]);
 	local headers = response.headers;
@@ -292,12 +322,21 @@ local function prepare_header(response)
 	return output;
 end
 _M.prepare_header = prepare_header;
+function _M.send_head_response(response)
+	if response.finished then return; end
+	local output = prepare_header(response);
+	response.conn:write(t_concat(output));
+	response:done();
+end
 function _M.send_response(response, body)
 	if response.finished then return; end
 	body = body or response.body or "";
 	-- Per RFC 7230, informational (1xx) and 204 (no content) should have no c-l header
 	if response.status_code > 199 and response.status_code ~= 204 then
-		response.headers.content_length = #body;
+		response.headers.content_length = ("%d"):format(#body);
+	end
+	if response.is_head_request then
+		return _M.send_head_response(response)
 	end
 	local output = prepare_header(response);
 	t_insert(output, body);
@@ -305,6 +344,10 @@ function _M.send_response(response, body)
 	response:done();
 end
 function _M.send_file(response, f)
+	if response.is_head_request then
+		if f.close then f:close(); end
+		return _M.send_head_response(response);
+	end
 	if response.finished then return; end
 	local chunked = not response.headers.content_length;
 	if chunked then response.headers.transfer_encoding = "chunked"; end
diff --git a/net/resolvers/basic.lua b/net/resolvers/basic.lua
index 867ccf60..a00fbce1 100644
--- a/net/resolvers/basic.lua
+++ b/net/resolvers/basic.lua
@@ -2,10 +2,13 @@ local adns = require "net.adns";
 local inet_pton = require "util.net".pton;
 local inet_ntop = require "util.net".ntop;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local methods = {};
 local resolver_mt = { __index = methods };
 
+-- FIXME RFC 6724
+
 -- Find the next target to connect to, and
 -- pass it to cb()
 function methods:next(cb)
@@ -36,23 +39,32 @@ function methods:next(cb)
 
 	-- Resolve DNS to target list
 	local dns_resolver = adns.resolver();
-	dns_resolver:lookup(function (answer)
-		if answer then
-			for _, record in ipairs(answer) do
-				table.insert(targets, { self.conn_type.."4", record.a, self.port, self.extra });
+
+	if not self.extra or self.extra.use_ipv4 ~= false then
+		dns_resolver:lookup(function (answer)
+			if answer then
+				for _, record in ipairs(answer) do
+					table.insert(targets, { self.conn_type.."4", record.a, self.port, self.extra });
+				end
 			end
-		end
+			ready();
+		end, self.hostname, "A", "IN");
+	else
 		ready();
-	end, self.hostname, "A", "IN");
+	end
 
-	dns_resolver:lookup(function (answer)
-		if answer then
-			for _, record in ipairs(answer) do
-				table.insert(targets, { self.conn_type.."6", record.aaaa, self.port, self.extra });
+	if not self.extra or self.extra.use_ipv6 ~= false then
+		dns_resolver:lookup(function (answer)
+			if answer then
+				for _, record in ipairs(answer) do
+					table.insert(targets, { self.conn_type.."6", record.aaaa, self.port, self.extra });
+				end
 			end
-		end
+			ready();
+		end, self.hostname, "AAAA", "IN");
+	else
 		ready();
-	end, self.hostname, "AAAA", "IN");
+	end
 end
 
 local function new(hostname, port, conn_type, extra)
diff --git a/net/resolvers/manual.lua b/net/resolvers/manual.lua
index c0d4e5d5..dbc40256 100644
--- a/net/resolvers/manual.lua
+++ b/net/resolvers/manual.lua
@@ -1,5 +1,6 @@
 local methods = {};
 local resolver_mt = { __index = methods };
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 -- Find the next target to connect to, and
 -- pass it to cb()
diff --git a/net/resolvers/service.lua b/net/resolvers/service.lua
index 34f14cba..d74adf06 100644
--- a/net/resolvers/service.lua
+++ b/net/resolvers/service.lua
@@ -1,6 +1,8 @@
 local adns = require "net.adns";
 local basic = require "net.resolvers.basic";
+local inet_pton = require "util.net".pton;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local methods = {};
 local resolver_mt = { __index = methods };
@@ -9,14 +11,17 @@ local resolver_mt = { __index = methods };
 -- pass it to cb()
 function methods:next(cb)
 	if self.targets then
-		if #self.targets == 0 then
-			cb(nil);
-			return;
+		if not self.resolver then
+			if #self.targets == 0 then
+				cb(nil);
+				return;
+			end
+			local next_target = table.remove(self.targets, 1);
+			self.resolver = basic.new(unpack(next_target, 1, 4));
 		end
-		local next_target = table.remove(self.targets, 1);
-		self.resolver = basic.new(unpack(next_target, 1, 4));
 		self.resolver:next(function (...)
 			if ... == nil then
+				self.resolver = nil;
 				self:next(cb);
 			else
 				cb(...);
@@ -39,7 +44,11 @@ function methods:next(cb)
 
 	-- Resolve DNS to target list
 	local dns_resolver = adns.resolver();
-	dns_resolver:lookup(function (answer)
+	dns_resolver:lookup(function (answer, err)
+		if not answer and not err then
+			-- net.adns returns nil if there are zero records or nxdomain
+			answer = {};
+		end
 		if answer then
 			if #answer == 0 then
 				if self.extra and self.extra.default_port then
@@ -64,6 +73,14 @@ function methods:next(cb)
 end
 
 local function new(hostname, service, conn_type, extra)
+	local is_ip = inet_pton(hostname);
+	if not is_ip and hostname:sub(1,1) == '[' then
+		is_ip = inet_pton(hostname:sub(2,-2));
+	end
+	if is_ip and extra and extra.default_port then
+		return basic.new(hostname, extra.default_port, conn_type, extra);
+	end
+
 	return setmetatable({
 		hostname = idna_to_ascii(hostname);
 		service = service;
diff --git a/net/server.lua b/net/server.lua
index abbb421d..f5666594 100644
--- a/net/server.lua
+++ b/net/server.lua
@@ -13,7 +13,11 @@ if not (prosody and prosody.config_loaded) then
 end
 
 local log = require "util.logger".init("net.server");
-local server_type = require "core.configmanager".get("*", "network_backend") or "select";
+
+local have_util_poll = pcall(require, "util.poll");
+local default_backend = have_util_poll and "epoll" or "select";
+
+local server_type = require "core.configmanager".get("*", "network_backend") or default_backend;
 
 if require "core.configmanager".get("*", "use_libevent") then
 	server_type = "event";
diff --git a/net/server_epoll.lua b/net/server_epoll.lua
index 953bbb11..32b2f032 100644
--- a/net/server_epoll.lua
+++ b/net/server_epoll.lua
@@ -9,20 +9,24 @@
 local t_insert = table.insert;
 local t_concat = table.concat;
 local setmetatable = setmetatable;
-local tostring = tostring;
 local pcall = pcall;
 local type = type;
 local next = next;
 local pairs = pairs;
-local log = require "util.logger".init("server_epoll");
+local traceback = debug.traceback;
+local logger = require "util.logger";
+local log = logger.init("server_epoll");
 local socket = require "socket";
 local luasec = require "ssl";
-local gettime = require "util.time".now;
+local realtime = require "util.time".now;
+local monotonic = require "util.time".monotonic;
 local indexedbheap = require "util.indexedbheap";
 local createtable = require "util.table".create;
 local inet = require "util.net";
 local inet_pton = inet.pton;
 local _SOCKETINVALID = socket._SOCKETINVALID or -1;
+local new_id = require "util.id".medium;
+local xpcall = require "util.xpcall".xpcall;
 
 local poller = require "util.poll"
 local EEXIST = poller.EEXIST;
@@ -38,7 +42,10 @@ local default_config = { __index = {
 	read_timeout = 14 * 60;
 
 	-- How long to wait for a socket to become writable after queuing data to send
-	send_timeout = 60;
+	send_timeout = 180;
+
+	-- How long to wait for a socket to become writable after creation
+	connect_timeout = 20;
 
 	-- Some number possibly influencing how many pending connections can be accepted
 	tcp_backlog = 128;
@@ -58,6 +65,20 @@ local default_config = { __index = {
 	-- Maximum and minimum amount of time to sleep waiting for events (adjusted for pending timers)
 	max_wait = 86400;
 	min_wait = 1e-06;
+
+	-- Enable extra noisy debug logging
+	-- TODO disable once considered stable
+	verbose = true;
+
+	-- EXPERIMENTAL
+	-- Whether to kill connections in case of callback errors.
+	fatal_errors = false;
+
+	-- Or disable protection (like server_select) for potential performance gains
+	protect_listeners = true;
+
+	-- Attempt writes instantly
+	opportunistic_writes = false;
 }};
 local cfg = default_config.__index;
 
@@ -68,48 +89,50 @@ local fds = createtable(10, 0); -- FD -> conn
 local timers = indexedbheap.create();
 
 local function noop() end
-local function closetimer(t)
-	t[1] = 0;
-	t[2] = noop;
-	timers:remove(t.id);
+local function closetimer(id)
+	timers:remove(id);
 end
 
-local function reschedule(t, time)
-	t[1] = time;
-	timers:reprioritize(t.id, time);
-end
-
--- Add absolute timer
-local function at(time, f)
-	local timer = { time, f, close = closetimer, reschedule = reschedule, id = nil };
-	timer.id = timers:insert(timer, time);
-	return timer;
+local function reschedule(id, time)
+	time = monotonic() + time;
+	timers:reprioritize(id, time);
 end
 
 -- Add relative timer
-local function addtimer(timeout, f)
-	return at(gettime() + timeout, f);
+local function addtimer(timeout, f, param)
+	local time = monotonic() + timeout;
+	if param ~= nil then
+		local timer_callback = f
+		function f(current_time, timer_id)
+			local t = timer_callback(current_time, timer_id, param)
+			return t;
+		end
+	end
+	local id = timers:insert(f, time);
+	return id;
 end
 
 -- Run callbacks of expired timers
 -- Return time until next timeout
 local function runtimers(next_delay, min_wait)
 	-- Any timers at all?
-	local now = gettime();
+	local elapsed = monotonic();
+	local now = realtime();
 	local peek = timers:peek();
 	while peek do
 
-		if peek > now then
-			next_delay = peek - now;
+		if peek > elapsed then
+			next_delay = peek - elapsed;
 			break;
 		end
 
 		local _, timer, id = timers:pop();
-		local ok, ret = pcall(timer[2], now);
+		local ok, ret = xpcall(timer, traceback, now, id);
 		if ok and type(ret) == "number"  then
-			local next_time = now+ret;
-			timer[1] = next_time;
+			local next_time = elapsed+ret;
 			timers:insert(timer, next_time);
+		elseif not ok then
+			log("error", "Error in timer: %s", ret);
 		end
 
 		peek = timers:peek();
@@ -138,6 +161,22 @@ function interface_mt:__tostring()
 	return ("FD %d"):format(self:getfd());
 end
 
+interface.log = log;
+function interface:debug(msg, ...) --luacheck: ignore 212/self
+	self.log("debug", msg, ...);
+end
+
+interface.noise = interface.debug;
+function interface:noise(msg, ...) --luacheck: ignore 212/self
+	if cfg.verbose then
+		return self:debug(msg, ...);
+	end
+end
+
+function interface:error(msg, ...) --luacheck: ignore 212/self
+	self.log("error", msg, ...);
+end
+
 -- Replace the listener and tell the old one
 function interface:setlistener(listeners, data)
 	self:on("detach");
@@ -148,21 +187,36 @@ end
 -- Call a listener callback
 function interface:on(what, ...)
 	if not self.listeners then
-		log("error", "%s has no listeners", self);
+		self:error("Interface is missing listener callbacks");
 		return;
 	end
 	local listener = self.listeners["on"..what];
 	if not listener then
-		-- log("debug", "Missing listener 'on%s'", what); -- uncomment for development and debugging
+		self:noise("Missing listener 'on%s'", what); -- uncomment for development and debugging
 		return;
 	end
-	local ok, err = pcall(listener, self, ...);
+	if not cfg.protect_listeners then
+		return listener(self, ...);
+	end
+	local onerror = self.listeners.onerror or traceback;
+	local ok, err = xpcall(listener, onerror, self, ...);
 	if not ok then
-		log("error", "Error calling on%s: %s", what, err);
+		if cfg.fatal_errors then
+			self:error("Closing due to error calling on%s: %s", what, err);
+			self:destroy();
+		else
+			self:debug("Error calling on%s: %s", what, err);
+		end
+		return nil, err;
 	end
 	return err;
 end
 
+-- Allow this one to be overridden
+function interface:onincoming(...)
+	return self:on("incoming", ...);
+end
+
 -- Return the file descriptor number
 function interface:getfd()
 	if self.conn then
@@ -219,19 +273,21 @@ end
 function interface:setreadtimeout(t)
 	if t == false then
 		if self._readtimeout then
-			self._readtimeout:close();
+			closetimer(self._readtimeout);
 			self._readtimeout = nil;
 		end
 		return
 	end
 	t = t or cfg.read_timeout;
 	if self._readtimeout then
-		self._readtimeout:reschedule(gettime() + t);
+		reschedule(self._readtimeout, t);
 	else
 		self._readtimeout = addtimer(t, function ()
 			if self:on("readtimeout") then
+				self:noise("Read timeout handled");
 				return cfg.read_timeout;
 			else
+				self:debug("Read timeout not handled, disconnecting");
 				self:on("disconnect", "read timeout");
 				self:destroy();
 			end
@@ -243,17 +299,18 @@ end
 function interface:setwritetimeout(t)
 	if t == false then
 		if self._writetimeout then
-			self._writetimeout:close();
+			closetimer(self._writetimeout);
 			self._writetimeout = nil;
 		end
 		return
 	end
 	t = t or cfg.send_timeout;
 	if self._writetimeout then
-		self._writetimeout:reschedule(gettime() + t);
+		reschedule(self._writetimeout, t);
 	else
 		self._writetimeout = addtimer(t, function ()
-			self:on("disconnect", "write timeout");
+			self:noise("Write timeout");
+			self:on("disconnect", self._connected and "write timeout" or "connection timeout");
 			self:destroy();
 		end);
 	end
@@ -269,15 +326,15 @@ function interface:add(r, w)
 	local ok, err, errno = poll:add(fd, r, w);
 	if not ok then
 		if errno == EEXIST then
-			log("debug", "%s already registered!", self);
+			self:debug("FD already registered in poller! (EEXIST)");
 			return self:set(r, w); -- So try to change its flags
 		end
-		log("error", "Could not register %s: %s(%d)", self, err, errno);
+		self:debug("Could not register in poller: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = r, w;
 	fds[fd] = self;
-	log("debug", "Watching %s", self);
+	self:noise("Registered in poller");
 	return true;
 end
 
@@ -290,7 +347,7 @@ function interface:set(r, w)
 	if w == nil then w = self._wantwrite; end
 	local ok, err, errno = poll:set(fd, r, w);
 	if not ok then
-		log("error", "Could not update poller state %s: %s(%d)", self, err, errno);
+		self:debug("Could not update poller state: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = r, w;
@@ -307,12 +364,12 @@ function interface:del()
 	end
 	local ok, err, errno = poll:del(fd);
 	if not ok and errno ~= ENOENT then
-		log("error", "Could not unregister %s: %s(%d)", self, err, errno);
+		self:debug("Could not unregister: %s(%d)", err, errno);
 		return ok, err;
 	end
 	self._wantread, self._wantwrite = nil, nil;
 	fds[fd] = nil;
-	log("debug", "Unwatched %s", self);
+	self:noise("Unregistered from poller");
 	return true;
 end
 
@@ -334,7 +391,7 @@ function interface:onreadable()
 	local data, err, partial = self.conn:receive(self.read_size or cfg.read_size);
 	if data then
 		self:onconnect();
-		self:on("incoming", data);
+		self:onincoming(data);
 	else
 		if err == "wantread" then
 			self:set(true, nil);
@@ -345,15 +402,28 @@ function interface:onreadable()
 		end
 		if partial and partial ~= "" then
 			self:onconnect();
-			self:on("incoming", partial, err);
+			self:onincoming(partial, err);
 		end
 		if err ~= "timeout" then
+			if err == "closed" then
+				self:debug("Connection closed by remote");
+			else
+				self:debug("Read error, closing (%s)", err);
+			end
 			self:on("disconnect", err);
 			self:destroy()
 			return;
 		end
 	end
 	if not self.conn then return; end
+	if self._limit and (data or partial) then
+		local cost = self._limit * #(data or partial);
+		if cost > cfg.min_wait then
+			self:setreadtimeout(false);
+			self:pausefor(cost);
+			return;
+		end
+	end
 	if self._wantread and self.conn:dirty() then
 		self:setreadtimeout(false);
 		self:pausefor(cfg.read_retry_delay);
@@ -367,7 +437,7 @@ function interface:onwritable()
 	self:onconnect();
 	if not self.conn then return; end -- could have been closed in onconnect
 	local buffer = self.writebuffer;
-	local data = t_concat(buffer);
+	local data = #buffer == 1 and buffer[1] or t_concat(buffer);
 	local ok, err, partial = self.conn:send(data);
 	if ok then
 		self:set(nil, false);
@@ -378,10 +448,12 @@ function interface:onwritable()
 		self:ondrain(); -- Be aware of writes in ondrain
 		return;
 	elseif partial then
+		self:debug("Sent %d out of %d buffered bytes", partial, #data);
 		buffer[1] = data:sub(partial+1);
 		for i = #buffer, 2, -1 do
 			buffer[i] = nil;
 		end
+		self:set(nil, true);
 		self:setwritetimeout();
 	end
 	if err == "wantwrite" or err == "timeout" then
@@ -407,8 +479,14 @@ function interface:write(data)
 	else
 		self.writebuffer = { data };
 	end
-	self:setwritetimeout();
-	self:set(nil, true);
+	if not self._write_lock then
+		if cfg.opportunistic_writes then
+			self:onwritable();
+			return #data;
+		end
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
 	return #data;
 end
 interface.send = interface.write;
@@ -418,10 +496,10 @@ function interface:close()
 	if self.writebuffer and self.writebuffer[1] then
 		self:set(false, true); -- Flush final buffer contents
 		self.write, self.send = noop, noop; -- No more writing
-		log("debug", "Close %s after writing", self);
+		self:debug("Close after writing remaining buffered data");
 		self.ondrain = interface.close;
 	else
-		log("debug", "Close %s now", self);
+		self:debug("Closing now");
 		self.write, self.send = noop, noop;
 		self.close = noop;
 		self:on("disconnect");
@@ -450,31 +528,32 @@ function interface:starttls(tls_ctx)
 	if tls_ctx then self.tls_ctx = tls_ctx; end
 	self.starttls = false;
 	if self.writebuffer and self.writebuffer[1] then
-		log("debug", "Start TLS on %s after write", self);
+		self:debug("Start TLS after write");
 		self.ondrain = interface.starttls;
 		self:set(nil, true); -- make sure wantwrite is set
 	else
 		if self.ondrain == interface.starttls then
 			self.ondrain = nil;
 		end
-		self.onwritable = interface.tlshandskake;
-		self.onreadable = interface.tlshandskake;
+		self.onwritable = interface.tlshandshake;
+		self.onreadable = interface.tlshandshake;
 		self:set(true, true);
-		log("debug", "Prepare to start TLS on %s", self);
+		self:debug("Prepared to start TLS");
 	end
 end
 
-function interface:tlshandskake()
+function interface:tlshandshake()
 	self:setwritetimeout(false);
 	self:setreadtimeout(false);
 	if not self._tls then
 		self._tls = true;
-		log("debug", "Start TLS on %s now", self);
+		self:debug("Starting TLS now");
 		self:del();
+		self:updatenames(); -- Can't getpeer/sockname after wrap()
 		local ok, conn, err = pcall(luasec.wrap, self.conn, self.tls_ctx);
 		if not ok then
 			conn, err = ok, conn;
-			log("error", "Failed to initialize TLS: %s", err);
+			self:debug("Failed to initialize TLS: %s", err);
 		end
 		if not conn then
 			self:on("disconnect", err);
@@ -483,33 +562,43 @@ function interface:tlshandskake()
 		end
 		conn:settimeout(0);
 		self.conn = conn;
-		if conn.sni and self.servername then
-			conn:sni(self.servername);
+		if conn.sni then
+			if self.servername then
+				conn:sni(self.servername);
+			elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+				conn:sni(self._server.hosts, true);
+			end
 		end
 		self:on("starttls");
 		self.ondrain = nil;
-		self.onwritable = interface.tlshandskake;
-		self.onreadable = interface.tlshandskake;
+		self.onwritable = interface.tlshandshake;
+		self.onreadable = interface.tlshandshake;
 		return self:init();
 	end
+	self:noise("Continuing TLS handshake");
 	local ok, err = self.conn:dohandshake();
 	if ok then
-		log("debug", "TLS handshake on %s complete", self);
+		local info = self.conn.info and self.conn:info();
+		if type(info) == "table" then
+			self:debug("TLS handshake complete (%s with %s)", info.protocol, info.cipher);
+		else
+			self:debug("TLS handshake complete");
+		end
 		self.onwritable = nil;
 		self.onreadable = nil;
 		self:on("status", "ssl-handshake-complete");
 		self:setwritetimeout();
 		self:set(true, true);
 	elseif err == "wantread" then
-		log("debug", "TLS handshake on %s to wait until readable", self);
+		self:noise("TLS handshake to wait until readable");
 		self:set(true, false);
 		self:setreadtimeout(cfg.ssl_handshake_timeout);
 	elseif err == "wantwrite" then
-		log("debug", "TLS handshake on %s to wait until writable", self);
+		self:noise("TLS handshake to wait until writable");
 		self:set(false, true);
 		self:setwritetimeout(cfg.ssl_handshake_timeout);
 	else
-		log("debug", "TLS handshake error on %s: %s", self, err);
+		self:debug("TLS handshake error: %s", err);
 		self:on("disconnect", err);
 		self:destroy();
 	end
@@ -517,15 +606,18 @@ end
 
 local function wrapsocket(client, server, read_size, listeners, tls_ctx, extra) -- luasocket object -> interface object
 	client:settimeout(0);
+	local conn_id = ("conn%s"):format(new_id());
 	local conn = setmetatable({
 		conn = client;
 		_server = server;
-		created = gettime();
+		created = realtime();
 		listeners = listeners;
 		read_size = read_size or (server and server.read_size);
 		writebuffer = {};
 		tls_ctx = tls_ctx or (server and server.tls_ctx);
 		tls_direct = server and server.tls_direct;
+		id = conn_id;
+		log = logger.init(conn_id);
 		extra = extra;
 	}, interface_mt);
 
@@ -542,12 +634,12 @@ end
 function interface:updatenames()
 	local conn = self.conn;
 	local ok, peername, peerport = pcall(conn.getpeername, conn);
-	if ok then
-		self.peername, self.peerport = peername, peerport;
+	if ok and peername then
+		self.peername, self.peerport = peername, peerport or 0;
 	end
 	local ok, sockname, sockport = pcall(conn.getsockname, conn);
-	if ok then
-		self.sockname, self.sockport = sockname, sockport;
+	if ok and sockname then
+		self.sockname, self.sockport = sockname, sockport or 0;
 	end
 end
 
@@ -556,76 +648,129 @@ end
 function interface:onacceptable()
 	local conn, err = self.conn:accept();
 	if not conn then
-		log("debug", "Error accepting new client: %s, server will be paused for %ds", err, cfg.accept_retry_interval);
+		self:debug("Error accepting new client: %s, server will be paused for %ds", err, cfg.accept_retry_interval);
 		self:pausefor(cfg.accept_retry_interval);
 		return;
 	end
 	local client = wrapsocket(conn, self, nil, self.listeners);
-	log("debug", "New connection %s", tostring(client));
+	client:debug("New connection %s on server %s", client, self);
 	client:init();
 	if self.tls_direct then
 		client:starttls(self.tls_ctx);
+	else
+		client:onconnect();
 	end
 end
 
 -- Initialization
 function interface:init()
-	self:setwritetimeout();
+	self:setwritetimeout(cfg.connect_timeout);
 	return self:add(true, true);
 end
 
 function interface:pause()
+	self:noise("Pause reading");
 	return self:set(false);
 end
 
 function interface:resume()
+	self:noise("Resume reading");
 	return self:set(true);
 end
 
 -- Pause connection for some time
 function interface:pausefor(t)
+	self:noise("Pause for %fs", t);
 	if self._pausefor then
-		self._pausefor:close();
+		closetimer(self._pausefor);
+		self._pausefor = nil;
 	end
 	if t == false then return; end
 	self:set(false);
 	self._pausefor = addtimer(t, function ()
 		self._pausefor = nil;
 		self:set(true);
+		self:noise("Resuming after pause, connection is %s", not self.conn and "missing" or self.conn:dirty() and "dirty" or "clean");
 		if self.conn and self.conn:dirty() then
 			self:onreadable();
 		end
 	end);
 end
 
+function interface:setlimit(Bps)
+	if Bps > 0 then
+		self._limit = 1/Bps;
+	else
+		self._limit = nil;
+	end
+end
+
+function interface:pause_writes()
+	if self._write_lock then
+		return
+	end
+	self:noise("Pause writes");
+	self._write_lock = true;
+	self:setwritetimeout(false);
+	self:set(nil, false);
+end
+
+function interface:resume_writes()
+	if not self._write_lock then
+		return
+	end
+	self:noise("Resume writes");
+	self._write_lock = nil;
+	if self.writebuffer[1] then
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
+end
+
 -- Connected!
 function interface:onconnect()
-	if self.conn and not self.peername and self.conn.getpeername then
-		self.peername, self.peerport = self.conn:getpeername();
-	end
+	self._connected = true;
+	self:updatenames();
+	self:debug("Connected (%s)", self);
 	self.onconnect = noop;
 	self:on("connect");
 end
 
-local function addserver(addr, port, listeners, read_size, tls_ctx)
-	local conn, err = socket.bind(addr, port, cfg.tcp_backlog);
-	if not conn then return conn, err; end
-	conn:settimeout(0);
+local function wrapserver(conn, addr, port, listeners, config)
 	local server = setmetatable({
 		conn = conn;
-		created = gettime();
+		created = realtime();
 		listeners = listeners;
-		read_size = read_size;
+		read_size = config and config.read_size;
 		onreadable = interface.onacceptable;
-		tls_ctx = tls_ctx;
-		tls_direct = tls_ctx and true or false;
+		tls_ctx = config and config.tls_ctx;
+		tls_direct = config and config.tls_direct;
+		hosts = config and config.sni_hosts;
 		sockname = addr;
 		sockport = port;
+		log = logger.init(("serv%s"):format(new_id()));
 	}, interface_mt);
+	server:debug("Server %s created", server);
 	server:add(true, false);
 	return server;
 end
 
+local function listen(addr, port, listeners, config)
+	local conn, err = socket.bind(addr, port, cfg.tcp_backlog);
+	if not conn then return conn, err; end
+	conn:settimeout(0);
+	return wrapserver(conn, addr, port, listeners, config);
+end
+
+-- COMPAT
+local function addserver(addr, port, listeners, read_size, tls_ctx)
+	return listen(addr, port, listeners, {
+		read_size = read_size;
+		tls_ctx = tls_ctx;
+		tls_direct = tls_ctx and true or false;
+	});
+end
+
 -- COMPAT
 local function wrapclient(conn, addr, port, listeners, read_size, tls_ctx, extra)
 	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx, extra);
@@ -659,13 +804,19 @@ local function addclient(addr, port, listeners, read_size, tls_ctx, typ, extra)
 		return nil, "invalid socket type";
 	end
 	local conn, err = create();
+	if not conn then return conn, err; end
 	local ok, err = conn:settimeout(0);
 	if not ok then return ok, err; end
 	local ok, err = conn:setpeername(addr, port);
 	if not ok and err ~= "timeout" then return ok, err; end
 	local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx, extra)
 	local ok, err = client:init();
+	if not client.peername then
+		-- otherwise not set until connected
+		client.peername, client.peerport = addr, port;
+	end
 	if not ok then return ok, err; end
+	client:debug("Client %s created", client);
 	if tls_ctx then
 		client:starttls(tls_ctx);
 	end
@@ -687,23 +838,23 @@ local function watchfd(fd, onreadable, onwritable)
 		end;
 		-- Otherwise it'll need to be something LuaSocket-compatible
 	end
+	conn.id = new_id();
+	conn.log = logger.init(("fdwatch%s"):format(conn.id));
 	conn:add(onreadable, onwritable);
 	return conn;
 end;
 
 -- Dump all data from one connection into another
-local function link(from, to)
-	from.listeners = setmetatable({
-		onincoming = function (_, data)
-			from:pause();
-			to:write(data);
-		end,
-	}, {__index=from.listeners});
-	to.listeners = setmetatable({
-		ondrain = function ()
-			from:resume();
-		end,
-	}, {__index=to.listeners});
+local function link(from, to, read_size)
+	from:debug("Linking to %s", to.id);
+	function from:onincoming(data)
+		self:pause();
+		to:write(data);
+	end
+	function to:ondrain() -- luacheck: ignore 212/self
+		from:resume();
+	end
+	from:set_mode(read_size);
 	from:set(true, nil);
 	to:set(nil, true);
 end
@@ -762,11 +913,21 @@ return {
 	addserver = addserver;
 	addclient = addclient;
 	add_task = addtimer;
-	at = at;
+	timer = {
+		-- API-compatible with util.timer
+		add_task = addtimer;
+		stop = closetimer;
+		reschedule = reschedule;
+		to_absolute_time = function (t)
+			return t-monotonic()+realtime();
+		end;
+	};
+	listen = listen;
 	loop = loop;
 	closeall = closeall;
 	setquitting = setquitting;
 	wrapclient = wrapclient;
+	wrapserver = wrapserver;
 	watchfd = watchfd;
 	link = link;
 	set_config = function (newconfig)
@@ -776,6 +937,7 @@ return {
 	-- libevent emulation
 	event = { EV_READ = "r", EV_WRITE = "w", EV_READWRITE = "rw", EV_LEAVE = -1 };
 	addevent = function (fd, mode, callback)
+		log("warn", "Using deprecated libevent emulation, please update code to use watchfd API instead");
 		local function onevent(self)
 			local ret = self:callback();
 			if ret == -1 then
@@ -795,6 +957,8 @@ return {
 				fds[fd] = nil;
 			end;
 		}, interface_mt);
+		conn.id = conn:getfd();
+		conn.log = logger.init(("fdwatch%d"):format(conn.id));
 		local ok, err = conn:add(mode == "r" or mode == "rw", mode == "w" or mode == "rw");
 		if not ok then return ok, err; end
 		return conn;
diff --git a/net/server_event.lua b/net/server_event.lua
index 746526ce..f7e1f448 100644
--- a/net/server_event.lua
+++ b/net/server_event.lua
@@ -165,8 +165,12 @@ function interface_mt:_start_ssl(call_onconnect) -- old socket will be destroyed
 		return false
 	end
 
-	if self.conn.sni and self.servername then
-		self.conn:sni(self.servername);
+	if self.conn.sni then
+		if self.servername then
+			self.conn:sni(self.servername);
+		elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+			self.conn:sni(self._server.hosts, true);
+		end
 	end
 
 	self.conn:settimeout( 0 )  -- set non blocking
@@ -258,6 +262,7 @@ end
 
 --TODO: Deprecate
 function interface_mt:lock_read(switch)
+	log("warn", ":lock_read is deprecated, use :pasue() and :resume()");
 	if switch then
 		return self:pause();
 	else
@@ -277,6 +282,19 @@ function interface_mt:resume()
 	end
 end
 
+function interface_mt:pause_writes()
+	return self:_lock(self.nointerface, self.noreading, true);
+end
+
+function interface_mt:resume_writes()
+	self:_lock(self.nointerface, self.noreading, false);
+	if self.writecallback and not self.eventwrite then
+		self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT );  -- register callback
+		return true;
+	end
+end
+
+
 function interface_mt:counter(c)
 	if c then
 		self._connections = self._connections + c
@@ -286,7 +304,7 @@ end
 
 -- Public methods
 function interface_mt:write(data)
-	if self.nowriting then return nil, "locked" end
+	if self.nointerface then return nil, "locked"; end
 	--vdebug( "try to send data to client, id/data:", self.id, data )
 	data = tostring( data )
 	local len = #data
@@ -298,7 +316,7 @@ function interface_mt:write(data)
 	end
 	t_insert(self.writebuffer, data) -- new buffer
 	self.writebufferlen = total
-	if not self.eventwrite then  -- register new write event
+	if not self.eventwrite and not self.nowriting  then  -- register new write event
 		--vdebug( "register new write event" )
 		self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT )
 	end
@@ -445,10 +463,6 @@ end
 function interface_mt:ontimeout()
 end
 function interface_mt:onreadtimeout()
-	self.fatalerror = "timeout during receiving"
-	debug( "connection failed:", self.fatalerror )
-	self:_close()
-	self.eventread = nil
 end
 function interface_mt:ondrain()
 end
@@ -642,7 +656,7 @@ local function handleclient( client, ip, port, server, pattern, listener, sslctx
 	return interface
 end
 
-local function handleserver( server, addr, port, pattern, listener, sslctx )  -- creates an server interface
+local function handleserver( server, addr, port, pattern, listener, sslctx, startssl )  -- creates a server interface
 	debug "creating server interface..."
 	local interface = {
 		_connections = 0;
@@ -658,6 +672,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 
 		_ip = addr, _port = port, _pattern = pattern,
 		_sslctx = sslctx;
+		hosts = {};
 	}
 	interface.id = tostring(interface):match("%x+$");
 	interface.readcallback = function( event )  -- server handler, called on incoming connections
@@ -677,6 +692,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 			end
 		end
 		--vdebug("max connection check ok, accepting...")
+		-- luacheck: ignore 231/err
 		local client, err = server:accept()    -- try to accept; TODO: check err
 		while client do
 			if interface._connections >= cfg.MAX_CONNECTIONS then
@@ -688,7 +704,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 			interface._connections = interface._connections + 1  -- increase connection count
 			local clientinterface = handleclient( client, client_ip, client_port, interface, pattern, listener, sslctx )
 			--vdebug( "client id:", clientinterface, "startssl:", startssl )
-			if has_luasec and sslctx then
+			if has_luasec and startssl then
 				clientinterface:starttls(sslctx, true)
 			else
 				clientinterface:_start_session( true )
@@ -707,9 +723,9 @@ local function handleserver( server, addr, port, pattern, listener, sslctx )  --
 	return interface
 end
 
-local function addserver( addr, port, listener, pattern, sslctx, startssl )  -- TODO: check arguments
-	--vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil")
-	if sslctx and not has_luasec then
+local function listen(addr, port, listener, config)
+	config = config or {}
+	if config.sslctx and not has_luasec then
 		debug "fatal error: luasec not found"
 		return nil, "luasec not found"
 	end
@@ -718,11 +734,20 @@ local function addserver( addr, port, listener, pattern, sslctx, startssl )  --
 		debug( "creating server socket on "..addr.." port "..port.." failed:", err )
 		return nil, err
 	end
-	local interface = handleserver( server, addr, port, pattern, listener, sslctx, startssl )  -- new server handler
+	local interface = handleserver( server, addr, port, config.read_size, listener, config.tls_ctx, config.tls_direct)  -- new server handler
 	debug( "new server created with id:", tostring(interface))
 	return interface
 end
 
+local function addserver( addr, port, listener, pattern, sslctx )  -- TODO: check arguments
+	--vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil")
+	return listen( addr, port, listener, {
+		read_size = pattern,
+		tls_ctx = sslctx,
+		tls_direct = not not sslctx,
+	});
+end
+
 local function wrapclient( client, ip, port, listeners, pattern, sslctx, extra )
 	local interface = handleclient( client, ip, port, nil, pattern, listeners, sslctx, extra )
 	interface:_start_connection(sslctx)
@@ -756,6 +781,7 @@ local function addclient( addr, serverport, listener, pattern, sslctx, typ, extr
 	client:settimeout( 0 )  -- set nonblocking
 	local res, err = client:setpeername( addr, serverport )  -- connect
 	if res or ( err == "timeout" ) then
+		-- luacheck: ignore 211/port
 		local ip, port = client:getsockname( )
 		local interface = wrapclient( client, ip, serverport, listener, pattern, sslctx, extra )
 		debug( "new connection id:", interface.id )
@@ -883,6 +909,7 @@ return {
 	event_base = base,
 	addevent = newevent,
 	addserver = addserver,
+	listen = listen,
 	addclient = addclient,
 	wrapclient = wrapclient,
 	setquitting = setquitting,
diff --git a/net/server_select.lua b/net/server_select.lua
index deb8fe48..09c1c027 100644
--- a/net/server_select.lua
+++ b/net/server_select.lua
@@ -68,6 +68,7 @@ local idfalse
 local closeall
 local addsocket
 local addserver
+local listen
 local addtimer
 local getserver
 local wrapserver
@@ -123,7 +124,7 @@ local _maxsslhandshake
 
 _server = { } -- key = port, value = table; list of listening servers
 _readlist = { } -- array with sockets to read from
-_sendlist = { } -- arrary with sockets to write to
+_sendlist = { } -- array with sockets to write to
 _timerlist = { } -- array of timer functions
 _socketlist = { } -- key = socket, value = wrapped socket (handlers)
 _readtimes = { } -- key = handler, value = timestamp of last data reading
@@ -149,7 +150,7 @@ _checkinterval = 30 -- interval in secs to check idle clients
 _sendtimeout = 60000 -- allowed send idle time in secs
 _readtimeout = 14 * 60 -- allowed read idle time in secs
 
-local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to detemine whether this is Windows
+local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to determine whether this is Windows
 _maxfd = (is_windows and math.huge) or luasocket._SETSIZE or 1024 -- max fd number, limit to 1024 by default to prevent glibc buffer overflow, but not on Windows
 _maxselectlen = luasocket._SETSIZE or 1024 -- But this still applies on Windows
 
@@ -157,7 +158,7 @@ _maxsslhandshake = 30 -- max handshake round-trips
 
 ----------------------------------// PRIVATE //--
 
-wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- this function wraps a server -- FIXME Make sure FD < _maxfd
+wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx, ssldirect ) -- this function wraps a server -- FIXME Make sure FD < _maxfd
 
 	if socket:getfd() >= _maxfd then
 		out_error("server.lua: Disallowed FD number: "..socket:getfd())
@@ -183,6 +184,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 	handler.sslctx = function( )
 		return sslctx
 	end
+	handler.hosts = {} -- sni
 	handler.remove = function( )
 		connections = connections - 1
 		if handler then
@@ -244,13 +246,13 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 		local client, err = accept( socket )	-- try to accept
 		if client then
 			local ip, clientport = client:getpeername( )
-			local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx ) -- wrap new client socket
+			local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx, ssldirect ) -- wrap new client socket
 			if err then -- error while wrapping ssl socket
 				return false
 			end
 			connections = connections + 1
 			out_put( "server.lua: accepted new client connection from ", tostring(ip), ":", tostring(clientport), " to ", tostring(serverport))
-			if dispatch and not sslctx then -- SSL connections will notify onconnect when handshake completes
+			if dispatch and not ssldirect then -- SSL connections will notify onconnect when handshake completes
 				return dispatch( handler );
 			end
 			return;
@@ -264,7 +266,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t
 	return handler
 end
 
-wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx, extra ) -- this function wraps a client to a handler object
+wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx, ssldirect, extra ) -- this function wraps a client to a handler object
 
 	if socket:getfd() >= _maxfd then
 		out_error("server.lua: Disallowed FD number: "..socket:getfd()) -- PROTIP: Switch to libevent
@@ -287,6 +289,8 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 
 	local ssl
 
+	local pending
+
 	local dispatch = listeners.onincoming
 	local status = listeners.onstatus
 	local disconnect = listeners.ondisconnect
@@ -341,6 +345,9 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 			listeners.onattach(self, data)
 		end
 	end
+	handler._setpending = function( )
+		pending = true
+	end
 	handler.getstats = function( )
 		return readtraffic, sendtraffic
 	end
@@ -377,7 +384,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 		_readlistlen = removesocket( _readlist, socket, _readlistlen )
 		_readtimes[ handler ] = nil
 		if bufferqueuelen ~= 0 then
-			handler.sendbuffer() -- Try now to send any outstanding data
+			handler:sendbuffer() -- Try now to send any outstanding data
 			if bufferqueuelen ~= 0 then -- Still not empty, so we'll try again later
 				if handler then
 					handler.write = nil -- ... but no further writing allowed
@@ -429,9 +436,8 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 		bufferlen = bufferlen + #data
 		if bufferlen > maxsendlen then
 			_closelist[ handler ] = "send buffer exceeded"	 -- cannot close the client at the moment, have to wait to the end of the cycle
-			handler.write = idfalse -- don't write anymore
 			return false
-		elseif socket and not _sendlist[ socket ] then
+		elseif not nosend and socket and not _sendlist[ socket ] then
 			_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
 		end
 		bufferqueuelen = bufferqueuelen + 1
@@ -461,49 +467,55 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 		maxreadlen = readlen or maxreadlen
 		return bufferlen, maxreadlen, maxsendlen
 	end
-	--TODO: Deprecate
 	handler.lock_read = function (self, switch)
+		out_error( "server.lua, lock_read() is deprecated, use pause() and resume()" )
 		if switch == true then
-			local tmp = _readlistlen
-			_readlistlen = removesocket( _readlist, socket, _readlistlen )
-			_readtimes[ handler ] = nil
-			if _readlistlen ~= tmp then
-				noread = true
-			end
+			return self:pause()
 		elseif switch == false then
-			if noread then
-				noread = false
-				_readlistlen = addsocket(_readlist, socket, _readlistlen)
-				_readtimes[ handler ] = _currenttime
-			end
+			return self:resume()
 		end
 		return noread
 	end
 	handler.pause = function (self)
-		return self:lock_read(true);
+		local tmp = _readlistlen
+		_readlistlen = removesocket( _readlist, socket, _readlistlen )
+		_readtimes[ handler ] = nil
+		if _readlistlen ~= tmp then
+			noread = true
+		end
+		return noread;
 	end
 	handler.resume = function (self)
-		return self:lock_read(false);
+		if noread then
+			noread = false
+			_readlistlen = addsocket(_readlist, socket, _readlistlen)
+			_readtimes[ handler ] = _currenttime
+		end
+		return noread;
 	end
 	handler.lock = function( self, switch )
-		handler.lock_read (switch)
+		out_error( "server.lua, lock() is deprecated" )
+		handler.lock_read (self, switch)
 		if switch == true then
-			handler.write = idfalse
-			local tmp = _sendlistlen
-			_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
-			_writetimes[ handler ] = nil
-			if _sendlistlen ~= tmp then
-				nosend = true
-			end
+			handler.pause_writes (self)
 		elseif switch == false then
-			handler.write = write
-			if nosend then
-				nosend = false
-				write( "" )
-			end
+			handler.resume_writes (self)
 		end
 		return noread, nosend
 	end
+	handler.pause_writes = function (self)
+		local tmp = _sendlistlen
+		_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
+		_writetimes[ handler ] = nil
+		nosend = true
+	end
+	handler.resume_writes = function (self)
+		nosend = false
+		if bufferlen > 0 and socket then
+			_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
+		end
+	end
+
 	local _readbuffer = function( ) -- this function reads data
 		local buffer, err, part = receive( socket, pattern )	-- receive buffer with "pattern"
 		if not err or (err == "wantread" or err == "timeout") then -- received something
@@ -518,6 +530,12 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 			_readtraffic = _readtraffic + count
 			_readtimes[ handler ] = _currenttime
 			--out_put( "server.lua: read data '", buffer:gsub("[^%w%p ]", "."), "', error: ", err )
+			if pending then -- connection established
+				pending = nil
+				if listeners.onconnect then
+					listeners.onconnect(handler)
+				end
+			end
 			return dispatch( handler, buffer, err )
 		else	-- connections was closed or fatal error
 			out_put( "server.lua: client ", tostring(ip), ":", tostring(clientport), " read error: ", tostring(err) )
@@ -528,6 +546,12 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 	local _sendbuffer = function( ) -- this function sends data
 		local succ, err, byte, buffer, count;
 		if socket then
+			if pending then
+				pending = nil
+				if listeners.onconnect then
+					listeners.onconnect(handler);
+				end
+			end
 			buffer = table_concat( bufferqueue, "", 1, bufferqueuelen )
 			succ, err, byte = send( socket, buffer, 1, bufferlen )
 			count = ( succ or byte or 0 ) * STAT_UNIT
@@ -604,7 +628,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 						coroutine_yield( ) -- handshake not finished
 					end
 				end
-				err = "ssl handshake error: " .. ( err or "handshake too long" );
+				err = ( err or "handshake too long" );
 				out_put( "server.lua: ", err );
 				_ = handler and handler:force_close(err)
 				return false, err -- handshake failed
@@ -624,13 +648,18 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 			out_put( "server.lua: attempting to start tls on " .. tostring( socket ) )
 			local oldsocket, err = socket
 			socket, err = ssl_wrap( socket, sslctx )	-- wrap socket
+
 			if not socket then
 				out_put( "server.lua: error while starting tls on client: ", tostring(err or "unknown error") )
 				return nil, err -- fatal error
 			end
 
-			if socket.sni and self.servername then
-				socket:sni(self.servername);
+			if socket.sni then
+				if self.servername then
+					socket:sni(self.servername);
+				elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then
+					socket:sni(self.server().hosts, true);
+				end
 			end
 
 			socket:settimeout( 0 )
@@ -668,7 +697,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport
 	_socketlist[ socket ] = handler
 	_readlistlen = addsocket(_readlist, socket, _readlistlen)
 
-	if sslctx and has_luasec then
+	if sslctx and ssldirect and has_luasec then
 		out_put "server.lua: auto-starting ssl negotiation..."
 		handler.autostart_ssl = true;
 		local ok, err = handler:starttls(sslctx);
@@ -723,7 +752,7 @@ local function link(sender, receiver, buffersize)
 	local sender_locked;
 	local _sendbuffer = receiver.sendbuffer;
 	function receiver.sendbuffer()
-		_sendbuffer();
+		_sendbuffer(receiver);
 		if sender_locked and receiver.bufferlen() < buffersize then
 			sender:lock_read(false); -- Unlock now
 			sender_locked = nil;
@@ -743,9 +772,13 @@ end
 
 ----------------------------------// PUBLIC //--
 
-addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server
+listen = function ( addr, port, listeners, config )
 	addr = addr or "*"
+	config = config or {}
 	local err
+	local sslctx = config.tls_ctx;
+	local ssldirect = config.tls_direct;
+	local pattern = config.read_size;
 	if type( listeners ) ~= "table" then
 		err = "invalid listener table"
 	elseif type ( addr ) ~= "string" then
@@ -766,7 +799,7 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function
 		out_error( "server.lua, [", addr, "]:", port, ": ", err )
 		return nil, err
 	end
-	local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx ) -- wrap new server socket
+	local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx, ssldirect ) -- wrap new server socket
 	if not handler then
 		server:close( )
 		return nil, err
@@ -779,6 +812,14 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function
 	return handler
 end
 
+addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server
+	return listen(addr, port, listeners, {
+		read_size = pattern;
+		tls_ctx = sslctx;
+		tls_direct = sslctx and true or false;
+	});
+end
+
 getserver = function ( addr, port )
 	return _server[ addr..":"..port ];
 end
@@ -921,7 +962,7 @@ loop = function(once) -- this is the main loop of the program
 		for _, socket in ipairs( read ) do -- receive data
 			local handler = _socketlist[ socket ]
 			if handler then
-				handler.readbuffer( )
+				handler:readbuffer( )
 			else
 				closesocket( socket )
 				out_put "server.lua: found no handler and closed socket (readlist)" -- this can happen
@@ -930,7 +971,7 @@ loop = function(once) -- this is the main loop of the program
 		for _, socket in ipairs( write ) do -- send data waiting in writequeues
 			local handler = _socketlist[ socket ]
 			if handler then
-				handler.sendbuffer( )
+				handler:sendbuffer( )
 			else
 				closesocket( socket )
 				out_put "server.lua: found no handler and closed socket (writelist)"	-- this should not happen
@@ -987,21 +1028,13 @@ end
 --// EXPERIMENTAL //--
 
 local wrapclient = function( socket, ip, serverport, listeners, pattern, sslctx, extra )
-	local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx, extra)
+	local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx, sslctx, extra)
 	if not handler then return nil, err end
 	_socketlist[ socket ] = handler
 	if not sslctx then
+		handler._setpending()
 		_readlistlen = addsocket(_readlist, socket, _readlistlen)
 		_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
-		if listeners.onconnect then
-			-- When socket is writeable, call onconnect
-			local _sendbuffer = handler.sendbuffer;
-			handler.sendbuffer = function ()
-				handler.sendbuffer = _sendbuffer;
-				listeners.onconnect(handler);
-				return _sendbuffer(); -- Send any queued outgoing data
-			end
-		end
 	end
 	return handler, socket
 end
@@ -1123,6 +1156,7 @@ return {
 	stats = stats,
 	closeall = closeall,
 	addserver = addserver,
+	listen = listen,
 	getserver = getserver,
 	setlogger = setlogger,
 	getsettings = getsettings,
diff --git a/net/unbound.lua b/net/unbound.lua
new file mode 100644
index 00000000..49f220e1
--- /dev/null
+++ b/net/unbound.lua
@@ -0,0 +1,208 @@
+-- libunbound based net.adns replacement for Prosody IM
+-- Copyright (C) 2013-2015 Kim Alvefur
+--
+-- This file is MIT licensed.
+--
+-- luacheck: ignore prosody
+
+local setmetatable = setmetatable;
+local tostring = tostring;
+local t_concat = table.concat;
+local s_format = string.format;
+local s_lower = string.lower;
+local s_upper = string.upper;
+local noop = function() end;
+
+local log = require "util.logger".init("unbound");
+local net_server = require "net.server";
+local libunbound = require"lunbound";
+local promise = require"util.promise";
+
+local gettime = require"socket".gettime;
+local dns_utils = require"util.dns";
+local classes, types, errors = dns_utils.classes, dns_utils.types, dns_utils.errors;
+local parsers = dns_utils.parsers;
+
+local function add_defaults(conf)
+	if conf then
+		for option, default in pairs(libunbound.config) do
+			if conf[option] == nil then
+				conf[option] = default;
+			end
+		end
+	end
+	return conf;
+end
+
+local unbound_config;
+if prosody then
+	local config = require"core.configmanager";
+	unbound_config = add_defaults(config.get("*", "unbound"));
+	prosody.events.add_handler("config-reloaded", function()
+		unbound_config = add_defaults(config.get("*", "unbound"));
+	end);
+end
+-- Note: libunbound will default to using root hints if resolvconf is unset
+
+local function connect_server(unbound, server)
+	return server.watchfd(unbound, function ()
+		unbound:process()
+	end);
+end
+
+local unbound = libunbound.new(unbound_config);
+
+local server_conn = connect_server(unbound, net_server);
+
+local answer_mt = {
+	__tostring = function(self)
+		if self._string then return self._string end
+		local h = s_format("Status: %s", errors[self.status]);
+		if self.secure then
+			h = h .. ", Secure";
+		elseif self.bogus then
+			h = h .. s_format(", Bogus: %s", self.bogus);
+		end
+		local t = { h };
+		for i = 1, #self do
+			t[i+1]=self.qname.."\t"..classes[self.qclass].."\t"..types[self.qtype].."\t"..tostring(self[i]);
+		end
+		local _string = t_concat(t, "\n");
+		self._string = _string;
+		return _string;
+	end;
+};
+
+local waiting_queries = {};
+
+local function prep_answer(a)
+	if not a then return end
+	local status = errors[a.rcode];
+	local qclass = classes[a.qclass];
+	local qtype = types[a.qtype];
+	a.status, a.class, a.type = status, qclass, qtype;
+
+	local t = s_lower(qtype);
+	local rr_mt = { __index = a, __tostring = function(self) return tostring(self[t]) end };
+	local parser = parsers[qtype];
+	for i = 1, #a do
+		if a.bogus then
+			-- Discard bogus data
+			a[i] = nil;
+		else
+			a[i] = setmetatable({[t] = parser(a[i])}, rr_mt);
+		end
+	end
+	return setmetatable(a, answer_mt);
+end
+
+local function lookup(callback, qname, qtype, qclass)
+	qtype = qtype and s_upper(qtype) or "A";
+	qclass = qclass and s_upper(qclass) or "IN";
+	local ntype, nclass = types[qtype], classes[qclass];
+	local startedat = gettime();
+	local ret;
+	local function callback_wrapper(a, err)
+		local gotdataat = gettime();
+		waiting_queries[ret] = nil;
+		if a then
+			prep_answer(a);
+			log("debug", "Results for %s %s %s: %s (%s, %f sec)", qname, qclass, qtype, a.rcode == 0 and (#a .. " items") or a.status,
+				a.secure and "Secure" or a.bogus or "Insecure", gotdataat - startedat); -- Insecure as in unsigned
+		else
+			log("error", "Results for %s %s %s: %s", qname, qclass, qtype, tostring(err));
+		end
+		local ok, cerr = pcall(callback, a, err);
+		if not ok then log("error", "Error in callback: %s", cerr); end
+	end
+	log("debug", "Resolve %s %s %s", qname, qclass, qtype);
+	local err;
+	ret, err = unbound:resolve_async(callback_wrapper, qname, ntype, nclass);
+	if ret then
+		waiting_queries[ret] = callback;
+	else
+		log("warn", err);
+	end
+	return ret, err;
+end
+
+local function lookup_sync(qname, qtype, qclass)
+	qtype = qtype and s_upper(qtype) or "A";
+	qclass = qclass and s_upper(qclass) or "IN";
+	local ntype, nclass = types[qtype], classes[qclass];
+	local a, err = unbound:resolve(qname, ntype, nclass);
+	if not a then return a, err; end
+	return prep_answer(a);
+end
+
+local function cancel(id)
+	local cb = waiting_queries[id];
+	unbound:cancel(id);
+	if cb then
+		cb(nil, "canceled");
+		waiting_queries[id] = nil;
+	end
+	return true;
+end
+
+-- Reinitiate libunbound context, drops cache
+local function purge()
+	for id in pairs(waiting_queries) do cancel(id); end
+	if server_conn then server_conn:close(); end
+	unbound = libunbound.new(unbound_config);
+	server_conn = connect_server(unbound, net_server);
+	return true;
+end
+
+local function not_implemented()
+	error "not implemented";
+end
+-- Public API
+local _M = {
+	lookup = lookup;
+	cancel = cancel;
+	new_async_socket = not_implemented;
+	dns = {
+		lookup = lookup_sync;
+		cancel = cancel;
+		cache = noop;
+		socket_wrapper_set = noop;
+		settimeout = noop;
+		query = noop;
+		purge = purge;
+		random = noop;
+		peek = noop;
+
+		types = types;
+		classes = classes;
+	};
+};
+
+local function lookup_promise(_, qname, qtype, qclass)
+	return promise.new(function (resolve, reject)
+		local function callback(answer, err)
+			if err then
+				return reject(err);
+			else
+				return resolve(answer);
+			end
+		end
+		local ret, err = lookup(callback, qname, qtype, qclass)
+		if not ret then reject(err); end
+	end);
+end
+
+local wrapper = {
+	lookup = function (_, callback, qname, qtype, qclass)
+		return lookup(callback, qname, qtype, qclass)
+	end;
+	lookup_promise = lookup_promise;
+	_resolver = {
+		settimeout = function () end;
+		closeall = function () end;
+	};
+}
+
+function _M.resolver() return wrapper; end
+
+return _M;
diff --git a/net/websocket.lua b/net/websocket.lua
index 469c6a58..193cd556 100644
--- a/net/websocket.lua
+++ b/net/websocket.lua
@@ -23,6 +23,7 @@ local websockets = {};
 local websocket_listeners = {};
 function websocket_listeners.ondisconnect(conn, err)
 	local s = websockets[conn];
+	if not s then return; end
 	websockets[conn] = nil;
 	if s.close_timer then
 		timer.stop(s.close_timer);
@@ -113,7 +114,7 @@ function websocket_listeners.onincoming(conn, buffer, err) -- luacheck: ignore 2
 				frame.MASK = true; -- RFC 6455 6.1.5: If the data is being sent by the client, the frame(s) MUST be masked
 				conn:write(frames.build(frame));
 			elseif frame.opcode == 0xA then -- Pong frame
-				log("debug", "Received unexpected pong frame: " .. tostring(frame.data));
+				log("debug", "Received unexpected pong frame: %s", frame.data);
 			else
 				return fail(s, 1002, "Reserved opcode");
 			end
@@ -131,7 +132,7 @@ end
 function websocket_methods:close(code, reason)
 	if self.readyState < 2 then
 		code = code or 1000;
-		log("debug", "closing WebSocket with code %i: %s" , code , tostring(reason));
+		log("debug", "closing WebSocket with code %i: %s" , code , reason);
 		self.readyState = 2;
 		local conn = self.conn;
 		conn:write(frames.build_close(code, reason, true));
@@ -245,7 +246,7 @@ local function connect(url, ex, listeners)
 		   or (protocol and not protocol[r.headers["sec-websocket-protocol"]])
 		   then
 			s.readyState = 3;
-			log("warn", "WebSocket connection to %s failed: %s", url, tostring(b));
+			log("warn", "WebSocket connection to %s failed: %s", url, b);
 			if s.onerror then s:onerror("connecting-failed"); end
 			return;
 		end
diff --git a/net/websocket/frames.lua b/net/websocket/frames.lua
index 1d0ac06f..03ce21a8 100644
--- a/net/websocket/frames.lua
+++ b/net/websocket/frames.lua
@@ -9,8 +9,7 @@
 local softreq = require "util.dependencies".softreq;
 local random_bytes = require "util.random".bytes;
 
-local bit = assert(softreq"bit" or softreq"bit32",
-	"No bit module found. See https://prosody.im/doc/depends#bitop");
+local bit = require "util.bitcompat";
 local band = bit.band;
 local bor = bit.bor;
 local lshift = bit.lshift;
@@ -19,8 +18,8 @@ local sbit = require "util.strbitop";
 local sxor = sbit.sxor;
 
 local s_char= string.char;
-local s_pack = string.pack; -- luacheck: ignore 143
-local s_unpack = string.unpack; -- luacheck: ignore 143
+local s_pack = string.pack;
+local s_unpack = string.unpack;
 
 if not s_pack and softreq"struct" then
 	s_pack = softreq"struct".pack;
diff --git a/plugins/adhoc/adhoc.lib.lua b/plugins/adhoc/adhoc.lib.lua
index 0b910299..0c61a636 100644
--- a/plugins/adhoc/adhoc.lib.lua
+++ b/plugins/adhoc/adhoc.lib.lua
@@ -21,7 +21,13 @@ local function _cmdtag(desc, status, sessionid, action)
 end
 
 function _M.new(name, node, handler, permission)
-	return { name = name, node = node, handler = handler, cmdtag = _cmdtag, permission = (permission or "user") };
+	if not permission then
+		error "adhoc.new() expects a permission argument, none given"
+	end
+	if permission == "user" then
+		error "the permission mode 'user' has been renamed 'any', please update your code"
+	end
+	return { name = name, node = node, handler = handler, cmdtag = _cmdtag, permission = permission };
 end
 
 function _M.handle_cmd(command, origin, stanza)
diff --git a/plugins/adhoc/mod_adhoc.lua b/plugins/adhoc/mod_adhoc.lua
index bf1775b4..188d05e8 100644
--- a/plugins/adhoc/mod_adhoc.lua
+++ b/plugins/adhoc/mod_adhoc.lua
@@ -8,7 +8,7 @@
 local it = require "util.iterators";
 local st = require "util.stanza";
 local is_admin = require "core.usermanager".is_admin;
-local jid_split = require "util.jid".split;
+local jid_host = require "util.jid".host;
 local adhoc_handle_cmd = module:require "adhoc".handle_cmd;
 local xmlns_cmd = "http://jabber.org/protocol/commands";
 local commands = {};
@@ -21,12 +21,12 @@ module:hook("host-disco-info-node", function (event)
 		local from = stanza.attr.from;
 		local privileged = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		local command = commands[node];
 		if (command.permission == "admin" and privileged)
 		    or (command.permission == "global_admin" and global_admin)
 		    or (command.permission == "local_user" and hostname == module.host)
-		    or (command.permission == "user") then
+		    or (command.permission == "any") then
 			reply:tag("identity", { name = command.name,
 			    category = "automation", type = "command-node" }):up();
 			reply:tag("feature", { var = xmlns_cmd }):up();
@@ -52,12 +52,12 @@ module:hook("host-disco-items-node", function (event)
 	local from = stanza.attr.from;
 	local admin = is_admin(from, stanza.attr.to);
 	local global_admin = is_admin(from);
-	local username, hostname = jid_split(from);
+	local hostname = jid_host(from);
 	for node, command in it.sorted_pairs(commands) do
 		if (command.permission == "admin" and admin)
 		    or (command.permission == "global_admin" and global_admin)
 		    or (command.permission == "local_user" and hostname == module.host)
-		    or (command.permission == "user") then
+		    or (command.permission == "any") then
 			reply:tag("item", { name = command.name,
 			    node = node, jid = module:get_host() });
 			reply:up();
@@ -74,7 +74,7 @@ module:hook("iq-set/host/"..xmlns_cmd..":command", function (event)
 		local from = stanza.attr.from;
 		local admin = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		if (command.permission == "admin" and not admin)
 		    or (command.permission == "global_admin" and not global_admin)
 		    or (command.permission == "local_user" and hostname ~= module.host) then
diff --git a/plugins/mod_admin_adhoc.lua b/plugins/mod_admin_adhoc.lua
index 37e77ab0..674b3339 100644
--- a/plugins/mod_admin_adhoc.lua
+++ b/plugins/mod_admin_adhoc.lua
@@ -59,7 +59,7 @@ local add_user_command_handler = adhoc_simple(add_user_layout, function(fields,
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return { status = "completed", error = { message = "Trying to add a user on " .. host .. " but command was sent to " .. module_host}};
 	end
@@ -94,7 +94,7 @@ local change_user_password_command_handler = adhoc_simple(change_user_password_l
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return {
 			status = "completed",
@@ -136,7 +136,7 @@ local delete_user_command_handler = adhoc_simple(delete_user_layout, function(fi
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and usermanager_delete_user(username, host) then
 			module:log("debug", "User %s has been deleted", aJID);
 			succeeded[#succeeded+1] = aJID;
@@ -180,7 +180,7 @@ local end_user_session_handler = adhoc_simple(end_user_session_layout, function(
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and disconnect_user(aJID) then
 			succeeded[#succeeded+1] = aJID;
 		else
@@ -212,7 +212,7 @@ local get_user_password_handler = adhoc_simple(get_user_password_layout, functio
 	if err then
 		return generate_error_message(err);
 	end
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	local accountjid;
 	local password;
 	if host ~= module_host then
@@ -243,7 +243,7 @@ local get_user_roster_handler = adhoc_simple(get_user_roster_layout, function(fi
 		return generate_error_message(err);
 	end
 
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	if host ~= module_host then
 		return { status = "completed", error = { message = "Tried to get roster for a user on " .. host .. " but command was sent to " .. module_host } };
 	elseif not usermanager_user_exists(user, host) then
@@ -392,6 +392,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		flags[#flags+1] = "authenticated";
 	end
+	if session.dialback_key then
+		flags[#flags+1] = "dialback";
+	end
+	if session.external_auth then
+		flags[#flags+1] = "SASL";
+	end
 	if session.secure then
 		flags[#flags+1] = "encrypted";
 	end
@@ -404,6 +410,12 @@ local function session_flags(session, line)
 	if session.ip and session.ip:match(":") then
 		flags[#flags+1] = "IPv6";
 	end
+	if session.incoming and session.outgoing then
+		flags[#flags+1] = "bidi";
+	elseif session.is_bidi or session.bidi_session then
+		flags[#flags+1] = "bidi";
+	end
+
 	line[#line+1] = "("..t_concat(flags, ", ")..")";
 
 	return t_concat(line, " ");
diff --git a/plugins/mod_admin_shell.lua b/plugins/mod_admin_shell.lua
new file mode 100644
index 00000000..5b4c2a61
--- /dev/null
+++ b/plugins/mod_admin_shell.lua
@@ -0,0 +1,1665 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+-- luacheck: ignore 212/self
+
+module:set_global();
+module:depends("admin_socket");
+
+local hostmanager = require "core.hostmanager";
+local modulemanager = require "core.modulemanager";
+local s2smanager = require "core.s2smanager";
+local portmanager = require "core.portmanager";
+local helpers = require "util.helpers";
+local server = require "net.server";
+local st = require "util.stanza";
+
+local _G = _G;
+
+local prosody = _G.prosody;
+
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
+local iterators = require "util.iterators";
+local keys, values = iterators.keys, iterators.values;
+local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
+local set, array = require "util.set", require "util.array";
+local cert_verify_identity = require "util.x509".verify_identity;
+local envload = require "util.envload".envload;
+local envloadfile = require "util.envload".envloadfile;
+local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local serialization = require "util.serialization";
+local serialize_config = serialization.new ({ fatal = false, unquoted = true});
+local time = require "util.time";
+
+local format_number = require "util.human.units".format;
+
+local commands = module:shared("commands")
+local def_env = module:shared("env");
+local default_env_mt = { __index = def_env };
+
+local function redirect_output(target, session)
+	local env = setmetatable({ print = session.print }, { __index = function (_, k) return rawget(target, k); end });
+	env.dofile = function(name)
+		local f, err = envloadfile(name, env);
+		if not f then return f, err; end
+		return f();
+	end;
+	return env;
+end
+
+console = {};
+
+local runner_callbacks = {};
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[shell]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+local function send_repl_output(session, line)
+	return session.send(st.stanza("repl-output"):text(tostring(line)));
+end
+
+function console:new_session(admin_session)
+	local session = {
+			send = function (t)
+				return send_repl_output(admin_session, t);
+			end;
+			print = function (...)
+				local t = {};
+				for i=1,select("#", ...) do
+					t[i] = tostring(select(i, ...));
+				end
+				return send_repl_output(admin_session, table.concat(t, "\t"));
+			end;
+			serialize = tostring;
+			disconnect = function () admin_session:close(); end;
+			};
+	session.env = setmetatable({}, default_env_mt);
+
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+	end, runner_callbacks, session);
+
+	-- Load up environment with helper objects
+	for name, t in pairs(def_env) do
+		if type(t) == "table" then
+			session.env[name] = setmetatable({ session = session }, { __index = t });
+		end
+	end
+
+	session.env.output:configure();
+
+	return session;
+end
+
+local function handle_line(event)
+	local session = event.origin.shell_session;
+	if not session then
+		session = console:new_session(event.origin);
+		event.origin.shell_session = session;
+	end
+	local line = event.stanza:get_text();
+	local useglobalenv;
+
+	local result = st.stanza("repl-result");
+
+	if line:match("^>") then
+		line = line:gsub("^>", "");
+		useglobalenv = true;
+	else
+		local command = line:match("^%w+") or line:match("%p");
+		if commands[command] then
+			commands[command](session, line);
+			event.origin.send(result);
+			return;
+		end
+	end
+
+	session.env._ = line;
+
+	if not useglobalenv and commands[line:lower()] then
+		commands[line:lower()](session, line);
+		event.origin.send(result);
+		return;
+	end
+
+	local chunkname = "=console";
+	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
+	-- luacheck: ignore 311/err
+	local chunk, err = envload("return "..line, chunkname, env);
+	if not chunk then
+		chunk, err = envload(line, chunkname, env);
+		if not chunk then
+			err = err:gsub("^%[string .-%]:%d+: ", "");
+			err = err:gsub("^:%d+: ", "");
+			err = err:gsub("'<eof>'", "the end of the line");
+			result.attr.type = "error";
+			result:text("Sorry, I couldn't understand that... "..err);
+			event.origin.send(result);
+			return;
+		end
+	end
+
+	local taskok, message = chunk();
+
+	if not message then
+		if type(taskok) ~= "string" and useglobalenv then
+			taskok = session.serialize(taskok);
+		end
+		result:text("Result: "..tostring(taskok));
+	elseif (not taskok) and message then
+		result.attr.type = "error";
+		result:text("Error: "..tostring(message));
+	else
+		result:text("OK: "..tostring(message));
+	end
+
+	event.origin.send(result);
+end
+
+module:hook("admin/repl-input", function (event)
+	local ok, err = pcall(handle_line, event);
+	if not ok then
+		event.origin.send(st.stanza("repl-result", { type = "error" }):text(err));
+	end
+end);
+
+-- Console commands --
+-- These are simple commands, not valid standalone in Lua
+
+function commands.help(session, data)
+	local print = session.print;
+	local section = data:match("^help (%w+)");
+	if not section then
+		print [[Commands are divided into multiple sections. For help on a particular section, ]]
+		print [[type: help SECTION (for example, 'help c2s'). Sections are: ]]
+		print [[]]
+		print [[c2s - Commands to manage local client-to-server sessions]]
+		print [[s2s - Commands to manage sessions between this server and others]]
+		print [[http - Commands to inspect HTTP services]] -- XXX plural but there is only one so far
+		print [[module - Commands to load/reload/unload modules/plugins]]
+		print [[host - Commands to activate, deactivate and list virtual hosts]]
+		print [[user - Commands to create and delete users, and change their passwords]]
+		print [[server - Uptime, version, shutting down, etc.]]
+		print [[port - Commands to manage ports the server is listening on]]
+		print [[dns - Commands to manage and inspect the internal DNS resolver]]
+		print [[xmpp - Commands for sending XMPP stanzas]]
+		print [[debug - Commands for debugging the server]]
+		print [[config - Reloading the configuration, etc.]]
+		print [[console - Help regarding the console itself]]
+	elseif section == "c2s" then
+		print [[c2s:show(jid) - Show all client sessions with the specified JID (or all if no JID given)]]
+		print [[c2s:show_insecure() - Show all unencrypted client connections]]
+		print [[c2s:show_secure() - Show all encrypted client connections]]
+		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
+		print [[c2s:count() - Count sessions without listing them]]
+		print [[c2s:close(jid) - Close all sessions for the specified JID]]
+		print [[c2s:closeall() - Close all active c2s connections ]]
+	elseif section == "s2s" then
+		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
+		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
+		print [[s2s:close(from, to) - Close a connection from one domain to another]]
+		print [[s2s:closeall(host) - Close all the incoming/outgoing s2s sessions to specified host]]
+	elseif section == "http" then
+		print [[http:list(hosts) - Show HTTP endpoints]]
+	elseif section == "module" then
+		print [[module:load(module, host) - Load the specified module on the specified host (or all hosts if none given)]]
+		print [[module:reload(module, host) - The same, but unloads and loads the module (saving state if the module supports it)]]
+		print [[module:unload(module, host) - The same, but just unloads the module from memory]]
+		print [[module:list(host) - List the modules loaded on the specified host]]
+	elseif section == "host" then
+		print [[host:activate(hostname) - Activates the specified host]]
+		print [[host:deactivate(hostname) - Disconnects all clients on this host and deactivates]]
+		print [[host:list() - List the currently-activated hosts]]
+	elseif section == "user" then
+		print [[user:create(jid, password) - Create the specified user account]]
+		print [[user:password(jid, password) - Set the password for the specified user account]]
+		print [[user:delete(jid) - Permanently remove the specified user account]]
+		print [[user:list(hostname, pattern) - List users on the specified host, optionally filtering with a pattern]]
+	elseif section == "server" then
+		print [[server:version() - Show the server's version number]]
+		print [[server:uptime() - Show how long the server has been running]]
+		print [[server:memory() - Show details about the server's memory usage]]
+		print [[server:shutdown(reason) - Shut down the server, with an optional reason to be broadcast to all connections]]
+	elseif section == "port" then
+		print [[port:list() - Lists all network ports prosody currently listens on]]
+		print [[port:close(port, interface) - Close a port]]
+	elseif section == "dns" then
+		print [[dns:lookup(name, type, class) - Do a DNS lookup]]
+		print [[dns:addnameserver(nameserver) - Add a nameserver to the list]]
+		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
+		print [[dns:purge() - Clear the DNS cache]]
+		print [[dns:cache() - Show cached records]]
+	elseif section == "xmpp" then
+		print [[xmpp:ping(localhost, remotehost) -- Sends a ping to a remote XMPP server and reports the response]]
+	elseif section == "config" then
+		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
+		print [[config:get([host,] option) - Show the value of a config option.]]
+	elseif section == "stats" then -- luacheck: ignore 542
+		-- TODO describe how stats:show() works
+	elseif section == "debug" then
+		print [[debug:logevents(host) - Enable logging of fired events on host]]
+		print [[debug:events(host, event) - Show registered event handlers]]
+		print [[debug:timers() - Show information about scheduled timers]]
+	elseif section == "console" then
+		print [[Hey! Welcome to Prosody's admin console.]]
+		print [[First thing, if you're ever wondering how to get out, simply type 'quit'.]]
+		print [[Secondly, note that we don't support the full telnet protocol yet (it's coming)]]
+		print [[so you may have trouble using the arrow keys, etc. depending on your system.]]
+		print [[]]
+		print [[For now we offer a couple of handy shortcuts:]]
+		print [[!! - Repeat the last command]]
+		print [[!old!new! - repeat the last command, but with 'old' replaced by 'new']]
+		print [[]]
+		print [[For those well-versed in Prosody's internals, or taking instruction from those who are,]]
+		print [[you can prefix a command with > to escape the console sandbox, and access everything in]]
+		print [[the running server. Great fun, but be careful not to break anything :)]]
+	end
+end
+
+-- Session environment --
+-- Anything in def_env will be accessible within the session as a global variable
+
+--luacheck: ignore 212/self
+local serialize_defaults = module:get_option("console_prettyprint_settings", { fatal = false, unquoted = true, maxdepth = 2})
+
+def_env.output = {};
+function def_env.output:configure(opts)
+	if type(opts) ~= "table" then
+		opts = { preset = opts };
+	end
+	if not opts.fallback then
+		-- XXX Error message passed to fallback is lost, does it matter?
+		opts.fallback = tostring;
+	end
+	for k,v in pairs(serialize_defaults) do
+		if opts[k] == nil then
+			opts[k] = v;
+		end
+	end
+	self.session.serialize = serialization.new(opts);
+end
+
+def_env.server = {};
+
+function def_env.server:insane_reload()
+	prosody.unlock_globals();
+	dofile "prosody"
+	prosody = _G.prosody;
+	return true, "Server reloaded";
+end
+
+function def_env.server:version()
+	return true, tostring(prosody.version or "unknown");
+end
+
+function def_env.server:uptime()
+	local t = os.time()-prosody.start_time;
+	local seconds = t%60;
+	t = (t - seconds)/60;
+	local minutes = t%60;
+	t = (t - minutes)/60;
+	local hours = t%24;
+	t = (t - hours)/24;
+	local days = t;
+	return true, string.format("This server has been running for %d day%s, %d hour%s and %d minute%s (since %s)",
+		days, (days ~= 1 and "s") or "", hours, (hours ~= 1 and "s") or "",
+		minutes, (minutes ~= 1 and "s") or "", os.date("%c", prosody.start_time));
+end
+
+function def_env.server:shutdown(reason)
+	prosody.shutdown(reason);
+	return true, "Shutdown initiated";
+end
+
+local function human(kb)
+	return format_number(kb*1024, "B", "b");
+end
+
+function def_env.server:memory()
+	if not has_pposix or not pposix.meminfo then
+		return true, "Lua is using "..human(collectgarbage("count"));
+	end
+	local mem, lua_mem = pposix.meminfo(), collectgarbage("count");
+	local print = self.session.print;
+	print("Process: "..human((mem.allocated+mem.allocated_mmap)/1024));
+	print("   Used: "..human(mem.used/1024).." ("..human(lua_mem).." by Lua)");
+	print("   Free: "..human(mem.unused/1024).." ("..human(mem.returnable/1024).." returnable)");
+	return true, "OK";
+end
+
+def_env.module = {};
+
+local function get_hosts_set(hosts)
+	if type(hosts) == "table" then
+		if hosts[1] then
+			return set.new(hosts);
+		elseif hosts._items then
+			return hosts;
+		end
+	elseif type(hosts) == "string" then
+		return set.new { hosts };
+	elseif hosts == nil then
+		return set.new(array.collect(keys(prosody.hosts)));
+	end
+end
+
+-- Hosts with a module or all virtualhosts if no module given
+-- matching modules_enabled in the global section
+local function get_hosts_with_module(hosts, module)
+	local hosts_set = get_hosts_set(hosts)
+	/ function (host)
+			if module then
+				-- Module given, filter in hosts with this module loaded
+				if modulemanager.is_loaded(host, module) then
+					return host;
+				else
+					return nil;
+				end
+			end
+			if not hosts then
+				-- No hosts given, filter in VirtualHosts
+				if prosody.hosts[host].type == "local" then
+					return host;
+				else
+					return nil
+				end
+			end;
+			-- No module given, but hosts are, don't filter at all
+			return host;
+		end;
+	if module and modulemanager.get_module("*", module) then
+		hosts_set:add("*");
+	end
+	return hosts_set;
+end
+
+function def_env.module:load(name, hosts, config)
+	hosts = get_hosts_with_module(hosts);
+
+	-- Load the module for each host
+	local ok, err, count, mod = true, nil, 0;
+	for host in hosts do
+		if (not modulemanager.is_loaded(host, name)) then
+			mod, err = modulemanager.load(host, name, config);
+			if not mod then
+				ok = false;
+				if err == "global-module-already-loaded" then
+					if count > 0 then
+						ok, err, count = true, nil, 1;
+					end
+					break;
+				end
+				self.session.print(err or "Unknown error loading module");
+			else
+				count = count + 1;
+				self.session.print("Loaded for "..mod.module.host);
+			end
+		end
+	end
+
+	return ok, (ok and "Module loaded onto "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+function def_env.module:unload(name, hosts)
+	hosts = get_hosts_with_module(hosts, name);
+
+	-- Unload the module for each host
+	local ok, err, count = true, nil, 0;
+	for host in hosts do
+		if modulemanager.is_loaded(host, name) then
+			ok, err = modulemanager.unload(host, name);
+			if not ok then
+				ok = false;
+				self.session.print(err or "Unknown error unloading module");
+			else
+				count = count + 1;
+				self.session.print("Unloaded from "..host);
+			end
+		end
+	end
+	return ok, (ok and "Module unloaded from "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+local function _sort_hosts(a, b)
+	if a == "*" then return true
+	elseif b == "*" then return false
+	else return a:gsub("[^.]+", string.reverse):reverse() < b:gsub("[^.]+", string.reverse):reverse(); end
+end
+
+function def_env.module:reload(name, hosts)
+	hosts = array.collect(get_hosts_with_module(hosts, name)):sort(_sort_hosts)
+
+	-- Reload the module for each host
+	local ok, err, count = true, nil, 0;
+	for _, host in ipairs(hosts) do
+		if modulemanager.is_loaded(host, name) then
+			ok, err = modulemanager.reload(host, name);
+			if not ok then
+				ok = false;
+				self.session.print(err or "Unknown error reloading module");
+			else
+				count = count + 1;
+				if ok == nil then
+					ok = true;
+				end
+				self.session.print("Reloaded on "..host);
+			end
+		end
+	end
+	return ok, (ok and "Module reloaded on "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+function def_env.module:list(hosts)
+	hosts = array.collect(set.new({ not hosts and "*" or nil }) + get_hosts_set(hosts)):sort(_sort_hosts);
+
+	local print = self.session.print;
+	for _, host in ipairs(hosts) do
+		print((host == "*" and "Global" or host)..":");
+		local modules = array.collect(keys(modulemanager.get_modules(host) or {})):sort();
+		if #modules == 0 then
+			if prosody.hosts[host] then
+				print("    No modules loaded");
+			else
+				print("    Host not found");
+			end
+		else
+			for _, name in ipairs(modules) do
+				local status, status_text = modulemanager.get_module(host, name).module:get_status();
+				local status_summary = "";
+				if status == "warn" or status == "error" then
+					status_summary = (" (%s: %s)"):format(status, status_text);
+				end
+				print(("    %s%s"):format(name, status_summary));
+			end
+		end
+	end
+end
+
+def_env.config = {};
+function def_env.config:load(filename, format)
+	local config_load = require "core.configmanager".load;
+	local ok, err = config_load(filename, format);
+	if not ok then
+		return false, err or "Unknown error loading config";
+	end
+	return true, "Config loaded";
+end
+
+function def_env.config:get(host, key)
+	if key == nil then
+		host, key = "*", host;
+	end
+	local config_get = require "core.configmanager".get
+	return true, serialize_config(config_get(host, key));
+end
+
+function def_env.config:reload()
+	local ok, err = prosody.reload_config();
+	return ok, (ok and "Config reloaded (you may need to reload modules to take effect)") or tostring(err);
+end
+
+local function common_info(session, line)
+	if session.id then
+		line[#line+1] = "["..session.id.."]"
+	else
+		line[#line+1] = "["..session.type..(tostring(session):match("%x*$")).."]"
+	end
+end
+
+local function session_flags(session, line)
+	line = line or {};
+	common_info(session, line);
+	if session.type == "c2s" then
+		local status, priority = "unavailable", tostring(session.priority or "-");
+		if session.presence then
+			status = session.presence:get_child_text("show") or "available";
+		end
+		line[#line+1] = status.."("..priority..")";
+	end
+	if session.cert_identity_status == "valid" then
+		line[#line+1] = "(authenticated)";
+	end
+	if session.dialback_key then
+		line[#line+1] = "(dialback)";
+	end
+	if session.external_auth then
+		line[#line+1] = "(SASL)";
+	end
+	if session.secure then
+		line[#line+1] = "(encrypted)";
+	end
+	if session.compressed then
+		line[#line+1] = "(compressed)";
+	end
+	if session.smacks then
+		line[#line+1] = "(sm)";
+	end
+	if session.state then
+		if type(session.csi_counter) == "number" then
+			line[#line+1] = string.format("(csi:%s queue #%d)", session.state, session.csi_counter);
+		else
+			line[#line+1] = string.format("(csi:%s)", session.state);
+		end
+	end
+	if session.ip and session.ip:match(":") then
+		line[#line+1] = "(IPv6)";
+	end
+	if session.remote then
+		line[#line+1] = "(remote)";
+	end
+	if session.incoming and session.outgoing then
+		line[#line+1] = "(bidi)";
+	elseif session.is_bidi or session.bidi_session then
+		line[#line+1] = "(bidi)";
+	end
+	if session.bosh_version then
+		line[#line+1] = "(bosh)";
+	end
+	if session.websocket_request then
+		line[#line+1] = "(websocket)";
+	end
+	return table.concat(line, " ");
+end
+
+local function tls_info(session, line)
+	line = line or {};
+	common_info(session, line);
+	if session.secure then
+		local sock = session.conn and session.conn.socket and session.conn:socket();
+		if sock then
+			local info = sock.info and sock:info();
+			if info then
+				line[#line+1] = ("(%s with %s)"):format(info.protocol, info.cipher);
+			else
+				-- TLS session might not be ready yet
+				line[#line+1] = "(cipher info unavailable)";
+			end
+			if sock.getsniname then
+				local name = sock:getsniname();
+				if name then
+					line[#line+1] = ("(SNI:%q)"):format(name);
+				end
+			end
+			if sock.getalpn then
+				local proto = sock:getalpn();
+				if proto then
+					line[#line+1] = ("(ALPN:%q)"):format(proto);
+				end
+			end
+		end
+	else
+		line[#line+1] = "(insecure)";
+	end
+	return table.concat(line, " ");
+end
+
+def_env.c2s = {};
+
+local function get_jid(session)
+	if session.username then
+		return session.full_jid or jid_join(session.username, session.host, session.resource);
+	end
+
+	local conn = session.conn;
+	local ip = session.ip or "?";
+	local clientport = conn and conn:clientport() or "?";
+	local serverip = conn and conn.server and conn:server():ip() or "?";
+	local serverport = conn and conn:serverport() or "?"
+	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
+end
+
+local function get_c2s()
+	local c2s = array.collect(values(prosody.full_sessions));
+	c2s:append(array.collect(values(module:shared"/*/c2s/sessions")));
+	c2s:append(array.collect(values(module:shared"/*/bosh/sessions")));
+	c2s:unique();
+	return c2s;
+end
+
+local function show_c2s(callback)
+	get_c2s():sort(function(a, b)
+		if a.host == b.host then
+			if a.username == b.username then
+				return (a.resource or "") > (b.resource or "");
+			end
+			return (a.username or "") > (b.username or "");
+		end
+		return _sort_hosts(a.host or "", b.host or "");
+	end):map(function (session)
+		callback(get_jid(session), session)
+	end);
+end
+
+function def_env.c2s:count()
+	local c2s = get_c2s();
+	return true, "Total: "..  #c2s .." clients";
+end
+
+function def_env.c2s:show(match_jid, annotate)
+	local print, count = self.session.print, 0;
+	annotate = annotate or session_flags;
+	local curr_host = false;
+	show_c2s(function (jid, session)
+		if curr_host ~= session.host then
+			curr_host = session.host;
+			print(curr_host or "(not connected to any host yet)");
+		end
+		if (not match_jid) or jid:match(match_jid) then
+			count = count + 1;
+			print(annotate(session, { "  ", jid }));
+		end
+	end);
+	return true, "Total: "..count.." clients";
+end
+
+function def_env.c2s:show_insecure(match_jid)
+	local print, count = self.session.print, 0;
+	show_c2s(function (jid, session)
+		if ((not match_jid) or jid:match(match_jid)) and not session.secure then
+			count = count + 1;
+			print(jid);
+		end
+	end);
+	return true, "Total: "..count.." insecure client connections";
+end
+
+function def_env.c2s:show_secure(match_jid)
+	local print, count = self.session.print, 0;
+	show_c2s(function (jid, session)
+		if ((not match_jid) or jid:match(match_jid)) and session.secure then
+			count = count + 1;
+			print(jid);
+		end
+	end);
+	return true, "Total: "..count.." secure client connections";
+end
+
+function def_env.c2s:show_tls(match_jid)
+	return self:show(match_jid, tls_info);
+end
+
+local function build_reason(text, condition)
+	if text or condition then
+		return {
+			text = text,
+			condition = condition or "undefined-condition",
+		};
+	end
+end
+
+function def_env.c2s:close(match_jid, text, condition)
+	local count = 0;
+	show_c2s(function (jid, session)
+		if jid == match_jid or jid_bare(jid) == match_jid then
+			count = count + 1;
+			session:close(build_reason(text, condition));
+		end
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
+function def_env.c2s:closeall(text, condition)
+	local count = 0;
+	--luacheck: ignore 212/jid
+	show_c2s(function (jid, session)
+		count = count + 1;
+		session:close(build_reason(text, condition));
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
+
+def_env.s2s = {};
+function def_env.s2s:show(match_jid, annotate)
+	local print = self.session.print;
+	annotate = annotate or session_flags;
+
+	local count_in, count_out = 0,0;
+	local s2s_list = { };
+
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	for _, session in pairs(s2s_sessions) do
+		local remotehost, localhost, direction;
+		if session.direction == "outgoing" then
+			direction = "->";
+			count_out = count_out + 1;
+			remotehost, localhost = session.to_host or "?", session.from_host or "?";
+		else
+			direction = "<-";
+			count_in = count_in + 1;
+			remotehost, localhost = session.from_host or "?", session.to_host or "?";
+		end
+		local sess_lines = { l = localhost, r = remotehost,
+			annotate(session, { "", direction, remotehost or "?" })};
+
+		if (not match_jid) or remotehost:match(match_jid) or localhost:match(match_jid) then
+			table.insert(s2s_list, sess_lines);
+			-- luacheck: ignore 421/print
+			local print = function (s) table.insert(sess_lines, "        "..s); end
+			if session.sendq then
+				print("There are "..#session.sendq.." queued outgoing stanzas for this connection");
+			end
+			if session.type == "s2sout_unauthed" then
+				if session.connecting then
+					print("Connection not yet established");
+					if not session.srv_hosts then
+						if not session.conn then
+							print("We do not yet have a DNS answer for this host's SRV records");
+						else
+							print("This host has no SRV records, using A record instead");
+						end
+					elseif session.srv_choice then
+						print("We are on SRV record "..session.srv_choice.." of "..#session.srv_hosts);
+						local srv_choice = session.srv_hosts[session.srv_choice];
+						print("Using "..(srv_choice.target or ".")..":"..(srv_choice.port or 5269));
+					end
+				elseif session.notopen then
+					print("The <stream> has not yet been opened");
+				elseif not session.dialback_key then
+					print("Dialback has not been initiated yet");
+				elseif session.dialback_key then
+					print("Dialback has been requested, but no result received");
+				end
+			end
+			if session.type == "s2sin_unauthed" then
+				print("Connection not yet authenticated");
+			elseif session.type == "s2sin" then
+				for name in pairs(session.hosts) do
+					if name ~= session.from_host then
+						print("also hosts "..tostring(name));
+					end
+				end
+			end
+		end
+	end
+
+	-- Sort by local host, then remote host
+	table.sort(s2s_list, function(a,b)
+		if a.l == b.l then return _sort_hosts(a.r, b.r); end
+		return _sort_hosts(a.l, b.l);
+	end);
+	local lasthost;
+	for _, sess_lines in ipairs(s2s_list) do
+		if sess_lines.l ~= lasthost then print(sess_lines.l); lasthost=sess_lines.l end
+		for _, line in ipairs(sess_lines) do print(line); end
+	end
+	return true, "Total: "..count_out.." outgoing, "..count_in.." incoming connections";
+end
+
+function def_env.s2s:show_tls(match_jid)
+	return self:show(match_jid, tls_info);
+end
+
+local function print_subject(print, subject)
+	for _, entry in ipairs(subject) do
+		print(
+			("    %s: %q"):format(
+				entry.name or entry.oid,
+				entry.value:gsub("[\r\n%z%c]", " ")
+			)
+		);
+	end
+end
+
+-- As much as it pains me to use the 0-based depths that OpenSSL does,
+-- I think there's going to be more confusion among operators if we
+-- break from that.
+local function print_errors(print, errors)
+	for depth, t in pairs(errors) do
+		print(
+			("    %d: %s"):format(
+				depth-1,
+				table.concat(t, "\n|        ")
+			)
+		);
+	end
+end
+
+function def_env.s2s:showcert(domain)
+	local print = self.session.print;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	local domain_sessions = set.new(array.collect(values(s2s_sessions)))
+		/function(session) return (session.to_host == domain or session.from_host == domain) and session or nil; end;
+	local cert_set = {};
+	for session in domain_sessions do
+		local conn = session.conn;
+		conn = conn and conn:socket();
+		if not conn.getpeerchain then
+			if conn.dohandshake then
+				error("This version of LuaSec does not support certificate viewing");
+			end
+		else
+			local cert = conn:getpeercertificate();
+			if cert then
+				local certs = conn:getpeerchain();
+				local digest = cert:digest("sha1");
+				if not cert_set[digest] then
+					local chain_valid, chain_errors = conn:getpeerverification();
+					cert_set[digest] = {
+						{
+						  from = session.from_host,
+						  to = session.to_host,
+						  direction = session.direction
+						};
+						chain_valid = chain_valid;
+						chain_errors = chain_errors;
+						certs = certs;
+					};
+				else
+					table.insert(cert_set[digest], {
+						from = session.from_host,
+						to = session.to_host,
+						direction = session.direction
+					});
+				end
+			end
+		end
+	end
+	local domain_certs = array.collect(values(cert_set));
+	-- Phew. We now have a array of unique certificates presented by domain.
+	local n_certs = #domain_certs;
+
+	if n_certs == 0 then
+		return "No certificates found for "..domain;
+	end
+
+	local function _capitalize_and_colon(byte)
+		return string.upper(byte)..":";
+	end
+	local function pretty_fingerprint(hash)
+		return hash:gsub("..", _capitalize_and_colon):sub(1, -2);
+	end
+
+	for cert_info in values(domain_certs) do
+		local certs = cert_info.certs;
+		local cert = certs[1];
+		print("---")
+		print("Fingerprint (SHA1): "..pretty_fingerprint(cert:digest("sha1")));
+		print("");
+		local n_streams = #cert_info;
+		print("Currently used on "..n_streams.." stream"..(n_streams==1 and "" or "s")..":");
+		for _, stream in ipairs(cert_info) do
+			if stream.direction == "incoming" then
+				print("    "..stream.to.." <- "..stream.from);
+			else
+				print("    "..stream.from.." -> "..stream.to);
+			end
+		end
+		print("");
+		local chain_valid, errors = cert_info.chain_valid, cert_info.chain_errors;
+		local valid_identity = cert_verify_identity(domain, "xmpp-server", cert);
+		if chain_valid then
+			print("Trusted certificate: Yes");
+		else
+			print("Trusted certificate: No");
+			print_errors(print, errors);
+		end
+		print("");
+		print("Issuer: ");
+		print_subject(print, cert:issuer());
+		print("");
+		print("Valid for "..domain..": "..(valid_identity and "Yes" or "No"));
+		print("Subject:");
+		print_subject(print, cert:subject());
+	end
+	print("---");
+	return ("Showing "..n_certs.." certificate"
+		..(n_certs==1 and "" or "s")
+		.." presented by "..domain..".");
+end
+
+function def_env.s2s:close(from, to, text, condition)
+	local print, count = self.session.print, 0;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+
+	local match_id;
+	if from and not to then
+		match_id, from = from, nil;
+	elseif not to then
+		return false, "Syntax: s2s:close('from', 'to') - Closes all s2s sessions from 'from' to 'to'";
+	elseif from == to then
+		return false, "Both from and to are the same... you can't do that :)";
+	end
+
+	for _, session in pairs(s2s_sessions) do
+		local id = session.id or (session.type..tostring(session):match("[a-f0-9]+$"));
+		if (match_id and match_id == id)
+		or (session.from_host == from and session.to_host == to) then
+			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
+			(session.close or s2smanager.destroy_session)(session, build_reason(text, condition));
+			count = count + 1 ;
+		end
+	end
+	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
+end
+
+function def_env.s2s:closeall(host, text, condition)
+	local count = 0;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	for _,session in pairs(s2s_sessions) do
+		if not host or session.from_host == host or session.to_host == host then
+			session:close(build_reason(text, condition));
+			count = count + 1;
+		end
+	end
+	if count == 0 then return false, "No sessions to close.";
+	else return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s"); end
+end
+
+def_env.host = {}; def_env.hosts = def_env.host;
+
+function def_env.host:activate(hostname, config)
+	return hostmanager.activate(hostname, config);
+end
+function def_env.host:deactivate(hostname, reason)
+	return hostmanager.deactivate(hostname, reason);
+end
+
+function def_env.host:list()
+	local print = self.session.print;
+	local i = 0;
+	local type;
+	for host, host_session in iterators.sorted_pairs(prosody.hosts, _sort_hosts) do
+		i = i + 1;
+		type = host_session.type;
+		if type == "local" then
+			print(host);
+		else
+			type = module:context(host):get_option_string("component_module", type);
+			if type ~= "component" then
+				type = type .. " component";
+			end
+			print(("%s (%s)"):format(host, type));
+		end
+	end
+	return true, i.." hosts";
+end
+
+def_env.port = {};
+
+function def_env.port:list()
+	local print = self.session.print;
+	local services = portmanager.get_active_services().data;
+	local n_services, n_ports = 0, 0;
+	for service, interfaces in iterators.sorted_pairs(services) do
+		n_services = n_services + 1;
+		local ports_list = {};
+		for interface, ports in pairs(interfaces) do
+			for port in pairs(ports) do
+				table.insert(ports_list, "["..interface.."]:"..port);
+			end
+		end
+		n_ports = n_ports + #ports_list;
+		print(service..": "..table.concat(ports_list, ", "));
+	end
+	return true, n_services.." services listening on "..n_ports.." ports";
+end
+
+function def_env.port:close(close_port, close_interface)
+	close_port = assert(tonumber(close_port), "Invalid port number");
+	local n_closed = 0;
+	local services = portmanager.get_active_services().data;
+	for service, interfaces in pairs(services) do -- luacheck: ignore 213
+		for interface, ports in pairs(interfaces) do
+			if not close_interface or close_interface == interface then
+				if ports[close_port] then
+					self.session.print("Closing ["..interface.."]:"..close_port.."...");
+					local ok, err = portmanager.close(interface, close_port)
+					if not ok then
+						self.session.print("Failed to close "..interface.." "..close_port..": "..err);
+					else
+						n_closed = n_closed + 1;
+					end
+				end
+			end
+		end
+	end
+	return true, "Closed "..n_closed.." ports";
+end
+
+def_env.muc = {};
+
+local console_room_mt = {
+	__index = function (self, k) return self.room[k]; end;
+	__tostring = function (self)
+		return "MUC room <"..self.room.jid..">";
+	end;
+};
+
+local function check_muc(jid)
+	local room_name, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not prosody.hosts[host].modules.muc then
+		return nil, "Host '"..host.."' is not a MUC service";
+	end
+	return room_name, host;
+end
+
+function def_env.muc:create(room_jid, config)
+	local room_name, host = check_muc(room_jid);
+	if not room_name then
+		return room_name, host;
+	end
+	if not room_name then return nil, host end
+	if config ~= nil and type(config) ~= "table" then return nil, "Config must be a table"; end
+	if prosody.hosts[host].modules.muc.get_room_from_jid(room_jid) then return nil, "Room exists already" end
+	return prosody.hosts[host].modules.muc.create_room(room_jid, config);
+end
+
+function def_env.muc:room(room_jid)
+	local room_name, host = check_muc(room_jid);
+	if not room_name then
+		return room_name, host;
+	end
+	local room_obj = prosody.hosts[host].modules.muc.get_room_from_jid(room_jid);
+	if not room_obj then
+		return nil, "No such room: "..room_jid;
+	end
+	return setmetatable({ room = room_obj }, console_room_mt);
+end
+
+function def_env.muc:list(host)
+	local host_session = prosody.hosts[host];
+	if not host_session or not host_session.modules.muc then
+		return nil, "Please supply the address of a local MUC component";
+	end
+	local print = self.session.print;
+	local c = 0;
+	for room in host_session.modules.muc.each_room() do
+		print(room.jid);
+		c = c + 1;
+	end
+	return true, c.." rooms";
+end
+
+local um = require"core.usermanager";
+
+def_env.user = {};
+function def_env.user:create(jid, password)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif um.user_exists(username, host) then
+		return nil, "User exists";
+	end
+	local ok, err = um.create_user(username, password, host);
+	if ok then
+		return true, "User created";
+	else
+		return nil, "Could not create user: "..err;
+	end
+end
+
+function def_env.user:delete(jid)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not um.user_exists(username, host) then
+		return nil, "No such user";
+	end
+	local ok, err = um.delete_user(username, host);
+	if ok then
+		return true, "User deleted";
+	else
+		return nil, "Could not delete user: "..err;
+	end
+end
+
+function def_env.user:password(jid, password)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not um.user_exists(username, host) then
+		return nil, "No such user";
+	end
+	local ok, err = um.set_password(username, password, host, nil);
+	if ok then
+		return true, "User password changed";
+	else
+		return nil, "Could not change password for user: "..err;
+	end
+end
+
+function def_env.user:list(host, pat)
+	if not host then
+		return nil, "No host given";
+	elseif not prosody.hosts[host] then
+		return nil, "No such host";
+	end
+	local print = self.session.print;
+	local total, matches = 0, 0;
+	for user in um.users(host) do
+		if not pat or user:match(pat) then
+			print(user.."@"..host);
+			matches = matches + 1;
+		end
+		total = total + 1;
+	end
+	return true, "Showing "..(pat and (matches.." of ") or "all " )..total.." users";
+end
+
+def_env.xmpp = {};
+
+local new_id = require "util.id".medium;
+function def_env.xmpp:ping(localhost, remotehost, timeout)
+	localhost = select(2, jid_split(localhost));
+	remotehost = select(2, jid_split(remotehost));
+	if not localhost then
+		return nil, "Invalid sender hostname";
+	elseif not prosody.hosts[localhost] then
+		return nil, "No such local host";
+	end
+	if not remotehost then
+		return nil, "Invalid destination hostname";
+	elseif prosody.hosts[remotehost] then
+		return nil, "Both hosts are local";
+	end
+	local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()}
+			:tag("ping", {xmlns="urn:xmpp:ping"});
+	local time_start = time.now();
+	local ret, err = async.wait_for(module:context(localhost):send_iq(iq, nil, timeout));
+	if ret then
+		return true, ("pong from %s in %gs"):format(ret.stanza.attr.from, time.now() - time_start);
+	else
+		return false, tostring(err);
+	end
+end
+
+def_env.dns = {};
+local adns = require"net.adns";
+
+local function get_resolver(session)
+	local resolver = session.dns_resolver;
+	if not resolver then
+		resolver = adns.resolver();
+		session.dns_resolver = resolver;
+	end
+	return resolver;
+end
+
+function def_env.dns:lookup(name, typ, class)
+	local resolver = get_resolver(self.session);
+	local ret, err = async.wait_for(resolver:lookup_promise(name, typ, class));
+	if ret then
+		return true, ret;
+	elseif err then
+		return false, err;
+	end
+end
+
+function def_env.dns:addnameserver(...)
+	local resolver = get_resolver(self.session);
+	resolver._resolver:addnameserver(...)
+	return true
+end
+
+function def_env.dns:setnameserver(...)
+	local resolver = get_resolver(self.session);
+	resolver._resolver:setnameserver(...)
+	return true
+end
+
+function def_env.dns:purge()
+	local resolver = get_resolver(self.session);
+	resolver._resolver:purge()
+	return true
+end
+
+function def_env.dns:cache()
+	local resolver = get_resolver(self.session);
+	return true, "Cache:\n"..tostring(resolver._resolver.cache)
+end
+
+def_env.http = {};
+
+function def_env.http:list(hosts)
+	local print = self.session.print;
+
+	for host in get_hosts_set(hosts) do
+		local http_apps = modulemanager.get_items("http-provider", host);
+		if #http_apps > 0 then
+			local http_host = module:context(host):get_option_string("http_host");
+			print("HTTP endpoints on "..host..(http_host and (" (using "..http_host.."):") or ":"));
+			for _, provider in ipairs(http_apps) do
+				local url = module:context(host):http_url(provider.name, provider.default_path);
+				print("", url);
+			end
+			print("");
+		end
+	end
+
+	local default_host = module:get_option_string("http_default_host");
+	if not default_host then
+		print("HTTP requests to unknown hosts will return 404 Not Found");
+	else
+		print("HTTP requests to unknown hosts will be handled by "..default_host);
+	end
+	return true;
+end
+
+def_env.debug = {};
+
+function def_env.debug:logevents(host)
+	helpers.log_host_events(host);
+	return true;
+end
+
+function def_env.debug:events(host, event)
+	local events_obj;
+	if host and host ~= "*" then
+		if host == "http" then
+			events_obj = require "net.http.server"._events;
+		elseif not prosody.hosts[host] then
+			return false, "Unknown host: "..host;
+		else
+			events_obj = prosody.hosts[host].events;
+		end
+	else
+		events_obj = prosody.events;
+	end
+	return true, helpers.show_events(events_obj, event);
+end
+
+function def_env.debug:timers()
+	local print = self.session.print;
+	local add_task = require"util.timer".add_task;
+	local h, params = add_task.h, add_task.params;
+	local function normalize_time(t)
+		return t;
+	end
+	local function format_time(t)
+		return os.date("%F %T", math.floor(normalize_time(t)));
+	end
+	if h then
+		print("-- util.timer");
+	elseif server.timer then
+		print("-- net.server.timer");
+		h = server.timer.add_task.timers;
+		normalize_time = server.timer.to_absolute_time or normalize_time;
+	end
+	if h then
+		for i, id in ipairs(h.ids) do
+			local t, cb = h.priorities[i], h.items[id];
+			if not params then
+				local param = cb.param;
+				if param then
+					cb = param.callback;
+				else
+					cb = cb.timer_callback or cb;
+				end
+			elseif params[id] then
+				cb = params[id].callback or cb;
+			end
+			print(format_time(t), cb);
+		end
+	end
+	if server.event_base then
+		local count = 0;
+		for _, v in pairs(debug.getregistry()) do
+			if type(v) == "function" and v.callback and v.callback == add_task._on_timer then
+				count = count + 1;
+			end
+		end
+		print(count .. " libevent callbacks");
+	end
+	if h then
+		local next_time = h:peek();
+		if next_time then
+			return true, ("Next event at %s (in %.6fs)"):format(format_time(next_time), normalize_time(next_time) - time.now());
+		end
+	end
+	return true;
+end
+
+-- COMPAT: debug:timers() was timer:info() for some time in trunk
+def_env.timer = { info = def_env.debug.timers };
+
+def_env.stats = {};
+
+local short_units = {
+	seconds = "s",
+	bytes = "B",
+};
+
+local function format_stat(type, unit, value, ref_value)
+	ref_value = ref_value or value;
+	--do return tostring(value) end
+	if not unit then
+		if type == "duration" then
+			unit = "seconds"
+		elseif type == "size" then
+			unit = "bytes";
+		elseif type == "rate" then
+			unit = " events/sec"
+			if ref_value < 0.9 then
+				unit = "events/min"
+				value = value*60;
+				if ref_value < 0.6/60 then
+					unit = "events/h"
+					value = value*60;
+				end
+			end
+			return ("%.3g %s"):format(value, unit);
+		end
+	end
+	return format_number(value, short_units[unit] or unit or "", unit == "bytes" and 'b' or nil);
+end
+
+local stats_methods = {};
+function stats_methods:bounds(_lower, _upper)
+	for _, stat_info in ipairs(self) do
+		local data = stat_info[4];
+		if data then
+			local lower = _lower or data.min;
+			local upper = _upper or data.max;
+			local new_data = {
+				min = lower;
+				max = upper;
+				samples = {};
+				sample_count = 0;
+				count = data.count;
+				units = data.units;
+			};
+			local sum = 0;
+			for _, v in ipairs(data.samples) do
+				if v > upper then
+					break;
+				elseif v>=lower then
+					table.insert(new_data.samples, v);
+					sum = sum + v;
+				end
+			end
+			new_data.sample_count = #new_data.samples;
+			stat_info[4] = new_data;
+			stat_info[3] = sum/new_data.sample_count;
+		end
+	end
+	return self;
+end
+
+function stats_methods:trim(lower, upper)
+	upper = upper or (100-lower);
+	local statistics = require "util.statistics";
+	for _, stat_info in ipairs(self) do
+		-- Strip outliers
+		local data = stat_info[4];
+		if data then
+			local new_data = {
+				min = statistics.get_percentile(data, lower);
+				max = statistics.get_percentile(data, upper);
+				samples = {};
+				sample_count = 0;
+				count = data.count;
+				units = data.units;
+			};
+			local sum = 0;
+			for _, v in ipairs(data.samples) do
+				if v > new_data.max then
+					break;
+				elseif v>=new_data.min then
+					table.insert(new_data.samples, v);
+					sum = sum + v;
+				end
+			end
+			new_data.sample_count = #new_data.samples;
+			stat_info[4] = new_data;
+			stat_info[3] = sum/new_data.sample_count;
+		end
+	end
+	return self;
+end
+
+function stats_methods:max(upper)
+	return self:bounds(nil, upper);
+end
+
+function stats_methods:min(lower)
+	return self:bounds(lower, nil);
+end
+
+function stats_methods:summary()
+	local statistics = require "util.statistics";
+	for _, stat_info in ipairs(self) do
+		local type, value, data = stat_info[2], stat_info[3], stat_info[4];
+		if data and data.samples then
+			table.insert(stat_info.output, string.format("Count: %d (%d captured)",
+				data.count,
+				data.sample_count
+			));
+			table.insert(stat_info.output, string.format("Min: %s  Mean: %s  Max: %s",
+				format_stat(type, data.units, data.min),
+				format_stat(type, data.units, value),
+				format_stat(type, data.units, data.max)
+			));
+			table.insert(stat_info.output, string.format("Q1: %s  Median: %s  Q3: %s",
+				format_stat(type, data.units, statistics.get_percentile(data, 25)),
+				format_stat(type, data.units, statistics.get_percentile(data, 50)),
+				format_stat(type, data.units, statistics.get_percentile(data, 75))
+			));
+		end
+	end
+	return self;
+end
+
+function stats_methods:cfgraph()
+	for _, stat_info in ipairs(self) do
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local function print(s)
+			table.insert(stat_info.output, s);
+		end
+
+		if data and data.sample_count and data.sample_count > 0 then
+			local raw_histogram = require "util.statistics".get_histogram(data);
+
+			local graph_width, graph_height = 50, 10;
+			local eighth_chars = "   ▁▂▃▄▅▆▇█";
+
+			local range = data.max - data.min;
+
+			if range > 0 then
+				local x_scaling = #raw_histogram/graph_width;
+				local histogram = {};
+				for i = 1, graph_width do
+					histogram[i] = math.max(raw_histogram[i*x_scaling-1] or 0, raw_histogram[i*x_scaling] or 0);
+				end
+
+				print("");
+				print(("_"):rep(52)..format_stat(type, data.units, data.max));
+				for row = graph_height, 1, -1 do
+					local row_chars = {};
+					local min_eighths, max_eighths = 8, 0;
+					for i = 1, #histogram do
+						local char_eighths = math.ceil(math.max(math.min((graph_height/(data.max/histogram[i]))-(row-1), 1), 0)*8);
+						if char_eighths < min_eighths then
+							min_eighths = char_eighths;
+						end
+						if char_eighths > max_eighths then
+							max_eighths = char_eighths;
+						end
+						if char_eighths == 0 then
+							row_chars[i] = "-";
+						else
+							local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
+							row_chars[i] = char;
+						end
+					end
+					print(table.concat(row_chars).."|-"..format_stat(type, data.units, data.max/(graph_height/(row-0.5))));
+				end
+				print(("\\    "):rep(11));
+				local x_labels = {};
+				for i = 1, 11 do
+					local s = ("%-4s"):format((i-1)*10);
+					if #s > 4 then
+						s = s:sub(1, 3).."…";
+					end
+					x_labels[i] = s;
+				end
+				print(" "..table.concat(x_labels, " "));
+				local units = "%";
+				local margin = math.floor((graph_width-#units)/2);
+				print((" "):rep(margin)..units);
+			else
+				print("[range too small to graph]");
+			end
+			print("");
+		end
+	end
+	return self;
+end
+
+function stats_methods:histogram()
+	for _, stat_info in ipairs(self) do
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local function print(s)
+			table.insert(stat_info.output, s);
+		end
+
+		if not data then
+			print("[no data]");
+			return self;
+		elseif not data.sample_count then
+			print("[not a sampled metric type]");
+			return self;
+		end
+
+		local graph_width, graph_height = 50, 10;
+		local eighth_chars = "   ▁▂▃▄▅▆▇█";
+
+		local range = data.max - data.min;
+
+		if range > 0 then
+			local n_buckets = graph_width;
+
+			local histogram = {};
+			for i = 1, n_buckets do
+				histogram[i] = 0;
+			end
+			local max_bin_samples = 0;
+			for _, d in ipairs(data.samples) do
+				local bucket = math.floor(1+(n_buckets-1)/(range/(d-data.min)));
+				histogram[bucket] = histogram[bucket] + 1;
+				if histogram[bucket] > max_bin_samples then
+					max_bin_samples = histogram[bucket];
+				end
+			end
+
+			print("");
+			print(("_"):rep(52)..max_bin_samples);
+			for row = graph_height, 1, -1 do
+				local row_chars = {};
+				local min_eighths, max_eighths = 8, 0;
+				for i = 1, #histogram do
+					local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/histogram[i]))-(row-1), 1), 0)*8);
+					if char_eighths < min_eighths then
+						min_eighths = char_eighths;
+					end
+					if char_eighths > max_eighths then
+						max_eighths = char_eighths;
+					end
+					if char_eighths == 0 then
+						row_chars[i] = "-";
+					else
+						local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
+						row_chars[i] = char;
+					end
+				end
+				print(table.concat(row_chars).."|-"..math.ceil((max_bin_samples/graph_height)*(row-0.5)));
+			end
+			print(("\\    "):rep(11));
+			local x_labels = {};
+			for i = 1, 11 do
+				local s = ("%-4s"):format(format_stat(type, data.units, data.min+range*i/11, data.min):match("^%S+"));
+				if #s > 4 then
+					s = s:sub(1, 3).."…";
+				end
+				x_labels[i] = s;
+			end
+			print(" "..table.concat(x_labels, " "));
+			local units = format_stat(type, data.units, data.min):match("%s+(.+)$") or data.units or "";
+			local margin = math.floor((graph_width-#units)/2);
+			print((" "):rep(margin)..units);
+		else
+			print("[range too small to graph]");
+		end
+		print("");
+	end
+	return self;
+end
+
+local function stats_tostring(stats)
+	local print = stats.session.print;
+	for _, stat_info in ipairs(stats) do
+		if #stat_info.output > 0 then
+			print("\n#"..stat_info[1]);
+			print("");
+			for _, v in ipairs(stat_info.output) do
+				print(v);
+			end
+			print("");
+		else
+			print(("%-50s %s"):format(stat_info[1], format_stat(stat_info[2], (stat_info[4] or {}).units, stat_info[3])));
+		end
+	end
+	return #stats.." statistics displayed";
+end
+
+local stats_mt = {__index = stats_methods, __tostring = stats_tostring }
+local function new_stats_context(self)
+	return setmetatable({ session = self.session, stats = true }, stats_mt);
+end
+
+function def_env.stats:show(filter)
+	-- luacheck: ignore 211/changed
+	local stats, changed, extra = require "core.statsmanager".get_stats();
+	local available, displayed = 0, 0;
+	local displayed_stats = new_stats_context(self);
+	for name, value in iterators.sorted_pairs(stats) do
+		available = available + 1;
+		if not filter or name:match(filter) then
+			displayed = displayed + 1;
+			local type = name:match(":(%a+)$");
+			table.insert(displayed_stats, {
+				name, type, value, extra[name];
+				output = {};
+			});
+		end
+	end
+	return displayed_stats;
+end
+
+
+
+-------------
+
+function printbanner(session)
+	local option = module:get_option_string("console_banner", "full");
+	if option == "full" or option == "graphic" then
+		session.print [[
+                   ____                \   /     _
+                    |  _ \ _ __ ___  ___  _-_   __| |_   _
+                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
+                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
+                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
+                    A study in simplicity            |___/
+
+]]
+	end
+	if option == "short" or option == "full" then
+	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
+	session.print("You may find more help on using this console in our online documentation at ");
+	session.print("https://prosody.im/doc/console\n");
+	end
+	if option ~= "short" and option ~= "full" and option ~= "graphic" then
+		session.print(option);
+	end
+end
diff --git a/plugins/mod_admin_socket.lua b/plugins/mod_admin_socket.lua
new file mode 100644
index 00000000..b197adae
--- /dev/null
+++ b/plugins/mod_admin_socket.lua
@@ -0,0 +1,73 @@
+module:set_global();
+
+local have_unix, unix = pcall(require, "socket.unix");
+
+if not have_unix or type(unix) ~= "table" then
+	module:log_status("error", "LuaSocket unix socket support not available or incompatible, ensure it is up to date");
+	return;
+end
+
+local server = require "net.server";
+
+local adminstream = require "util.adminstream";
+
+local socket_path = module:get_option_path("admin_socket", "prosody.sock", "data");
+
+local sessions = module:shared("sessions");
+
+local function fire_admin_event(session, stanza)
+	local event_data = {
+		origin = session, stanza = stanza;
+	};
+	local event_name;
+	if stanza.attr.xmlns then
+		event_name = "admin/"..stanza.attr.xmlns..":"..stanza.name;
+	else
+		event_name = "admin/"..stanza.name;
+	end
+	module:log("debug", "Firing %s", event_name);
+	return module:fire_event(event_name, event_data);
+end
+
+module:hook("server-stopping", function ()
+	for _, session in pairs(sessions) do
+		session:close("system-shutdown");
+	end
+	os.remove(socket_path);
+end);
+
+--- Unix domain socket management
+
+local conn, sock;
+
+local listeners = adminstream.server(sessions, fire_admin_event).listeners;
+
+local function accept_connection()
+	module:log("debug", "accepting...");
+	local client = sock:accept();
+	if not client then return; end
+	server.wrapclient(client, "unix", 0, listeners, "*a");
+end
+
+function module.load()
+	sock = unix.stream();
+	sock:settimeout(0);
+	os.remove(socket_path);
+	assert(sock:bind(socket_path));
+	assert(sock:listen());
+	if server.wrapserver then
+		conn = server.wrapserver(sock, socket_path, 0, listeners);
+	else
+		conn = server.watchfd(sock:getfd(), accept_connection);
+	end
+end
+
+function module.unload()
+	if conn then
+		conn:close();
+	end
+	if sock then
+		sock:close();
+	end
+	os.remove(socket_path);
+end
diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua
index b0e349da..15220ec9 100644
--- a/plugins/mod_admin_telnet.lua
+++ b/plugins/mod_admin_telnet.lua
@@ -8,49 +8,68 @@
 -- luacheck: ignore 212/self
 
 module:set_global();
-
-local hostmanager = require "core.hostmanager";
-local modulemanager = require "core.modulemanager";
-local s2smanager = require "core.s2smanager";
-local portmanager = require "core.portmanager";
-local helpers = require "util.helpers";
-local server = require "net.server";
-
-local _G = _G;
-
-local prosody = _G.prosody;
+module:depends("admin_shell");
 
 local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" };
 
-local iterators = require "util.iterators";
-local keys, values = iterators.keys, iterators.values;
-local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
-local set, array = require "util.set", require "util.array";
-local cert_verify_identity = require "util.x509".verify_identity;
-local envload = require "util.envload".envload;
-local envloadfile = require "util.envload".envloadfile;
-local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local st = require "util.stanza";
 
-local commands = module:shared("commands")
-local def_env = module:shared("env");
+local def_env = module:shared("admin_shell/env");
 local default_env_mt = { __index = def_env };
 
-local function redirect_output(target, session)
-	local env = setmetatable({ print = session.print }, { __index = function (_, k) return rawget(target, k); end });
-	env.dofile = function(name)
-		local f, err = envloadfile(name, env);
-		if not f then return f, err; end
-		return f();
-	end;
-	return env;
+local function printbanner(session)
+	local option = module:get_option_string("console_banner", "full");
+	if option == "full" or option == "graphic" then
+		session.print [[
+                   ____                \   /     _
+                    |  _ \ _ __ ___  ___  _-_   __| |_   _
+                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
+                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
+                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
+                    A study in simplicity            |___/
+
+]]
+	end
+	if option == "short" or option == "full" then
+	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
+	session.print("You may find more help on using this console in our online documentation at ");
+	session.print("https://prosody.im/doc/console\n");
+	end
+	if option ~= "short" and option ~= "full" and option ~= "graphic" then
+		session.print(option);
+	end
 end
 
 console = {};
 
+local runner_callbacks = {};
+
+function runner_callbacks:ready()
+	self.data.conn:resume();
+end
+
+function runner_callbacks:waiting()
+	self.data.conn:pause();
+end
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[telnet]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+
 function console:new_session(conn)
 	local w = function(s) conn:write(s:gsub("\n", "\r\n")); end;
 	local session = { conn = conn;
-			send = function (t) w(tostring(t)); end;
+			send = function (t)
+				if st.is_stanza(t) and (t.name == "repl-result" or t.name == "repl-output") then
+					t = "| "..t:get_text().."\n";
+				end
+				w(tostring(t));
+			end;
 			print = function (...)
 				local t = {};
 				for i=1,select("#", ...) do
@@ -58,10 +77,16 @@ function console:new_session(conn)
 				end
 				w("| "..table.concat(t, "\t").."\n");
 			end;
+			serialize = tostring;
 			disconnect = function () conn:close(); end;
 			};
 	session.env = setmetatable({}, default_env_mt);
 
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+		session.send(string.char(0));
+	end, runner_callbacks, session);
+
 	-- Load up environment with helper objects
 	for name, t in pairs(def_env) do
 		if type(t) == "table" then
@@ -69,69 +94,37 @@ function console:new_session(conn)
 		end
 	end
 
+	session.env.output:configure();
+
 	return session;
 end
 
 function console:process_line(session, line)
-	local useglobalenv;
-
-	if line:match("^>") then
-		line = line:gsub("^>", "");
-		useglobalenv = true;
-	elseif line == "\004" then
-		commands["bye"](session, line);
+	line = line:gsub("\r?\n$", "");
+	if line == "bye" or line == "quit" or line == "exit" or line:byte() == 4 then
+		session.print("See you!");
+		session:disconnect();
 		return;
-	else
-		local command = line:match("^%w+") or line:match("%p");
-		if commands[command] then
-			commands[command](session, line);
-			return;
-		end
-	end
-
-	session.env._ = line;
-
-	local chunkname = "=console";
-	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
-	local chunk, err = envload("return "..line, chunkname, env);
-	if not chunk then
-		chunk, err = envload(line, chunkname, env);
-		if not chunk then
-			err = err:gsub("^%[string .-%]:%d+: ", "");
-			err = err:gsub("^:%d+: ", "");
-			err = err:gsub("'<eof>'", "the end of the line");
-			session.print("Sorry, I couldn't understand that... "..err);
-			return;
-		end
 	end
+	return module:fire_event("admin/repl-input", { origin = session, stanza = st.stanza("repl-input"):text(line) });
+end
 
-	local ranok, taskok, message = pcall(chunk);
-
-	if not (ranok or message or useglobalenv) and commands[line:lower()] then
-		commands[line:lower()](session, line);
-		return;
-	end
+local sessions = {};
 
-	if not ranok then
-		session.print("Fatal error while running command, it did not complete");
-		session.print("Error: "..taskok);
-		return;
-	end
+function module.save()
+	return { sessions = sessions }
+end
 
-	if not message then
-		session.print("Result: "..tostring(taskok));
-		return;
-	elseif (not taskok) and message then
-		session.print("Command completed with a problem");
-		session.print("Message: "..tostring(message));
-		return;
+function module.restore(data)
+	if data.sessions then
+		for conn in pairs(data.sessions) do
+			conn:setlistener(console_listener);
+			local session = console:new_session(conn);
+			sessions[conn] = session;
+		end
 	end
-
-	session.print("OK: "..tostring(message));
 end
 
-local sessions = {};
-
 function console_listener.onconnect(conn)
 	-- Handle new connection
 	local session = console:new_session(conn);
@@ -150,8 +143,7 @@ function console_listener.onincoming(conn, data)
 
 	for line in data:gmatch("[^\n]*[\n\004]") do
 		if session.closed then return end
-		console:process_line(session, line);
-		session.send(string.char(0));
+		session.thread:run(line);
 	end
 	session.partial_data = data:match("[^\n]+$");
 end
@@ -176,1382 +168,6 @@ function console_listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
--- Console commands --
--- These are simple commands, not valid standalone in Lua
-
-function commands.bye(session)
-	session.print("See you! :)");
-	session.closed = true;
-	session.disconnect();
-end
-commands.quit, commands.exit = commands.bye, commands.bye;
-
-commands["!"] = function (session, data)
-	if data:match("^!!") and session.env._ then
-		session.print("!> "..session.env._);
-		return console_listener.onincoming(session.conn, session.env._);
-	end
-	local old, new = data:match("^!(.-[^\\])!(.-)!$");
-	if old and new then
-		local ok, res = pcall(string.gsub, session.env._, old, new);
-		if not ok then
-			session.print(res)
-			return;
-		end
-		session.print("!> "..res);
-		return console_listener.onincoming(session.conn, res);
-	end
-	session.print("Sorry, not sure what you want");
-end
-
-
-function commands.help(session, data)
-	local print = session.print;
-	local section = data:match("^help (%w+)");
-	if not section then
-		print [[Commands are divided into multiple sections. For help on a particular section, ]]
-		print [[type: help SECTION (for example, 'help c2s'). Sections are: ]]
-		print [[]]
-		print [[c2s - Commands to manage local client-to-server sessions]]
-		print [[s2s - Commands to manage sessions between this server and others]]
-		print [[module - Commands to load/reload/unload modules/plugins]]
-		print [[host - Commands to activate, deactivate and list virtual hosts]]
-		print [[user - Commands to create and delete users, and change their passwords]]
-		print [[server - Uptime, version, shutting down, etc.]]
-		print [[port - Commands to manage ports the server is listening on]]
-		print [[dns - Commands to manage and inspect the internal DNS resolver]]
-		print [[config - Reloading the configuration, etc.]]
-		print [[console - Help regarding the console itself]]
-	elseif section == "c2s" then
-		print [[c2s:show(jid) - Show all client sessions with the specified JID (or all if no JID given)]]
-		print [[c2s:show_insecure() - Show all unencrypted client connections]]
-		print [[c2s:show_secure() - Show all encrypted client connections]]
-		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
-		print [[c2s:close(jid) - Close all sessions for the specified JID]]
-	elseif section == "s2s" then
-		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
-		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
-		print [[s2s:close(from, to) - Close a connection from one domain to another]]
-		print [[s2s:closeall(host) - Close all the incoming/outgoing s2s sessions to specified host]]
-	elseif section == "module" then
-		print [[module:load(module, host) - Load the specified module on the specified host (or all hosts if none given)]]
-		print [[module:reload(module, host) - The same, but unloads and loads the module (saving state if the module supports it)]]
-		print [[module:unload(module, host) - The same, but just unloads the module from memory]]
-		print [[module:list(host) - List the modules loaded on the specified host]]
-	elseif section == "host" then
-		print [[host:activate(hostname) - Activates the specified host]]
-		print [[host:deactivate(hostname) - Disconnects all clients on this host and deactivates]]
-		print [[host:list() - List the currently-activated hosts]]
-	elseif section == "user" then
-		print [[user:create(jid, password) - Create the specified user account]]
-		print [[user:password(jid, password) - Set the password for the specified user account]]
-		print [[user:delete(jid) - Permanently remove the specified user account]]
-		print [[user:list(hostname, pattern) - List users on the specified host, optionally filtering with a pattern]]
-	elseif section == "server" then
-		print [[server:version() - Show the server's version number]]
-		print [[server:uptime() - Show how long the server has been running]]
-		print [[server:memory() - Show details about the server's memory usage]]
-		print [[server:shutdown(reason) - Shut down the server, with an optional reason to be broadcast to all connections]]
-	elseif section == "port" then
-		print [[port:list() - Lists all network ports prosody currently listens on]]
-		print [[port:close(port, interface) - Close a port]]
-	elseif section == "dns" then
-		print [[dns:lookup(name, type, class) - Do a DNS lookup]]
-		print [[dns:addnameserver(nameserver) - Add a nameserver to the list]]
-		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
-		print [[dns:purge() - Clear the DNS cache]]
-		print [[dns:cache() - Show cached records]]
-	elseif section == "config" then
-		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
-	elseif section == "console" then
-		print [[Hey! Welcome to Prosody's admin console.]]
-		print [[First thing, if you're ever wondering how to get out, simply type 'quit'.]]
-		print [[Secondly, note that we don't support the full telnet protocol yet (it's coming)]]
-		print [[so you may have trouble using the arrow keys, etc. depending on your system.]]
-		print [[]]
-		print [[For now we offer a couple of handy shortcuts:]]
-		print [[!! - Repeat the last command]]
-		print [[!old!new! - repeat the last command, but with 'old' replaced by 'new']]
-		print [[]]
-		print [[For those well-versed in Prosody's internals, or taking instruction from those who are,]]
-		print [[you can prefix a command with > to escape the console sandbox, and access everything in]]
-		print [[the running server. Great fun, but be careful not to break anything :)]]
-	end
-	print [[]]
-end
-
--- Session environment --
--- Anything in def_env will be accessible within the session as a global variable
-
---luacheck: ignore 212/self
-
-def_env.server = {};
-
-function def_env.server:insane_reload()
-	prosody.unlock_globals();
-	dofile "prosody"
-	prosody = _G.prosody;
-	return true, "Server reloaded";
-end
-
-function def_env.server:version()
-	return true, tostring(prosody.version or "unknown");
-end
-
-function def_env.server:uptime()
-	local t = os.time()-prosody.start_time;
-	local seconds = t%60;
-	t = (t - seconds)/60;
-	local minutes = t%60;
-	t = (t - minutes)/60;
-	local hours = t%24;
-	t = (t - hours)/24;
-	local days = t;
-	return true, string.format("This server has been running for %d day%s, %d hour%s and %d minute%s (since %s)",
-		days, (days ~= 1 and "s") or "", hours, (hours ~= 1 and "s") or "",
-		minutes, (minutes ~= 1 and "s") or "", os.date("%c", prosody.start_time));
-end
-
-function def_env.server:shutdown(reason)
-	prosody.shutdown(reason);
-	return true, "Shutdown initiated";
-end
-
-local function human(kb)
-	local unit = "K";
-	if kb > 1024 then
-		kb, unit = kb/1024, "M";
-	end
-	return ("%0.2f%sB"):format(kb, unit);
-end
-
-function def_env.server:memory()
-	if not has_pposix or not pposix.meminfo then
-		return true, "Lua is using "..human(collectgarbage("count"));
-	end
-	local mem, lua_mem = pposix.meminfo(), collectgarbage("count");
-	local print = self.session.print;
-	print("Process: "..human((mem.allocated+mem.allocated_mmap)/1024));
-	print("   Used: "..human(mem.used/1024).." ("..human(lua_mem).." by Lua)");
-	print("   Free: "..human(mem.unused/1024).." ("..human(mem.returnable/1024).." returnable)");
-	return true, "OK";
-end
-
-def_env.module = {};
-
-local function get_hosts_set(hosts, module)
-	if type(hosts) == "table" then
-		if hosts[1] then
-			return set.new(hosts);
-		elseif hosts._items then
-			return hosts;
-		end
-	elseif type(hosts) == "string" then
-		return set.new { hosts };
-	elseif hosts == nil then
-		local hosts_set = set.new(array.collect(keys(prosody.hosts)))
-			/ function (host) return (prosody.hosts[host].type == "local" or module and modulemanager.is_loaded(host, module)) and host or nil; end;
-		if module and modulemanager.get_module("*", module) then
-			hosts_set:add("*");
-		end
-		return hosts_set;
-	end
-end
-
-function def_env.module:load(name, hosts, config)
-	hosts = get_hosts_set(hosts);
-
-	-- Load the module for each host
-	local ok, err, count, mod = true, nil, 0;
-	for host in hosts do
-		if (not modulemanager.is_loaded(host, name)) then
-			mod, err = modulemanager.load(host, name, config);
-			if not mod then
-				ok = false;
-				if err == "global-module-already-loaded" then
-					if count > 0 then
-						ok, err, count = true, nil, 1;
-					end
-					break;
-				end
-				self.session.print(err or "Unknown error loading module");
-			else
-				count = count + 1;
-				self.session.print("Loaded for "..mod.module.host);
-			end
-		end
-	end
-
-	return ok, (ok and "Module loaded onto "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-function def_env.module:unload(name, hosts)
-	hosts = get_hosts_set(hosts, name);
-
-	-- Unload the module for each host
-	local ok, err, count = true, nil, 0;
-	for host in hosts do
-		if modulemanager.is_loaded(host, name) then
-			ok, err = modulemanager.unload(host, name);
-			if not ok then
-				ok = false;
-				self.session.print(err or "Unknown error unloading module");
-			else
-				count = count + 1;
-				self.session.print("Unloaded from "..host);
-			end
-		end
-	end
-	return ok, (ok and "Module unloaded from "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-local function _sort_hosts(a, b)
-	if a == "*" then return true
-	elseif b == "*" then return false
-	else return a < b; end
-end
-
-function def_env.module:reload(name, hosts)
-	hosts = array.collect(get_hosts_set(hosts, name)):sort(_sort_hosts)
-
-	-- Reload the module for each host
-	local ok, err, count = true, nil, 0;
-	for _, host in ipairs(hosts) do
-		if modulemanager.is_loaded(host, name) then
-			ok, err = modulemanager.reload(host, name);
-			if not ok then
-				ok = false;
-				self.session.print(err or "Unknown error reloading module");
-			else
-				count = count + 1;
-				if ok == nil then
-					ok = true;
-				end
-				self.session.print("Reloaded on "..host);
-			end
-		end
-	end
-	return ok, (ok and "Module reloaded on "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-function def_env.module:list(hosts)
-	if hosts == nil then
-		hosts = array.collect(keys(prosody.hosts));
-		table.insert(hosts, 1, "*");
-	end
-	if type(hosts) == "string" then
-		hosts = { hosts };
-	end
-	if type(hosts) ~= "table" then
-		return false, "Please supply a host or a list of hosts you would like to see";
-	end
-
-	local print = self.session.print;
-	for _, host in ipairs(hosts) do
-		print((host == "*" and "Global" or host)..":");
-		local modules = array.collect(keys(modulemanager.get_modules(host) or {})):sort();
-		if #modules == 0 then
-			if prosody.hosts[host] then
-				print("    No modules loaded");
-			else
-				print("    Host not found");
-			end
-		else
-			for _, name in ipairs(modules) do
-				print("    "..name);
-			end
-		end
-	end
-end
-
-def_env.config = {};
-function def_env.config:load(filename, format)
-	local config_load = require "core.configmanager".load;
-	local ok, err = config_load(filename, format);
-	if not ok then
-		return false, err or "Unknown error loading config";
-	end
-	return true, "Config loaded";
-end
-
-function def_env.config:get(host, section, key)
-	local config_get = require "core.configmanager".get
-	return true, tostring(config_get(host, section, key));
-end
-
-function def_env.config:reload()
-	local ok, err = prosody.reload_config();
-	return ok, (ok and "Config reloaded (you may need to reload modules to take effect)") or tostring(err);
-end
-
-local function common_info(session, line)
-	if session.id then
-		line[#line+1] = "["..session.id.."]"
-	else
-		line[#line+1] = "["..session.type..(tostring(session):match("%x*$")).."]"
-	end
-end
-
-local function session_flags(session, line)
-	line = line or {};
-	common_info(session, line);
-	if session.type == "c2s" then
-		local status, priority = "unavailable", tostring(session.priority or "-");
-		if session.presence then
-			status = session.presence:get_child_text("show") or "available";
-		end
-		line[#line+1] = status.."("..priority..")";
-	end
-	if session.cert_identity_status == "valid" then
-		line[#line+1] = "(authenticated)";
-	end
-	if session.secure then
-		line[#line+1] = "(encrypted)";
-	end
-	if session.compressed then
-		line[#line+1] = "(compressed)";
-	end
-	if session.smacks then
-		line[#line+1] = "(sm)";
-	end
-	if session.ip and session.ip:match(":") then
-		line[#line+1] = "(IPv6)";
-	end
-	if session.remote then
-		line[#line+1] = "(remote)";
-	end
-	return table.concat(line, " ");
-end
-
-local function tls_info(session, line)
-	line = line or {};
-	common_info(session, line);
-	if session.secure then
-		local sock = session.conn and session.conn.socket and session.conn:socket();
-		if sock then
-			local info = sock.info and sock:info();
-			if info then
-				line[#line+1] = ("(%s with %s)"):format(info.protocol, info.cipher);
-			else
-				-- TLS session might not be ready yet
-				line[#line+1] = "(cipher info unavailable)";
-			end
-		end
-	else
-		line[#line+1] = "(insecure)";
-	end
-	return table.concat(line, " ");
-end
-
-def_env.c2s = {};
-
-local function get_jid(session)
-	if session.username then
-		return session.full_jid or jid_join(session.username, session.host, session.resource);
-	end
-
-	local conn = session.conn;
-	local ip = session.ip or "?";
-	local clientport = conn and conn:clientport() or "?";
-	local serverip = conn and conn.server and conn:server():ip() or "?";
-	local serverport = conn and conn:serverport() or "?"
-	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
-end
-
-local function show_c2s(callback)
-	local c2s = array.collect(values(module:shared"/*/c2s/sessions"));
-	c2s:sort(function(a, b)
-		if a.host == b.host then
-			if a.username == b.username then
-				return (a.resource or "") > (b.resource or "");
-			end
-			return (a.username or "") > (b.username or "");
-		end
-		return (a.host or "") > (b.host or "");
-	end):map(function (session)
-		callback(get_jid(session), session)
-	end);
-end
-
-function def_env.c2s:count()
-	return true, "Total: "..  iterators.count(values(module:shared"/*/c2s/sessions")) .." clients";
-end
-
-function def_env.c2s:show(match_jid, annotate)
-	local print, count = self.session.print, 0;
-	annotate = annotate or session_flags;
-	local curr_host = false;
-	show_c2s(function (jid, session)
-		if curr_host ~= session.host then
-			curr_host = session.host;
-			print(curr_host or "(not connected to any host yet)");
-		end
-		if (not match_jid) or jid:match(match_jid) then
-			count = count + 1;
-			print(annotate(session, { "  ", jid }));
-		end
-	end);
-	return true, "Total: "..count.." clients";
-end
-
-function def_env.c2s:show_insecure(match_jid)
-	local print, count = self.session.print, 0;
-	show_c2s(function (jid, session)
-		if ((not match_jid) or jid:match(match_jid)) and not session.secure then
-			count = count + 1;
-			print(jid);
-		end
-	end);
-	return true, "Total: "..count.." insecure client connections";
-end
-
-function def_env.c2s:show_secure(match_jid)
-	local print, count = self.session.print, 0;
-	show_c2s(function (jid, session)
-		if ((not match_jid) or jid:match(match_jid)) and session.secure then
-			count = count + 1;
-			print(jid);
-		end
-	end);
-	return true, "Total: "..count.." secure client connections";
-end
-
-function def_env.c2s:show_tls(match_jid)
-	return self:show(match_jid, tls_info);
-end
-
-function def_env.c2s:close(match_jid)
-	local count = 0;
-	show_c2s(function (jid, session)
-		if jid == match_jid or jid_bare(jid) == match_jid then
-			count = count + 1;
-			session:close();
-		end
-	end);
-	return true, "Total: "..count.." sessions closed";
-end
-
-
-def_env.s2s = {};
-function def_env.s2s:show(match_jid, annotate)
-	local print = self.session.print;
-	annotate = annotate or session_flags;
-
-	local count_in, count_out = 0,0;
-	local s2s_list = { };
-
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	for _, session in pairs(s2s_sessions) do
-		local remotehost, localhost, direction;
-		if session.direction == "outgoing" then
-			direction = "->";
-			count_out = count_out + 1;
-			remotehost, localhost = session.to_host or "?", session.from_host or "?";
-		else
-			direction = "<-";
-			count_in = count_in + 1;
-			remotehost, localhost = session.from_host or "?", session.to_host or "?";
-		end
-		local sess_lines = { l = localhost, r = remotehost,
-			annotate(session, { "", direction, remotehost or "?" })};
-
-		if (not match_jid) or remotehost:match(match_jid) or localhost:match(match_jid) then
-			table.insert(s2s_list, sess_lines);
-			-- luacheck: ignore 421/print
-			local print = function (s) table.insert(sess_lines, "        "..s); end
-			if session.sendq then
-				print("There are "..#session.sendq.." queued outgoing stanzas for this connection");
-			end
-			if session.type == "s2sout_unauthed" then
-				if session.connecting then
-					print("Connection not yet established");
-					if not session.srv_hosts then
-						if not session.conn then
-							print("We do not yet have a DNS answer for this host's SRV records");
-						else
-							print("This host has no SRV records, using A record instead");
-						end
-					elseif session.srv_choice then
-						print("We are on SRV record "..session.srv_choice.." of "..#session.srv_hosts);
-						local srv_choice = session.srv_hosts[session.srv_choice];
-						print("Using "..(srv_choice.target or ".")..":"..(srv_choice.port or 5269));
-					end
-				elseif session.notopen then
-					print("The <stream> has not yet been opened");
-				elseif not session.dialback_key then
-					print("Dialback has not been initiated yet");
-				elseif session.dialback_key then
-					print("Dialback has been requested, but no result received");
-				end
-			end
-			if session.type == "s2sin_unauthed" then
-				print("Connection not yet authenticated");
-			elseif session.type == "s2sin" then
-				for name in pairs(session.hosts) do
-					if name ~= session.from_host then
-						print("also hosts "..tostring(name));
-					end
-				end
-			end
-		end
-	end
-
-	-- Sort by local host, then remote host
-	table.sort(s2s_list, function(a,b)
-		if a.l == b.l then return a.r < b.r; end
-		return a.l < b.l;
-	end);
-	local lasthost;
-	for _, sess_lines in ipairs(s2s_list) do
-		if sess_lines.l ~= lasthost then print(sess_lines.l); lasthost=sess_lines.l end
-		for _, line in ipairs(sess_lines) do print(line); end
-	end
-	return true, "Total: "..count_out.." outgoing, "..count_in.." incoming connections";
-end
-
-function def_env.s2s:show_tls(match_jid)
-	return self:show(match_jid, tls_info);
-end
-
-local function print_subject(print, subject)
-	for _, entry in ipairs(subject) do
-		print(
-			("    %s: %q"):format(
-				entry.name or entry.oid,
-				entry.value:gsub("[\r\n%z%c]", " ")
-			)
-		);
-	end
-end
-
--- As much as it pains me to use the 0-based depths that OpenSSL does,
--- I think there's going to be more confusion among operators if we
--- break from that.
-local function print_errors(print, errors)
-	for depth, t in pairs(errors) do
-		print(
-			("    %d: %s"):format(
-				depth-1,
-				table.concat(t, "\n|        ")
-			)
-		);
-	end
-end
-
-function def_env.s2s:showcert(domain)
-	local print = self.session.print;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	local domain_sessions = set.new(array.collect(values(s2s_sessions)))
-		/function(session) return (session.to_host == domain or session.from_host == domain) and session or nil; end;
-	local cert_set = {};
-	for session in domain_sessions do
-		local conn = session.conn;
-		conn = conn and conn:socket();
-		if not conn.getpeerchain then
-			if conn.dohandshake then
-				error("This version of LuaSec does not support certificate viewing");
-			end
-		else
-			local cert = conn:getpeercertificate();
-			if cert then
-				local certs = conn:getpeerchain();
-				local digest = cert:digest("sha1");
-				if not cert_set[digest] then
-					local chain_valid, chain_errors = conn:getpeerverification();
-					cert_set[digest] = {
-						{
-						  from = session.from_host,
-						  to = session.to_host,
-						  direction = session.direction
-						};
-						chain_valid = chain_valid;
-						chain_errors = chain_errors;
-						certs = certs;
-					};
-				else
-					table.insert(cert_set[digest], {
-						from = session.from_host,
-						to = session.to_host,
-						direction = session.direction
-					});
-				end
-			end
-		end
-	end
-	local domain_certs = array.collect(values(cert_set));
-	-- Phew. We now have a array of unique certificates presented by domain.
-	local n_certs = #domain_certs;
-
-	if n_certs == 0 then
-		return "No certificates found for "..domain;
-	end
-
-	local function _capitalize_and_colon(byte)
-		return string.upper(byte)..":";
-	end
-	local function pretty_fingerprint(hash)
-		return hash:gsub("..", _capitalize_and_colon):sub(1, -2);
-	end
-
-	for cert_info in values(domain_certs) do
-		local certs = cert_info.certs;
-		local cert = certs[1];
-		print("---")
-		print("Fingerprint (SHA1): "..pretty_fingerprint(cert:digest("sha1")));
-		print("");
-		local n_streams = #cert_info;
-		print("Currently used on "..n_streams.." stream"..(n_streams==1 and "" or "s")..":");
-		for _, stream in ipairs(cert_info) do
-			if stream.direction == "incoming" then
-				print("    "..stream.to.." <- "..stream.from);
-			else
-				print("    "..stream.from.." -> "..stream.to);
-			end
-		end
-		print("");
-		local chain_valid, errors = cert_info.chain_valid, cert_info.chain_errors;
-		local valid_identity = cert_verify_identity(domain, "xmpp-server", cert);
-		if chain_valid then
-			print("Trusted certificate: Yes");
-		else
-			print("Trusted certificate: No");
-			print_errors(print, errors);
-		end
-		print("");
-		print("Issuer: ");
-		print_subject(print, cert:issuer());
-		print("");
-		print("Valid for "..domain..": "..(valid_identity and "Yes" or "No"));
-		print("Subject:");
-		print_subject(print, cert:subject());
-	end
-	print("---");
-	return ("Showing "..n_certs.." certificate"
-		..(n_certs==1 and "" or "s")
-		.." presented by "..domain..".");
-end
-
-function def_env.s2s:close(from, to)
-	local print, count = self.session.print, 0;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-
-	local match_id;
-	if from and not to then
-		match_id, from = from, nil;
-	elseif not to then
-		return false, "Syntax: s2s:close('from', 'to') - Closes all s2s sessions from 'from' to 'to'";
-	elseif from == to then
-		return false, "Both from and to are the same... you can't do that :)";
-	end
-
-	for _, session in pairs(s2s_sessions) do
-		local id = session.type..tostring(session):match("[a-f0-9]+$");
-		if (match_id and match_id == id)
-		or (session.from_host == from and session.to_host == to) then
-			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
-			(session.close or s2smanager.destroy_session)(session);
-			count = count + 1 ;
-		end
-	end
-	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
-end
-
-function def_env.s2s:closeall(host)
-	local count = 0;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	for _,session in pairs(s2s_sessions) do
-		if not host or session.from_host == host or session.to_host == host then
-			session:close();
-			count = count + 1;
-		end
-	end
-	if count == 0 then return false, "No sessions to close.";
-	else return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s"); end
-end
-
-def_env.host = {}; def_env.hosts = def_env.host;
-
-function def_env.host:activate(hostname, config)
-	return hostmanager.activate(hostname, config);
-end
-function def_env.host:deactivate(hostname, reason)
-	return hostmanager.deactivate(hostname, reason);
-end
-
-function def_env.host:list()
-	local print = self.session.print;
-	local i = 0;
-	local type;
-	for host, host_session in iterators.sorted_pairs(prosody.hosts) do
-		i = i + 1;
-		type = host_session.type;
-		if type == "local" then
-			print(host);
-		else
-			type = module:context(host):get_option_string("component_module", type);
-			if type ~= "component" then
-				type = type .. " component";
-			end
-			print(("%s (%s)"):format(host, type));
-		end
-	end
-	return true, i.." hosts";
-end
-
-def_env.port = {};
-
-function def_env.port:list()
-	local print = self.session.print;
-	local services = portmanager.get_active_services().data;
-	local n_services, n_ports = 0, 0;
-	for service, interfaces in iterators.sorted_pairs(services) do
-		n_services = n_services + 1;
-		local ports_list = {};
-		for interface, ports in pairs(interfaces) do
-			for port in pairs(ports) do
-				table.insert(ports_list, "["..interface.."]:"..port);
-			end
-		end
-		n_ports = n_ports + #ports_list;
-		print(service..": "..table.concat(ports_list, ", "));
-	end
-	return true, n_services.." services listening on "..n_ports.." ports";
-end
-
-function def_env.port:close(close_port, close_interface)
-	close_port = assert(tonumber(close_port), "Invalid port number");
-	local n_closed = 0;
-	local services = portmanager.get_active_services().data;
-	for service, interfaces in pairs(services) do -- luacheck: ignore 213
-		for interface, ports in pairs(interfaces) do
-			if not close_interface or close_interface == interface then
-				if ports[close_port] then
-					self.session.print("Closing ["..interface.."]:"..close_port.."...");
-					local ok, err = portmanager.close(interface, close_port)
-					if not ok then
-						self.session.print("Failed to close "..interface.." "..close_port..": "..err);
-					else
-						n_closed = n_closed + 1;
-					end
-				end
-			end
-		end
-	end
-	return true, "Closed "..n_closed.." ports";
-end
-
-def_env.muc = {};
-
-local console_room_mt = {
-	__index = function (self, k) return self.room[k]; end;
-	__tostring = function (self)
-		return "MUC room <"..self.room.jid..">";
-	end;
-};
-
-local function check_muc(jid)
-	local room_name, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not prosody.hosts[host].modules.muc then
-		return nil, "Host '"..host.."' is not a MUC service";
-	end
-	return room_name, host;
-end
-
-function def_env.muc:create(room_jid, config)
-	local room_name, host = check_muc(room_jid);
-	if not room_name then
-		return room_name, host;
-	end
-	if not room_name then return nil, host end
-	if config ~= nil and type(config) ~= "table" then return nil, "Config must be a table"; end
-	if prosody.hosts[host].modules.muc.get_room_from_jid(room_jid) then return nil, "Room exists already" end
-	return prosody.hosts[host].modules.muc.create_room(room_jid, config);
-end
-
-function def_env.muc:room(room_jid)
-	local room_name, host = check_muc(room_jid);
-	if not room_name then
-		return room_name, host;
-	end
-	local room_obj = prosody.hosts[host].modules.muc.get_room_from_jid(room_jid);
-	if not room_obj then
-		return nil, "No such room: "..room_jid;
-	end
-	return setmetatable({ room = room_obj }, console_room_mt);
-end
-
-function def_env.muc:list(host)
-	local host_session = prosody.hosts[host];
-	if not host_session or not host_session.modules.muc then
-		return nil, "Please supply the address of a local MUC component";
-	end
-	local print = self.session.print;
-	local c = 0;
-	for room in host_session.modules.muc.each_room() do
-		print(room.jid);
-		c = c + 1;
-	end
-	return true, c.." rooms";
-end
-
-local um = require"core.usermanager";
-
-def_env.user = {};
-function def_env.user:create(jid, password)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif um.user_exists(username, host) then
-		return nil, "User exists";
-	end
-	local ok, err = um.create_user(username, password, host);
-	if ok then
-		return true, "User created";
-	else
-		return nil, "Could not create user: "..err;
-	end
-end
-
-function def_env.user:delete(jid)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not um.user_exists(username, host) then
-		return nil, "No such user";
-	end
-	local ok, err = um.delete_user(username, host);
-	if ok then
-		return true, "User deleted";
-	else
-		return nil, "Could not delete user: "..err;
-	end
-end
-
-function def_env.user:password(jid, password)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not um.user_exists(username, host) then
-		return nil, "No such user";
-	end
-	local ok, err = um.set_password(username, password, host, nil);
-	if ok then
-		return true, "User password changed";
-	else
-		return nil, "Could not change password for user: "..err;
-	end
-end
-
-function def_env.user:list(host, pat)
-	if not host then
-		return nil, "No host given";
-	elseif not prosody.hosts[host] then
-		return nil, "No such host";
-	end
-	local print = self.session.print;
-	local total, matches = 0, 0;
-	for user in um.users(host) do
-		if not pat or user:match(pat) then
-			print(user.."@"..host);
-			matches = matches + 1;
-		end
-		total = total + 1;
-	end
-	return true, "Showing "..(pat and (matches.." of ") or "all " )..total.." users";
-end
-
-def_env.xmpp = {};
-
-local st = require "util.stanza";
-function def_env.xmpp:ping(localhost, remotehost)
-	if prosody.hosts[localhost] then
-		module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" }
-				:tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]);
-		return true, "Sent ping";
-	else
-		return nil, "No such host";
-	end
-end
-
-def_env.dns = {};
-local adns = require"net.adns";
-
-local function get_resolver(session)
-	local resolver = session.dns_resolver;
-	if not resolver then
-		resolver = adns.resolver();
-		session.dns_resolver = resolver;
-	end
-	return resolver;
-end
-
-function def_env.dns:lookup(name, typ, class)
-	local resolver = get_resolver(self.session);
-	local ret = "Query sent";
-	local print = self.session.print;
-	local function handler(...)
-		ret = "Got response";
-		print(...);
-	end
-	resolver:lookup(handler, name, typ, class);
-	return true, ret;
-end
-
-function def_env.dns:addnameserver(...)
-	local resolver = get_resolver(self.session);
-	resolver._resolver:addnameserver(...)
-	return true
-end
-
-function def_env.dns:setnameserver(...)
-	local resolver = get_resolver(self.session);
-	resolver._resolver:setnameserver(...)
-	return true
-end
-
-function def_env.dns:purge()
-	local resolver = get_resolver(self.session);
-	resolver._resolver:purge()
-	return true
-end
-
-function def_env.dns:cache()
-	local resolver = get_resolver(self.session);
-	return true, "Cache:\n"..tostring(resolver._resolver.cache)
-end
-
-def_env.http = {};
-
-function def_env.http:list()
-	local print = self.session.print;
-
-	for host in pairs(prosody.hosts) do
-		local http_apps = modulemanager.get_items("http-provider", host);
-		if #http_apps > 0 then
-			local http_host = module:context(host):get_option_string("http_host");
-			print("HTTP endpoints on "..host..(http_host and (" (using "..http_host.."):") or ":"));
-			for _, provider in ipairs(http_apps) do
-				local url = module:context(host):http_url(provider.name, provider.default_path);
-				print("", url);
-			end
-			print("");
-		end
-	end
-
-	local default_host = module:get_option_string("http_default_host");
-	if not default_host then
-		print("HTTP requests to unknown hosts will return 404 Not Found");
-	else
-		print("HTTP requests to unknown hosts will be handled by "..default_host);
-	end
-	return true;
-end
-
-def_env.debug = {};
-
-function def_env.debug:logevents(host)
-	helpers.log_host_events(host);
-	return true;
-end
-
-function def_env.debug:events(host, event)
-	local events_obj;
-	if host and host ~= "*" then
-		if host == "http" then
-			events_obj = require "net.http.server"._events;
-		elseif not prosody.hosts[host] then
-			return false, "Unknown host: "..host;
-		else
-			events_obj = prosody.hosts[host].events;
-		end
-	else
-		events_obj = prosody.events;
-	end
-	return true, helpers.show_events(events_obj, event);
-end
-
-function def_env.debug:timers()
-	local socket = require "socket";
-	local print = self.session.print;
-	local add_task = require"util.timer".add_task;
-	local h, params = add_task.h, add_task.params;
-	if h then
-		print("-- util.timer");
-		for i, id in ipairs(h.ids) do
-			if not params[id] then
-				print(os.date("%F %T", h.priorities[i]), h.items[id]);
-			elseif not params[id].callback then
-				print(os.date("%F %T", h.priorities[i]), h.items[id], unpack(params[id]));
-			else
-				print(os.date("%F %T", h.priorities[i]), params[id].callback, unpack(params[id]));
-			end
-		end
-	end
-	if server.event_base then
-		local count = 0;
-		for _, v in pairs(debug.getregistry()) do
-			if type(v) == "function" and v.callback and v.callback == add_task._on_timer then
-				count = count + 1;
-			end
-		end
-		print(count .. " libevent callbacks");
-	end
-	if h then
-		local next_time = h:peek();
-		if next_time then
-			return true, os.date("Next event at %F %T (in %%.6fs)", next_time):format(next_time - socket.gettime());
-		end
-	end
-	return true;
-end
-
--- COMPAT: debug:timers() was timer:info() for some time in trunk
-def_env.timer = { info = def_env.debug.timers };
-
-module:hook("server-stopping", function(event)
-	for _, session in pairs(sessions) do
-		session.print("Shutting down: "..(event.reason or "unknown reason"));
-	end
-end);
-
-def_env.stats = {};
-
-local function format_stat(type, value, ref_value)
-	ref_value = ref_value or value;
-	--do return tostring(value) end
-	if type == "duration" then
-		if ref_value < 0.001 then
-			return ("%d µs"):format(value*1000000);
-		elseif ref_value < 0.9 then
-			return ("%0.2f ms"):format(value*1000);
-		end
-		return ("%0.2f"):format(value);
-	elseif type == "size" then
-		if ref_value > 1048576 then
-			return ("%d MB"):format(value/1048576);
-		elseif ref_value > 1024 then
-			return ("%d KB"):format(value/1024);
-		end
-		return ("%d bytes"):format(value);
-	elseif type == "rate" then
-		if ref_value < 0.9 then
-			return ("%0.2f/min"):format(value*60);
-		end
-		return ("%0.2f/sec"):format(value);
-	end
-	return tostring(value);
-end
-
-local stats_methods = {};
-function stats_methods:bounds(_lower, _upper)
-	for _, stat_info in ipairs(self) do
-		local data = stat_info[4];
-		if data then
-			local lower = _lower or data.min;
-			local upper = _upper or data.max;
-			local new_data = {
-				min = lower;
-				max = upper;
-				samples = {};
-				sample_count = 0;
-				count = data.count;
-				units = data.units;
-			};
-			local sum = 0;
-			for _, v in ipairs(data.samples) do
-				if v > upper then
-					break;
-				elseif v>=lower then
-					table.insert(new_data.samples, v);
-					sum = sum + v;
-				end
-			end
-			new_data.sample_count = #new_data.samples;
-			stat_info[4] = new_data;
-			stat_info[3] = sum/new_data.sample_count;
-		end
-	end
-	return self;
-end
-
-function stats_methods:trim(lower, upper)
-	upper = upper or (100-lower);
-	local statistics = require "util.statistics";
-	for _, stat_info in ipairs(self) do
-		-- Strip outliers
-		local data = stat_info[4];
-		if data then
-			local new_data = {
-				min = statistics.get_percentile(data, lower);
-				max = statistics.get_percentile(data, upper);
-				samples = {};
-				sample_count = 0;
-				count = data.count;
-				units = data.units;
-			};
-			local sum = 0;
-			for _, v in ipairs(data.samples) do
-				if v > new_data.max then
-					break;
-				elseif v>=new_data.min then
-					table.insert(new_data.samples, v);
-					sum = sum + v;
-				end
-			end
-			new_data.sample_count = #new_data.samples;
-			stat_info[4] = new_data;
-			stat_info[3] = sum/new_data.sample_count;
-		end
-	end
-	return self;
-end
-
-function stats_methods:max(upper)
-	return self:bounds(nil, upper);
-end
-
-function stats_methods:min(lower)
-	return self:bounds(lower, nil);
-end
-
-function stats_methods:summary()
-	local statistics = require "util.statistics";
-	for _, stat_info in ipairs(self) do
-		local type, value, data = stat_info[2], stat_info[3], stat_info[4];
-		if data and data.samples then
-			table.insert(stat_info.output, string.format("Count: %d (%d captured)",
-				data.count,
-				data.sample_count
-			));
-			table.insert(stat_info.output, string.format("Min: %s  Mean: %s  Max: %s",
-				format_stat(type, data.min),
-				format_stat(type, value),
-				format_stat(type, data.max)
-			));
-			table.insert(stat_info.output, string.format("Q1: %s  Median: %s  Q3: %s",
-				format_stat(type, statistics.get_percentile(data, 25)),
-				format_stat(type, statistics.get_percentile(data, 50)),
-				format_stat(type, statistics.get_percentile(data, 75))
-			));
-		end
-	end
-	return self;
-end
-
-function stats_methods:cfgraph()
-	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
-		local function print(s)
-			table.insert(stat_info.output, s);
-		end
-
-		if data and data.sample_count and data.sample_count > 0 then
-			local raw_histogram = require "util.statistics".get_histogram(data);
-
-			local graph_width, graph_height = 50, 10;
-			local eighth_chars = "   ▁▂▃▄▅▆▇█";
-
-			local range = data.max - data.min;
-
-			if range > 0 then
-				local x_scaling = #raw_histogram/graph_width;
-				local histogram = {};
-				for i = 1, graph_width do
-					histogram[i] = math.max(raw_histogram[i*x_scaling-1] or 0, raw_histogram[i*x_scaling] or 0);
-				end
-
-				print("");
-				print(("_"):rep(52)..format_stat(type, data.max));
-				for row = graph_height, 1, -1 do
-					local row_chars = {};
-					local min_eighths, max_eighths = 8, 0;
-					for i = 1, #histogram do
-						local char_eighths = math.ceil(math.max(math.min((graph_height/(data.max/histogram[i]))-(row-1), 1), 0)*8);
-						if char_eighths < min_eighths then
-							min_eighths = char_eighths;
-						end
-						if char_eighths > max_eighths then
-							max_eighths = char_eighths;
-						end
-						if char_eighths == 0 then
-							row_chars[i] = "-";
-						else
-							local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-							row_chars[i] = char;
-						end
-					end
-					print(table.concat(row_chars).."|-"..format_stat(type, data.max/(graph_height/(row-0.5))));
-				end
-				print(("\\    "):rep(11));
-				local x_labels = {};
-				for i = 1, 11 do
-					local s = ("%-4s"):format((i-1)*10);
-					if #s > 4 then
-						s = s:sub(1, 3).."…";
-					end
-					x_labels[i] = s;
-				end
-				print(" "..table.concat(x_labels, " "));
-				local units = "%";
-				local margin = math.floor((graph_width-#units)/2);
-				print((" "):rep(margin)..units);
-			else
-				print("[range too small to graph]");
-			end
-			print("");
-		end
-	end
-	return self;
-end
-
-function stats_methods:histogram()
-	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
-		local function print(s)
-			table.insert(stat_info.output, s);
-		end
-
-		if not data then
-			print("[no data]");
-			return self;
-		elseif not data.sample_count then
-			print("[not a sampled metric type]");
-			return self;
-		end
-
-		local graph_width, graph_height = 50, 10;
-		local eighth_chars = "   ▁▂▃▄▅▆▇█";
-
-		local range = data.max - data.min;
-
-		if range > 0 then
-			local n_buckets = graph_width;
-
-			local histogram = {};
-			for i = 1, n_buckets do
-				histogram[i] = 0;
-			end
-			local max_bin_samples = 0;
-			for _, d in ipairs(data.samples) do
-				local bucket = math.floor(1+(n_buckets-1)/(range/(d-data.min)));
-				histogram[bucket] = histogram[bucket] + 1;
-				if histogram[bucket] > max_bin_samples then
-					max_bin_samples = histogram[bucket];
-				end
-			end
-
-			print("");
-			print(("_"):rep(52)..max_bin_samples);
-			for row = graph_height, 1, -1 do
-				local row_chars = {};
-				local min_eighths, max_eighths = 8, 0;
-				for i = 1, #histogram do
-					local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/histogram[i]))-(row-1), 1), 0)*8);
-					if char_eighths < min_eighths then
-						min_eighths = char_eighths;
-					end
-					if char_eighths > max_eighths then
-						max_eighths = char_eighths;
-					end
-					if char_eighths == 0 then
-						row_chars[i] = "-";
-					else
-						local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-						row_chars[i] = char;
-					end
-				end
-				print(table.concat(row_chars).."|-"..math.ceil((max_bin_samples/graph_height)*(row-0.5)));
-			end
-			print(("\\    "):rep(11));
-			local x_labels = {};
-			for i = 1, 11 do
-				local s = ("%-4s"):format(format_stat(type, data.min+range*i/11, data.min):match("^%S+"));
-				if #s > 4 then
-					s = s:sub(1, 3).."…";
-				end
-				x_labels[i] = s;
-			end
-			print(" "..table.concat(x_labels, " "));
-			local units = format_stat(type, data.min):match("%s+(.+)$") or data.units or "";
-			local margin = math.floor((graph_width-#units)/2);
-			print((" "):rep(margin)..units);
-		else
-			print("[range too small to graph]");
-		end
-		print("");
-	end
-	return self;
-end
-
-local function stats_tostring(stats)
-	local print = stats.session.print;
-	for _, stat_info in ipairs(stats) do
-		if #stat_info.output > 0 then
-			print("\n#"..stat_info[1]);
-			print("");
-			for _, v in ipairs(stat_info.output) do
-				print(v);
-			end
-			print("");
-		else
-			print(("%-50s %s"):format(stat_info[1], format_stat(stat_info[2], stat_info[3])));
-		end
-	end
-	return #stats.." statistics displayed";
-end
-
-local stats_mt = {__index = stats_methods, __tostring = stats_tostring }
-local function new_stats_context(self)
-	return setmetatable({ session = self.session, stats = true }, stats_mt);
-end
-
-function def_env.stats:show(filter)
-	local stats, changed, extra = require "core.statsmanager".get_stats();
-	local available, displayed = 0, 0;
-	local displayed_stats = new_stats_context(self);
-	for name, value in pairs(stats) do
-		available = available + 1;
-		if not filter or name:match(filter) then
-			displayed = displayed + 1;
-			local type = name:match(":(%a+)$");
-			table.insert(displayed_stats, {
-				name, type, value, extra[name];
-				output = {};
-			});
-		end
-	end
-	return displayed_stats;
-end
-
-
-
--------------
-
-function printbanner(session)
-	local option = module:get_option_string("console_banner", "full");
-	if option == "full" or option == "graphic" then
-		session.print [[
-                   ____                \   /     _
-                    |  _ \ _ __ ___  ___  _-_   __| |_   _
-                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
-                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
-                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
-                    A study in simplicity            |___/
-
-]]
-	end
-	if option == "short" or option == "full" then
-	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
-	session.print("You may find more help on using this console in our online documentation at ");
-	session.print("https://prosody.im/doc/console\n");
-	end
-	if option ~= "short" and option ~= "full" and option ~= "graphic" then
-		session.print(option);
-	end
-end
-
 module:provides("net", {
 	name = "console";
 	listener = console_listener;
diff --git a/plugins/mod_announce.lua b/plugins/mod_announce.lua
index 970a273a..c742ebb8 100644
--- a/plugins/mod_announce.lua
+++ b/plugins/mod_announce.lua
@@ -38,6 +38,7 @@ end
 -- Old <message>-based jabberd-style announcement sending
 function handle_announcement(event)
 	local stanza = event.stanza;
+	-- luacheck: ignore 211/node
 	local node, host, resource = jid.split(stanza.attr.to);
 
 	if resource ~= "announce/online" then
diff --git a/plugins/mod_auth_anonymous.lua b/plugins/mod_auth_anonymous.lua
index 1f2bceb3..90646e71 100644
--- a/plugins/mod_auth_anonymous.lua
+++ b/plugins/mod_auth_anonymous.lua
@@ -11,6 +11,8 @@ local new_sasl = require "util.sasl".new;
 local datamanager = require "util.datamanager";
 local hosts = prosody.hosts;
 
+local allow_storage = module:get_option_boolean("allow_anonymous_storage", false);
+
 -- define auth provider
 local provider = {};
 
@@ -62,10 +64,14 @@ if not module:get_option_boolean("allow_anonymous_s2s", false) then
 end
 
 function module.load()
-	datamanager.add_callback(dm_callback);
+	if not allow_storage then
+		datamanager.add_callback(dm_callback);
+	end
 end
 function module.unload()
-	datamanager.remove_callback(dm_callback);
+	if not allow_storage then
+		datamanager.remove_callback(dm_callback);
+	end
 end
 
 module:provides("auth", provider);
diff --git a/plugins/mod_auth_internal_hashed.lua b/plugins/mod_auth_internal_hashed.lua
index 15058098..4fd0bd13 100644
--- a/plugins/mod_auth_internal_hashed.lua
+++ b/plugins/mod_auth_internal_hashed.lua
@@ -9,7 +9,7 @@
 
 local max = math.max;
 
-local getAuthenticationDatabaseSHA1 = require "util.sasl.scram".getAuthenticationDatabaseSHA1;
+local scram_hashers = require "util.sasl.scram".hashers;
 local usermanager = require "core.usermanager";
 local generate_uuid = require "util.uuid".generate;
 local new_sasl = require "util.sasl".new;
@@ -22,7 +22,9 @@ local host = module.host;
 
 local accounts = module:open_store("accounts");
 
-
+local hash_name = module:get_option_string("password_hash", "SHA-1");
+local get_auth_db = assert(scram_hashers[hash_name], "SCRAM-"..hash_name.." not supported by SASL library");
+local scram_name = "scram_"..hash_name:gsub("%-","_"):lower();
 
 -- Default; can be set per-user
 local default_iteration_count = 4096;
@@ -54,7 +56,7 @@ function provider.test_password(username, password)
 		return nil, "Auth failed. Stored salt and iteration count information is not complete.";
 	end
 
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, credentials.salt, credentials.iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, credentials.salt, credentials.iteration_count);
 
 	local stored_key_hex = to_hex(stored_key);
 	local server_key_hex = to_hex(server_key);
@@ -72,7 +74,7 @@ function provider.set_password(username, password)
 	if account then
 		account.salt = generate_uuid();
 		account.iteration_count = max(account.iteration_count or 0, default_iteration_count);
-		local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, account.salt, account.iteration_count);
+		local valid, stored_key, server_key = get_auth_db(password, account.salt, account.iteration_count);
 		if not valid then
 			return valid, stored_key;
 		end
@@ -106,7 +108,7 @@ function provider.create_user(username, password)
 		return accounts:set(username, {});
 	end
 	local salt = generate_uuid();
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, default_iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, salt, default_iteration_count);
 	if not valid then
 		return valid, stored_key;
 	end
@@ -127,7 +129,7 @@ function provider.get_sasl_handler()
 		plain_test = function(_, username, password, realm)
 			return usermanager.test_password(username, realm, password), true;
 		end,
-		scram_sha_1 = function(_, username)
+		[scram_name] = function(_, username)
 			local credentials = accounts:get(username);
 			if not credentials then return; end
 			if credentials.password then
diff --git a/plugins/mod_authz_internal.lua b/plugins/mod_authz_internal.lua
new file mode 100644
index 00000000..0f6e4873
--- /dev/null
+++ b/plugins/mod_authz_internal.lua
@@ -0,0 +1,22 @@
+local normalize = require "util.jid".prep;
+local admin_jids = module:get_option_inherited_set("admins", {}) / normalize;
+local host = module.host;
+local role_store = module:open_store("roles");
+
+local admin_role = { ["prosody:admin"] = true };
+
+function get_user_roles(user)
+	if admin_jids:contains(user.."@"..host) then
+		return admin_role;
+	end
+	return role_store:get(user);
+end
+
+function get_jid_roles(jid)
+	if admin_jids:contains(jid) then
+		return admin_role;
+	end
+	return nil;
+end
+
+
diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua
index ee48ffad..dad06b62 100644
--- a/plugins/mod_blocklist.lua
+++ b/plugins/mod_blocklist.lua
@@ -67,7 +67,7 @@ local function migrate_privacy_list(username)
 		if item.type == "jid" and item.action == "deny" then
 			local jid = jid_prep(item.value);
 			if not jid then
-				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, tostring(item.value));
+				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, item.value);
 			else
 				migrated_data[jid] = true;
 			end
@@ -162,7 +162,7 @@ local function edit_blocklist(event)
 	local blocklist = cache[username] or get_blocklist(username);
 
 	local new_blocklist = {
-		-- We set the [false] key to someting as a signal not to migrate privacy lists
+		-- We set the [false] key to something as a signal not to migrate privacy lists
 		[false] = blocklist[false] or { created = now; };
 	};
 	if type(blocklist[false]) == "table" then
@@ -189,6 +189,7 @@ local function edit_blocklist(event)
 
 	if is_blocking then
 		for jid in pairs(send_unavailable) do
+			-- Check that this JID isn't already blocked, i.e. this is not a change
 			if not blocklist[jid] then
 				for _, session in pairs(sessions[username].sessions) do
 					if session.presence then
diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua
index 52168670..6393a73e 100644
--- a/plugins/mod_bosh.lua
+++ b/plugins/mod_bosh.lua
@@ -44,19 +44,41 @@ local bosh_max_polling = module:get_option_number("bosh_max_polling", 5);
 local bosh_max_wait = module:get_option_number("bosh_max_wait", 120);
 
 local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure");
-local cross_domain = module:get_option("cross_domain_bosh", false);
+local cross_domain = module:get_option("cross_domain_bosh");
 
-if cross_domain == true then cross_domain = "*"; end
-if type(cross_domain) == "table" then cross_domain = table.concat(cross_domain, ", "); end
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_bosh' option has been deprecated");
+end
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 
 -- All sessions, and sessions that have no requests open
 local sessions = module:shared("sessions");
 
+local measure_active = module:measure("active_sessions", "amount");
+local measure_inactive = module:measure("inactive_sessions", "amount");
+local report_bad_host = module:measure("bad_host", "rate");
+local report_bad_sid = module:measure("bad_sid", "rate");
+local report_new_sid = module:measure("new_sid", "rate");
+local report_timeout = module:measure("timeout", "rate");
+
+module:hook("stats-update", function ()
+	local active = 0;
+	local inactive = 0;
+	for _, session in pairs(sessions) do
+		if #session.requests > 0 then
+			active = active + 1;
+		else
+			inactive = inactive + 1;
+		end
+	end
+	measure_active(active);
+	measure_inactive(inactive);
+end);
+
 -- Used to respond to idle sessions (those with waiting requests)
 function on_destroy_request(request)
-	log("debug", "Request destroyed: %s", tostring(request));
+	log("debug", "Request destroyed: %s", request);
 	local session = sessions[request.context.sid];
 	if session then
 		local requests = session.requests;
@@ -73,7 +95,7 @@ function on_destroy_request(request)
 			if session.inactive_timer then
 				session.inactive_timer:stop();
 			end
-			session.inactive_timer = module:add_timer(max_inactive, check_inactive, session, request.context,
+			session.inactive_timer = module:add_timer(max_inactive, session_timeout, session, request.context,
 				"BOSH client silent for over "..max_inactive.." seconds");
 			(session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive);
 		end
@@ -84,31 +106,16 @@ function on_destroy_request(request)
 	end
 end
 
-function check_inactive(now, session, context, reason) -- luacheck: ignore 212/now
+function session_timeout(now, session, context, reason) -- luacheck: ignore 212/now
 	if not session.destroyed then
+		report_timeout();
 		sessions[context.sid] = nil;
 		sm_destroy_session(session, reason);
 	end
 end
 
-local function set_cross_domain_headers(response)
-	local headers = response.headers;
-	headers.access_control_allow_methods = "GET, POST, OPTIONS";
-	headers.access_control_allow_headers = "Content-Type";
-	headers.access_control_max_age = "7200";
-	headers.access_control_allow_origin = cross_domain;
-	return response;
-end
-
-function handle_OPTIONS(event)
-	if cross_domain and event.request.headers.origin then
-		set_cross_domain_headers(event.response);
-	end
-	return "";
-end
-
 function handle_POST(event)
-	log("debug", "Handling new request %s: %s\n----------", tostring(event.request), tostring(event.request.body));
+	log("debug", "Handling new request %s: %s\n----------", event.request, event.request.body);
 
 	local request, response = event.request, event.response;
 	response.on_destroy = on_destroy_request;
@@ -121,10 +128,6 @@ function handle_POST(event)
 	local headers = response.headers;
 	headers.content_type = "text/xml; charset=utf-8";
 
-	if cross_domain and request.headers.origin then
-		set_cross_domain_headers(response);
-	end
-
 	-- stream:feed() calls the stream_callbacks, so all stanzas in
 	-- the body are processed in this next line before it returns.
 	-- In particular, the streamopened() stream callback is where
@@ -205,6 +208,7 @@ function handle_POST(event)
 		return;
 	end
 	module:log("warn", "Unable to associate request with a session (incomplete request?)");
+	report_bad_sid();
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams, condition = "item-not-found" });
 	return tostring(close_reply) .. "\n";
@@ -220,7 +224,7 @@ local function bosh_reset_stream(session) session.notopen = true; end
 
 local stream_xmlns_attr = { xmlns = "urn:ietf:params:xml:ns:xmpp-streams" };
 local function bosh_close_stream(session, reason)
-	(session.log or log)("info", "BOSH client disconnected: %s", tostring((reason and reason.condition or reason) or "session close"));
+	(session.log or log)("info", "BOSH client disconnected: %s", (reason and reason.condition or reason) or "session close");
 
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams });
@@ -245,7 +249,7 @@ local function bosh_close_stream(session, reason)
 				close_reply = reason;
 			end
 		end
-		log("info", "Disconnecting client, <stream:error> is: %s", tostring(close_reply));
+		log("info", "Disconnecting client, <stream:error> is: %s", close_reply);
 	end
 
 	local response_body = tostring(close_reply);
@@ -268,9 +272,19 @@ function stream_callbacks.streamopened(context, attr)
 		-- New session request
 		context.notopen = nil; -- Signals that we accept this opening tag
 
+		if not attr.to then
+			log("debug", "BOSH client tried to connect without specifying a host");
+			report_bad_host();
+			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
+				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
+			response:send(tostring(close_reply));
+			return;
+		end
+
 		local to_host = nameprep(attr.to);
 		if not to_host then
-			log("debug", "BOSH client tried to connect to invalid host: %s", tostring(attr.to));
+			log("debug", "BOSH client tried to connect to invalid host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -279,6 +293,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		if not prosody.hosts[to_host] then
 			log("debug", "BOSH client tried to connect to non-existant host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -287,6 +302,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		if prosody.hosts[to_host].type ~= "local" then
 			log("debug", "BOSH client tried to connect to %s host: %s", prosody.hosts[to_host].type, attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -295,7 +311,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		local wait = tonumber(attr.wait);
 		if not rid or (not attr.wait or not wait or wait < 0 or wait % 1 ~= 0) then
-			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", tostring(attr.rid), tostring(attr.wait));
+			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", attr.rid, attr.wait);
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "bad-request" });
 			response:send(tostring(close_reply));
@@ -326,6 +342,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		session.log("debug", "BOSH session created for request from %s", session.ip);
 		log("info", "New BOSH session, assigned it sid '%s'", sid);
+		report_new_sid();
 
 		module:fire_event("bosh-session", { session = session, request = request });
 
@@ -340,7 +357,7 @@ function stream_callbacks.streamopened(context, attr)
 				s.attr.xmlns = "jabber:client";
 			end
 			s = filter("stanzas/out", s);
-			--log("debug", "Sending BOSH data: %s", tostring(s));
+			--log("debug", "Sending BOSH data: %s", s);
 			if not s then return true end
 			t_insert(session.send_buffer, tostring(s));
 
@@ -380,6 +397,7 @@ function stream_callbacks.streamopened(context, attr)
 	if not session then
 		-- Unknown sid
 		log("info", "Client tried to use sid '%s' which we don't know about", sid);
+		report_bad_sid();
 		response:send(tostring(st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", condition = "item-not-found" })));
 		context.notopen = nil;
 		return;
@@ -442,7 +460,7 @@ function stream_callbacks.streamopened(context, attr)
 	end
 end
 
-local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(err, 2)); end
 
 function runner_callbacks:error(err) -- luacheck: ignore 212/self
 	return handleerr(err);
@@ -512,14 +530,16 @@ function stream_callbacks.error(context, error)
 	end
 end
 
+local GET_response_body = [[<html><body>
+	<p>It works! Now point your BOSH client to this URL to connect to Prosody.</p>
+	<p>For more information see <a href="https://prosody.im/doc/setting_up_bosh">Prosody: Setting up BOSH</a>.</p>
+	</body></html>]];
+
 local GET_response = {
 	headers = {
 		content_type = "text/html";
 	};
-	body = [[<html><body>
-	<p>It works! Now point your BOSH client to this URL to connect to Prosody.</p>
-	<p>For more information see <a href="https://prosody.im/doc/setting_up_bosh">Prosody: Setting up BOSH</a>.</p>
-	</body></html>]];
+	body = module:get_option_string("bosh_get_response_body", GET_response_body);
 };
 
 module:depends("http");
@@ -528,8 +548,6 @@ module:provides("http", {
 	route = {
 		["GET"] = GET_response;
 		["GET /"] = GET_response;
-		["OPTIONS"] = handle_OPTIONS;
-		["OPTIONS /"] = handle_OPTIONS;
 		["POST"] = handle_POST;
 		["POST /"] = handle_POST;
 	};
diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua
index c648850f..ef4bd4b3 100644
--- a/plugins/mod_c2s.lua
+++ b/plugins/mod_c2s.lua
@@ -56,7 +56,17 @@ end);
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
 
 function stream_callbacks.streamopened(session, attr)
+	-- run _streamopened in async context
+	session.thread:run({ stream = "opened", attr = attr });
+end
+
+function stream_callbacks._streamopened(session, attr)
 	local send = session.send;
+	if not attr.to then
+		session:close{ condition = "improper-addressing",
+			text = "A 'to' attribute is required on stream headers" };
+		return;
+	end
 	local host = nameprep(attr.to);
 	if not host then
 		session:close{ condition = "improper-addressing",
@@ -98,7 +108,6 @@ function stream_callbacks.streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -107,12 +116,23 @@ function stream_callbacks.streamopened(session, attr)
 	if features.tags[1] or session.full_jid then
 		send(features);
 	else
-		(session.log or log)("warn", "No stream features to offer");
+		if session.secure then
+			-- Here SASL should be offered
+			(session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings.");
+		else
+			-- Normally STARTTLS would be offered
+			(session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings.");
+		end
 		session:close{ condition = "undefined-condition", text = "No stream features to proceed with" };
 	end
 end
 
-function stream_callbacks.streamclosed(session)
+function stream_callbacks.streamclosed(session, attr)
+	-- run _streamclosed in async context
+	session.thread:run({ stream = "closed", attr = attr });
+end
+
+function stream_callbacks._streamclosed(session)
 	session.log("debug", "Received </stream:stream>");
 	session:close(false);
 end
@@ -122,7 +142,7 @@ function stream_callbacks.error(session, error, data)
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		(session.log or log)("debug", "Client XML parse error: %s", tostring(data));
+		(session.log or log)("debug", "Client XML parse error: %s", data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -252,8 +272,6 @@ function listener.onconnect(conn)
 		local sock = conn:socket();
 		if sock.info then
 			session.compressed = sock:info"compression";
-		elseif sock.compression then
-			session.compressed = sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -273,7 +291,13 @@ function listener.onconnect(conn)
 	end
 
 	session.thread = runner(function (stanza)
-		core_process_stanza(session, stanza);
+		if st.is_stanza(stanza) then
+			core_process_stanza(session, stanza);
+		elseif stanza.stream == "opened" then
+			stream_callbacks._streamopened(session, stanza.attr);
+		elseif stanza.stream == "closed" then
+			stream_callbacks._streamclosed(session, stanza.attr);
+		end
 	end, runner_callbacks, session);
 
 	local filter = session.filter;
@@ -284,8 +308,12 @@ function listener.onconnect(conn)
 			if data then
 				local ok, err = stream:feed(data);
 				if not ok then
-					log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
-					session:close("not-well-formed");
+					log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+					if err == "stanza-too-large" then
+						session:close({ condition = "policy-violation", text = "XML stanza is too big" });
+					else
+						session:close("not-well-formed");
+					end
 				end
 			end
 		end
@@ -328,6 +356,13 @@ function listener.onreadtimeout(conn)
 	end
 end
 
+function listener.ondrain(conn)
+	local session = sessions[conn];
+	if session then
+		return (hosts[session.host] or prosody).events.fire_event("c2s-ondrain", { session = session });
+	end
+end
+
 local function keepalive(event)
 	local session = event.session;
 	if not session.notopen then
@@ -360,6 +395,7 @@ module:provides("net", {
 	default_port = 5222;
 	encryption = "starttls";
 	multiplex = {
+		protocol = "xmpp-client";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:client%1.*>";
 	};
 });
diff --git a/plugins/mod_carbons.lua b/plugins/mod_carbons.lua
index 79d3e737..e0f0a711 100644
--- a/plugins/mod_carbons.lua
+++ b/plugins/mod_carbons.lua
@@ -5,10 +5,15 @@
 
 local st = require "util.stanza";
 local jid_bare = require "util.jid".bare;
+local jid_resource = require "util.jid".resource;
 local xmlns_carbons = "urn:xmpp:carbons:2";
 local xmlns_forward = "urn:xmpp:forward:0";
 local full_sessions, bare_sessions = prosody.full_sessions, prosody.bare_sessions;
 
+local function is_bare(jid)
+	return not jid_resource(jid);
+end
+
 local function toggle_carbons(event)
 	local origin, stanza = event.origin, event.stanza;
 	local state = stanza.tags[1].name;
@@ -20,6 +25,50 @@ end
 module:hook("iq-set/self/"..xmlns_carbons..":disable", toggle_carbons);
 module:hook("iq-set/self/"..xmlns_carbons..":enable", toggle_carbons);
 
+local function should_copy(stanza, c2s, user_bare) --> boolean, reason: string
+	local st_type = stanza.attr.type or "normal";
+	if stanza:get_child("private", xmlns_carbons) then
+		return false, "private";
+	end
+
+	if stanza:get_child("no-copy", "urn:xmpp:hints") then
+		return false, "hint";
+	end
+
+	if not c2s and stanza.attr.to ~= user_bare and stanza:get_child("x", "http://jabber.org/protocol/muc#user") then
+		-- MUC PMs are normally sent to full JIDs
+		return false, "muc-pm";
+	end
+
+	if st_type == "chat" then
+		return true, "type";
+	end
+
+	if st_type == "normal" and stanza:get_child("body") then
+		return true, "type";
+	end
+
+	-- Normal outgoing chat messages are sent to=bare JID. This clause should
+	-- match the error bounces from those, which would have from=bare JID and
+	-- be incoming (not c2s).
+	if st_type == "error" and not c2s and is_bare(stanza.attr.from) then
+		return true, "bounce";
+	end
+
+	if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
+		-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+		return true, "jingle call";
+	end
+
+	for archived in stanza:childtags("stanza-id", "urn:xmpp:sid:0") do
+		if archived and archived.attr.by == user_bare then
+			return true, "archived";
+		end
+	end
+
+	return false, "default";
+end
+
 local function message_handler(event, c2s)
 	local origin, stanza = event.origin, event.stanza;
 	local orig_type = stanza.attr.type or "normal";
@@ -28,10 +77,6 @@ local function message_handler(event, c2s)
 	local orig_to = stanza.attr.to;
 	local bare_to = jid_bare(orig_to);
 
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body"))) then
-		return -- Only chat type messages
-	end
-
 	-- Stanza sent by a local client
 	local bare_jid = bare_from; -- JID of the local user
 	local target_session = origin;
@@ -56,35 +101,21 @@ local function message_handler(event, c2s)
 		return -- No use in sending carbons to an offline user
 	end
 
-	if stanza:get_child("private", xmlns_carbons) then
-		if not c2s then
+	local should, why = should_copy(stanza, c2s, bare_jid);
+
+	if not should then
+		module:log("debug", "Not copying stanza: %s (%s)", stanza:top_tag(), why);
+		if why == "private" and not c2s then
 			stanza:maptags(function(tag)
 				if not ( tag.attr.xmlns == xmlns_carbons and tag.name == "private" ) then
 					return tag;
 				end
 			end);
 		end
-		module:log("debug", "Message tagged private, ignoring");
-		return
-	elseif stanza:get_child("no-copy", "urn:xmpp:hints") then
-		module:log("debug", "Message has no-copy hint, ignoring");
-		return
-	elseif not c2s and bare_jid ~= orig_to and stanza:get_child("x", "http://jabber.org/protocol/muc#user") then
-		module:log("debug", "MUC PM, ignoring");
-		return
-	end
-
-	-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
-	local copy = st.clone(stanza);
-	if c2s and not orig_to then
-		stanza.attr.to = bare_from;
+		return;
 	end
-	copy.attr.xmlns = "jabber:client";
-	local carbon = st.message{ from = bare_jid, type = orig_type, }
-		:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
-			:tag("forwarded", { xmlns = xmlns_forward })
-				:add_child(copy):reset();
 
+	local carbon;
 	user_sessions = user_sessions and user_sessions.sessions;
 	for _, session in pairs(user_sessions) do
 		-- Carbons are sent to resources that have enabled it
@@ -93,6 +124,20 @@ local function message_handler(event, c2s)
 		and session ~= target_session
 		-- and isn't among the top resources that would receive the message per standard routing rules
 		and (c2s or session.priority ~= top_priority) then
+			if not carbon then
+				-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
+				local copy = st.clone(stanza);
+				if c2s and not orig_to then
+					stanza.attr.to = bare_from;
+				end
+				copy.attr.xmlns = "jabber:client";
+				carbon = st.message{ from = bare_jid, type = orig_type, }
+					:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
+						:tag("forwarded", { xmlns = xmlns_forward })
+							:add_child(copy):reset();
+
+			end
+
 			carbon.attr.to = session.full_jid;
 			module:log("debug", "Sending carbon to %s", session.full_jid);
 			session.send(carbon);
diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua
index b41204a2..d06df71c 100644
--- a/plugins/mod_component.lua
+++ b/plugins/mod_component.lua
@@ -49,6 +49,7 @@ function module.add_host(module)
 	local send;
 
 	local function on_destroy(session, err) --luacheck: ignore 212/err
+		module:set_status("warn", err and ("Disconnected: "..err) or "Disconnected");
 		env.connected = false;
 		env.session = false;
 		send = nil;
@@ -102,6 +103,7 @@ function module.add_host(module)
 		module:log("info", "External component successfully authenticated");
 		session.send(st.stanza("handshake"));
 		module:fire_event("component-authenticated", { session = session });
+		module:set_status("info", "Connected");
 
 		return true;
 	end
@@ -130,7 +132,8 @@ function module.add_host(module)
 			end
 			module:log("warn", "Component not connected, bouncing error for: %s", stanza:top_tag());
 			if stanza.attr.type ~= "error" and stanza.attr.type ~= "result" then
-				event.origin.send(st.error_reply(stanza, "wait", "service-unavailable", "Component unavailable"));
+				event.origin.send(st.error_reply(stanza, "wait", "remote-server-timeout", "Component unavailable", module.host)
+					:tag("not-connected", { xmlns = "xmpp:prosody.im/protocol/component" }));
 			end
 		end
 		return true;
@@ -165,11 +168,11 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.error(session, error, data)
 	if session.destroyed then return; end
-	module:log("warn", "Error processing component stream: %s", tostring(error));
+	module:log("warn", "Error processing component stream: %s", error);
 	if error == "no-stream" then
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		session.log("warn", "External component %s XML parse error: %s", tostring(session.host), tostring(data));
+		session.log("warn", "External component %s XML parse error: %s", session.host, data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -206,7 +209,7 @@ function stream_callbacks.streamclosed(session)
 	session:close();
 end
 
-local function handleerr(err) log("error", "Traceback[component]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[component]: %s", traceback(err, 2)); end
 function stream_callbacks.handlestanza(session, stanza)
 	-- Namespaces are icky.
 	if not stanza.attr.xmlns and stanza.name == "handshake" then
@@ -266,10 +269,10 @@ local function session_close(session, reason)
 					if reason.extra then
 						stanza:add_child(reason.extra);
 					end
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(stanza));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", stanza);
 					session.send(stanza);
 				elseif reason.name then -- a stanza
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(reason));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", reason);
 					session.send(reason);
 				end
 			end
@@ -310,7 +313,7 @@ function listener.onconnect(conn)
 	function session.data(_, data)
 		local ok, err = stream:feed(data);
 		if ok then return; end
-		module:log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+		log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 		session:close("not-well-formed");
 	end
 
@@ -325,7 +328,7 @@ end
 function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
-		(session.log or log)("info", "component disconnected: %s (%s)", tostring(session.host), tostring(err));
+		(session.log or log)("info", "component disconnected: %s (%s)", session.host, err);
 		if session.host then
 			module:context(session.host):fire_event("component-disconnected", { session = session, reason = err });
 		end
diff --git a/plugins/mod_csi.lua b/plugins/mod_csi.lua
index 84476cac..458ff491 100644
--- a/plugins/mod_csi.lua
+++ b/plugins/mod_csi.lua
@@ -2,8 +2,9 @@ local st = require "util.stanza";
 local xmlns_csi = "urn:xmpp:csi:0";
 local csi_feature = st.stanza("csi", { xmlns = xmlns_csi });
 
+local csi_handler_available = nil;
 module:hook("stream-features", function (event)
-	if event.origin.username then
+	if event.origin.username and csi_handler_available then
 		event.features:add_child(csi_feature);
 	end
 end);
@@ -21,3 +22,14 @@ end
 module:hook("stanza/"..xmlns_csi..":active", refire_event("csi-client-active"));
 module:hook("stanza/"..xmlns_csi..":inactive", refire_event("csi-client-inactive"));
 
+function module.load()
+	if prosody.hosts[module.host].events._handlers["csi-client-active"] then
+		csi_handler_available = true;
+		module:set_status("core", "CSI handler module loaded");
+	else
+		csi_handler_available = false;
+		module:set_status("warn", "No CSI handler module loaded");
+	end
+end
+module:hook("module-loaded", module.load);
+module:hook("module-unloaded", module.load);
diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua
index 7fb6b41f..d814f083 100644
--- a/plugins/mod_csi_simple.lua
+++ b/plugins/mod_csi_simple.lua
@@ -1,4 +1,4 @@
--- Copyright (C) 2016-2018 Kim Alvefur
+-- Copyright (C) 2016-2020 Kim Alvefur
 --
 -- This project is MIT/X11 licensed. Please see the
 -- COPYING file in the source package for more information.
@@ -9,115 +9,209 @@ module:depends"csi"
 local jid = require "util.jid";
 local st = require "util.stanza";
 local dt = require "util.datetime";
-local new_queue = require "util.queue".new;
-
-local function new_pump(output, ...)
-	-- luacheck: ignore 212/self
-	local q = new_queue(...);
-	local flush = true;
-	function q:pause()
-		flush = false;
-	end
-	function q:resume()
-		flush = true;
-		return q:flush();
-	end
-	local push = q.push;
-	function q:push(item)
-		local ok = push(self, item);
-		if not ok then
-			q:flush();
-			output(item, self);
-		elseif flush then
-			return q:flush();
-		end
-		return true;
-	end
-	function q:flush()
-		local item = self:pop();
-		while item do
-			output(item, self);
-			item = self:pop();
-		end
-		return true;
-	end
-	return q;
-end
+local filters = require "util.filters";
 
 local queue_size = module:get_option_number("csi_queue_size", 256);
 
-module:hook("csi-is-stanza-important", function (event)
-	local stanza = event.stanza;
-	if not st.is_stanza(stanza) then
-		return true;
+local important_payloads = module:get_option_set("csi_important_payloads", { });
+
+function is_important(stanza) --> boolean, reason: string
+	if stanza == " " then
+		return true, "whitespace keepalive";
+	elseif type(stanza) == "string" then
+		return true, "raw data";
+	elseif not st.is_stanza(stanza) then
+		-- This should probably never happen
+		return true, type(stanza);
+	end
+	if stanza.attr.xmlns ~= nil then
+		-- stream errors, stream management etc
+		return true, "nonza";
 	end
 	local st_name = stanza.name;
 	if not st_name then return false; end
 	local st_type = stanza.attr.type;
 	if st_name == "presence" then
-		if st_type == nil or st_type == "unavailable" then
-			return false;
+		if st_type == nil or st_type == "unavailable" or st_type == "error" then
+			return false, "presence update";
 		end
-		return true;
+		-- TODO Some MUC awareness, e.g. check for the 'this relates to you' status code
+		return true, "subscription request";
 	elseif st_name == "message" then
 		if st_type == "headline" then
-			return false;
+			-- Headline messages are ephemeral by definition
+			return false, "headline";
+		end
+		if st_type == "error" then
+			return true, "delivery failure";
 		end
 		if stanza:get_child("sent", "urn:xmpp:carbons:2") then
-			return true;
+			return true, "carbon";
 		end
 		local forwarded = stanza:find("{urn:xmpp:carbons:2}received/{urn:xmpp:forward:0}/{jabber:client}message");
 		if forwarded then
 			stanza = forwarded;
 		end
 		if stanza:get_child("body") then
-			return true;
+			return true, "body";
 		end
 		if stanza:get_child("subject") then
-			return true;
+			-- Last step of a MUC join
+			return true, "subject";
 		end
 		if stanza:get_child("encryption", "urn:xmpp:eme:0") then
-			return true;
+			-- Since we can't know what an encrypted message contains, we assume it's important
+			-- XXX Experimental XEP
+			return true, "encrypted";
+		end
+		if stanza:get_child("x", "jabber:x:conference") or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
+			return true, "invite";
 		end
 		if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
-			return true;
+			-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+			return true, "jingle call";
+		end
+		for important in important_payloads do
+			if stanza:find(important) then
+				return true;
+			end
 		end
 		return false;
+	elseif st_name == "iq" then
+		return true;
 	end
-	return true;
+end
+
+module:hook("csi-is-stanza-important", function (event)
+	local important, why = is_important(event.stanza);
+	event.reason = why;
+	return important;
 end, -1);
 
-module:hook("csi-client-inactive", function (event)
-	local session = event.origin;
-	if session.pump then
-		session.pump:pause();
-	else
-		local bare_jid = jid.join(session.username, session.host);
-		local send = session.send;
-		session._orig_send = send;
-		local pump = new_pump(session.send, queue_size);
-		pump:pause();
-		session.pump = pump;
-		function session.send(stanza)
-			if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
-				pump:flush();
-				send(stanza);
-			else
-				if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
-					stanza = st.clone(stanza);
-					stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = bare_jid, stamp = dt.datetime()}));
-				end
-				pump:push(stanza);
-			end
-			return true;
+local function should_flush(stanza, session, ctr) --> boolean, reason: string
+	if ctr >= queue_size then
+		return true, "queue size limit reached";
+	end
+	local event = { stanza = stanza, session = session };
+	local ret = module:fire_event("csi-is-stanza-important", event)
+	return ret, event.reason;
+end
+
+local function with_timestamp(stanza, from)
+	if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
+		stanza = st.clone(stanza);
+		stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = from, stamp = dt.datetime()}));
+	end
+	return stanza;
+end
+
+local measure_buffer_hold = module:measure("buffer_hold", "times");
+
+local flush_reasons = setmetatable({}, {
+		__index = function (t, reason)
+			local m = module:measure("flush_reason."..reason:gsub("%W", "_"), "rate");
+			t[reason] = m;
+			return m;
+		end;
+	});
+
+
+local function manage_buffer(stanza, session)
+	local ctr = session.csi_counter or 0;
+	local flush, why = should_flush(stanza, session, ctr);
+	if flush then
+		if session.csi_measure_buffer_hold then
+			session.csi_measure_buffer_hold();
+			session.csi_measure_buffer_hold = nil;
 		end
+		flush_reasons[why or "important"]();
+		session.log("debug", "Flushing buffer (%s; queue size is %d)", why or "important", session.csi_counter);
+		session.conn:resume_writes();
+	else
+		session.log("debug", "Holding buffer (%s; queue size is %d)", why or "unimportant", session.csi_counter);
+		stanza = with_timestamp(stanza, jid.join(session.username, session.host))
+	end
+	session.csi_counter = ctr + 1;
+	return stanza;
+end
+
+local function flush_buffer(data, session)
+	session.log("debug", "Flushing buffer (%s; queue size is %d)", "client activity", session.csi_counter);
+	flush_reasons["client activity"]();
+	if session.csi_measure_buffer_hold then
+		session.csi_measure_buffer_hold();
+		session.csi_measure_buffer_hold = nil;
+	end
+	session.conn:resume_writes();
+	return data;
+end
+
+function enable_optimizations(session)
+	if session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.csi_measure_buffer_hold = measure_buffer_hold();
+		session.csi_counter = 0;
+		filters.add_filter(session, "stanzas/out", manage_buffer);
+		filters.add_filter(session, "bytes/in", flush_buffer);
+	else
+		session.log("warn", "Session connection does not support write pausing");
+	end
+end
+
+function disable_optimizations(session)
+	filters.remove_filter(session, "stanzas/out", manage_buffer);
+	filters.remove_filter(session, "bytes/in", flush_buffer);
+	session.csi_counter = nil;
+	if session.csi_measure_buffer_hold then
+		session.csi_measure_buffer_hold();
+		session.csi_measure_buffer_hold = nil;
 	end
+	if session.conn and session.conn.resume_writes then
+		session.conn:resume_writes();
+	end
+end
+
+module:hook("csi-client-inactive", function (event)
+	local session = event.origin;
+	enable_optimizations(session);
 end);
 
 module:hook("csi-client-active", function (event)
 	local session = event.origin;
-	if session.pump then
-		session.pump:resume();
+	disable_optimizations(session);
+end);
+
+module:hook("pre-resource-unbind", function (event)
+	local session = event.session;
+	disable_optimizations(session);
+end, 1);
+
+module:hook("c2s-ondrain", function (event)
+	local session = event.session;
+	if session.state == "inactive" and session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.csi_measure_buffer_hold = measure_buffer_hold();
+		session.log("debug", "Buffer flushed, resuming inactive mode (queue size was %d)", session.csi_counter);
+		session.csi_counter = 0;
 	end
 end);
 
+function module.load()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				enable_optimizations(session);
+			end
+		end
+	end
+end
+
+function module.unload()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				disable_optimizations(session);
+			end
+		end
+	end
+end
diff --git a/plugins/mod_dialback.lua b/plugins/mod_dialback.lua
index eddc3209..f580d948 100644
--- a/plugins/mod_dialback.lua
+++ b/plugins/mod_dialback.lua
@@ -93,6 +93,11 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 		-- he wants to be identified through dialback
 		-- We need to check the key with the Authoritative server
 		local attr = stanza.attr;
+		if not attr.to or not attr.from then
+			origin.log("debug", "Missing Dialback addressing (from=%q, to=%q)", attr.from, attr.to);
+			origin:close("improper-addressing");
+			return true;
+		end
 		local to, from = nameprep(attr.to), nameprep(attr.from);
 
 		if not hosts[to] then
@@ -102,6 +107,7 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 			return true;
 		elseif not from then
 			origin:close("improper-addressing");
+			return true;
 		end
 
 		if dwd and origin.secure then
diff --git a/plugins/mod_external_services.lua b/plugins/mod_external_services.lua
new file mode 100644
index 00000000..e18e7c3e
--- /dev/null
+++ b/plugins/mod_external_services.lua
@@ -0,0 +1,233 @@
+
+local dt = require "util.datetime";
+local base64 = require "util.encodings".base64;
+local hashes = require "util.hashes";
+local st = require "util.stanza";
+local jid = require "util.jid";
+local array = require "util.array";
+
+local default_host = module:get_option_string("external_service_host", module.host);
+local default_port = module:get_option_number("external_service_port");
+local default_secret = module:get_option_string("external_service_secret");
+local default_ttl = module:get_option_number("external_service_ttl", 86400);
+
+local configured_services = module:get_option_array("external_services", {});
+
+local access = module:get_option_set("external_service_access", {});
+
+-- https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
+local function behave_turn_rest_credentials(srv, item, secret)
+	local ttl = default_ttl;
+	if type(item.ttl) == "number" then
+		ttl = item.ttl;
+	end
+	local expires = srv.expires or os.time() + ttl;
+	local username;
+	if type(item.username) == "string" then
+		username = string.format("%d:%s", expires, item.username);
+	else
+		username = string.format("%d", expires);
+	end
+	srv.username = username;
+	srv.password = base64.encode(hashes.hmac_sha1(secret, srv.username));
+end
+
+local algorithms = {
+	turn = behave_turn_rest_credentials;
+}
+
+-- filter config into well-defined service records
+local function prepare(item)
+	if type(item) ~= "table" then
+		module:log("error", "Service definition is not a table: %q", item);
+		return nil;
+	end
+
+	local srv = {
+		type = nil;
+		transport = nil;
+		host = default_host;
+		port = default_port;
+		username = nil;
+		password = nil;
+		restricted = nil;
+		expires = nil;
+	};
+
+	if type(item.type) == "string" then
+		srv.type = item.type;
+	else
+		module:log("error", "Service missing mandatory 'type' field: %q", item);
+		return nil;
+	end
+	if type(item.transport) == "string" then
+		srv.transport = item.transport;
+	end
+	if type(item.host) == "string" then
+		srv.host = item.host;
+	end
+	if type(item.port) == "number" then
+		srv.port = item.port;
+	end
+	if type(item.username) == "string" then
+		srv.username = item.username;
+	end
+	if type(item.password) == "string" then
+		srv.password = item.password;
+		srv.restricted = true;
+	end
+	if item.restricted == true then
+		srv.restricted = true;
+	end
+	if type(item.expires) == "number" then
+		srv.expires = item.expires;
+	elseif type(item.ttl) == "number" then
+		srv.expires = os.time() + item.ttl;
+	end
+	if (item.secret == true and default_secret) or type(item.secret) == "string" then
+		local secret_cb = item.credentials_cb or algorithms[item.algorithm] or algorithms[srv.type];
+		local secret = item.secret;
+		if secret == true then
+			secret = default_secret;
+		end
+		if secret_cb then
+			secret_cb(srv, item, secret);
+			srv.restricted = true;
+		end
+	end
+	return srv;
+end
+
+function module.load()
+	-- Trigger errors on startup
+	local services = configured_services / prepare;
+	if #services == 0 then
+		module:log("warn", "No services configured or all had errors");
+	end
+end
+
+-- Ensure only valid items are added in events
+local services_mt = {
+	__index = getmetatable(array()).__index;
+	__newindex = function (self, i, v)
+		rawset(self, i, assert(prepare(v), "Invalid service entry added"));
+	end;
+}
+
+local function handle_services(event)
+	local origin, stanza = event.origin, event.stanza;
+	local action = stanza.tags[1];
+
+	local user_bare = jid.bare(stanza.attr.from);
+	local user_host = jid.host(user_bare);
+	if not ((access:empty() and origin.type == "c2s") or access:contains(user_bare) or access:contains(user_host)) then
+		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		return true;
+	end
+
+	local reply = st.reply(stanza):tag("services", { xmlns = action.attr.xmlns });
+	local extras = module:get_host_items("external_service");
+	local services = ( configured_services + extras ) / prepare;
+
+	local requested_type = action.attr.type;
+	if requested_type then
+		services:filter(function(item)
+			return item.type == requested_type;
+		end);
+	end
+
+	setmetatable(services, services_mt);
+
+	module:fire_event("external_service/services", {
+			origin = origin;
+			stanza = stanza;
+			reply = reply;
+			requested_type = requested_type;
+			services = services;
+		});
+
+	for _, srv in ipairs(services) do
+		reply:tag("service", {
+				type = srv.type;
+				transport = srv.transport;
+				host = srv.host;
+				port = srv.port and string.format("%d", srv.port) or nil;
+				username = srv.username;
+				password = srv.password;
+				expires = srv.expires and dt.datetime(srv.expires) or nil;
+				restricted = srv.restricted and "1" or nil;
+			}):up();
+	end
+
+	origin.send(reply);
+	return true;
+end
+
+local function handle_credentials(event)
+	local origin, stanza = event.origin, event.stanza;
+	local action = stanza.tags[1];
+
+	if origin.type ~= "c2s" then
+		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		return true;
+	end
+
+	local reply = st.reply(stanza):tag("credentials", { xmlns = action.attr.xmlns });
+	local extras = module:get_host_items("external_service");
+	local services = ( configured_services + extras ) / prepare;
+	services:filter(function (item)
+		return item.restricted;
+	end)
+
+	local requested_credentials = {};
+	for service in action:childtags("service") do
+		table.insert(requested_credentials, {
+				type = service.attr.type;
+				host = service.attr.host;
+				port = tonumber(service.attr.port);
+			});
+	end
+
+	setmetatable(services, services_mt);
+	setmetatable(requested_credentials, services_mt);
+
+	module:fire_event("external_service/credentials", {
+			origin = origin;
+			stanza = stanza;
+			reply = reply;
+			requested_credentials = requested_credentials;
+			services = services;
+		});
+
+	for req_srv in action:childtags("service") do
+		for _, srv in ipairs(services) do
+			if srv.type == req_srv.attr.type and srv.host == req_srv.attr.host
+				and not req_srv.attr.port or srv.port == tonumber(req_srv.attr.port) then
+				reply:tag("service", {
+						type = srv.type;
+						transport = srv.transport;
+						host = srv.host;
+						port = srv.port and string.format("%d", srv.port) or nil;
+						username = srv.username;
+						password = srv.password;
+						expires = srv.expires and dt.datetime(srv.expires) or nil;
+						restricted = srv.restricted and "1" or nil;
+					}):up();
+			end
+		end
+	end
+
+	origin.send(reply);
+	return true;
+end
+
+-- XEP-0215 v0.7
+module:add_feature("urn:xmpp:extdisco:2");
+module:hook("iq-get/host/urn:xmpp:extdisco:2:services", handle_services);
+module:hook("iq-get/host/urn:xmpp:extdisco:2:credentials", handle_credentials);
+
+-- COMPAT XEP-0215 v0.6
+-- Those still on the old version gets to deal with undefined attributes until they upgrade.
+module:add_feature("urn:xmpp:extdisco:1");
+module:hook("iq-get/host/urn:xmpp:extdisco:1:services", handle_services);
+module:hook("iq-get/host/urn:xmpp:extdisco:1:credentials", handle_credentials);
diff --git a/plugins/mod_groups.lua b/plugins/mod_groups.lua
index 646b7408..0c44f481 100644
--- a/plugins/mod_groups.lua
+++ b/plugins/mod_groups.lua
@@ -25,7 +25,7 @@ function inject_roster_contacts(event)
 	local function import_jids_to_roster(group_name)
 		for jid in pairs(groups[group_name]) do
 			-- Add them to roster
-			--module:log("debug", "processing jid %s in group %s", tostring(jid), tostring(group_name));
+			--module:log("debug", "processing jid %s in group %s", jid, group_name);
 			if jid ~= bare_jid then
 				if not roster[jid] then roster[jid] = {}; end
 				roster[jid].subscription = "both";
@@ -99,7 +99,7 @@ function module.load()
 				end
 				members[false][#members[false]+1] = curr_group; -- Is a public group
 			end
-			module:log("debug", "New group: %s", tostring(curr_group));
+			module:log("debug", "New group: %s", curr_group);
 			groups[curr_group] = groups[curr_group] or {};
 		else
 			-- Add JID
@@ -108,7 +108,7 @@ function module.load()
 			local jid;
 			jid = jid_prep(entryjid:match("%S+"));
 			if jid then
-				module:log("debug", "New member of %s: %s", tostring(curr_group), tostring(jid));
+				module:log("debug", "New member of %s: %s", curr_group, jid);
 				groups[curr_group][jid] = name or false;
 				members[jid] = members[jid] or {};
 				members[jid][#members[jid]+1] = curr_group;
diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua
index a1d409bd..84e2d18f 100644
--- a/plugins/mod_http.lua
+++ b/plugins/mod_http.lua
@@ -7,13 +7,21 @@
 --
 
 module:set_global();
-module:depends("http_errors");
+pcall(function ()
+	module:depends("http_errors");
+end);
 
 local portmanager = require "core.portmanager";
 local moduleapi = require "core.moduleapi";
 local url_parse = require "socket.url".parse;
 local url_build = require "socket.url".build;
 local normalize_path = require "util.http".normalize_path;
+local set = require "util.set";
+
+local ip_util = require "util.ip";
+local new_ip = ip_util.new_ip;
+local match_ip = ip_util.match;
+local parse_cidr = ip_util.parse_cidr;
 
 local server = require "net.http.server";
 
@@ -22,6 +30,12 @@ server.set_default_host(module:get_option_string("http_default_host"));
 server.set_option("body_size_limit", module:get_option_number("http_max_content_size"));
 server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size"));
 
+-- CORS settigs
+local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "OPTIONS" });
+local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" });
+local opt_credentials = module:get_option_boolean("access_control_allow_credentials", false);
+local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60);
+
 local function get_http_event(host, app_path, key)
 	local method, path = key:match("^(%S+)%s+(.+)$");
 	if not method then -- No path specified, default to "" (base path)
@@ -79,10 +93,22 @@ function moduleapi.http_url(module, app_name, default_path)
 			return url_build(url);
 		end
 	end
-	module:log("warn", "No http ports enabled, can't generate an external URL");
+	if prosody.process_type == "prosody" then
+		module:log("warn", "No http ports enabled, can't generate an external URL");
+	end
 	return "http://disabled.invalid/";
 end
 
+local function apply_cors_headers(response, methods, headers, max_age, allow_credentials, origin)
+	response.headers.access_control_allow_methods = tostring(methods);
+	response.headers.access_control_allow_headers = tostring(headers);
+	response.headers.access_control_max_age = tostring(max_age)
+	response.headers.access_control_allow_origin = origin or "*";
+	if allow_credentials then
+		response.headers.access_control_allow_credentials = "true";
+	end
+end
+
 function module.add_host(module)
 	local host = module.host;
 	if host ~= "*" then
@@ -101,9 +127,29 @@ function module.add_host(module)
 		end
 		apps[app_name] = apps[app_name] or {};
 		local app_handlers = apps[app_name];
+
+		local app_methods = opt_methods;
+
+		local function cors_handler(event_data)
+			local request, response = event_data.request, event_data.response;
+			apply_cors_headers(response, app_methods, opt_headers, opt_max_age, opt_credentials, request.headers.origin);
+		end
+
+		local function options_handler(event_data)
+			cors_handler(event_data);
+			return "";
+		end
+
+		local streaming = event.item.streaming_uploads;
+
 		for key, handler in pairs(event.item.route or {}) do
 			local event_name = get_http_event(host, app_path, key);
 			if event_name then
+				local method = event_name:match("^%S+");
+				if not app_methods:contains(method) then
+					app_methods = app_methods + set.new{ method };
+				end
+				local options_event_name = event_name:gsub("^%S+", "OPTIONS");
 				if type(handler) ~= "function" then
 					local data = handler;
 					handler = function () return data; end
@@ -118,9 +164,24 @@ function module.add_host(module)
 				elseif event_name:sub(-1, -1) == "/" then
 					module:hook_object_event(server, event_name:sub(1, -2), redir_handler, -1);
 				end
+				if not streaming then
+					-- COMPAT Modules not compatible with streaming uploads behave as before.
+					local _handler = handler;
+					function handler(event) -- luacheck: ignore 432/event
+						if event.request.body ~= false then
+							return _handler(event);
+						end
+					end
+				end
 				if not app_handlers[event_name] then
-					app_handlers[event_name] = handler;
+					app_handlers[event_name] = {
+						main = handler;
+						cors = cors_handler;
+						options = options_handler;
+					};
 					module:hook_object_event(server, event_name, handler);
+					module:hook_object_event(server, event_name, cors_handler, 1);
+					module:hook_object_event(server, options_event_name, options_handler, -1);
 				else
 					module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name);
 				end
@@ -130,8 +191,8 @@ function module.add_host(module)
 		end
 		local services = portmanager.get_active_services();
 		if services:get("https") or services:get("http") then
-			module:log("debug", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
-		else
+			module:log("info", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
+		elseif prosody.process_type == "prosody" then
 			module:log("warn", "Not listening on any ports, '%s' will be unreachable", app_name);
 		end
 	end
@@ -139,8 +200,11 @@ function module.add_host(module)
 	local function http_app_removed(event)
 		local app_handlers = apps[event.item.name];
 		apps[event.item.name] = nil;
-		for event_name, handler in pairs(app_handlers) do
-			module:unhook_object_event(server, event_name, handler);
+		for event_name, handlers in pairs(app_handlers) do
+			module:unhook_object_event(server, event_name, handlers.main);
+			module:unhook_object_event(server, event_name, handlers.cors);
+			local options_event_name = event_name:gsub("^%S+", "OPTIONS");
+			module:unhook_object_event(server, options_event_name, handlers.options);
 		end
 	end
 
@@ -158,13 +222,31 @@ module.add_host(module); -- set up handling on global context too
 
 local trusted_proxies = module:get_option_set("trusted_proxies", { "127.0.0.1", "::1" })._items;
 
+local function is_trusted_proxy(ip)
+	local parsed_ip = new_ip(ip)
+	for trusted_proxy in trusted_proxies do
+		if match_ip(parsed_ip, parse_cidr(trusted_proxy)) then
+			return true;
+		end
+	end
+	return false
+end
+
 local function get_ip_from_request(request)
 	local ip = request.conn:ip();
 	local forwarded_for = request.headers.x_forwarded_for;
 	if forwarded_for then
+		-- luacheck: ignore 631
+		-- This logic looks weird at first, but it makes sense.
+		-- The for loop will take the last non-trusted-proxy IP from `forwarded_for`.
+		-- We append the original request IP to the header. Then, since the last IP wins, there are two cases:
+		-- Case a) The original request IP is *not* in trusted proxies, in which case the X-Forwarded-For header will, effectively, be ineffective; the original request IP will win because it overrides any other IP in the header.
+		-- Case b) The original request IP is in trusted proxies. In that case, the if branch in the for loop will skip the last IP, causing it to be ignored. The second-to-last IP will be taken instead.
+		-- Case c) If the second-to-last IP is also a trusted proxy, it will also be ignored, iteratively, up to the last IP which isn’t in trusted proxies.
+		-- Case d) If all IPs are in trusted proxies, something went obviously wrong and the logic never overwrites `ip`, leaving it at the original request IP.
 		forwarded_for = forwarded_for..", "..ip;
 		for forwarded_ip in forwarded_for:gmatch("[^%s,]+") do
-			if not trusted_proxies[forwarded_ip] then
+			if not is_trusted_proxy(forwarded_ip) then
 				ip = forwarded_ip;
 			end
 		end
@@ -195,10 +277,8 @@ module:provides("net", {
 	listener = server.listener;
 	default_port = 5281;
 	encryption = "ssl";
-	ssl_config = {
-		verify = "none";
-	};
 	multiplex = {
+		protocol = "http/1.1";
 		pattern = "^[A-Z]";
 	};
 });
diff --git a/plugins/mod_http_errors.lua b/plugins/mod_http_errors.lua
index 13473219..a2138d88 100644
--- a/plugins/mod_http_errors.lua
+++ b/plugins/mod_http_errors.lua
@@ -26,28 +26,38 @@ local html = [[
 <meta charset="utf-8">
 <title>{title}</title>
 <style>
-body{
-	margin-top:14%;
-	text-align:center;
-	background-color:#F8F8F8;
-	font-family:sans-serif;
+body {
+	margin-top : 14%;
+	text-align : center;
+	background-color : #F8F8F8;
+	font-family : sans-serif
 }
-h1{
-	font-size:xx-large;
+
+h1 {
+	font-size : xx-large
 }
-p{
-	font-size:x-large;
+
+p {
+	font-size : x-large
 }
-p+p {
-	font-size:large;
-	font-family:courier;
+
+p.extra {
+	font-size : large;
+	font-family : courier
+}
+
+@media(prefers-color-scheme: dark) {
+	body {
+		background-color: #161616;
+		color: #eee
+	}
 }
 </style>
 </head>
 <body>
 <h1>{title}</h1>
 <p>{message}</p>
-<p>{extra?}</p>
+{extra&<p class="extra">{extra?}</p>}
 </body>
 </html>
 ]];
@@ -70,5 +80,17 @@ module:hook_object_event(server, "http-error", function (event)
 	if event.response then
 		event.response.headers.content_type = "text/html; charset=utf-8";
 	end
-	return get_page(event.code, (show_private and event.private_message) or event.message);
+	return get_page(event.code, (show_private and event.private_message) or event.message or (event.error and event.error.text));
 end);
+
+module:hook_object_event(server, "http-error", function (event)
+	local request, response = event.request, event.response;
+	if request and response and request.path == "/" and response.status_code == 404 then
+		response.headers.content_type = "text/html; charset=utf-8";
+		return render(html, {
+				title = "Prosody is running!";
+				message = "Welcome to the XMPP world!";
+			});
+	end
+end, 1);
+
diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua
index a8398c01..4d0b14cd 100644
--- a/plugins/mod_http_files.lua
+++ b/plugins/mod_http_files.lua
@@ -7,14 +7,9 @@
 --
 
 module:depends("http");
-local server = require"net.http.server";
-local lfs = require "lfs";
 
-local os_date = os.date;
 local open = io.open;
-local stat = lfs.attributes;
-local build_path = require"socket.url".build_path;
-local path_sep = package.config:sub(1,1);
+local fileserver = require"net.http.files";
 
 local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path"));
 local cache_size = module:get_option_number("http_files_cache_size", 128);
@@ -38,7 +33,9 @@ if not mime_map then
 	module:shared("/*/http_files/mime").types = mime_map;
 
 	local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r");
-	if mime_types then
+	if not mime_types then
+		module:log("debug", "Could not open MIME database: %s", err);
+	else
 		local mime_data = mime_types:read("*a");
 		mime_types:close();
 		setmetatable(mime_map, {
@@ -51,148 +48,56 @@ if not mime_map then
 	end
 end
 
-local forbidden_chars_pattern = "[/%z]";
-if prosody.platform == "windows" then
-	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+local function get_calling_module()
+	local info = debug.getinfo(3, "S");
+	if not info then return "An unknown module"; end
+	return info.source:match"mod_[^/\\.]+" or info.short_src;
 end
 
-local urldecode = require "util.http".urldecode;
-function sanitize_path(path)
-	if not path then return end
-	local out = {};
-
-	local c = 0;
-	for component in path:gmatch("([^/]+)") do
-		component = urldecode(component);
-		if component:find(forbidden_chars_pattern) then
-			return nil;
-		elseif component == ".." then
-			if c <= 0 then
-				return nil;
-			end
-			out[c] = nil;
-			c = c - 1;
-		elseif component ~= "." then
-			c = c + 1;
-			out[c] = component;
-		end
-	end
-	if path:sub(-1,-1) == "/" then
-		out[c+1] = "";
-	end
-	return "/"..table.concat(out, "/");
-end
-
-local cache = require "util.cache".new(cache_size);
-
+-- COMPAT -- TODO deprecate
 function serve(opts)
 	if type(opts) ~= "table" then -- assume path string
 		opts = { path = opts };
 	end
-	-- luacheck: ignore 431
-	local base_path = opts.path;
-	local dir_indices = opts.index_files or dir_indices;
-	local directory_index = opts.directory_index;
-	local function serve_file(event, path)
-		local request, response = event.request, event.response;
-		local sanitized_path = sanitize_path(path);
-		if path and not sanitized_path then
-			return 400;
-		end
-		path = sanitized_path;
-		local orig_path = sanitize_path(request.path);
-		local full_path = base_path .. (path or ""):gsub("/", path_sep);
-		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
-		if not attr then
-			return 404;
-		end
-
-		local request_headers, response_headers = request.headers, response.headers;
-
-		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
-		response_headers.last_modified = last_modified;
-
-		local etag = ('"%x-%x-%x"'):format(attr.change or 0, attr.size or 0, attr.modification or 0);
-		response_headers.etag = etag;
-
-		local if_none_match = request_headers.if_none_match
-		local if_modified_since = request_headers.if_modified_since;
-		if etag == if_none_match
-		or (not if_none_match and last_modified == if_modified_since) then
-			return 304;
-		end
-
-		local data = cache:get(orig_path);
-		if data and data.etag == etag then
-			response_headers.content_type = data.content_type;
-			data = data.data;
-		elseif attr.mode == "directory" and path then
-			if full_path:sub(-1) ~= "/" then
-				local dir_path = { is_absolute = true, is_directory = true };
-				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
-				response_headers.location = build_path(dir_path);
-				return 301;
-			end
-			for i=1,#dir_indices do
-				if stat(full_path..dir_indices[i], "mode") == "file" then
-					return serve_file(event, path..dir_indices[i]);
-				end
-			end
-
-			if directory_index then
-				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
-			end
-			if not data then
-				return 403;
-			end
-			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
-			response_headers.content_type = mime_map.html;
-
-		else
-			local f, err = open(full_path, "rb");
-			if not f then
-				module:log("debug", "Could not open %s. Error was %s", full_path, err);
-				return 403;
-			end
-			local ext = full_path:match("%.([^./]+)$");
-			local content_type = ext and mime_map[ext];
-			response_headers.content_type = content_type;
-			if attr.size > cache_max_file_size then
-				response_headers.content_length = attr.size;
-				module:log("debug", "%d > cache_max_file_size", attr.size);
-				return response:send_file(f);
-			else
-				data = f:read("*a");
-				f:close();
-			end
-			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
-		end
-
-		return response:send(data);
+	if opts.directory_index == nil then
+		opts.directory_index = directory_index;
 	end
-
-	return serve_file;
+	if opts.mime_map == nil then
+		opts.mime_map = mime_map;
+	end
+	if opts.cache_size == nil then
+		opts.cache_size = cache_size;
+	end
+	if opts.cache_max_file_size == nil then
+		opts.cache_max_file_size = cache_max_file_size;
+	end
+	if opts.index_files == nil then
+		opts.index_files = dir_indices;
+	end
+	-- TODO Crank up to warning
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
+	return fileserver.serve(opts);
 end
 
 function wrap_route(routes)
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
 	for route,handler in pairs(routes) do
 		if type(handler) ~= "function" then
-			routes[route] = serve(handler);
+			routes[route] = fileserver.serve(handler);
 		end
 	end
 	return routes;
 end
 
-if base_path then
-	module:provides("http", {
-		route = {
-			["GET /*"] = serve {
-				path = base_path;
-				directory_index = directory_index;
-			}
-		};
-	});
-else
-	module:log("debug", "http_files_dir not set, assuming use by some other module");
-end
-
+module:provides("http", {
+	route = {
+		["GET /*"] = fileserver.serve({
+			path = base_path;
+			directory_index = directory_index;
+			mime_map = mime_map;
+			cache_size = cache_size;
+			cache_max_file_size = cache_max_file_size;
+			index_files = dir_indices;
+		});
+	};
+});
diff --git a/plugins/mod_lastactivity.lua b/plugins/mod_lastactivity.lua
index 575e66be..91d11bd2 100644
--- a/plugins/mod_lastactivity.lua
+++ b/plugins/mod_lastactivity.lua
@@ -30,7 +30,7 @@ module:hook("iq-get/bare/jabber:iq:last:query", function(event)
 	if not stanza.attr.to or is_contact_subscribed(username, module.host, jid_bare(stanza.attr.from)) then
 		local seconds, text = "0", "";
 		if map[username] then
-			seconds = tostring(os.difftime(os.time(), map[username].t));
+			seconds = string.format("%d", os.difftime(os.time(), map[username].t));
 			text = map[username].s;
 		end
 		origin.send(st.reply(stanza):tag('query', {xmlns='jabber:iq:last', seconds=seconds}):text(text));
diff --git a/plugins/mod_legacyauth.lua b/plugins/mod_legacyauth.lua
index 0f41d3e7..941806d3 100644
--- a/plugins/mod_legacyauth.lua
+++ b/plugins/mod_legacyauth.lua
@@ -78,8 +78,10 @@ module:hook("stanza/iq/jabber:iq:auth:query", function(event)
 					session:close(); -- FIXME undo resource bind and auth instead of closing the session?
 					return true;
 				end
+				session.send(st.reply(stanza));
+			else
+				session.send(st.error_reply(stanza, "auth", "not-authorized", err));
 			end
-			session.send(st.reply(stanza));
 		else
 			session.send(st.error_reply(stanza, "auth", "not-authorized"));
 		end
diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua
index 914d5c44..024ab686 100644
--- a/plugins/mod_limits.lua
+++ b/plugins/mod_limits.lua
@@ -32,7 +32,7 @@ local function parse_burst(burst, sess_type)
 	end
 	local n_burst = tonumber(burst);
 	if not n_burst then
-		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, tostring(burst), default_burst);
+		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, burst, default_burst);
 	end
 	return n_burst or default_burst;
 end
@@ -51,18 +51,18 @@ end
 local default_filter_set = {};
 
 function default_filter_set.bytes_in(bytes, session)
-  local sess_throttle = session.throttle;
-  if sess_throttle then
-    local ok, balance, outstanding = sess_throttle:poll(#bytes, true);
+	local sess_throttle = session.throttle;
+	if sess_throttle then
+		local ok, _, outstanding = sess_throttle:poll(#bytes, true);
 		if not ok then
-      session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
+			session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
 			outstanding = ceil(outstanding);
 			session.conn:pause(); -- Read no more data from the connection until there is no outstanding data
 			local outstanding_data = bytes:sub(-outstanding);
 			bytes = bytes:sub(1, #bytes-outstanding);
 			timer.add_task(limits_resolution, function ()
 				if not session.conn then return; end
-        if sess_throttle:peek(#outstanding_data) then
+				if sess_throttle:peek(#outstanding_data) then
 					session.log("debug", "Resuming paused session");
 					session.conn:resume();
 				end
@@ -84,8 +84,13 @@ local function filter_hook(session)
 	local session_type = session.type:match("^[^_]+");
 	local filter_set, opts = type_filters[session_type], limits[session_type];
 	if opts then
-		session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
-		filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		if session.conn and session.conn.setlimit then
+			session.conn:setlimit(opts.bytes_per_second);
+			-- Currently no burst support
+		else
+			session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
+			filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		end
 	end
 end
 
@@ -96,3 +101,25 @@ end
 function module.unload()
 	filters.remove_filter_hook(filter_hook);
 end
+
+function module.add_host(module)
+	local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {});
+
+	if not unlimited_jids:empty() then
+		module:hook("authentication-success", function (event)
+			local session = event.session;
+			local session_type = session.type:match("^[^_]+");
+			local jid = session.username .. "@" .. session.host;
+			if unlimited_jids:contains(jid) then
+				if session.conn and session.conn.setlimit then
+					session.conn:setlimit(0);
+					-- Currently no burst support
+				else
+					local filter_set = type_filters[session_type];
+					filters.remove_filter(session, "bytes/in", filter_set.bytes_in);
+					session.throttle = nil;
+				end
+			end
+		end);
+	end
+end
diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index e7d89a95..4fe714eb 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -25,6 +25,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prepped_split = require "util.jid".prepped_split;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 local host = module.host;
 
 local rm_load_roster = require "core.rostermanager".load_roster;
@@ -40,6 +41,11 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja
 local archive_store = module:get_option_string("archive_store", "archive");
 local archive = module:open_store(archive_store, "archive");
 
+local cleanup_after = module:get_option_string("archive_expires_after", "1w");
+local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+local archive_truncate = math.floor(archive_item_limit * 0.99);
+
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -98,7 +104,14 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local qwith, qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -117,10 +130,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query, id %s with %s from %s until %s",
-		tostring(qid), qwith or "anyone",
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qwith or "*",
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -128,6 +143,9 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local reverse = qset and qset.before or false;
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(origin.username, {
@@ -140,7 +158,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -189,13 +212,13 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
-
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -213,13 +236,13 @@ local function shall_store(user, who)
 	end
 	local prefs = get_prefs(user);
 	local rule = prefs[who];
-	module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
+	module:log("debug", "%s's rule for %s is %s", user, who, rule);
 	if rule ~= nil then
 		return rule;
 	end
 	-- Below could be done by a metatable
 	local default = prefs[false];
-	module:log("debug", "%s's default rule is %s", user, tostring(default));
+	module:log("debug", "%s's default rule is %s", user, default);
 	if default == "roster" then
 		return has_in_roster(user, who);
 	end
@@ -242,11 +265,70 @@ local function strip_stanza_id(stanza, user)
 	return stanza;
 end
 
+local function should_store(stanza, c2s) --> boolean, reason: string
+	local st_type = stanza.attr.type or "normal";
+	-- FIXME pass direction of stanza and use that along with bare/full JID addressing
+	-- for more accurate MUC / type=groupchat check
+
+	if st_type == "headline" then
+		-- Headline messages are ephemeral by definition
+		return false, "headline";
+	end
+	if st_type == "error" and not c2s then
+		-- Store delivery failure notifications so you know if your own messages were not delivered
+		return true, "bounce";
+	end
+	if st_type == "groupchat" then
+		-- MUC messages always go to the full JID, usually archived by the MUC
+		return false, "groupchat";
+	end
+	if stanza:get_child("no-store", "urn:xmpp:hints")
+	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		-- XXX Experimental XEP
+		return false, "hint";
+	end
+	if stanza:get_child("store", "urn:xmpp:hints") then
+		return true, "hint";
+	end
+	if stanza:get_child("body") then
+		return true, "body";
+	end
+	if stanza:get_child("subject") then
+		-- XXX Who would send a message with a subject but without a body?
+		return true, "subject";
+	end
+	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
+		-- Since we can't know what an encrypted message contains, we assume it's important
+		-- XXX Experimental XEP
+		return true, "encrypted";
+	end
+	if stanza:get_child(nil, "urn:xmpp:receipts") then
+		-- If it's important enough to ask for a receipt then it's important enough to archive
+		-- and the same applies to the receipt
+		return true, "receipt";
+	end
+	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		-- XXX Experimental XEP
+		return true, "marker";
+	end
+	if stanza:get_child("x", "jabber:x:conference")
+	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
+		return true, "invite";
+	end
+	if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
+		-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+		return true, "jingle call";
+	end
+
+	 -- The IM-NG thing to do here would be to return `not st_to_full`
+	 -- One day ...
+	return false, "default";
+end
+
 -- Handle messages
 local function message_handler(event, c2s)
 	local origin, stanza = event.origin, event.stanza;
 	local log = c2s and origin.log or module._log;
-	local orig_type = stanza.attr.type or "normal";
 	local orig_from = stanza.attr.from;
 	local orig_to = stanza.attr.to or orig_from;
 	-- Stanza without 'to' are treated as if it was to their own bare jid
@@ -259,21 +341,12 @@ local function message_handler(event, c2s)
 	-- Filter out <stanza-id> that claim to be from us
 	event.stanza = strip_stanza_id(stanza, store_user);
 
-	-- We store chat messages or normal messages that have a body
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
-		log("debug", "Not archiving stanza: %s (type)", stanza:top_tag());
+	local should, why = should_store(stanza, c2s);
+	if not should then
+		log("debug", "Not archiving stanza: %s (%s)", stanza:top_tag(), why);
 		return;
 	end
 
-	-- or if hints suggest we shouldn't
-	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
-		if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
-			or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
-			log("debug", "Not archiving stanza: %s (hint)", stanza:top_tag());
-			return;
-		end
-	end
-
 	local clone_for_storage;
 	if not strip_tags:empty() then
 		clone_for_storage = st.clone(stanza);
@@ -294,10 +367,31 @@ local function message_handler(event, c2s)
 
 	-- Check with the users preferences
 	if shall_store(store_user, with) then
-		log("debug", "Archiving stanza: %s", stanza:top_tag());
+		log("debug", "Archiving stanza: %s (%s)", stanza:top_tag(), why);
 
 		-- And stash it
-		local ok, err = archive:append(store_user, nil, clone_for_storage, time_now(), with);
+		local time = time_now();
+		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+		if not ok and err == "quota-limit" then
+			if type(cleanup_after) == "number" then
+				module:log("debug", "User '%s' over quota, cleaning archive", store_user);
+				local cleaned = archive:delete(store_user, {
+					["end"] = (os.time() - cleanup_after);
+				});
+				if cleaned then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+			if not ok and (archive.caps and archive.caps.truncate) then
+				module:log("debug", "User '%s' over quota, truncating archive", store_user);
+				local truncated = archive:delete(store_user, {
+					truncate = archive_truncate;
+				});
+				if truncated then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+		end
 		if ok then
 			local clone_for_other_handlers = st.clone(stanza);
 			local id = ok;
@@ -325,8 +419,6 @@ end
 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 
-local cleanup_after = module:get_option_string("archive_expires_after", "1w");
-local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("archive_cleanup");
 	local cleanup_map = module:open_store("archive_cleanup", "map");
@@ -361,9 +453,11 @@ if cleanup_after ~= "never" then
 			last_date:set(username, date);
 		end
 	end
+	local cleanup_time = module:measure("cleanup", "times");
 
 	local async = require "util.async";
 	cleanup_runner = async.runner(function ()
+		local cleanup_done = cleanup_time();
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -397,6 +491,7 @@ if cleanup_after ~= "never" then
 			wait();
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua
new file mode 100644
index 00000000..b586a70c
--- /dev/null
+++ b/plugins/mod_mimicking.lua
@@ -0,0 +1,85 @@
+-- Prosody IM
+-- Copyright (C) 2012 Florian Zeitz
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local encodings = require "util.encodings";
+assert(encodings.confusable, "This module requires that Prosody be built with ICU");
+local skeleton = encodings.confusable.skeleton;
+
+local usage = require "util.prosodyctl".show_usage;
+local usermanager = require "core.usermanager";
+local storagemanager = require "core.storagemanager";
+
+local skeletons
+function module.load()
+	if module.host ~= "*" then
+		skeletons = module:open_store("skeletons");
+	end
+end
+
+module:hook("user-registered", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, { username = user.username });
+	if not ok then
+		module:log("error", "Unable to store mimicry data (%q => %q): %s", user.username, skel, err);
+	end
+end);
+
+module:hook("user-deleted", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, nil);
+	if not ok and err then
+		module:log("error", "Unable to clear mimicry data (%q): %s", skel, err);
+	end
+end);
+
+module:hook("user-registering", function(user)
+	local existing, err = skeletons:get(skeleton(user.username));
+	if existing then
+		module:log("debug", "Attempt to register username '%s' which could be confused with '%s'", user.username, existing.username);
+		user.allowed = false;
+	elseif err then
+		module:log("error", "Unable to check if new username '%s' can be confused with any existing user: %s", err);
+	end
+end);
+
+function module.command(arg)
+	if (arg[1] ~= "bootstrap" or not arg[2]) then
+		usage("mod_mimicking bootstrap <host>", "Initialize username mimicry database");
+		return;
+	end
+
+	local host = arg[2];
+
+	local host_session = prosody.hosts[host];
+	if not host_session then
+		return "No such host";
+	end
+
+	storagemanager.initialize_host(host);
+	usermanager.initialize_host(host);
+
+	skeletons = storagemanager.open(host, "skeletons");
+
+	local count = 0;
+	for user in usermanager.users(host) do
+		local skel = skeleton(user);
+		local existing, err = skeletons:get(skel);
+		if existing and existing.username ~= user then
+			module:log("warn", "Existing usernames '%s' and '%s' are confusable", existing.username, user);
+		elseif err then
+			module:log("error", "Error checking for existing mimicry data (%q = %q): %s", user, skel, err);
+		end
+		local ok, err = skeletons:set(skel, { username = user });
+		if ok then
+			count = count + 1;
+		elseif err then
+			module:log("error", "Unable to store mimicry data (%q => %q): %s", user, skel, err);
+		end
+	end
+	module:log("info", "%d usernames indexed", count);
+end
diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua
index f0417889..65794cf4 100644
--- a/plugins/mod_muc_mam.lua
+++ b/plugins/mod_muc_mam.lua
@@ -4,7 +4,7 @@
 -- This file is MIT/X11 licensed.
 
 if module:get_host_type() ~= "component" then
-	module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
+	module:log_status("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
 	return;
 end
 
@@ -21,6 +21,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prep = require "util.jid".prep;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 
 local mod_muc = module:depends"muc";
 local get_room_from_jid = mod_muc.get_room_from_jid;
@@ -32,6 +33,9 @@ local m_min = math.min;
 local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 
+local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
+local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
+
 local default_history_length = 20;
 local max_history_length = module:get_option_number("max_history_messages", math.huge);
 
@@ -49,6 +53,9 @@ local log_by_default = module:get_option_boolean("muc_log_by_default", true);
 local archive_store = "muc_log";
 local archive = module:open_store(archive_store, "archive");
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+local archive_truncate = math.floor(archive_item_limit * 0.99);
+
 if archive.name == "null" or not archive.find then
 	if not archive.find then
 		module:log("error", "Attempt to open archive storage returned a driver without archive API support");
@@ -63,12 +70,15 @@ end
 
 local function archiving_enabled(room)
 	if log_all_rooms then
+		module:log("debug", "Archiving all rooms");
 		return true;
 	end
 	local enabled = room._data.archiving;
 	if enabled == nil then
+		module:log("debug", "Default is %s (for %s)", log_by_default, room.jid);
 		return log_by_default;
 	end
+	module:log("debug", "Logging in room %s is %s", room.jid, enabled);
 	return enabled;
 end
 
@@ -135,7 +145,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	local qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -153,10 +170,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query id %s from %s until %s)",
-		tostring(qid),
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -165,6 +183,9 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(room_node, {
@@ -176,7 +197,12 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -233,13 +259,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -274,7 +301,7 @@ module:hook("muc-get-history", function (event)
 	local data, err = archive:find(jid_split(room_jid), query);
 
 	if not data then
-		module:log("error", "Could not fetch history: %s", tostring(err));
+		module:log("error", "Could not fetch history: %s", err);
 		return
 	end
 
@@ -300,7 +327,7 @@ module:hook("muc-get-history", function (event)
 			maxchars = maxchars - chars;
 		end
 		history[i], i = item, i+1;
-		-- module:log("debug", tostring(item));
+		-- module:log("debug", item);
 	end
 	function event.next_stanza()
 		i = i - 1;
@@ -325,7 +352,7 @@ end, 1);
 
 -- Handle messages
 local function save_to_history(self, stanza)
-	local room_node, room_host = jid_split(self.jid);
+	local room_node = jid_split(self.jid);
 
 	local stored_stanza = stanza;
 
@@ -352,7 +379,29 @@ local function save_to_history(self, stanza)
 	end
 
 	-- And stash it
-	local id, err = archive:append(room_node, nil, stored_stanza, time_now(), with);
+	local time = time_now();
+	local id, err = archive:append(room_node, nil, stored_stanza, time, with);
+
+	if not id and err == "quota-limit" then
+		if type(cleanup_after) == "number" then
+			module:log("debug", "Room '%s' over quota, cleaning archive", room_node);
+			local cleaned = archive:delete(room_node, {
+				["end"] = (os.time() - cleanup_after);
+			});
+			if cleaned then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+		if not id and (archive.caps and archive.caps.truncate) then
+			module:log("debug", "User '%s' over quota, truncating archive", room_node);
+			local truncated = archive:delete(room_node, {
+				truncate = archive_truncate;
+			});
+			if truncated then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+	end
 
 	if id then
 		schedule_cleanup(room_node);
@@ -391,15 +440,14 @@ end
 module:add_feature(xmlns_mam);
 
 module:hook("muc-disco#info", function(event)
-	event.reply:tag("feature", {var=xmlns_mam}):up();
+	if archiving_enabled(event.room) then
+		event.reply:tag("feature", {var=xmlns_mam}):up();
+	end
 	event.reply:tag("feature", {var=xmlns_st_id}):up();
 end);
 
 -- Cleanup
 
-local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
-local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
-
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("muc_log_cleanup");
 	local cleanup_map = module:open_store("muc_log_cleanup", "map");
@@ -435,8 +483,11 @@ if cleanup_after ~= "never" then
 		end
 	end
 
+	local cleanup_time = module:measure("cleanup", "times");
+
 	local async = require "util.async";
 	cleanup_runner = async.runner(function ()
+		local cleanup_done = cleanup_time();
 		local rooms = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -470,6 +521,7 @@ if cleanup_after ~= "never" then
 			wait();
 		end
 		module:log("info", "Deleted %d expired messages for %d rooms", sum, num_rooms);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_net_multiplex.lua b/plugins/mod_net_multiplex.lua
index 8ef77883..ddd58463 100644
--- a/plugins/mod_net_multiplex.lua
+++ b/plugins/mod_net_multiplex.lua
@@ -1,22 +1,38 @@
 module:set_global();
 
+local array = require "util.array";
 local max_buffer_len = module:get_option_number("multiplex_buffer_size", 1024);
+local default_mode = module:get_option_number("network_default_read_size", 4096);
 
 local portmanager = require "core.portmanager";
 
 local available_services = {};
+local service_by_protocol = {};
+local available_protocols = array();
 
 local function add_service(service)
 	local multiplex_pattern = service.multiplex and service.multiplex.pattern;
+	local protocol_name = service.multiplex and service.multiplex.protocol;
+	if protocol_name then
+		module:log("debug", "Adding multiplex service %q with protocol %q", service.name, protocol_name);
+		service_by_protocol[protocol_name] = service;
+		available_protocols:push(protocol_name);
+	end
 	if multiplex_pattern then
 		module:log("debug", "Adding multiplex service %q with pattern %q", service.name, multiplex_pattern);
 		available_services[service] = multiplex_pattern;
-	else
+	elseif not protocol_name then
 		module:log("debug", "Service %q is not multiplex-capable", service.name);
 	end
 end
 module:hook("service-added", function (event) add_service(event.service); end);
-module:hook("service-removed", function (event)	available_services[event.service] = nil; end);
+module:hook("service-removed", function (event)
+	available_services[event.service] = nil;
+	if event.service.multiplex and event.service.multiplex.protocol then
+		available_protocols:filter(function (p) return p ~= event.service.multiplex.protocol end);
+		service_by_protocol[event.service.multiplex.protocol] = nil;
+	end
+end);
 
 for _, services in pairs(portmanager.get_registered_services()) do
 	for _, service in ipairs(services) do
@@ -26,9 +42,22 @@ end
 
 local buffers = {};
 
-local listener = { default_mode = "*a" };
+local listener = { default_mode = max_buffer_len };
 
-function listener.onconnect()
+function listener.onconnect(conn)
+	local sock = conn:socket();
+	if sock.getalpn then
+		local selected_proto = sock:getalpn();
+		local service = service_by_protocol[selected_proto];
+		if service then
+			module:log("debug", "Routing incoming connection to %s based on ALPN %q", service.name, selected_proto);
+			local next_listener = service.listener;
+			conn:setlistener(next_listener);
+			conn:set_mode(next_listener.default_mode or default_mode);
+			local onconnect = next_listener.onconnect;
+			if onconnect then return onconnect(conn) end
+		end
+	end
 end
 
 function listener.onincoming(conn, data)
@@ -40,6 +69,7 @@ function listener.onincoming(conn, data)
 			module:log("debug", "Routing incoming connection to %s", service.name);
 			local next_listener = service.listener;
 			conn:setlistener(next_listener);
+			conn:set_mode(next_listener.default_mode or default_mode);
 			local onconnect = next_listener.onconnect;
 			if onconnect then onconnect(conn) end
 			return next_listener.onincoming(conn, buf);
@@ -68,5 +98,10 @@ module:provides("net", {
 	name = "multiplex_ssl";
 	config_prefix = "ssl";
 	encryption = "ssl";
+	ssl_config = {
+		alpn = function ()
+			return available_protocols;
+		end;
+	};
 	listener = listener;
 });
diff --git a/plugins/mod_offline.lua b/plugins/mod_offline.lua
index 487098d1..dffe8357 100644
--- a/plugins/mod_offline.lua
+++ b/plugins/mod_offline.lua
@@ -24,11 +24,16 @@ module:hook("message/offline/handle", function(event)
 		node = origin.username;
 	end
 
-	return offline_messages:append(node, nil, stanza, os.time(), "");
+	local ok = offline_messages:append(node, nil, stanza, os.time(), "");
+	if ok then
+		module:log("debug", "Saved to offline storage: %s", stanza:top_tag());
+	end
+	return ok;
 end, -1);
 
 module:hook("message/offline/broadcast", function(event)
 	local origin = event.origin;
+	origin.log("debug", "Broadcasting offline messages");
 
 	local node, host = origin.username, origin.host;
 
@@ -38,6 +43,9 @@ module:hook("message/offline/broadcast", function(event)
 		stanza:tag("delay", {xmlns = "urn:xmpp:delay", from = host, stamp = datetime.datetime(when)}):up(); -- XEP-0203
 		origin.send(stanza);
 	end
-	offline_messages:delete(node);
+	local ok = offline_messages:delete(node);
+	if type(ok) == "number" and ok > 0 then
+		origin.log("debug", "%d offline messages consumed");
+	end
 	return true;
 end, -1);
diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua
index f8047a92..db4fdc87 100644
--- a/plugins/mod_pep.lua
+++ b/plugins/mod_pep.lua
@@ -8,6 +8,7 @@ local calculate_hash = require "util.caps".calculate_hash;
 local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed;
 local cache = require "util.cache";
 local set = require "util.set";
+local new_id = require "util.id".medium;
 local storagemanager = require "core.storagemanager";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
@@ -123,9 +124,6 @@ local function get_broadcaster(username)
 		if kind == "retract" then
 			kind = "items"; -- XEP-0060 signals retraction in an <items> container
 		end
-		local message = st.message({ from = user_bare, type = "headline" })
-			:tag("event", { xmlns = xmlns_pubsub_event })
-				:tag(kind, { node = node });
 		if item then
 			item = st.clone(item);
 			item.attr.xmlns = nil; -- Clear the pubsub namespace
@@ -134,10 +132,19 @@ local function get_broadcaster(username)
 					item:maptags(function () return nil; end);
 				end
 			end
+		end
+
+		local id = new_id();
+		local message = st.message({ from = user_bare, type = "headline", id = id })
+			:tag("event", { xmlns = xmlns_pubsub_event })
+				:tag(kind, { node = node });
+
+		if item then
 			message:add_child(item);
 		end
+
 		for jid in pairs(jids) do
-			module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item));
+			module:log("debug", "Sending notification to %s from %s for node %s", jid, user_bare, node);
 			message.attr.to = jid;
 			module:send(message);
 		end
@@ -166,12 +173,12 @@ local function get_subscriber_filter(username)
 end
 
 function get_pep_service(username)
-	module:log("debug", "get_pep_service(%q)", username);
 	local user_bare = jid_join(username, host);
 	local service = services[username];
 	if service then
 		return service;
 	end
+	module:log("debug", "Creating pubsub service for user %q", username);
 	service = pubsub.new({
 		pep_username = username;
 		node_defaults = {
@@ -238,8 +245,6 @@ end
 module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq);
 module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq);
 
-module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody"));
-module:add_feature("http://jabber.org/protocol/pubsub#publish");
 
 local function get_caps_hash_from_presence(stanza, current)
 	local t = stanza.attr.type;
diff --git a/plugins/mod_pep_simple.lua b/plugins/mod_pep_simple.lua
index f0b5d7ef..e686b99b 100644
--- a/plugins/mod_pep_simple.lua
+++ b/plugins/mod_pep_simple.lua
@@ -14,6 +14,7 @@ local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed
 local pairs = pairs;
 local next = next;
 local type = type;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local calculate_hash = require "util.caps".calculate_hash;
 local core_post_stanza = prosody.core_post_stanza;
 local bare_sessions = prosody.bare_sessions;
@@ -84,6 +85,7 @@ local function publish_all(user, recipient, session)
 	if d and notify then
 		for node in pairs(notify) do
 			if d[node] then
+				-- luacheck: ignore id
 				local id, item = unpack(d[node]);
 				session.send(st.message({from=user, to=recipient, type='headline'})
 					:tag('event', {xmlns='http://jabber.org/protocol/pubsub#event'})
@@ -229,13 +231,13 @@ module:hook("iq/bare/http://jabber.org/protocol/pubsub:pubsub", function(event)
 				return true;
 			else --invalid request
 				session.send(st.error_reply(stanza, 'modify', 'bad-request'));
-				module:log("debug", "Invalid request: %s", tostring(payload));
+				module:log("debug", "Invalid request: %s", payload);
 				return true;
 			end
 		else --no presence subscription
 			session.send(st.error_reply(stanza, 'auth', 'not-authorized')
 				:tag('presence-subscription-required', {xmlns='http://jabber.org/protocol/pubsub#errors'}));
-			module:log("debug", "Unauthorized request: %s", tostring(payload));
+			module:log("debug", "Unauthorized request: %s", payload);
 			return true;
 		end
 	end
diff --git a/plugins/mod_ping.lua b/plugins/mod_ping.lua
index 5fff58d1..b6ccc928 100644
--- a/plugins/mod_ping.lua
+++ b/plugins/mod_ping.lua
@@ -11,23 +11,9 @@ local st = require "util.stanza";
 module:add_feature("urn:xmpp:ping");
 
 local function ping_handler(event)
-	return event.origin.send(st.reply(event.stanza));
+	event.origin.send(st.reply(event.stanza));
+	return true;
 end
 
 module:hook("iq-get/bare/urn:xmpp:ping:ping", ping_handler);
 module:hook("iq-get/host/urn:xmpp:ping:ping", ping_handler);
-
--- Ad-hoc command
-
-local datetime = require "util.datetime".datetime;
-
-function ping_command_handler (self, data, state) -- luacheck: ignore 212
-	local now = datetime();
-	return { info = "Pong\n"..now, status = "completed" };
-end
-
-module:depends "adhoc";
-local adhoc_new = module:require "adhoc".new;
-local descriptor = adhoc_new("Ping", "ping", ping_command_handler);
-module:provides("adhoc", descriptor);
-
diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua
index fe826c22..03177530 100644
--- a/plugins/mod_posix.lua
+++ b/plugins/mod_posix.lua
@@ -20,7 +20,6 @@ if not have_signal then
 	module:log("warn", "Couldn't load signal library, won't respond to SIGTERM");
 end
 
-local format = require "util.format".format;
 local lfs = require "lfs";
 local stat = lfs.attributes;
 
@@ -31,39 +30,12 @@ module:set_global(); -- we're a global module
 local umask = module:get_option_string("umask", "027");
 pposix.umask(umask);
 
--- Allow switching away from root, some people like strange ports.
-module:hook("server-started", function ()
-	local uid = module:get_option("setuid");
-	local gid = module:get_option("setgid");
-	if gid then
-		local success, msg = pposix.setgid(gid);
-		if success then
-			module:log("debug", "Changed group to %s successfully.", gid);
-		else
-			module:log("error", "Failed to change group to %s. Error: %s", gid, msg);
-			prosody.shutdown("Failed to change group to %s", gid);
-		end
-	end
-	if uid then
-		local success, msg = pposix.setuid(uid);
-		if success then
-			module:log("debug", "Changed user to %s successfully.", uid);
-		else
-			module:log("error", "Failed to change user to %s. Error: %s", uid, msg);
-			prosody.shutdown("Failed to change user to %s", uid);
-		end
-	end
-end);
-
 -- Don't even think about it!
 if not prosody.start_time then -- server-starting
-	local suid = module:get_option("setuid");
-	if not suid or suid == 0 or suid == "root" then
-		if pposix.getuid() == 0 and not module:get_option_boolean("run_as_root") then
-			module:log("error", "Danger, Will Robinson! Prosody doesn't need to be run as root, so don't do it!");
-			module:log("error", "For more information on running Prosody as root, see https://prosody.im/doc/root");
-			prosody.shutdown("Refusing to run as root");
-		end
+	if pposix.getuid() == 0 and not module:get_option_boolean("run_as_root") then
+		module:log("error", "Danger, Will Robinson! Prosody doesn't need to be run as root, so don't do it!");
+		module:log("error", "For more information on running Prosody as root, see https://prosody.im/doc/root");
+		prosody.shutdown("Refusing to run as root");
 	end
 end
 
@@ -113,24 +85,15 @@ local function write_pidfile()
 	end
 end
 
-local syslog_opened;
-function syslog_sink_maker(config) -- luacheck: ignore 212/config
-	if not syslog_opened then
-		pposix.syslog_open("prosody", module:get_option_string("syslog_facility"));
-		syslog_opened = true;
-	end
-	local syslog = pposix.syslog_log;
-	return function (name, level, message, ...)
-		syslog(level, name, format(message, ...));
-	end;
-end
-require "core.loggingmanager".register_sink_type("syslog", syslog_sink_maker);
-
 local daemonize = prosody.opts.daemonize;
 
 if daemonize == nil then
 	-- Fall back to config file if not specified on command-line
-	daemonize = module:get_option("daemonize", prosody.installed);
+	daemonize = module:get_option_boolean("daemonize", nil);
+	if daemonize ~= nil then
+		module:log("warn", "The 'daemonize' option has been deprecated, specify -D or -F on the command line instead.");
+		-- TODO: Write some docs and include a link in the warning.
+	end
 end
 
 local function remove_log_sinks()
@@ -154,9 +117,7 @@ if daemonize then
 			write_pidfile();
 		end
 	end
-	if not prosody.start_time then -- server-starting
-		daemonize_server();
-	end
+	module:hook("server-started", daemonize_server)
 else
 	-- Not going to daemonize, so write the pid of this process
 	write_pidfile();
@@ -186,5 +147,20 @@ if have_signal then
 			prosody.shutdown("Received SIGINT");
 			prosody.lock_globals();
 		end);
+
+		signal.signal("SIGUSR1", function ()
+			module:log("info", "Received SIGUSR1");
+			module:fire_event("signal/SIGUSR1");
+		end);
+
+		signal.signal("SIGUSR2", function ()
+			module:log("info", "Received SIGUSR2");
+			module:fire_event("signal/SIGUSR2");
+		end);
 	end);
 end
+
+-- For other modules to reference
+features = {
+	signal_events = true;
+};
diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua
index 268a2f0c..3f9a0c12 100644
--- a/plugins/mod_presence.lua
+++ b/plugins/mod_presence.lua
@@ -14,6 +14,7 @@ local s_find = string.find;
 local tonumber = tonumber;
 
 local core_post_stanza = prosody.core_post_stanza;
+local core_process_stanza = prosody.core_process_stanza;
 local st = require "util.stanza";
 local jid_split = require "util.jid".split;
 local jid_bare = require "util.jid".bare;
@@ -30,6 +31,14 @@ local recalc_resource_map = require "util.presence".recalc_resource_map;
 
 local ignore_presence_priority = module:get_option_boolean("ignore_presence_priority", false);
 
+local pre_approval_stream_feature = st.stanza("sub", {xmlns="urn:xmpp:features:pre-approval"});
+module:hook("stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.username then
+		features:add_child(pre_approval_stream_feature);
+	end
+end);
+
 function handle_normal_presence(origin, stanza)
 	if ignore_presence_priority then
 		local priority = stanza:get_child("priority");
@@ -81,8 +90,14 @@ function handle_normal_presence(origin, stanza)
 				res.presence.attr.to = nil;
 			end
 		end
-		for jid in pairs(roster[false].pending) do -- resend incoming subscription requests
-			origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original?
+		for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests
+			if type(pending_request) == "table" then
+				local subscribe = st.deserialize(pending_request);
+				subscribe.attr.type, subscribe.attr.from = "subscribe", jid;
+				origin.send(subscribe);
+			else
+				origin.send(st.presence({type="subscribe", from=jid}));
+			end
 		end
 		local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host});
 		for jid, item in pairs(roster) do -- resend outgoing subscription requests
@@ -175,8 +190,10 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		if rostermanager.subscribed(node, host, to_bare) then
 			rostermanager.roster_push(node, host, to_bare);
 		end
-		core_post_stanza(origin, stanza);
-		send_presence_of_available_resources(node, host, to_bare, origin);
+		if rostermanager.is_contact_subscribed(node, host, to_bare) then
+			core_post_stanza(origin, stanza);
+			send_presence_of_available_resources(node, host, to_bare, origin);
+		end
 		if rostermanager.is_user_subscribed(node, host, to_bare) then
 			core_post_stanza(origin, st.presence({ type = "probe", from = from_bare, to = to_bare }));
 		end
@@ -184,6 +201,8 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		-- 1. send unavailable
 		-- 2. route stanza
 		-- 3. roster push (subscription = from or both)
+		-- luacheck: ignore 211/pending_in
+		-- Is pending_in meant to be used?
 		local success, pending_in, subscribed = rostermanager.unsubscribed(node, host, to_bare);
 		if success then
 			if subscribed then
@@ -223,10 +242,16 @@ function handle_inbound_presence_subscriptions_and_probes(origin, stanza, from_b
 			if 0 == send_presence_of_available_resources(node, host, from_bare, origin) then
 				core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- TODO send last activity
 			end
+		elseif rostermanager.is_contact_preapproved(node, host, from_bare) then
+			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
+					core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="subscribed"}), true);
+				end -- TODO else return error, unable to save
+			end
 		else
 			core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt
 			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
-				if rostermanager.set_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
 					sessionmanager.send_to_available_resources(node, host, stanza);
 				end -- TODO else return error, unable to save
 			end
@@ -346,7 +371,7 @@ module:hook("resource-unbind", function(event)
 		if err then
 			pres:tag("status"):text("Disconnected: "..err):up();
 		end
-		session:dispatch_stanza(pres);
+		core_process_stanza(session, pres);
 	elseif session.directed then
 		local pres = st.presence{ type = "unavailable", from = session.full_jid };
 		if err then
diff --git a/plugins/mod_proxy65.lua b/plugins/mod_proxy65.lua
index 00833772..bac36b55 100644
--- a/plugins/mod_proxy65.lua
+++ b/plugins/mod_proxy65.lua
@@ -12,7 +12,6 @@ module:set_global();
 local jid_compare, jid_prep = require "util.jid".compare, require "util.jid".prep;
 local st = require "util.stanza";
 local sha1 = require "util.hashes".sha1;
-local b64 = require "util.encodings".base64.encode;
 local server = require "net.server";
 local portmanager = require "core.portmanager";
 
@@ -45,7 +44,7 @@ function listener.onincoming(conn, data)
 		end -- else error, unexpected input
 		conn:write("\5\255"); -- send (SOCKS version 5, no acceptable method)
 		conn:close();
-		module:log("debug", "Invalid SOCKS5 greeting received: '%s'", b64(data));
+		module:log("debug", "Invalid SOCKS5 greeting received: %q", data:sub(1, 300));
 	else -- connection request
 		--local head = string.char( 0x05, 0x01, 0x00, 0x03, 40 ); -- ( VER=5=SOCKS5, CMD=1=CONNECT, RSV=0=RESERVED, ATYP=3=DOMAIMNAME, SHA-1 size )
 		if #data == 47 and data:sub(1,5) == "\5\1\0\3\40" and data:sub(-2) == "\0\0" then
@@ -67,7 +66,7 @@ function listener.onincoming(conn, data)
 		else -- error, unexpected input
 			conn:write("\5\1\0\3\0\0\0"); -- VER, REP, RSV, ATYP, BND.ADDR (sha), BND.PORT (2 Byte)
 			conn:close();
-			module:log("debug", "Invalid SOCKS5 negotiation received: '%s'", b64(data));
+			module:log("debug", "Invalid SOCKS5 negotiation received: %q", data:sub(1, 300));
 		end
 	end
 end
@@ -117,7 +116,7 @@ function module.add_host(module)
 				if jid_compare(jid, acl) then allow = true; break; end
 			end
 			if allow then break; end
-			module:log("warn", "Denying use of proxy for %s", tostring(stanza.attr.from));
+			module:log("warn", "Denying use of proxy for %s", stanza.attr.from);
 			origin.send(st.error_reply(stanza, "auth", "forbidden"));
 			return true;
 		end
diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua
index b6019c71..de1000b2 100644
--- a/plugins/mod_pubsub/mod_pubsub.lua
+++ b/plugins/mod_pubsub/mod_pubsub.lua
@@ -42,7 +42,7 @@ end
 
 local node_store = module:open_store(module.name.."_nodes");
 
-local function create_simple_itemstore(node_config, node_name)
+local function create_simple_itemstore(node_config, node_name) --> util.cache like object
 	local driver = storagemanager.get_driver(module.host, "pubsub_data");
 	local archive = driver:open("pubsub_"..node_name, "archive");
 	return lib_pubsub.archive_itemstore(archive, node_config, nil, node_name);
@@ -75,14 +75,13 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 	local msg_type = node_obj and node_obj.config.message_type or "headline";
 	local message = st.message({ from = module.host, type = msg_type, id = id })
 		:tag("event", { xmlns = xmlns_pubsub_event })
-			:tag(kind, { node = node })
+			:tag(kind, { node = node });
 
 	if item then
 		message:add_child(item);
 	end
 
 	local summary;
-	-- Compose a sensible textual representation of at least Atom payloads
 	if item and item.tags[1] then
 		local payload = item.tags[1];
 		summary = module:fire_event("pubsub-summary/"..payload.attr.xmlns, {
@@ -101,11 +100,12 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 end
 
 local max_max_items = module:get_option_number("pubsub_max_items", 256);
-function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node
+function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor
 	if (new_config["max_items"] or 1) > max_max_items then
 		return false;
 	end
-	if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then
+	if new_config["access_model"] ~= "whitelist"
+	and new_config["access_model"] ~= "open" then
 		return false;
 	end
 	return true;
@@ -115,6 +115,7 @@ function is_item_stanza(item)
 	return st.is_stanza(item) and item.attr.xmlns == xmlns_pubsub and item.name == "item" and #item.tags == 1;
 end
 
+-- Compose a textual representation of Atom payloads
 module:hook("pubsub-summary/http://www.w3.org/2005/Atom", function (event)
 	local payload = event.payload;
 	local title = payload:get_child_text("title");
diff --git a/plugins/mod_pubsub/pubsub.lib.lua b/plugins/mod_pubsub/pubsub.lib.lua
index 50ef7ddf..c84208a2 100644
--- a/plugins/mod_pubsub/pubsub.lib.lua
+++ b/plugins/mod_pubsub/pubsub.lib.lua
@@ -7,6 +7,7 @@ local st = require "util.stanza";
 local it = require "util.iterators";
 local uuid_generate = require "util.uuid".generate;
 local dataform = require"util.dataforms".new;
+local errors = require "util.error";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
 local xmlns_pubsub_errors = "http://jabber.org/protocol/pubsub#errors";
@@ -34,6 +35,9 @@ local pubsub_errors = {
 };
 local function pubsub_error_reply(stanza, error)
 	local e = pubsub_errors[error];
+	if not e and errors.is_err(error) then
+		e = { error.type, error.condition, error.text, error.pubsub_condition };
+	end
 	local reply = st.error_reply(stanza, t_unpack(e, 1, 3));
 	if e[4] then
 		reply:tag(e[4], { xmlns = xmlns_pubsub_errors }):up();
@@ -185,6 +189,14 @@ local node_metadata_form = dataform {
 		type = "text-single";
 		name = "pubsub#type";
 	};
+	{
+		type = "text-single";
+		name = "pubsub#access_model";
+	};
+	{
+		type = "text-single";
+		name = "pubsub#publish_model";
+	};
 };
 
 local service_method_feature_map = {
@@ -258,6 +270,8 @@ function _M.handle_disco_info_node(event, service)
 			["pubsub#title"] = node_obj.config.title;
 			["pubsub#description"] = node_obj.config.description;
 			["pubsub#type"] = node_obj.config.payload_type;
+			["pubsub#access_model"] = node_obj.config.access_model;
+			["pubsub#publish_model"] = node_obj.config.publish_model;
 		}, "result"));
 	end
 end
@@ -318,14 +332,9 @@ function handlers.get_items(origin, stanza, items, service)
 	for _, id in ipairs(results) do
 		data:add_child(results[id]);
 	end
-	local reply;
-	if data then
-		reply = st.reply(stanza)
-			:tag("pubsub", { xmlns = xmlns_pubsub })
-				:add_child(data);
-	else
-		reply = pubsub_error_reply(stanza, "item-not-found");
-	end
+	local reply = st.reply(stanza)
+		:tag("pubsub", { xmlns = xmlns_pubsub })
+			:add_child(data);
 	origin.send(reply);
 	return true;
 end
@@ -633,14 +642,13 @@ function handlers.set_retract(origin, stanza, retract, service)
 end
 
 function handlers.owner_set_purge(origin, stanza, purge, service)
-	local node, notify = purge.attr.node, purge.attr.notify;
-	notify = (notify == "1") or (notify == "true");
+	local node = purge.attr.node;
 	local reply;
 	if not node then
 		origin.send(pubsub_error_reply(stanza, "nodeid-required"));
 		return true;
 	end
-	local ok, ret = service:purge(node, stanza.attr.from, notify);
+	local ok, ret = service:purge(node, stanza.attr.from, true);
 	if ok then
 		reply = st.reply(stanza);
 	else
@@ -788,7 +796,7 @@ local function create_encapsulating_item(id, payload)
 end
 
 local function archive_itemstore(archive, config, user, node)
-	module:log("debug", "Creation of itemstore for node %s with config %s", node, config);
+	module:log("debug", "Creation of archive itemstore for node %s with config %q", node, config);
 	local get_set = {};
 	local max_items = config["max_items"];
 	function get_set:items() -- luacheck: ignore 212/self
@@ -802,6 +810,7 @@ local function archive_itemstore(archive, config, user, node)
 		end
 		module:log("debug", "Listed items %s", data);
 		return it.reverse(function()
+			-- luacheck: ignore 211/when
 			local id, payload, when, publisher = data();
 			if id == nil then
 				return;
@@ -863,7 +872,7 @@ local function archive_itemstore(archive, config, user, node)
 			return item.attr.id, item;
 		end
 	end
-	return setmetatable(get_set, archive);
+	return get_set;
 end
 _M.archive_itemstore = archive_itemstore;
 
diff --git a/plugins/mod_register.lua b/plugins/mod_register.lua
index 763e2fd9..bec3fb5a 100644
--- a/plugins/mod_register.lua
+++ b/plugins/mod_register.lua
@@ -11,6 +11,7 @@ local allow_registration = module:get_option_boolean("allow_registration", false
 
 if allow_registration then
 	module:depends("register_ibr");
+	module:depends("watchregistrations");
 end
 
 module:depends("user_account_management");
diff --git a/plugins/mod_register_ibr.lua b/plugins/mod_register_ibr.lua
index e04e6ecd..83d284c8 100644
--- a/plugins/mod_register_ibr.lua
+++ b/plugins/mod_register_ibr.lua
@@ -9,10 +9,12 @@
 
 local st = require "util.stanza";
 local dataform_new = require "util.dataforms".new;
-local usermanager_user_exists = require "core.usermanager".user_exists;
-local usermanager_create_user = require "core.usermanager".create_user;
-local usermanager_delete_user = require "core.usermanager".delete_user;
+local usermanager_user_exists  = require "core.usermanager".user_exists;
+local usermanager_create_user  = require "core.usermanager".create_user;
+local usermanager_set_password = require "core.usermanager".create_user;
+local usermanager_delete_user  = require "core.usermanager".delete_user;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
+local util_error = require "util.error";
 
 local additional_fields = module:get_option("additional_registration_fields", {});
 local require_encryption = module:get_option_boolean("c2s_require_encryption",
@@ -155,7 +157,7 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	local username, password = nodeprep(data.username), data.password;
+	local username, password = nodeprep(data.username, true), data.password;
 	data.username, data.password = nil, nil;
 	local host = module.host;
 	if not username or username == "" then
@@ -167,25 +169,44 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 	local user = { username = username, password = password, host = host, additional = data, ip = session.ip, session = session, allowed = true }
 	module:fire_event("user-registering", user);
 	if not user.allowed then
-		log("debug", "Registration disallowed by module: %s", user.reason or "no reason given");
-		session.send(st.error_reply(stanza, "modify", "not-acceptable", user.reason));
+		local error_type, error_condition, reason;
+		local err = user.error;
+		if err then
+			error_type, error_condition, reason = err.type, err.condition, err.text;
+		else
+			-- COMPAT pre-util.error
+			error_type, error_condition, reason = user.error_type, user.error_condition, user.reason;
+		end
+		log("debug", "Registration disallowed by module: %s", reason or "no reason given");
+		session.send(st.error_reply(stanza, error_type or "modify", error_condition or "not-acceptable", reason));
 		return true;
 	end
 
 	if usermanager_user_exists(username, host) then
-		log("debug", "Attempt to register with existing username");
-		session.send(st.error_reply(stanza, "cancel", "conflict", "The requested username already exists."));
-		return true;
+		if user.allow_reset == username then
+			local ok, err = util_error.coerce(usermanager_set_password(username, password, host));
+			if ok then
+				module:fire_event("user-password-reset", user);
+				session.send(st.reply(stanza)); -- reset ok!
+			else
+				session.log("error", "Unable to reset password for %s@%s: %s", username, host, err);
+				session.send(st.error_reply(stanza, err.type, err.condition, err.text));
+			end
+			return true;
+		else
+			log("debug", "Attempt to register with existing username");
+			session.send(st.error_reply(stanza, "cancel", "conflict", "The requested username already exists."));
+			return true;
+		end
 	end
 
-	-- TODO unable to write file, file may be locked, etc, what's the correct error?
-	local error_reply = st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk.");
-	if usermanager_create_user(username, password, host) then
+	local created, err = usermanager_create_user(username, password, host);
+	if created then
 		data.registered = os.time();
 		if not account_details:set(username, data) then
 			log("debug", "Could not store extra details");
 			usermanager_delete_user(username, host);
-			session.send(error_reply);
+			session.send(st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk."));
 			return true;
 		end
 		session.send(st.reply(stanza)); -- user created!
@@ -194,8 +215,8 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 			username = username, host = host, source = "mod_register",
 			session = session });
 	else
-		log("debug", "Could not create user");
-		session.send(error_reply);
+		log("debug", "Could not create user", err);
+		session.send(st.error_reply(stanza, "cancel", "feature-not-implemented", err));
 	end
 	return true;
 end);
diff --git a/plugins/mod_register_limits.lua b/plugins/mod_register_limits.lua
index 736282a5..7c80f18b 100644
--- a/plugins/mod_register_limits.lua
+++ b/plugins/mod_register_limits.lua
@@ -13,6 +13,7 @@ local ip_util = require "util.ip";
 local new_ip = ip_util.new_ip;
 local match_ip = ip_util.match;
 local parse_cidr = ip_util.parse_cidr;
+local errors = require "util.error";
 
 local min_seconds_between_registrations = module:get_option_number("min_seconds_between_registrations");
 local whitelist_only = module:get_option_boolean("whitelist_registration_only");
@@ -54,6 +55,24 @@ local function ip_in_set(set, ip)
 	return false;
 end
 
+local err_registry = {
+	blacklisted = {
+		text = "Your IP address is blacklisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	not_whitelisted = {
+		text = "Your IP address is not whitelisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	throttled = {
+		text = "Too many registrations from this IP address recently";
+		type = "wait";
+		condition = "policy-violation";
+	};
+}
+
 module:hook("user-registering", function (event)
 	local session = event.session;
 	local ip = event.ip or session and session.ip;
@@ -63,16 +82,22 @@ module:hook("user-registering", function (event)
 	elseif ip_in_set(blacklisted_ips, ip) then
 		log("debug", "Registration disallowed by blacklist");
 		event.allowed = false;
-		event.reason = "Your IP address is blacklisted";
+		event.error = errors.new("blacklisted", event, err_registry);
 	elseif (whitelist_only and not ip_in_set(whitelisted_ips, ip)) then
 		log("debug", "Registration disallowed by whitelist");
 		event.allowed = false;
-		event.reason = "Your IP address is not whitelisted";
+		event.error = errors.new("not_whitelisted", event, err_registry);
 	elseif throttle_max and not ip_in_set(whitelisted_ips, ip) then
 		if not check_throttle(ip) then
 			log("debug", "Registrations over limit for ip %s", ip or "?");
 			event.allowed = false;
-			event.reason = "Too many registrations from this IP address recently";
+			event.error = errors.new("throttled", event, err_registry);
 		end
 	end
+	if event.error then
+		-- COMPAT pre-util.error
+		event.reason = event.error.text;
+		event.error_type = event.error.type;
+		event.error_condition = event.error.condition;
+	end
 end);
diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s.lua
index 5439aa96..0674f981 100644
--- a/plugins/mod_s2s/mod_s2s.lua
+++ b/plugins/mod_s2s.lua
@@ -27,8 +27,10 @@ local s2s_destroy_session = require "core.s2smanager".destroy_session;
 local uuid_gen = require "util.uuid".generate;
 local fire_global_event = prosody.events.fire_event;
 local runner = require "util.async".runner;
-
-local s2sout = module:require("s2sout");
+local connect = require "net.connect".connect;
+local service = require "net.resolvers.service";
+local errors = require "util.error";
+local set = require "util.set";
 
 local connect_timeout = module:get_option_number("s2s_timeout", 90);
 local stream_close_timeout = module:get_option_number("s2s_close_timeout", 5);
@@ -46,8 +48,16 @@ local sessions = module:shared("sessions");
 
 local runner_callbacks = {};
 
+local listener = {};
+
 local log = module._log;
 
+local s2s_service_options = {
+	default_port = 5269;
+	use_ipv4 = module:get_option_boolean("use_ipv4", true);
+	use_ipv6 = module:get_option_boolean("use_ipv6", true);
+};
+
 module:hook("stats-update", function ()
 	local count = 0;
 	local ipv6 = 0;
@@ -78,15 +88,28 @@ local function bounce_sendq(session, reason)
 			(session.log or log)("error", "Attempting to close the dummy origin of s2s error replies, please report this! Traceback: %s", traceback());
 		end;
 	};
+	-- FIXME Allow for more specific error conditions
+	-- TODO use util.error ?
+	local error_type = "cancel";
+	local condition = "remote-server-not-found";
+	local reason_text;
+	if session.had_stream then -- set when a stream is opened by the remote
+		error_type, condition = "wait", "remote-server-timeout";
+	end
+	if errors.is_err(reason) then
+		error_type, condition, reason_text = reason.type, reason.condition, reason.text;
+	elseif type(reason) == "string" then
+		reason_text = reason;
+	end
 	for i, data in ipairs(sendq) do
 		local reply = data[2];
 		if reply and not(reply.attr.xmlns) and bouncy_stanzas[reply.name] then
 			reply.attr.type = "error";
-			reply:tag("error", {type = "cancel", by = session.from_host})
-				:tag("remote-server-not-found", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
-			if reason then
+			reply:tag("error", {type = error_type, by = session.from_host})
+				:tag(condition, {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
+			if reason_text then
 				reply:tag("text", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"})
-					:text("Server-to-server connection failed: "..reason):up();
+					:text("Server-to-server connection failed: "..reason_text):up();
 			end
 			core_process_stanza(dummy, reply);
 		end
@@ -107,38 +130,33 @@ function route_to_existing_session(event)
 		return false;
 	end
 	local host = hosts[from_host].s2sout[to_host];
-	if host then
-		-- We have a connection to this host already
-		if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
-			(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
-
-			-- Queue stanza until we are able to send it
-			local queued_item = {
-				tostring(stanza),
-				stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
-			};
-			if host.sendq then
-				t_insert(host.sendq, queued_item);
-			else
-				-- luacheck: ignore 122
-				host.sendq = { queued_item };
-			end
-			host.log("debug", "stanza [%s] queued ", stanza.name);
-			return true;
-		elseif host.type == "local" or host.type == "component" then
-			log("error", "Trying to send a stanza to ourselves??")
-			log("error", "Traceback: %s", traceback());
-			log("error", "Stanza: %s", tostring(stanza));
-			return false;
+	if not host then return end
+
+	-- We have a connection to this host already
+	if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
+		(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
+
+		-- Queue stanza until we are able to send it
+		local queued_item = {
+			tostring(stanza),
+			stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
+		};
+		if host.sendq then
+			t_insert(host.sendq, queued_item);
 		else
-			-- FIXME
-			if host.from_host ~= from_host then
-				log("error", "WARNING! This might, possibly, be a bug, but it might not...");
-				log("error", "We are going to send from %s instead of %s", host.from_host, from_host);
-			end
-			if host.sends2s(stanza) then
-				return true;
-			end
+			-- luacheck: ignore 122
+			host.sendq = { queued_item };
+		end
+		host.log("debug", "stanza [%s] queued ", stanza.name);
+		return true;
+	elseif host.type == "local" or host.type == "component" then
+		log("error", "Trying to send a stanza to ourselves??")
+		log("error", "Traceback: %s", traceback());
+		log("error", "Stanza: %s", stanza);
+		return false;
+	else
+		if host.sends2s(stanza) then
+			return true;
 		end
 	end
 end
@@ -148,17 +166,13 @@ function route_to_new_session(event)
 	local from_host, to_host, stanza = event.from_host, event.to_host, event.stanza;
 	log("debug", "opening a new outgoing connection for this stanza");
 	local host_session = s2s_new_outgoing(from_host, to_host);
+	host_session.version = 1;
 
 	-- Store in buffer
 	host_session.bounce_sendq = bounce_sendq;
 	host_session.sendq = { {tostring(stanza), stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza)} };
-	log("debug", "stanza [%s] queued until connection complete", tostring(stanza.name));
-	s2sout.initiate_connection(host_session);
-	if (not host_session.connecting) and (not host_session.conn) then
-		log("warn", "Connection to %s failed already, destroying session...", to_host);
-		s2s_destroy_session(host_session, "Connection failed");
-		return false;
-	end
+	log("debug", "stanza [%s] queued until connection complete", stanza.name);
+	connect(service.new(to_host, "xmpp-server", "tcp", s2s_service_options), listener, nil, { session = host_session });
 	return true;
 end
 
@@ -183,10 +197,20 @@ function module.add_host(module)
 			-- so the stream is ready for stanzas.  RFC 6120 Section 4.3
 			mark_connected(session);
 			return true;
+		elseif require_encryption and not session.secure then
+			session.log("warn", "Encrypted server-to-server communication is required but was not offered by %s", session.to_host);
+			session:close({
+					condition = "policy-violation",
+					text = "Encrypted server-to-server communication is required but was not offered",
+				}, nil, "Could not establish encrypted connection to remote server");
+			return true;
 		elseif not session.dialback_verifying then
 			session.log("warn", "No SASL EXTERNAL offer and Dialback doesn't seem to be enabled, giving up");
-			session:close();
-			return false;
+			session:close({
+					condition = "unsupported-feature",
+					text = "No viable authentication method offered",
+				}, nil, "No viable authentication method offered by remote server");
+			return true;
 		end
 	end, -1);
 end
@@ -204,7 +228,18 @@ function mark_connected(session)
 	if session.type == "s2sout" then
 		fire_global_event("s2sout-established", event_data);
 		hosts[from].events.fire_event("s2sout-established", event_data);
+
+		if session.incoming then
+			session.send = function(stanza)
+				return hosts[from].events.fire_event("route/remote", { from_host = from, to_host = to, stanza = stanza });
+			end;
+		end
+
 	else
+		if session.outgoing and not hosts[to].s2sout[from] then
+			session.log("debug", "Setting up to handle route from %s to %s", to, from);
+			hosts[to].s2sout[from] = session; -- luacheck: ignore 122
+		end
 		local host_session = hosts[to];
 		session.send = function(stanza)
 			return host_session.events.fire_event("route/remote", { from_host = to, to_host = from, stanza = stanza });
@@ -224,13 +259,6 @@ function mark_connected(session)
 			end
 			session.sendq = nil;
 		end
-
-		if session.resolver then
-			session.resolver._resolver:closeall()
-		end
-		session.resolver = nil;
-		session.ip_hosts = nil;
-		session.srv_hosts = nil;
 	end
 end
 
@@ -242,7 +270,7 @@ function make_authenticated(event)
 				condition = "policy-violation",
 				text = "Encrypted server-to-server communication is required but was not "
 				       ..((session.direction == "outgoing" and "offered") or "used")
-			});
+			}, nil, "Could not establish encrypted connection to remote server");
 		end
 	end
 	if hosts[host] then
@@ -252,15 +280,13 @@ function make_authenticated(event)
 		session.type = "s2sout";
 	elseif session.type == "s2sin_unauthed" then
 		session.type = "s2sin";
-		if host then
-			if not session.hosts[host] then session.hosts[host] = {}; end
-			session.hosts[host].authed = true;
-		end
-	elseif session.type == "s2sin" and host then
+	elseif session.type ~= "s2sin" and session.type ~= "s2sout" then
+		return false;
+	end
+
+	if session.incoming and host then
 		if not session.hosts[host] then session.hosts[host] = {}; end
 		session.hosts[host].authed = true;
-	else
-		return false;
 	end
 	session.log("debug", "connection %s->%s is now authenticated for %s", session.from_host, session.to_host, host);
 
@@ -297,11 +323,12 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.streamopened(session, attr)
 	-- run _streamopened in async context
-	session.thread:run({ attr = attr });
+	session.thread:run({ stream = "opened", attr = attr });
 end
 
 function stream_callbacks._streamopened(session, attr)
 	session.version = tonumber(attr.version) or 0;
+	session.had_stream = true; -- Had a stream opened at least once
 
 	-- TODO: Rename session.secure to session.encrypted
 	if session.secure == false then
@@ -315,7 +342,6 @@ function stream_callbacks._streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -323,7 +349,9 @@ function stream_callbacks._streamopened(session, attr)
 		-- Send a reply stream header
 
 		-- Validate to/from
-		local to, from = nameprep(attr.to), nameprep(attr.from);
+		local to, from = attr.to, attr.from;
+		if to then to = nameprep(attr.to); end
+		if from then from = nameprep(attr.from); end
 		if not to and attr.to then -- COMPAT: Some servers do not reliably set 'to' (especially on stream restarts)
 			session:close({ condition = "improper-addressing", text = "Invalid 'to' address" });
 			return;
@@ -417,20 +445,6 @@ function stream_callbacks._streamopened(session, attr)
 			end
 		end
 
-		-- Send unauthed buffer
-		-- (stanzas which are fine to send before dialback)
-		-- Note that this is *not* the stanza queue (which
-		-- we can only send if auth succeeds) :)
-		local send_buffer = session.send_buffer;
-		if send_buffer and #send_buffer > 0 then
-			log("debug", "Sending s2s send_buffer now...");
-			for i, data in ipairs(send_buffer) do
-				session.sends2s(tostring(data));
-				send_buffer[i] = nil;
-			end
-		end
-		session.send_buffer = nil;
-
 		-- If server is pre-1.0, don't wait for features, just do dialback
 		if session.version < 1.0 then
 			if not session.dialback_verifying then
@@ -442,11 +456,16 @@ function stream_callbacks._streamopened(session, attr)
 	end
 end
 
-function stream_callbacks.streamclosed(session)
+function stream_callbacks._streamclosed(session)
 	(session.log or log)("debug", "Received </stream:stream>");
 	session:close(false);
 end
 
+function stream_callbacks.streamclosed(session, attr)
+	-- run _streamclosed in async context
+	session.thread:run({ stream = "closed", attr = attr });
+end
+
 function stream_callbacks.error(session, error, data)
 	if error == "no-stream" then
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
@@ -472,11 +491,12 @@ function stream_callbacks.error(session, error, data)
 	end
 end
 
-local listener = {};
-
 --- Session methods
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
-local function session_close(session, reason, remote_reason)
+-- reason: stream error to send to the remote server
+-- remote_reason: stream error received from the remote server
+-- bounce_reason: stanza error to pass to bounce_sendq because stream- and stanza errors are different
+local function session_close(session, reason, remote_reason, bounce_reason)
 	local log = session.log or log;
 	if session.conn then
 		if session.notopen then
@@ -487,27 +507,23 @@ local function session_close(session, reason, remote_reason)
 			end
 		end
 		if reason then -- nil == no err, initiated by us, false == initiated by remote
+			local stream_error;
 			if type(reason) == "string" then -- assume stream error
-				log("debug", "Disconnecting %s[%s], <stream:error> is: %s", session.host or session.ip or "(unknown host)", session.type, reason);
-				session.sends2s(st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' }));
-			elseif type(reason) == "table" then
-				if reason.condition then
-					local stanza = st.stanza("stream:error"):tag(reason.condition, stream_xmlns_attr):up();
-					if reason.text then
-						stanza:tag("text", stream_xmlns_attr):text(reason.text):up();
-					end
-					if reason.extra then
-						stanza:add_child(reason.extra);
-					end
-					log("debug", "Disconnecting %s[%s], <stream:error> is: %s",
-					session.host or session.ip or "(unknown host)", session.type, stanza);
-					session.sends2s(stanza);
-				elseif reason.name then -- a stanza
-					log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
-						session.from_host or "(unknown host)", session.to_host or "(unknown host)",
-						session.type, reason);
-					session.sends2s(reason);
+				stream_error = st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' });
+			elseif type(reason) == "table" and not st.is_stanza(reason) then
+				stream_error = st.stanza("stream:error"):tag(reason.condition or "undefined-condition", stream_xmlns_attr):up();
+				if reason.text then
+					stream_error:tag("text", stream_xmlns_attr):text(reason.text):up();
 				end
+				if reason.extra then
+					stream_error:add_child(reason.extra);
+				end
+			end
+			if st.is_stanza(stream_error) then
+				-- to and from are never unknown on outgoing connections
+				log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
+					session.from_host or "(unknown host)" or session.ip, session.to_host or "(unknown host)", session.type, reason);
+				session.sends2s(stream_error);
 			end
 		end
 
@@ -522,16 +538,16 @@ local function session_close(session, reason, remote_reason)
 
 		-- Authenticated incoming stream may still be sending us stanzas, so wait for </stream:stream> from remote
 		local conn = session.conn;
-		if reason == nil and not session.notopen and session.type == "s2sin" then
+		if reason == nil and not session.notopen and session.incoming then
 			add_task(stream_close_timeout, function ()
 				if not session.destroyed then
 					session.log("warn", "Failed to receive a stream close response, closing connection anyway...");
-					s2s_destroy_session(session, reason);
+					s2s_destroy_session(session, reason, bounce_reason);
 					conn:close();
 				end
 			end);
 		else
-			s2s_destroy_session(session, reason);
+			s2s_destroy_session(session, reason, bounce_reason);
 			conn:close(); -- Close immediately, as this is an outgoing connection or is not authed
 		end
 	end
@@ -554,10 +570,12 @@ local function initialize_session(session)
 	local stream = new_xmpp_stream(session, stream_callbacks, stanza_size_limit);
 
 	session.thread = runner(function (stanza)
-		if stanza.name == nil then
-			stream_callbacks._streamopened(session, stanza.attr);
-		else
+		if st.is_stanza(stanza) then
 			core_process_stanza(session, stanza);
+		elseif stanza.stream == "opened" then
+			stream_callbacks._streamopened(session, stanza.attr);
+		elseif stanza.stream == "closed" then
+			stream_callbacks._streamclosed(session, stanza.attr);
 		end
 	end, runner_callbacks, session);
 
@@ -596,8 +614,12 @@ local function initialize_session(session)
 		if data then
 			local ok, err = stream:feed(data);
 			if ok then return; end
-			log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
-			session:close("not-well-formed");
+			log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+			if err == "stanza-too-large" then
+				session:close({ condition = "policy-violation", text = "XML stanza is too big" }, nil, "Received invalid XML from remote server");
+			else
+				session:close("not-well-formed", nil, "Received invalid XML from remote server");
+			end
 		end
 	end
 
@@ -672,11 +694,20 @@ function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
 		sessions[conn] = nil;
+		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
+		if session.secure == false and err then
+			-- TODO util.error-ify this
+			err = "Error during negotiation of encrypted connection: "..err;
+		end
+		s2s_destroy_session(session, err);
+	end
+end
+
+function listener.onfail(data, err)
+	local session = data and data.session;
+	if session then
 		if err and session.direction == "outgoing" and session.notopen then
 			(session.log or log)("debug", "s2s connection attempt failed: %s", err);
-			if s2sout.attempt_connection(session, err) then
-				return; -- Session lives for now
-			end
 		end
 		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
 		s2s_destroy_session(session, err);
@@ -700,6 +731,34 @@ function listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
+function listener.onattach(conn, data)
+	local session = data and data.session;
+	if session then
+		session.conn = conn;
+		sessions[conn] = session;
+		initialize_session(session);
+	end
+end
+
+-- Complete the sentence "Your certificate " with what's wrong
+local function friendly_cert_error(session) --> string
+	if session.cert_chain_status == "invalid" then
+		if session.cert_chain_errors then
+			local cert_errors = set.new(session.cert_chain_errors[1]);
+			if cert_errors:contains("certificate has expired") then
+				return "has expired";
+			elseif cert_errors:contains("self signed certificate") then
+				return "is self-signed";
+			end
+		end
+		return "is not trusted"; -- for some other reason
+	elseif session.cert_identity_status == "invalid" then
+		return "is not valid for this name";
+	end
+	-- this should normally be unreachable except if no s2s auth module was loaded
+	return "could not be validated";
+end
+
 function check_auth_policy(event)
 	local host, session = event.host, event.session;
 	local must_secure = secure_auth;
@@ -711,20 +770,21 @@ function check_auth_policy(event)
 	end
 
 	if must_secure and (session.cert_chain_status ~= "valid" or session.cert_identity_status ~= "valid") then
-		module:log("warn", "Forbidding insecure connection to/from %s", host or session.ip or "(unknown host)");
-		if session.direction == "incoming" then
-			session:close({ condition = "not-authorized", text = "Your server's certificate is invalid, expired, or not trusted by "..session.to_host });
-		else -- Close outgoing connections without warning
-			session:close(false);
-		end
+		local reason = friendly_cert_error(session);
+		session.log("warn", "Forbidding insecure connection to/from %s because its certificate %s", host or session.ip or "(unknown host)", reason);
+		-- XEP-0178 recommends closing outgoing connections without warning
+		-- but does not give a rationale for this.
+		-- In practice most cases are configuration mistakes or forgotten
+		-- certificate renewals. We think it's better to let the other party
+		-- know about the problem so that they can fix it.
+		session:close({ condition = "not-authorized", text = "Your server's certificate "..reason },
+			nil, "Remote server's certificate "..reason);
 		return false;
 	end
 end
 
 module:hook("s2s-check-certificate", check_auth_policy, -1);
 
-s2sout.set_listener(listener);
-
 module:hook("server-stopping", function(event)
 	local reason = event.reason;
 	for _, session in pairs(sessions) do
@@ -739,7 +799,11 @@ module:provides("net", {
 	listener = listener;
 	default_port = 5269;
 	encryption = "starttls";
+	ssl_config = { -- FIXME This is not used atm, see mod_tls
+		verify = { "peer", "client_once", };
+	};
 	multiplex = {
+		protocol = "xmpp-server";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>";
 	};
 });
diff --git a/plugins/mod_s2s/s2sout.lib.lua b/plugins/mod_s2s/s2sout.lib.lua
deleted file mode 100644
index 5f765da8..00000000
--- a/plugins/mod_s2s/s2sout.lib.lua
+++ /dev/null
@@ -1,349 +0,0 @@
--- Prosody IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
-
---- Module containing all the logic for connecting to a remote server
-
--- luacheck: ignore 432/err
-
-local portmanager = require "core.portmanager";
-local wrapclient = require "net.server".wrapclient;
-local initialize_filters = require "util.filters".initialize;
-local idna_to_ascii = require "util.encodings".idna.to_ascii;
-local new_ip = require "util.ip".new_ip;
-local rfc6724_dest = require "util.rfc6724".destination;
-local socket = require "socket";
-local adns = require "net.adns";
-local t_insert, t_sort, ipairs = table.insert, table.sort, ipairs;
-local local_addresses = require "util.net".local_addresses;
-
-local s2s_destroy_session = require "core.s2smanager".destroy_session;
-
-local default_mode = module:get_option("network_default_read_size", 4096);
-
-local log = module._log;
-
-local sources = {};
-local has_ipv4, has_ipv6;
-
-local dns_timeout = module:get_option_number("dns_timeout", 15);
-local resolvers = module:get_option_set("s2s_dns_resolvers")
-
-local s2sout = {};
-
-local s2s_listener;
-
-
-function s2sout.set_listener(listener)
-	s2s_listener = listener;
-end
-
-local function compare_srv_priorities(a,b)
-	return a.priority < b.priority or (a.priority == b.priority and a.weight > b.weight);
-end
-
-function s2sout.initiate_connection(host_session)
-	local log = host_session.log or log;
-
-	initialize_filters(host_session);
-	host_session.version = 1;
-
-	host_session.resolver = adns.resolver();
-	host_session.resolver._resolver:settimeout(dns_timeout);
-	if resolvers then
-		for resolver in resolvers do
-			host_session.resolver._resolver:addnameserver(resolver);
-		end
-	end
-
-	-- Kick the connection attempting machine into life
-	if not s2sout.attempt_connection(host_session) then
-		-- Intentionally not returning here, the
-		-- session is needed, connected or not
-		s2s_destroy_session(host_session);
-	end
-
-	if not host_session.sends2s then
-		-- A sends2s which buffers data (until the stream is opened)
-		-- note that data in this buffer will be sent before the stream is authed
-		-- and will not be ack'd in any way, successful or otherwise
-		local buffer;
-		function host_session.sends2s(data)
-			if not buffer then
-				buffer = {};
-				host_session.send_buffer = buffer;
-			end
-			log("debug", "Buffering data on unconnected s2sout to %s", host_session.to_host);
-			buffer[#buffer+1] = data;
-			log("debug", "Buffered item %d: %s", #buffer, data);
-		end
-	end
-end
-
-function s2sout.attempt_connection(host_session, err)
-	local to_host = host_session.to_host;
-	local connect_host, connect_port = to_host and idna_to_ascii(to_host), 5269;
-	local log = host_session.log or log;
-
-	if not connect_host then
-		return false;
-	end
-
-	if not err then -- This is our first attempt
-		log("debug", "First attempt to connect to %s, starting with SRV lookup...", to_host);
-		host_session.connecting = true;
-		host_session.resolver:lookup(function (answer)
-			local srv_hosts = { answer = answer };
-			host_session.srv_hosts = srv_hosts;
-			host_session.srv_choice = 0;
-			host_session.connecting = nil;
-			if answer and #answer > 0 then
-				log("debug", "%s has SRV records, handling...", to_host);
-				for _, record in ipairs(answer) do
-					t_insert(srv_hosts, record.srv);
-				end
-				if #srv_hosts == 1 and srv_hosts[1].target == "." then
-					log("debug", "%s does not provide a XMPP service", to_host);
-					s2s_destroy_session(host_session, err); -- Nothing to see here
-					return;
-				end
-				t_sort(srv_hosts, compare_srv_priorities);
-
-				local srv_choice = srv_hosts[1];
-				host_session.srv_choice = 1;
-				if srv_choice then
-					connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-					log("debug", "Best record found, will connect to %s:%d", connect_host, connect_port);
-				end
-			else
-				log("debug", "%s has no SRV records, falling back to A/AAAA", to_host);
-			end
-			-- Try with SRV, or just the plain hostname if no SRV
-			local ok, err = s2sout.try_connect(host_session, connect_host, connect_port);
-			if not ok then
-				if not s2sout.attempt_connection(host_session, err) then
-					-- No more attempts will be made
-					s2s_destroy_session(host_session, err);
-				end
-			end
-		end, "_xmpp-server._tcp."..connect_host..".", "SRV");
-
-		return true; -- Attempt in progress
-	elseif host_session.ip_hosts then
-		return s2sout.try_connect(host_session, connect_host, connect_port, err);
-	elseif host_session.srv_hosts and #host_session.srv_hosts > host_session.srv_choice then -- Not our first attempt, and we also have SRV
-		host_session.srv_choice = host_session.srv_choice + 1;
-		local srv_choice = host_session.srv_hosts[host_session.srv_choice];
-		connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-		host_session.log("info", "Connection failed (%s). Attempt #%d: This time to %s:%d", err, host_session.srv_choice, connect_host, connect_port);
-	else
-		host_session.log("info", "Failed in all attempts to connect to %s", host_session.to_host);
-		-- We're out of options
-		return false;
-	end
-
-	if not (connect_host and connect_port) then
-		-- Likely we couldn't resolve DNS
-		log("warn", "Hmm, we're without a host (%s) and port (%s) to connect to for %s, giving up :(", connect_host, connect_port, to_host);
-		return false;
-	end
-
-	return s2sout.try_connect(host_session, connect_host, connect_port);
-end
-
-function s2sout.try_next_ip(host_session)
-	host_session.connecting = nil;
-	host_session.ip_choice = host_session.ip_choice + 1;
-	local ip = host_session.ip_hosts[host_session.ip_choice];
-	local ok, err= s2sout.make_connect(host_session, ip.ip, ip.port);
-	if not ok then
-		if not s2sout.attempt_connection(host_session, err or "closed") then
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connection failed"..err);
-		end
-	end
-end
-
-function s2sout.try_connect(host_session, connect_host, connect_port, err)
-	host_session.connecting = true;
-	local log = host_session.log or log;
-
-	if not err then
-		local IPs = {};
-		host_session.ip_hosts = IPs;
-		-- luacheck: ignore 231/handle4 231/handle6
-		local handle4, handle6;
-		local have_other_result = not(has_ipv4) or not(has_ipv6) or false;
-
-		if has_ipv4 then
-			handle4 = host_session.resolver:lookup(function (reply, err)
-				handle4 = nil;
-
-				if reply and reply[#reply] and reply[#reply].a then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.a);
-						IPs[#IPs+1] = new_ip(ip.a, "IPv4");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "A", "IN");
-		else
-			have_other_result = true;
-		end
-
-		if has_ipv6 then
-			handle6 = host_session.resolver:lookup(function (reply, err)
-				handle6 = nil;
-
-				if reply and reply[#reply] and reply[#reply].aaaa then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.aaaa);
-						IPs[#IPs+1] = new_ip(ip.aaaa, "IPv6");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "AAAA", "IN");
-		else
-			have_other_result = true;
-		end
-		return true;
-	elseif host_session.ip_hosts and #host_session.ip_hosts > host_session.ip_choice then -- Not our first attempt, and we also have IPs left to try
-		s2sout.try_next_ip(host_session);
-	else
-		log("debug", "Out of IP addresses, trying next SRV record (if any)");
-		host_session.ip_hosts = nil;
-		if not s2sout.attempt_connection(host_session, "out of IP addresses") then -- Retry if we can
-			log("debug", "No other records to try for %s - destroying", host_session.to_host);
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connecting failed"..err); -- End of the line, we can't
-			return false;
-		end
-	end
-
-	return true;
-end
-
-function s2sout.make_connect(host_session, connect_host, connect_port)
-	local log = host_session.log or log;
-	log("debug", "Beginning new connection attempt to %s ([%s]:%d)", host_session.to_host, connect_host.addr, connect_port);
-
-	-- Reset secure flag in case this is another
-	-- connection attempt after a failed STARTTLS
-	host_session.secure = nil;
-	host_session.encrypted = nil;
-
-	local conn, handler;
-	local proto = connect_host.proto;
-	if proto == "IPv4" then
-		conn, handler = socket.tcp();
-	elseif proto == "IPv6" and socket.tcp6 then
-		conn, handler = socket.tcp6();
-	else
-		handler = "Unsupported protocol: "..tostring(proto);
-	end
-
-	if not conn then
-		log("warn", "Failed to create outgoing connection, system error: %s", handler);
-		return false, handler;
-	end
-
-	conn:settimeout(0);
-	local success, err = conn:connect(connect_host.addr, connect_port);
-	if not success and err ~= "timeout" then
-		log("warn", "s2s connect() to %s (%s:%d) failed: %s", host_session.to_host, connect_host.addr, connect_port, err);
-		return false, err;
-	end
-
-	conn = wrapclient(conn, connect_host.addr, connect_port, s2s_listener, default_mode);
-	host_session.conn = conn;
-
-	-- Register this outgoing connection so that xmppserver_listener knows about it
-	-- otherwise it will assume it is a new incoming connection
-	s2s_listener.register_outgoing(conn, host_session);
-
-	log("debug", "Connection attempt in progress...");
-	return true;
-end
-
-module:hook_global("service-added", function (event)
-	if event.name ~= "s2s" then return end
-
-	local s2s_sources = portmanager.get_active_services():get("s2s");
-	if not s2s_sources then
-		module:log("warn", "s2s not listening on any ports, outgoing connections may fail");
-		return;
-	end
-	for source, _ in pairs(s2s_sources) do
-		if source == "*" or source == "0.0.0.0" then
-			for _, addr in ipairs(local_addresses("ipv4", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv4");
-			end
-		elseif source == "::" then
-			for _, addr in ipairs(local_addresses("ipv6", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv6");
-			end
-		else
-			sources[#sources + 1] = new_ip(source, (source:find(":") and "IPv6") or "IPv4");
-		end
-	end
-	for i = 1,#sources do
-		if sources[i].proto == "IPv6" then
-			has_ipv6 = true;
-		elseif sources[i].proto == "IPv4" then
-			has_ipv4 = true;
-		end
-	end
-	if not (has_ipv4 or has_ipv6)  then
-		module:log("warn", "No local IPv4 or IPv6 addresses detected, outgoing connections may fail");
-	end
-end);
-
-return s2sout;
diff --git a/plugins/mod_s2s_auth_certs.lua b/plugins/mod_s2s_auth_certs.lua
index dd0eb3cb..37519aa1 100644
--- a/plugins/mod_s2s_auth_certs.lua
+++ b/plugins/mod_s2s_auth_certs.lua
@@ -17,9 +17,6 @@ module:hook("s2s-check-certificate", function(event)
 	local chain_valid, errors;
 	if conn.getpeerverification then
 		chain_valid, errors = conn:getpeerverification();
-	elseif conn.getpeerchainvalid then -- COMPAT mw/luasec-hg
-		chain_valid, errors = conn:getpeerchainvalid();
-		errors = (not chain_valid) and { { errors } } or nil;
 	else
 		chain_valid, errors = false, { { "Chain verification not supported by this version of LuaSec" } };
 	end
@@ -30,6 +27,7 @@ module:hook("s2s-check-certificate", function(event)
 			log("debug", "certificate error(s) at depth %d: %s", depth-1, table.concat(t, ", "))
 		end
 		session.cert_chain_status = "invalid";
+		session.cert_chain_errors = errors;
 	else
 		log("debug", "certificate chain validation result: valid");
 		session.cert_chain_status = "valid";
diff --git a/plugins/mod_s2s_bidi.lua b/plugins/mod_s2s_bidi.lua
new file mode 100644
index 00000000..28e047de
--- /dev/null
+++ b/plugins/mod_s2s_bidi.lua
@@ -0,0 +1,40 @@
+-- Prosody IM
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local xmlns_bidi_feature = "urn:xmpp:features:bidi"
+local xmlns_bidi = "urn:xmpp:bidi";
+
+local require_encryption = module:get_option_boolean("s2s_require_encryption", false);
+
+module:hook("s2s-stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.type == "s2sin_unauthed" and (not require_encryption or origin.secure) then
+		features:tag("bidi", { xmlns = xmlns_bidi_feature }):up();
+	end
+end);
+
+module:hook_tag("http://etherx.jabber.org/streams", "features", function (session, stanza)
+	if session.type == "s2sout_unauthed" and (not require_encryption or session.secure) then
+		local bidi = stanza:get_child("bidi", xmlns_bidi_feature);
+		if bidi then
+			session.incoming = true;
+			session.log("debug", "Requesting bidirectional stream");
+			session.sends2s(st.stanza("bidi", { xmlns = xmlns_bidi }));
+		end
+	end
+end, 200);
+
+module:hook_tag("urn:xmpp:bidi", "bidi", function(session)
+	if session.type == "s2sin_unauthed" and (not require_encryption or session.secure) then
+		session.log("debug", "Requested bidirectional stream");
+		session.outgoing = true;
+		return true;
+	end
+end);
+
diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua
index fba84ef8..2a5edcb2 100644
--- a/plugins/mod_saslauth.lua
+++ b/plugins/mod_saslauth.lua
@@ -12,9 +12,10 @@ local st = require "util.stanza";
 local sm_bind_resource = require "core.sessionmanager".bind_resource;
 local sm_make_authenticated = require "core.sessionmanager".make_authenticated;
 local base64 = require "util.encodings".base64;
+local set = require "util.set";
+local errors = require "util.error";
 
 local usermanager_get_sasl_handler = require "core.usermanager".get_sasl_handler;
-local tostring = tostring;
 
 local secure_auth_only = module:get_option_boolean("c2s_require_encryption", module:get_option_boolean("require_encryption", false));
 local allow_unencrypted_plain_auth = module:get_option_boolean("allow_unencrypted_plain_auth", false)
@@ -48,7 +49,7 @@ local function handle_status(session, status, ret, err_msg)
 		module:fire_event("authentication-failure", { session = session, condition = ret, text = err_msg });
 		session.sasl_handler = session.sasl_handler:clean_clone();
 	elseif status == "success" then
-		local ok, err = sm_make_authenticated(session, session.sasl_handler.username);
+		local ok, err = sm_make_authenticated(session, session.sasl_handler.username, session.sasl_handler.scope);
 		if ok then
 			module:fire_event("authentication-success", { session = session });
 			session.sasl_handler = nil;
@@ -67,7 +68,6 @@ local function sasl_process_cdata(session, stanza)
 	local text = stanza[1];
 	if text then
 		text = base64.decode(text);
-		--log("debug", "AUTH: %s", text:gsub("[%z\001-\008\011\012\014-\031]", " "));
 		if not text then
 			session.sasl_handler = nil;
 			session.send(build_reply("failure", "incorrect-encoding"));
@@ -77,7 +77,6 @@ local function sasl_process_cdata(session, stanza)
 	local status, ret, err_msg = session.sasl_handler:process(text);
 	status, ret, err_msg = handle_status(session, status, ret, err_msg);
 	local s = build_reply(status, ret, err_msg);
-	log("debug", "sasl reply: %s", tostring(s));
 	session.send(s);
 	return true;
 end
@@ -104,18 +103,27 @@ module:hook_tag(xmlns_sasl, "failure", function (session, stanza)
 			break;
 		end
 	end
-	if text and condition then
-		condition = condition .. ": " .. text;
-	end
-	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, condition);
+	local err = errors.new({
+			-- TODO type = what?
+			text = text,
+			condition = condition,
+		}, {
+			session = session,
+			stanza = stanza,
+		});
+
+	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, err);
 
 	session.external_auth = "failed"
-	session.external_auth_failure_reason = condition;
+	session.external_auth_failure_reason = err;
 end, 500)
 
 module:hook_tag(xmlns_sasl, "failure", function (session, stanza) -- luacheck: ignore 212/stanza
 	session.log("debug", "No fallback from SASL EXTERNAL failure, giving up");
-	session:close(nil, session.external_auth_failure_reason);
+	session:close(nil, session.external_auth_failure_reason, errors.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "Could not authenticate to remote server",
+		}, { session = session, sasl_failure = session.external_auth_failure_reason, }));
 	return true;
 end, 90)
 
@@ -248,37 +256,72 @@ module:hook("stream-features", function(event)
 		local sasl_handler = usermanager_get_sasl_handler(module.host, origin)
 		origin.sasl_handler = sasl_handler;
 		if origin.encrypted then
-			-- check wether LuaSec has the nifty binding to the function needed for tls-unique
+			-- check whether LuaSec has the nifty binding to the function needed for tls-unique
 			-- FIXME: would be nice to have this check only once and not for every socket
 			if sasl_handler.add_cb_handler then
 				local socket = origin.conn:socket();
 				if socket.getpeerfinished then
+					log("debug", "Channel binding 'tls-unique' supported");
 					sasl_handler:add_cb_handler("tls-unique", tls_unique);
+				else
+					log("debug", "Channel binding 'tls-unique' not supported (by LuaSec?)");
 				end
 				sasl_handler["userdata"] = {
 					["tls-unique"] = socket;
 				};
+			else
+				log("debug", "Channel binding not supported by SASL handler");
 			end
 		end
 		local mechanisms = st.stanza("mechanisms", mechanisms_attr);
 		local sasl_mechanisms = sasl_handler:mechanisms()
+		local available_mechanisms = set.new();
 		for mechanism in pairs(sasl_mechanisms) do
-			if disabled_mechanisms:contains(mechanism) then
-				log("debug", "Not offering disabled mechanism %s", mechanism);
-			elseif not origin.secure and insecure_mechanisms:contains(mechanism) then
-				log("debug", "Not offering mechanism %s on insecure connection", mechanism);
-			else
-				log("debug", "Offering mechanism %s", mechanism);
+			available_mechanisms:add(mechanism);
+		end
+		log("debug", "SASL mechanisms supported by handler: %s", available_mechanisms);
+
+		local usable_mechanisms = available_mechanisms - disabled_mechanisms;
+
+		local available_disabled = set.intersection(available_mechanisms, disabled_mechanisms);
+		if not available_disabled:empty() then
+			log("debug", "Not offering disabled mechanisms: %s", available_disabled);
+		end
+
+		local available_insecure = set.intersection(available_mechanisms, insecure_mechanisms);
+		if not origin.secure and not available_insecure:empty() then
+			log("debug", "Session is not secure, not offering insecure mechanisms: %s", available_insecure);
+			usable_mechanisms = usable_mechanisms - insecure_mechanisms;
+		end
+
+		if not usable_mechanisms:empty() then
+			log("debug", "Offering usable mechanisms: %s", usable_mechanisms);
+			for mechanism in usable_mechanisms do
 				mechanisms:tag("mechanism"):text(mechanism):up();
 			end
-		end
-		if mechanisms[1] then
 			features:add_child(mechanisms);
-		elseif not next(sasl_mechanisms) then
-			log("warn", "No available SASL mechanisms, verify that the configured authentication module is working");
-		else
-			log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection");
+			return;
+		end
+
+		local authmod = module:get_option_string("authentication", "internal_plain");
+		if available_mechanisms:empty() then
+			log("warn", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod);
+			return;
 		end
+
+		if not origin.secure and not available_insecure:empty() then
+			if not available_disabled:empty() then
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s) or disabled (%s)",
+					authmod, available_insecure, available_disabled);
+			else
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s)",
+					authmod, available_insecure);
+			end
+		elseif not available_disabled:empty() then
+			log("warn", "All SASL mechanisms provided by authentication module '%s' are disabled (%s)",
+				authmod, available_disabled);
+		end
+
 	else
 		features:tag("bind", bind_attr):tag("required"):up():up();
 		features:tag("session", xmpp_session_attr):tag("optional"):up():up();
diff --git a/plugins/mod_scansion_record.lua b/plugins/mod_scansion_record.lua
index 8d772b4e..0e109a0d 100644
--- a/plugins/mod_scansion_record.lua
+++ b/plugins/mod_scansion_record.lua
@@ -37,8 +37,7 @@ local function record_event(session, event)
 end
 
 local function record_stanza(stanza, session, verb)
-	local flattened = tostring(stanza):gsub("><", ">\n\t<");
-	-- TODO Proper prettyprinting with indentation
+	local flattened = tostring(stanza:indent(2, "\t"));
 	record(session.scansion_id.." "..verb..":\n\t"..flattened.."\n\n");
 end
 
diff --git a/plugins/mod_server_contact_info.lua b/plugins/mod_server_contact_info.lua
index 0110ceaf..9c916ebc 100644
--- a/plugins/mod_server_contact_info.lua
+++ b/plugins/mod_server_contact_info.lua
@@ -16,6 +16,7 @@ local form_layout = require "util.dataforms".new({
 	{ name = "feedback", var = "feedback-addresses", type = "list-multi" },
 	{ name = "sales", var = "sales-addresses", type = "list-multi" },
 	{ name = "security", var = "security-addresses", type = "list-multi" },
+	{ name = "status", var = "status-addresses", type = "list-multi" },
 	{ name = "support", var = "support-addresses", type = "list-multi" },
 });
 
diff --git a/plugins/mod_stanza_debug.lua b/plugins/mod_stanza_debug.lua
index 6dedb6f7..af98670c 100644
--- a/plugins/mod_stanza_debug.lua
+++ b/plugins/mod_stanza_debug.lua
@@ -1,18 +1,17 @@
 module:set_global();
 
-local tostring = tostring;
 local filters = require "util.filters";
 
 local function log_send(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "SEND: %s", tostring(t));
+		session.log("debug", "SEND: %s", t);
 	end
 	return t;
 end
 
 local function log_recv(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "RECV: %s", tostring(t));
+		session.log("debug", "RECV: %s", t);
 	end
 	return t;
 end
diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua
index 0becfc8f..28dc8921 100644
--- a/plugins/mod_storage_internal.lua
+++ b/plugins/mod_storage_internal.lua
@@ -1,12 +1,17 @@
+local cache = require "util.cache";
 local datamanager = require "core.storagemanager".olddm;
 local array = require "util.array";
 local datetime = require "util.datetime";
 local st = require "util.stanza";
 local now = require "util.time".now;
 local id = require "util.id".medium;
+local jid_join = require "util.jid".join;
 
 local host = module.host;
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
 local driver = {};
 
 function driver:open(store, typ)
@@ -43,6 +48,12 @@ end
 local archive = {};
 driver.archive = { __index = archive };
 
+archive.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive:append(username, key, value, when, with)
 	when = when or now();
 	if not st.is_stanza(value) then
@@ -54,28 +65,57 @@ function archive:append(username, key, value, when, with)
 	value.attr.stamp = datetime.datetime(when);
 	value.attr.stamp_legacy = datetime.legacy(when);
 
+	local cache_key = jid_join(username, host, self.store);
+	local item_count = archive_item_count_cache:get(cache_key);
+
 	if key then
 		local items, err = datamanager.list_load(username, host, self.store);
 		if not items and err then return items, err; end
+
+		-- Check the quota
+		item_count = items and #items or 0;
+		archive_item_count_cache:set(cache_key, item_count);
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
+
 		if items then
+			-- Filter out any item with the same key as the one being added
 			items = array(items);
 			items:filter(function (item)
 				return item.key ~= key;
 			end);
+
 			value.key = key;
 			items:push(value);
 			local ok, err = datamanager.list_store(username, host, self.store, items);
 			if not ok then return ok, err; end
+			archive_item_count_cache:set(cache_key, #items);
 			return key;
 		end
 	else
+		if not item_count then -- Item count not cached?
+			-- We need to load the list to get the number of items currently stored
+			local items, err = datamanager.list_load(username, host, self.store);
+			if not items and err then return items, err; end
+			item_count = items and #items or 0;
+			archive_item_count_cache:set(cache_key, item_count);
+		end
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
 		key = id();
 	end
 
+	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
+
 	value.key = key;
 
 	local ok, err = datamanager.list_append(username, host, self.store, value);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, item_count+1);
 	return key;
 end
 
@@ -84,12 +124,18 @@ function archive:find(username, query)
 	if not items then
 		if err then
 			return items, err;
-		else
-			return function () end, 0;
+		elseif query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
 		end
+		return function () end;
 	end
-	local count = #items;
-	local i = 0;
+	local count = nil;
+	local i, last_key = 0;
 	if query then
 		items = array(items);
 		if query.key then
@@ -114,24 +160,38 @@ function archive:find(username, query)
 				return when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
+		elseif query.before then
+			last_key = query.before;
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -140,7 +200,9 @@ function archive:find(username, query)
 	return function ()
 		i = i + 1;
 		local item = items[i];
-		if not item then return; end
+		if not item or (last_key and item.key == last_key) then
+			return;
+		end
 		local key = item.key or tostring(i);
 		local when = item.when or datetime.parse(item.attr.stamp);
 		local with = item.with;
@@ -152,14 +214,83 @@ function archive:find(username, query)
 	end, count;
 end
 
+function archive:get(username, wanted_key)
+	local iter, err = self:find(username, { key = wanted_key })
+	if not iter then return iter, err; end
+	for key, stanza, when, with in iter do
+		if key == wanted_key then
+			return stanza, when, with;
+		end
+	end
+	return nil, "item-not-found";
+end
+
+function archive:set(username, key, new_value, new_when, new_with)
+	local items, err = datamanager.list_load(username, host, self.store);
+	if not items then
+		if err then
+			return items, err;
+		else
+			return nil, "item-not-found";
+		end
+	end
+
+	for i = 1, #items do
+		local old_item = items[i];
+		if old_item.key == key then
+			local item = st.preserialize(st.clone(new_value));
+
+			local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp);
+			item.key = key;
+			item.when = when;
+			item.with = new_with or old_item.with;
+			item.attr.stamp = datetime.datetime(when);
+			item.attr.stamp_legacy = datetime.legacy(when);
+			items[i] = item;
+			return datamanager.list_store(username, host, self.store, items);
+		end
+	end
+
+	return nil, "item-not-found";
+end
+
 function archive:dates(username)
 	local items, err = datamanager.list_load(username, host, self.store);
 	if not items then return items, err; end
 	return array(items):pluck("when"):map(datetime.date):unique();
 end
 
+function archive:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	local body = {};
+	for _, stanza, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+		body[with] = stanza:get_child_text("body") or body[with];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+		body = body;
+	};
+end
+
+function archive:users()
+	return datamanager.users(host, self.store, "list");
+end
+
 function archive:delete(username, query)
+	local cache_key = jid_join(username, host, self.store);
 	if not query or next(query) == nil then
+		archive_item_count_cache:set(cache_key, nil);
 		return datamanager.list_store(username, host, self.store, nil);
 	end
 	local items, err = datamanager.list_load(username, host, self.store);
@@ -167,6 +298,7 @@ function archive:delete(username, query)
 		if err then
 			return items, err;
 		end
+		archive_item_count_cache:set(cache_key, 0);
 		-- Store is empty
 		return 0;
 	end
@@ -216,6 +348,7 @@ function archive:delete(username, query)
 	end
 	local ok, err = datamanager.list_store(username, host, self.store, items);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, #items);
 	return count;
 end
 
diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua
index 745e394b..d71dc0f0 100644
--- a/plugins/mod_storage_memory.lua
+++ b/plugins/mod_storage_memory.lua
@@ -8,6 +8,8 @@ local new_id = require "util.id".medium;
 local auto_purge_enabled = module:get_option_boolean("storage_memory_temporary", false);
 local auto_purge_stores = module:get_option_set("storage_memory_temporary_stores", {});
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000);
+
 local memory = setmetatable({}, {
 	__index = function(t, k)
 		local store = module:shared(k)
@@ -51,6 +53,12 @@ archive_store.__index = archive_store;
 
 archive_store.users = _users;
 
+archive_store.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive_store:append(username, key, value, when, with)
 	if is_stanza(value) then
 		value = st.preserialize(value);
@@ -70,6 +78,8 @@ function archive_store:append(username, key, value, when, with)
 	end
 	if a[key] then
 		table.remove(a, a[key]);
+	elseif #a >= archive_item_limit then
+		return nil, "quota-limit";
 	end
 	local i = #a+1;
 	a[i] = v;
@@ -80,10 +90,18 @@ end
 function archive_store:find(username, query)
 	local items = self.store[username or NULL];
 	if not items then
-		return function () end, 0;
+		if query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
+		end
+		return function () end;
 	end
-	local count = #items;
-	local i = 0;
+	local count = nil;
+	local i, last_key = 0;
 	if query then
 		items = array():append(items);
 		if query.key then
@@ -106,24 +124,38 @@ function archive_store:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
+		elseif query.before then
+			last_key = query.before;
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -132,11 +164,62 @@ function archive_store:find(username, query)
 	return function ()
 		i = i + 1;
 		local item = items[i];
-		if not item then return; end
+		if not item or (last_key and item.key == last_key) then return; end
 		return item.key, item.value(), item.when, item.with;
 	end, count;
 end
 
+function archive_store:get(username, wanted_key)
+	local items = self.store[username or NULL];
+	if not items then return nil, "item-not-found"; end
+	local i = items[wanted_key];
+	if not i then return nil, "item-not-found"; end
+	local item = items[i];
+	return item.value(), item.when, item.with;
+end
+
+function archive_store:set(username, wanted_key, new_value, new_when, new_with)
+	local items = self.store[username or NULL];
+	if not items then return nil, "item-not-found"; end
+	local i = items[wanted_key];
+	if not i then return nil, "item-not-found"; end
+	local item = items[i];
+
+	if is_stanza(new_value) then
+		new_value = st.preserialize(new_value);
+		item.value = envload("return xml"..serialize(new_value), "=(stanza)", { xml = st.deserialize })
+	else
+		item.value = envload("return "..serialize(new_value), "=(data)", {});
+	end
+	if new_when then
+		item.when = new_when;
+	end
+	if new_with then
+		item.with = new_when;
+	end
+	return true;
+end
+
+function archive_store:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	for _, _, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 
 function archive_store:delete(username, query)
 	if not query or next(query) == nil then
diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua
index 5b1c3603..8fc92410 100644
--- a/plugins/mod_storage_sql.lua
+++ b/plugins/mod_storage_sql.lua
@@ -1,17 +1,19 @@
 
 -- luacheck: ignore 212/self
 
+local cache = require "util.cache";
 local json = require "util.json";
 local sql = require "util.sql";
 local xml_parse = require "util.xml".parse;
 local uuid = require "util.uuid";
 local resolve_relative_path = require "util.paths".resolve_relative_path;
+local jid_join = require "util.jid".join;
 
 local is_stanza = require"util.stanza".is_stanza;
 local t_concat = table.concat;
 
 local noop = function() end
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local function iterator(result)
 	return function(result_)
 		local row = result_();
@@ -148,7 +150,13 @@ end
 
 --- Archive store API
 
--- luacheck: ignore 512 431/user 431/store
+local archive_item_limit = module:get_option_number("storage_archive_item_limit");
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
+local item_count_cache_hit = module:measure("item_count_cache_hit", "rate");
+local item_count_cache_miss = module:measure("item_count_cache_miss", "rate")
+
+-- luacheck: ignore 512 431/user 431/store 431/err
 local map_store = {};
 map_store.__index = map_store;
 map_store.remove = {};
@@ -225,13 +233,91 @@ function map_store:set_keys(username, keydatas)
 	return result;
 end
 
+function map_store:get_all(key)
+	if type(key) ~= "string" or key == "" then
+		return nil, "get_all only supports non-empty string keys";
+	end
+	local ok, result = engine:transaction(function()
+		local query = [[
+		SELECT "user", "type", "value"
+		FROM "prosody"
+		WHERE "host"=? AND "store"=? AND "key"=?
+		]];
+
+		local data;
+		for row in engine:select(query, host, self.store, key) do
+			local key_data, err = deserialize(row[2], row[3]);
+			assert(key_data ~= nil, err);
+			if data == nil then
+				data = {};
+			end
+			data[row[1]] = key_data;
+		end
+
+		return data;
+
+	end);
+	if not ok then return nil, result; end
+	return result;
+end
+
+function map_store:delete_all(key)
+	if type(key) ~= "string" or key == "" then
+		return nil, "delete_all only supports non-empty string keys";
+	end
+	local ok, result = engine:transaction(function()
+		local delete_sql = [[
+		DELETE FROM "prosody"
+		WHERE "host"=? AND "store"=? AND "key"=?;
+		]];
+		engine:delete(delete_sql, host, self.store, key);
+		return true;
+	end);
+	if not ok then return nil, result; end
+	return result;
+end
+
 local archive_store = {}
 archive_store.caps = {
 	total = true;
+	quota = archive_item_limit;
+	truncate = true;
 };
 archive_store.__index = archive_store
 function archive_store:append(username, key, value, when, with)
 	local user,store = username,self.store;
+	local cache_key = jid_join(username, host, store);
+	local item_count = archive_item_count_cache:get(cache_key);
+	if not item_count then
+		item_count_cache_miss();
+		local ok, ret = engine:transaction(function()
+			local count_sql = [[
+			SELECT COUNT(*) FROM "prosodyarchive"
+			WHERE "host"=? AND "user"=? AND "store"=?;
+			]];
+			local result = engine:select(count_sql, host, user, store);
+			if result then
+				for row in result do
+					item_count = row[1];
+				end
+			end
+		end);
+		if not ok or not item_count then
+			module:log("error", "Failed while checking quota for %s: %s", username, ret);
+			return nil, "Failure while checking quota";
+		end
+		archive_item_count_cache:set(cache_key, item_count);
+	else
+		item_count_cache_hit();
+	end
+
+	if archive_item_limit then
+		module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit);
+		if item_count >= archive_item_limit then
+			return nil, "quota-limit";
+		end
+	end
+
 	when = when or os.time();
 	with = with or "";
 	local ok, ret = engine:transaction(function()
@@ -245,12 +331,16 @@ function archive_store:append(username, key, value, when, with)
 		VALUES (?,?,?,?,?,?,?,?);
 		]];
 		if key then
-			engine:delete(delete_sql, host, user or "", store, key);
+			local result = engine:delete(delete_sql, host, user or "", store, key);
+			if result then
+				item_count = item_count - result:affected();
+			end
 		else
 			key = uuid.generate();
 		end
 		local t, encoded_value = assert(serialize(value));
 		engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value);
+		archive_item_count_cache:set(cache_key, item_count+1);
 		return key;
 	end);
 	if not ok then return ok, ret; end
@@ -287,45 +377,48 @@ local function archive_where(query, args, where)
 	end
 end
 local function archive_where_id_range(query, args, where)
-	local args_len = #args
 	-- Before or after specific item, exclusive
+	local id_lookup_sql = [[
+	SELECT "sort_id"
+	FROM "prosodyarchive"
+	WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
+	LIMIT 1;
+	]];
 	if query.after then  -- keys better be unique!
-		where[#where+1] = [[
-		"sort_id" > COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			), 0)
-		]];
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.after, args[1], args[2], args[3];
-		args_len = args_len + 4
+		local after_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			after_id = row[1];
+		end
+		if not after_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" > ?';
+		args[#args+1] = after_id;
 	end
 	if query.before then
-		where[#where+1] = [[
-		"sort_id" < COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			),
-			(
-				SELECT MAX("sort_id")+1
-				FROM "prosodyarchive"
-			)
-		)
-		]]
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.before, args[1], args[2], args[3];
+		local before_id = nil;
+		for row in engine:select(id_lookup_sql, query.before, args[1], args[2], args[3]) do
+			before_id = row[1];
+		end
+		if not before_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" < ?';
+		args[#args+1] = before_id;
 	end
+	return true;
 end
 
 function archive_store:find(username, query)
 	query = query or {};
 	local user,store = username,self.store;
-	local total;
-	local ok, result = engine:transaction(function()
+	local cache_key = jid_join(username, host, self.store);
+	local total = archive_item_count_cache:get(cache_key);
+	(total and item_count_cache_hit or item_count_cache_miss)();
+	if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+		return noop, total;
+	end
+	local ok, result, err = engine:transaction(function()
 		local sql_query = [[
 		SELECT "key", "type", "value", "when", "with"
 		FROM "prosodyarchive"
@@ -346,12 +439,16 @@ function archive_store:find(username, query)
 					total = row[1];
 				end
 			end
+			if query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+				archive_item_count_cache:set(cache_key, total);
+			end
 			if query.limit == 0 then -- Skip the real query
 				return noop, total;
 			end
 		end
 
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 
 		if query.limit then
 			args[#args+1] = query.limit;
@@ -361,7 +458,8 @@ function archive_store:find(username, query)
 			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
 		return engine:select(sql_query, unpack(args));
 	end);
-	if not ok then return ok, result end
+	if not ok then return ok, result; end
+	if not result then return nil, err; end
 	return function()
 		local row = result();
 		if row ~= nil then
@@ -372,6 +470,48 @@ function archive_store:find(username, query)
 	end, total;
 end
 
+function archive_store:summary(username, query)
+	query = query or {};
+	local user,store = username,self.store;
+	local ok, result = engine:transaction(function()
+		local sql_query = [[
+		SELECT DISTINCT "with", COUNT(*), MIN("when"), MAX("when")
+		FROM "prosodyarchive"
+		WHERE %s
+		GROUP BY "with"
+		ORDER BY "sort_id" %s%s;
+		]];
+		local args = { host, user or "", store, };
+		local where = { "\"host\" = ?", "\"user\" = ?", "\"store\" = ?", };
+
+		archive_where(query, args, where);
+
+		archive_where_id_range(query, args, where);
+
+		if query.limit then
+			args[#args+1] = query.limit;
+		end
+
+		sql_query = sql_query:format(t_concat(where, " AND "), query.reverse
+			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
+		return engine:select(sql_query, unpack(args));
+	end);
+	if not ok then return ok, result end
+	local counts = {};
+	local earliest, latest = {}, {};
+	for row in result do
+		local with, count = row[1], row[2];
+		counts[with] = count;
+		earliest[with] = row[3];
+		latest[with] = row[4];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 function archive_store:delete(username, query)
 	query = query or {};
 	local user,store = username,self.store;
@@ -384,7 +524,8 @@ function archive_store:delete(username, query)
 			table.remove(where, 2);
 		end
 		archive_where(query, args, where);
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 		if query.truncate == nil then
 			sql_query = sql_query:format(t_concat(where, " AND "));
 		else
@@ -423,9 +564,24 @@ function archive_store:delete(username, query)
 		end
 		return engine:delete(sql_query, unpack(args));
 	end);
+	local cache_key = jid_join(username, host, self.store);
+	archive_item_count_cache:set(cache_key, nil);
 	return ok and stmt:affected(), stmt;
 end
 
+function archive_store:users()
+	local ok, result = engine:transaction(function()
+		local select_sql = [[
+		SELECT DISTINCT "user"
+		FROM "prosodyarchive"
+		WHERE "host"=? AND "store"=?;
+		]];
+		return engine:select(select_sql, host, self.store);
+	end);
+	if not ok then error(result); end
+	return iterator(result);
+end
+
 local stores = {
 	keyval = keyval_store;
 	map = map_store;
@@ -610,9 +766,10 @@ function module.load()
 	if prosody.prosodyctl then return; end
 	local engines = module:shared("/*/sql/connections");
 	local params = normalize_params(module:get_option("sql", default_params));
-	engine = engines[sql.db2uri(params)];
+	local db_uri = sql.db2uri(params);
+	engine = engines[db_uri];
 	if not engine then
-		module:log("debug", "Creating new engine");
+		module:log("debug", "Creating new engine %s", db_uri);
 		engine = sql:create_engine(params, function (engine) -- luacheck: ignore 431/engine
 			if module:get_option("sql_manage_tables", true) then
 				-- Automatically create table, ignore failure (table probably already exists)
diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua
index 8c0c4e20..151add6f 100644
--- a/plugins/mod_tls.lua
+++ b/plugins/mod_tls.lua
@@ -35,9 +35,10 @@ local host = hosts[module.host];
 
 local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin;
 local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin;
+local err_c2s, err_s2sin, err_s2sout;
 
 function module.load(reload)
-	local NULL, err = {};
+	local NULL = {};
 	local modhost = module.host;
 	local parent = modhost:match("%.(.*)$");
 
@@ -53,16 +54,20 @@ function module.load(reload)
 	local host_s2s   = rawgetopt(modhost, "s2s_ssl") or parent_s2s;
 
 	module:log("debug", "Creating context for c2s");
-	ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
-	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end
+	local request_client_certs = { verify = { "peer", "client_once", }; };
 
 	module:log("debug", "Creating context for s2sout");
-	ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
-	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end
+	ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
+	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end
 
 	module:log("debug", "Creating context for s2sin");
-	ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
-	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end
+	-- for outgoing server connections
+	ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end
+
+	-- for incoming server connections
+	ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end
 
 	if reload then
 		module:log("info", "Certificates reloaded");
@@ -83,12 +88,21 @@ local function can_do_tls(session)
 		return session.ssl_ctx;
 	end
 	if session.type == "c2s_unauthed" then
+		if not ssl_ctx_c2s and c2s_require_encryption then
+			session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s);
+		end
 		session.ssl_ctx = ssl_ctx_c2s;
 		session.ssl_cfg = ssl_cfg_c2s;
 	elseif session.type == "s2sin_unauthed" and allow_s2s_tls then
+		if not ssl_ctx_s2sin and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin);
+		end
 		session.ssl_ctx = ssl_ctx_s2sin;
 		session.ssl_cfg = ssl_cfg_s2sin;
 	elseif session.direction == "outgoing" and allow_s2s_tls then
+		if not ssl_ctx_s2sout and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout);
+		end
 		session.ssl_ctx = ssl_ctx_s2sout;
 		session.ssl_cfg = ssl_cfg_s2sout;
 	else
diff --git a/plugins/mod_tokenauth.lua b/plugins/mod_tokenauth.lua
new file mode 100644
index 00000000..c04a1aa4
--- /dev/null
+++ b/plugins/mod_tokenauth.lua
@@ -0,0 +1,82 @@
+local id = require "util.id";
+local jid = require "util.jid";
+local base64 = require "util.encodings".base64;
+
+local token_store = module:open_store("auth_tokens", "map");
+
+function create_jid_token(actor_jid, token_jid, token_scope, token_ttl)
+	token_jid = jid.prep(token_jid);
+	if not actor_jid or token_jid ~= actor_jid and not jid.compare(token_jid, actor_jid) then
+		return nil, "not-authorized";
+	end
+
+	local token_username, token_host, token_resource = jid.split(token_jid);
+
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+
+	local token_info = {
+		owner = actor_jid;
+		created = os.time();
+		expires = token_ttl and (os.time() + token_ttl) or nil;
+		jid = token_jid;
+		session = {
+			username = token_username;
+			host = token_host;
+			resource = token_resource;
+
+			auth_scope = token_scope;
+		};
+	};
+
+	local token_id = id.long();
+	local token = base64.encode("1;"..jid.join(token_username, token_host)..";"..token_id);
+	token_store:set(token_username, token_id, token_info);
+
+	return token, token_info;
+end
+
+local function parse_token(encoded_token)
+	local token = base64.decode(encoded_token);
+	if not token then return nil; end
+	local token_jid, token_id = token:match("^1;([^;]+);(.+)$");
+	if not token_jid then return nil; end
+	local token_user, token_host = jid.split(token_jid);
+	return token_id, token_user, token_host;
+end
+
+function get_token_info(token)
+	local token_id, token_user, token_host = parse_token(token);
+	if not token_id then
+		return nil, "invalid-token-format";
+	end
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+
+	local token_info, err = token_store:get(token_user, token_id);
+	if not token_info then
+		if err then
+			return nil, "internal-error";
+		end
+		return nil, "not-authorized";
+	end
+
+	if token_info.expires and token_info.expires < os.time() then
+		return nil, "not-authorized";
+	end
+
+	return token_info
+end
+
+function revoke_token(token)
+	local token_id, token_user, token_host = parse_token(token);
+	if not token_id then
+		return nil, "invalid-token-format";
+	end
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+	return token_store:set(token_user, token_id, nil);
+end
diff --git a/plugins/mod_uptime.lua b/plugins/mod_uptime.lua
index ccd8e511..8a01fb17 100644
--- a/plugins/mod_uptime.lua
+++ b/plugins/mod_uptime.lua
@@ -16,7 +16,7 @@ module:add_feature("jabber:iq:last");
 
 module:hook("iq-get/host/jabber:iq:last:query", function(event)
 	local origin, stanza = event.origin, event.stanza;
-	origin.send(st.reply(stanza):tag("query", {xmlns = "jabber:iq:last", seconds = tostring(os.difftime(os.time(), start_time))}));
+	origin.send(st.reply(stanza):tag("query", {xmlns = "jabber:iq:last", seconds = tostring(("%d"):format(os.difftime(os.time(), start_time)))}));
 	return true;
 end);
 
@@ -42,6 +42,6 @@ function uptime_command_handler ()
 	return { info = uptime_text(), status = "completed" };
 end
 
-local descriptor = adhoc_new("Get uptime", "uptime", uptime_command_handler);
+local descriptor = adhoc_new("Get uptime", "uptime", uptime_command_handler, "any");
 
 module:provides("adhoc", descriptor);
diff --git a/plugins/mod_user_account_management.lua b/plugins/mod_user_account_management.lua
index 615c1ed6..130ed089 100644
--- a/plugins/mod_user_account_management.lua
+++ b/plugins/mod_user_account_management.lua
@@ -53,9 +53,10 @@ local function handle_registration_stanza(event)
 			log("info", "User removed their account: %s@%s", username, host);
 			module:fire_event("user-deregistered", { username = username, host = host, source = "mod_register", session = session });
 		else
-			local username = nodeprep(query:get_child_text("username"));
+			local username = query:get_child_text("username");
 			local password = query:get_child_text("password");
 			if username and password then
+				username = nodeprep(username);
 				if username == session.username then
 					if usermanager_set_password(username, password, session.host, session.resource) then
 						session.send(st.reply(stanza));
diff --git a/plugins/mod_vcard.lua b/plugins/mod_vcard.lua
index b1a4c6e8..c3d6fb8b 100644
--- a/plugins/mod_vcard.lua
+++ b/plugins/mod_vcard.lua
@@ -19,7 +19,7 @@ local function handle_vcard(event)
 	if stanza.attr.type == "get" then
 		local vCard;
 		if to then
-			local node, host = jid_split(to);
+			local node = jid_split(to);
 			vCard = st.deserialize(vcards:get(node)); -- load vCard for user or server
 		else
 			vCard = st.deserialize(vcards:get(session.username));-- load user's own vCard
diff --git a/plugins/mod_vcard_legacy.lua b/plugins/mod_vcard_legacy.lua
index 5e75947a..92b1d5e9 100644
--- a/plugins/mod_vcard_legacy.lua
+++ b/plugins/mod_vcard_legacy.lua
@@ -38,7 +38,7 @@ local simple_map = {
 module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 	local origin, stanza = event.origin, event.stanza;
 	local pep_service = mod_pep.get_pep_service(jid_split(stanza.attr.to) or origin.username);
-	local ok, id, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
+	local ok, _, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
 
 	local vcard_temp = st.stanza("vCard", { xmlns = "vcard-temp" });
 	if ok and vcard4_item then
@@ -105,26 +105,46 @@ module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 					vcard_temp:tag("WORK"):up();
 				end
 				vcard_temp:up();
+			elseif tag.name == "impp" then
+				local uri = tag:get_child_text("uri");
+				if uri and uri:sub(1, 5) == "xmpp:" then
+					vcard_temp:text_tag("JABBERID", uri:sub(6))
+				end
+			elseif tag.name == "org" then
+				vcard_temp:tag("ORG")
+					:text_tag("ORGNAME", tag:get_child_text("text"))
+				:up();
+			end
+		end
+	else
+		local ok, _, nick_item = pep_service:get_last_item("http://jabber.org/protocol/nick", stanza.attr.from);
+		if ok and nick_item then
+			local nickname = nick_item:get_child_text("nick", "http://jabber.org/protocol/nick");
+			if nickname then
+				vcard_temp:text_tag("NICKNAME", nickname);
 			end
 		end
 	end
 
-	local meta_ok, avatar_meta = pep_service:get_items("urn:xmpp:avatar:metadata", stanza.attr.from);
-	local data_ok, avatar_data = pep_service:get_items("urn:xmpp:avatar:data", stanza.attr.from);
+	local ok, avatar_hash, meta = pep_service:get_last_item("urn:xmpp:avatar:metadata", true);
+	if ok and avatar_hash then
 
-	if data_ok then
-		for _, hash in ipairs(avatar_data) do
-			local meta = meta_ok and avatar_meta[hash];
-			local data = avatar_data[hash];
-			local info = meta and meta.tags[1]:get_child("info");
+		local info = meta.tags[1]:get_child("info");
+		if info then
 			vcard_temp:tag("PHOTO");
-			if info and info.attr.type then
+
+			if info.attr.type then
 				vcard_temp:text_tag("TYPE", info.attr.type);
 			end
-			if data then
-				vcard_temp:text_tag("BINVAL", data.tags[1]:get_text());
-			elseif info and info.attr.url then
+
+			if info.attr.url then
 				vcard_temp:text_tag("EXTVAL", info.attr.url);
+			elseif info.attr.id then
+				local data_ok, avatar_data = pep_service:get_items("urn:xmpp:avatar:data", stanza.attr.from, { info.attr.id });
+				if data_ok and avatar_data and avatar_data[info.attr.id]  then
+					local data = avatar_data[info.attr.id];
+					vcard_temp:text_tag("BINVAL", data.tags[1]:get_text());
+				end
 			end
 			vcard_temp:up();
 		end
@@ -140,7 +160,7 @@ local node_defaults = {
 };
 
 function vcard_to_pep(vcard_temp)
-	local avatars = {};
+	local avatar = {};
 
 	local vcard4 = st.stanza("item", { xmlns = "http://jabber.org/protocol/pubsub", id = "current" })
 		:tag("vcard", { xmlns = 'urn:ietf:params:xml:ns:vcard-4.0' });
@@ -216,6 +236,10 @@ function vcard_to_pep(vcard_temp)
 				vcard4:text_tag("text", "work");
 			end
 			vcard4:up():up():up();
+		elseif tag.name == "JABBERID" then
+			vcard4:tag("impp")
+				:text_tag("uri", "xmpp:" .. tag:get_text())
+			:up();
 		elseif tag.name == "PHOTO" then
 			local avatar_type = tag:get_child_text("TYPE");
 			local avatar_payload = tag:get_child_text("BINVAL");
@@ -225,7 +249,9 @@ function vcard_to_pep(vcard_temp)
 				local avatar_raw = base64_decode(avatar_payload);
 				local avatar_hash = sha1(avatar_raw, true);
 
-				local avatar_meta = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
+				avatar.hash = avatar_hash;
+
+				avatar.meta = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
 					:tag("metadata", { xmlns="urn:xmpp:avatar:metadata" })
 						:tag("info", {
 							bytes = tostring(#avatar_raw),
@@ -233,36 +259,27 @@ function vcard_to_pep(vcard_temp)
 							type = avatar_type,
 						});
 
-				local avatar_data = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
+				avatar.data = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
 					:tag("data", { xmlns="urn:xmpp:avatar:data" })
 						:text(avatar_payload);
 
-				table.insert(avatars, { hash = avatar_hash, meta = avatar_meta, data = avatar_data });
 			end
 		end
 	end
-	return vcard4, avatars;
+	return vcard4, avatar;
 end
 
-function save_to_pep(pep_service, actor, vcard4, avatars)
-	if avatars then
+function save_to_pep(pep_service, actor, vcard4, avatar)
+	if avatar then
 
 		if pep_service:purge("urn:xmpp:avatar:metadata", actor) then
 			pep_service:purge("urn:xmpp:avatar:data", actor);
 		end
 
-		local avatar_defaults = node_defaults;
-		if #avatars > 1 then
-			avatar_defaults = {};
-			for k,v in pairs(node_defaults) do
-				avatar_defaults[k] = v;
-			end
-			avatar_defaults.max_items = #avatars;
-		end
-		for _, avatar in ipairs(avatars) do
-			local ok, err = pep_service:publish("urn:xmpp:avatar:data", actor, avatar.hash, avatar.data, avatar_defaults);
+		if avatar.data and avatar.meta then
+			local ok, err = assert(pep_service:publish("urn:xmpp:avatar:data", actor, avatar.hash, avatar.data, node_defaults));
 			if ok then
-				ok, err = pep_service:publish("urn:xmpp:avatar:metadata", actor, avatar.hash, avatar.meta, avatar_defaults);
+				ok, err = assert(pep_service:publish("urn:xmpp:avatar:metadata", actor, avatar.hash, avatar.meta, node_defaults));
 			end
 			if not ok then
 				return ok, err;
diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua
index a613672b..7a5f2d03 100644
--- a/plugins/mod_websocket.lua
+++ b/plugins/mod_websocket.lua
@@ -33,18 +33,10 @@ local frame_buffer_limit = module:get_option_number("websocket_frame_buffer_limi
 local frame_fragment_limit = module:get_option_number("websocket_frame_fragment_limit", 8);
 local stream_close_timeout = module:get_option_number("c2s_close_timeout", 5);
 local consider_websocket_secure = module:get_option_boolean("consider_websocket_secure");
-local cross_domain = module:get_option_set("cross_domain_websocket", {});
-if cross_domain:contains("*") or cross_domain:contains(true) then
-	cross_domain = true;
+local cross_domain = module:get_option("cross_domain_websocket");
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_websocket' option has been deprecated");
 end
-
-local function check_origin(origin)
-	if cross_domain == true then
-		return true;
-	end
-	return cross_domain:contains(origin);
-end
-
 local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing";
 local xmlns_streams = "http://etherx.jabber.org/streams";
 local xmlns_client = "jabber:client";
@@ -92,7 +84,7 @@ local function session_close(session, reason)
 					stream_error = reason;
 				end
 			end
-			log("debug", "Disconnecting client, <stream:error> is: %s", tostring(stream_error));
+			log("debug", "Disconnecting client, <stream:error> is: %s", stream_error);
 			session.send(stream_error);
 		end
 
@@ -143,6 +135,11 @@ local function filter_open_close(data)
 	return data;
 end
 
+local default_get_response_body = [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
+<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
+</body></html>]]
+local websocket_get_response_body = module:get_option_string("websocket_get_response_body", default_get_response_body)
+
 local function validate_frame(frame, max_length)
 	local opcode, length = frame.opcode, frame.length;
 
@@ -207,11 +204,9 @@ function handle_request(event)
 
 	conn.starttls = false; -- Prevent mod_tls from believing starttls can be done
 
-	if not request.headers.sec_websocket_key then
+	if not request.headers.sec_websocket_key or request.method ~= "GET" then
 		response.headers.content_type = "text/html";
-		return [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
-			<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
-			</body></html>]];
+		return websocket_get_response_body;
 	end
 
 	local wants_xmpp = contains_token(request.headers.sec_websocket_protocol or "", "xmpp");
@@ -221,11 +216,6 @@ function handle_request(event)
 		return 501;
 	end
 
-	if not check_origin(request.headers.origin or "") then
-		module:log("debug", "Origin %s is not allowed by 'cross_domain_websocket' [ %s ]", request.headers.origin or "(missing header)", cross_domain);
-		return 403;
-	end
-
 	local function websocket_close(code, message)
 		conn:write(build_close(code, message));
 		conn:close();
@@ -364,27 +354,4 @@ module:provides("http", {
 
 function module.add_host(module)
 	module:hook("c2s-read-timeout", keepalive, -0.9);
-
-	if cross_domain ~= true then
-		local url = require "socket.url";
-		local ws_url = module:http_url("websocket", "xmpp-websocket");
-		local url_components = url.parse(ws_url);
-		-- The 'Origin' consists of the base URL without path
-		url_components.path = nil;
-		local this_origin = url.build(url_components);
-		local local_cross_domain = module:get_option_set("cross_domain_websocket", { this_origin });
-		if local_cross_domain:contains(true) then
-			module:log("error", "cross_domain_websocket = true only works in the global section");
-			return;
-		end
-
-		-- Don't add / remove something added by another host
-		-- This might be weird with random load order
-		local_cross_domain:exclude(cross_domain);
-		cross_domain:include(local_cross_domain);
-		module:log("debug", "cross_domain = %s", tostring(cross_domain));
-		function module.unload()
-			cross_domain:exclude(local_cross_domain);
-		end
-	end
 end
diff --git a/plugins/muc/hats.lib.lua b/plugins/muc/hats.lib.lua
new file mode 100644
index 00000000..77051af2
--- /dev/null
+++ b/plugins/muc/hats.lib.lua
@@ -0,0 +1,24 @@
+local st = require "util.stanza";
+local muc_util = module:require "muc/util";
+
+local xmlns_hats = "xmpp:prosody.im/protocol/hats:1";
+
+-- Strip any hats claimed by the client (to prevent spoofing)
+muc_util.add_filtered_namespace(xmlns_hats);
+
+module:hook("muc-build-occupant-presence", function (event)
+	local aff_data = event.room:get_affiliation_data(event.occupant.bare_jid);
+	local hats = aff_data and aff_data.hats;
+	if not hats then return; end
+	local hats_el;
+	for hat_id, hat_data in pairs(hats) do
+		if hat_data.active then
+			if not hats_el then
+				hats_el = st.stanza("hats", { xmlns = xmlns_hats });
+			end
+			hats_el:tag("hat", { uri = hat_id, title = hat_data.title }):up();
+		end
+	end
+	if not hats_el then return; end
+	event.stanza:add_direct_child(hats_el);
+end);
diff --git a/plugins/muc/history.lib.lua b/plugins/muc/history.lib.lua
index 0d69c97d..fdf65365 100644
--- a/plugins/muc/history.lib.lua
+++ b/plugins/muc/history.lib.lua
@@ -48,16 +48,18 @@ module:hook("muc-config-form", function(event)
 	table.insert(event.form, {
 		name = "muc#roomconfig_historylength";
 		type = "text-single";
+		datatype = "xs:integer";
 		label = "Maximum number of history messages returned by room";
 		desc = "Specify the maximum number of previous messages that should be sent to users when they join the room";
-		value = tostring(get_historylength(event.room));
+		value = get_historylength(event.room);
 	});
 	table.insert(event.form, {
 		name = 'muc#roomconfig_defaulthistorymessages',
 		type = 'text-single',
+		datatype = "xs:integer";
 		label = 'Default number of history messages returned by room',
 		desc = "Specify the number of previous messages sent to new users when they join the room";
-		value = tostring(get_defaulthistorymessages(event.room))
+		value = get_defaulthistorymessages(event.room);
 	});
 end, 70-5);
 
@@ -180,9 +182,6 @@ module:hook("muc-add-history", function(event)
 	stanza:tag("delay", { -- XEP-0203
 		xmlns = "urn:xmpp:delay", from = room.jid, stamp = stamp
 	}):up();
-	stanza:tag("x", { -- XEP-0091 (deprecated)
-		xmlns = "jabber:x:delay", from = room.jid, stamp = datetime.legacy()
-	}):up();
 	local entry = { stanza = stanza, timestamp = ts };
 	table.insert(history, entry);
 	while #history > get_historylength(room) do table.remove(history, 1) end
@@ -198,7 +197,27 @@ module:hook("muc-broadcast-message", function(event)
 end);
 
 module:hook("muc-message-is-historic", function (event)
-	return event.stanza:get_child("body");
+	local stanza = event.stanza;
+	if stanza:get_child("no-store", "urn:xmpp:hints")
+	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		-- XXX Experimental XEP
+		return false, "hint";
+	end
+	if stanza:get_child("store", "urn:xmpp:hints") then
+		return true, "hint";
+	end
+	if stanza:get_child("body") then
+		return true;
+	end
+	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
+		-- Since we can't know what an encrypted message contains, we assume it's important
+		-- XXX Experimental XEP
+		return true, "encrypted";
+	end
+	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		-- XXX Experimental XEP
+		return true, "marker";
+	end
 end, -1);
 
 return {
diff --git a/plugins/muc/language.lib.lua b/plugins/muc/language.lib.lua
index ee80806b..2ee2ba0f 100644
--- a/plugins/muc/language.lib.lua
+++ b/plugins/muc/language.lib.lua
@@ -32,6 +32,7 @@ local function add_form_option(event)
 		label = "Language tag for room (e.g. 'en', 'de', 'fr' etc.)";
 		type = "text-single";
 		desc = "Indicate the primary language spoken in this room";
+		datatype = "xs:language";
 		value = get_language(event.room) or "";
 	});
 end
diff --git a/plugins/muc/lock.lib.lua b/plugins/muc/lock.lib.lua
index 062ab615..32f2647b 100644
--- a/plugins/muc/lock.lib.lua
+++ b/plugins/muc/lock.lib.lua
@@ -43,7 +43,7 @@ end
 module:hook("muc-occupant-pre-join", function(event)
 	if not event.is_new_room and is_locked(event.room) then -- Deny entry
 		module:log("debug", "Room is locked, denying entry");
-		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found"));
+		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found", nil, module.host));
 		return true;
 	end
 end, -30);
diff --git a/plugins/muc/members_only.lib.lua b/plugins/muc/members_only.lib.lua
index 4194c5c7..79077153 100644
--- a/plugins/muc/members_only.lib.lua
+++ b/plugins/muc/members_only.lib.lua
@@ -113,7 +113,7 @@ module:hook("muc-occupant-pre-join", function(event)
 		local stanza = event.stanza;
 		local affiliation = room:get_affiliation(stanza.attr.from);
 		if valid_affiliations[affiliation or "none"] <= valid_affiliations.none then
-			local reply = st.error_reply(stanza, "auth", "registration-required"):up();
+			local reply = st.error_reply(stanza, "auth", "registration-required", nil, room.jid):up();
 			reply.tags[1].attr.code = "407";
 			event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 			return true;
@@ -131,7 +131,7 @@ module:hook("muc-pre-invite", function(event)
 		local inviter_affiliation = room:get_affiliation(stanza.attr.from) or "none";
 		local required_affiliation = room._data.allow_member_invites and "member" or "admin";
 		if valid_affiliations[inviter_affiliation] < valid_affiliations[required_affiliation] then
-			event.origin.send(st.error_reply(stanza, "auth", "forbidden"));
+			event.origin.send(st.error_reply(stanza, "auth", "forbidden", nil, room.jid));
 			return true;
 		end
 	end
diff --git a/plugins/muc/mod_muc.lua b/plugins/muc/mod_muc.lua
index b9c7c859..e9fd1521 100644
--- a/plugins/muc/mod_muc.lua
+++ b/plugins/muc/mod_muc.lua
@@ -86,7 +86,14 @@ room_mt.get_registered_nick = register.get_registered_nick;
 room_mt.get_registered_jid = register.get_registered_jid;
 room_mt.handle_register_iq = register.handle_register_iq;
 
+local presence_broadcast = module:require "muc/presence_broadcast";
+room_mt.get_presence_broadcast = presence_broadcast.get;
+room_mt.set_presence_broadcast = presence_broadcast.set;
+room_mt.get_valid_broadcast_roles = presence_broadcast.get_valid_broadcast_roles;
+
+
 local jid_split = require "util.jid".split;
+local jid_prep = require "util.jid".prep;
 local jid_bare = require "util.jid".bare;
 local st = require "util.stanza";
 local cache = require "util.cache";
@@ -98,6 +105,7 @@ module:depends("disco");
 module:add_identity("conference", "text", module:get_option_string("name", "Prosody Chatrooms"));
 module:add_feature("http://jabber.org/protocol/muc");
 module:depends "muc_unique"
+module:require "muc/hats";
 module:require "muc/lock";
 
 local function is_admin(jid)
@@ -129,7 +137,12 @@ local room_items_cache = {};
 local function room_save(room, forced, savestate)
 	local node = jid_split(room.jid);
 	local is_persistent = persistent.get(room);
-	room_items_cache[room.jid] = room:get_public() and room:get_name() or nil;
+	if room:get_public() then
+		room_items_cache[room.jid] = room:get_name() or "";
+	else
+		room_items_cache[room.jid] = nil;
+	end
+
 	if is_persistent or savestate then
 		persistent_rooms:set(nil, room.jid, true);
 		local data, state = room:freeze(savestate);
@@ -155,7 +168,11 @@ local rooms = cache.new(max_rooms or max_live_rooms, function (jid, room)
 	end
 	module:log("debug", "Evicting room %s", jid);
 	room_eviction();
-	room_items_cache[room.jid] = room:get_public() and room:get_name() or nil;
+	if room:get_public() then
+		room_items_cache[room.jid] = room:get_name() or "";
+	else
+		room_items_cache[room.jid] = nil;
+	end
 	local ok, err = room_save(room, nil, true); -- Force to disk
 	if not ok then
 		module:log("error", "Failed to swap inactive room %s to disk: %s", jid, err);
@@ -185,7 +202,7 @@ end
 
 local function handle_broken_room(room, origin, stanza)
 	module:log("debug", "Returning error from broken room %s", room.jid);
-	origin.send(st.error_reply(stanza, "wait", "internal-server-error"));
+	origin.send(st.error_reply(stanza, "wait", "internal-server-error", nil, room.jid));
 	return true;
 end
 
@@ -264,9 +281,13 @@ local function set_room_defaults(room, lang)
 	room:set_changesubject(module:get_option_boolean("muc_room_default_change_subject", room:get_changesubject()));
 	room:set_historylength(module:get_option_number("muc_room_default_history_length", room:get_historylength()));
 	room:set_language(lang or module:get_option_string("muc_room_default_language"));
+	room:set_presence_broadcast(module:get_option("muc_room_default_presence_broadcast", room:get_presence_broadcast()));
 end
 
 function create_room(room_jid, config)
+	if jid_bare(room_jid) ~= room_jid or not jid_prep(room_jid, true) then
+		return nil, "invalid-jid";
+	end
 	local exists = get_room_from_jid(room_jid);
 	if exists then
 		return nil, "room-exists";
@@ -325,13 +346,14 @@ module:hook("host-disco-items", function(event)
 	module:log("debug", "host-disco-items called");
 	if next(room_items_cache) ~= nil then
 		for jid, room_name in pairs(room_items_cache) do
+			if room_name == "" then room_name = nil; end
 			reply:tag("item", { jid = jid, name = room_name }):up();
 		end
 	else
 		for room in all_rooms() do
 			if not room:get_hidden() then
 				local jid, room_name = room.jid, room:get_name();
-				room_items_cache[jid] = room_name;
+				room_items_cache[jid] = room_name or "";
 				reply:tag("item", { jid = jid, name = room_name }):up();
 			end
 		end
@@ -345,7 +367,7 @@ end, 1);
 module:hook("muc-room-pre-create", function(event)
 	local origin, stanza = event.origin, event.stanza;
 	if not track_room(event.room) then
-		origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+		origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 		return true;
 	end
 end, -1000);
@@ -396,7 +418,7 @@ do
 				restrict_room_creation == "local" and
 				select(2, jid_split(user_jid)) == host_suffix
 			) then
-				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted"));
+				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted", module.host));
 				return true;
 			end
 		end);
@@ -441,7 +463,7 @@ for event_name, method in pairs {
 				room = nil;
 			else
 				if stanza.attr.type ~= "error" then
-					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason)
+					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason, module.host)
 					if room._data.newjid then
 						local uri = "xmpp:"..room._data.newjid.."?join";
 						reply:get_child("error"):child_with_name("gone"):text(uri);
@@ -454,17 +476,21 @@ for event_name, method in pairs {
 
 		if room == nil then
 			-- Watch presence to create rooms
-			if stanza.attr.type == nil and stanza.name == "presence" then
+			if not jid_prep(room_jid, true) then
+				origin.send(st.error_reply(stanza, "modify", "jid-malformed", nil, module.host));
+				return true;
+			end
+			if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then
 				room = muclib.new_room(room_jid);
 				return room:handle_first_presence(origin, stanza);
 			elseif stanza.attr.type ~= "error" then
-				origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+				origin.send(st.error_reply(stanza, "cancel", "item-not-found", nil, module.host));
 				return true;
 			else
 				return;
 			end
 		elseif room == false then -- Error loading room
-			origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+			origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 			return true;
 		end
 		return room[method](room, origin, stanza);
@@ -483,6 +509,7 @@ do -- Ad-hoc commands
 	local t_concat = table.concat;
 	local adhoc_new = module:require "adhoc".new;
 	local adhoc_initial = require "util.adhoc".new_initial_data_form;
+	local adhoc_simple = require "util.adhoc".new_simple_form;
 	local array = require "util.array";
 	local dataforms_new = require "util.dataforms".new;
 
@@ -513,4 +540,46 @@ do -- Ad-hoc commands
 		"http://prosody.im/protocol/muc#destroy", destroy_rooms_handler, "admin");
 
 	module:provides("adhoc", destroy_rooms_desc);
+
+
+	local set_affiliation_layout = dataforms_new {
+		-- FIXME wordsmith title, instructions, labels etc
+		title = "Set affiliation";
+
+		{ name = "FORM_TYPE", type = "hidden", value = "http://prosody.im/protocol/muc#set-affiliation" };
+		{ name = "room", type = "jid-single", required = true, label = "Room"};
+		{ name = "jid", type = "jid-single", required = true, label = "JID"};
+		{ name = "affiliation", type = "list-single", required = true, label = "Affiliation",
+			options = { "owner"; "admin"; "member"; "none"; "outcast"; },
+		};
+		{ name = "reason", type = "text-single", "Reason", }
+	};
+
+	local set_affiliation_handler = adhoc_simple(set_affiliation_layout, function (fields, errors)
+		if errors then
+			local errmsg = {};
+			for field, err in pairs(errors) do
+				errmsg[#errmsg + 1] = field .. ": " .. err;
+			end
+			return { status = "completed", error = { message = t_concat(errmsg, "\n") } };
+		end
+
+		local room = get_room_from_jid(fields.room);
+		if not room then
+			return { status = "canceled", error = { message =  "No such room"; }; };
+		end
+		local ok, err, condition = room:set_affiliation(true, fields.jid, fields.affiliation, fields.reason);
+
+		if not ok then
+			return { status = "canceled", error = { message =  "Affiliation change failed: "..err..":"..condition; }; };
+		end
+
+		return { status = "completed", info = "Affiliation updated",
+		};
+	end);
+
+	local set_affiliation_desc = adhoc_new("Set affiliation in room",
+		"http://prosody.im/protocol/muc#set-affiliation", set_affiliation_handler, "admin");
+
+	module:provides("adhoc", set_affiliation_desc);
 end
diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua
index 3139c131..6b330131 100644
--- a/plugins/muc/muc.lib.lua
+++ b/plugins/muc/muc.lib.lua
@@ -23,6 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep;
 local st = require "util.stanza";
 local base64 = require "util.encodings".base64;
 local md5 = require "util.hashes".md5;
+local new_id = require "util.id".medium;
 
 local log = module._log;
 
@@ -39,7 +40,7 @@ function room_mt:__tostring()
 end
 
 function room_mt.save()
-	-- overriden by mod_muc.lua
+	-- overridden by mod_muc.lua
 end
 
 function room_mt:get_occupant_jid(real_jid)
@@ -217,13 +218,13 @@ end
 
 -- Broadcasts an occupant's presence to the whole room
 -- Takes the x element that goes into the stanzas
-function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
+function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason, prev_role, force_unavailable)
 	local base_x = x.base or x;
 	-- Build real jid and (optionally) occupant jid template presences
 	local base_presence do
 		-- Try to use main jid's presence
 		local pr = occupant:get_presence();
-		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") then
+		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") and not force_unavailable then
 			base_presence = st.clone(pr);
 		else -- user is leaving but didn't send a leave presence. make one for them
 			base_presence = st.presence {from = occupant.nick; type = "unavailable";};
@@ -236,6 +237,7 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		occupant = occupant; nick = nick; actor = actor;
 		reason = reason;
 	}
+	module:fire_event("muc-build-occupant-presence", event);
 	module:fire_event("muc-broadcast-presence", event);
 
 	-- Allow muc-broadcast-presence listeners to change things
@@ -279,7 +281,9 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		self_p = st.clone(base_presence):add_child(self_x);
 	end
 
-	-- General populance
+	local broadcast_roles = self:get_presence_broadcast();
+
+	-- General populace
 	for occupant_nick, n_occupant in self:each_occupant() do
 		if occupant_nick ~= occupant.nick then
 			local pr;
@@ -290,7 +294,13 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 			else
 				pr = get_anon_p();
 			end
-			self:route_to_occupant(n_occupant, pr);
+			if broadcast_roles[occupant.role or "none"] or force_unavailable then
+				self:route_to_occupant(n_occupant, pr);
+			elseif prev_role and broadcast_roles[prev_role] then
+				pr.attr.type = 'unavailable';
+				self:route_to_occupant(n_occupant, pr);
+			end
+
 		end
 	end
 
@@ -303,6 +313,7 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		-- use their own presences as templates
 		for full_jid, pr in occupant:each_session() do
 			pr = st.clone(pr);
+			module:fire_event("muc-build-occupant-presence", { room = self, occupant = occupant, stanza = pr });
 			pr.attr.to = full_jid;
 			pr:add_child(self_x);
 			self:route_stanza(pr);
@@ -314,6 +325,7 @@ function room_mt:send_occupant_list(to, filter)
 	local to_bare = jid_bare(to);
 	local is_anonymous = false;
 	local whois = self:get_whois();
+	local broadcast_roles = self:get_presence_broadcast();
 	if whois ~= "anyone" then
 		local affiliation = self:get_affiliation(to);
 		if affiliation ~= "admin" and affiliation ~= "owner" then
@@ -323,14 +335,38 @@ function room_mt:send_occupant_list(to, filter)
 			end
 		end
 	end
+	local broadcast_bare_jids = {}; -- Track which bare JIDs we have sent presence for
 	for occupant_jid, occupant in self:each_occupant() do
+		broadcast_bare_jids[occupant.bare_jid] = true;
 		if filter == nil or filter(occupant_jid, occupant) then
 			local x = st.stanza("x", {xmlns='http://jabber.org/protocol/muc#user'});
 			self:build_item_list(occupant, x, is_anonymous and to_bare ~= occupant.bare_jid); -- can always see your own jids
 			local pres = st.clone(occupant:get_presence());
 			pres.attr.to = to;
 			pres:add_child(x);
-			self:route_stanza(pres);
+			module:fire_event("muc-build-occupant-presence", { room = self, occupant = occupant, stanza = pres });
+			if to_bare == occupant.bare_jid or broadcast_roles[occupant.role or "none"] then
+				self:route_stanza(pres);
+			end
+		end
+	end
+	if broadcast_roles.none then
+		-- Broadcast stanzas for affiliated users not currently in the MUC
+		for affiliated_jid, affiliation, affiliation_data in self:each_affiliation() do
+			local nick = affiliation_data and affiliation_data.reserved_nickname;
+			if (nick or not is_anonymous) and not broadcast_bare_jids[affiliated_jid]
+			and (filter == nil or filter(affiliated_jid, nil)) then
+				local from = nick and (self.jid.."/"..nick) or self.jid;
+				local pres = st.presence({ to = to, from = from, type = "unavailable" })
+					:tag("x", { xmlns = 'http://jabber.org/protocol/muc#user' })
+						:tag("item", {
+							affiliation = affiliation;
+							role = "none";
+							nick = nick;
+							jid = not is_anonymous and affiliated_jid or nil }):up()
+						:up();
+				self:route_stanza(pres);
+			end
 		end
 	end
 end
@@ -373,13 +409,14 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	local real_jid = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(real_jid);
 	if occupant == nil then return nil; end
-	local type, condition, text = stanza:get_error();
+	local _, condition, text = stanza:get_error();
 	local error_message = "Kicked: "..(condition and condition:gsub("%-", " ") or "presence error");
 	if text and self:get_whois() == "anyone" then
 		error_message = error_message..": "..text;
 	end
 	occupant:set_session(real_jid, st.presence({type="unavailable"})
 		:tag('status'):text(error_message));
+	local orig_role = occupant.role;
 	local is_last_session = occupant.jid == real_jid;
 	if is_last_session then
 		occupant.role = nil;
@@ -389,9 +426,13 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	if is_last_session then
 		x:tag("status", {code = "333"});
 	end
-	self:publicise_occupant_status(new_occupant or occupant, x);
+	self:publicise_occupant_status(new_occupant or occupant, x, nil, nil, nil, orig_role);
 	if is_last_session then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -406,7 +447,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	local room, stanza = event.room, event.stanza;
 	local affiliation = room:get_affiliation(stanza.attr.from);
 	if affiliation == "outcast" then
-		local reply = st.error_reply(stanza, "auth", "forbidden"):up();
+		local reply = st.error_reply(stanza, "auth", "forbidden", nil, room.jid):up();
 		reply.tags[1].attr.code = "403";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -414,28 +455,41 @@ module:hook("muc-occupant-pre-join", function(event)
 end, -10);
 
 module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
 	local nick = jid_resource(event.occupant.nick);
 	if not nick:find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
 module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
 	if not jid_resource(event.dest_occupant.nick):find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
-function room_mt:handle_first_presence(origin, stanza)
-	if not stanza:get_child("x", "http://jabber.org/protocol/muc") then
-		module:log("debug", "Room creation without <x>, possibly desynced");
+module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
+		return true;
+	end
+end, 2);
 
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.dest_occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
 		return true;
 	end
+end, 2);
 
+function room_mt:handle_first_presence(origin, stanza)
 	local real_jid = stanza.attr.from;
 	local dest_jid = stanza.attr.to;
 	local bare_jid = jid_bare(real_jid);
@@ -505,7 +559,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	if orig_occupant == nil and not muc_x and stanza.attr.type == nil then
 		module:log("debug", "Attempted join without <x>, possibly desynced");
 		origin.send(st.error_reply(stanza, "cancel", "item-not-found",
-			"You must join the room before sending presence updates"));
+			"You are not currently connected to this chat", self.jid));
 		return true;
 	end
 
@@ -567,7 +621,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 		and bare_jid ~= jid_bare(dest_occupant.bare_jid) then
 		-- new nick or has different bare real jid
 		log("debug", "%s couldn't join due to nick conflict: %s", real_jid, dest_occupant.nick);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, self.jid):up();
 		reply.tags[1].attr.code = "409";
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -576,6 +630,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	-- Send presence stanza about original occupant
 	if orig_occupant ~= nil and orig_occupant ~= dest_occupant then
 		local orig_x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local orig_role = orig_occupant.role;
 		local dest_nick;
 		if dest_occupant == nil then -- Session is leaving
 			log("debug", "session %s is leaving occupant %s", real_jid, orig_occupant.nick);
@@ -613,12 +668,12 @@ function room_mt:handle_normal_presence(origin, stanza)
 				x:tag("status", {code = "303";}):up();
 				x:tag("status", {code = "110";}):up();
 				self:route_stanza(generated_unavail:add_child(x));
-				dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant
+				dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant
 			end
 		end
 
 		self:save_occupant(orig_occupant);
-		self:publicise_occupant_status(orig_occupant, orig_x, dest_nick);
+		self:publicise_occupant_status(orig_occupant, orig_x, dest_nick, nil, nil, orig_role);
 
 		if is_last_orig_session then
 			module:fire_event("muc-occupant-left", {
@@ -639,7 +694,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 			-- Send occupant list to newly joined or desynced user
 			self:send_occupant_list(real_jid, function(nick, occupant) -- luacheck: ignore 212
 				-- Don't include self
-				return occupant:get_presence(real_jid) == nil;
+				return (not occupant) or occupant:get_presence(real_jid) == nil;
 			end)
 		end
 		local dest_x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
@@ -650,7 +705,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 		if nick_changed then
 			self_x:tag("status", {code="210"}):up();
 		end
-		self:publicise_occupant_status(dest_occupant, {base=dest_x,self=self_x});
+		self:publicise_occupant_status(dest_occupant, {base=dest_x,self=self_x}, nil, nil, nil, orig_occupant and orig_occupant.role or nil);
 
 		if orig_occupant ~= nil and orig_occupant ~= dest_occupant and not is_last_orig_session then
 			-- If user is swapping and wasn't last original session
@@ -696,7 +751,7 @@ function room_mt:handle_presence_to_occupant(origin, stanza)
 		return self:handle_normal_presence(origin, stanza);
 	elseif type ~= 'result' then -- bad type
 		if type ~= 'visible' and type ~= 'invisible' then -- COMPAT ejabberd can broadcast or forward XEP-0018 presences
-			origin.send(st.error_reply(stanza, "modify", "bad-request")); -- FIXME correct error?
+			origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid)); -- FIXME correct error?
 		end
 	end
 	return true;
@@ -731,11 +786,11 @@ function room_mt:handle_iq_to_occupant(origin, stanza)
 	else -- Type is "get" or "set"
 		local current_nick = self:get_occupant_jid(from);
 		if not current_nick then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 			return true;
 		end
 		if not occupant then -- recipient not in room
-			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 			return true;
 		end
 		-- XEP-0410 MUC Self-Ping #1220
@@ -764,12 +819,12 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 	local type = stanza.attr.type;
 	if not current_nick then -- not in room
 		if type ~= "error" then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 		end
 		return true;
 	end
 	if type == "groupchat" then -- groupchat messages not allowed in PM
-		origin.send(st.error_reply(stanza, "modify", "bad-request"));
+		origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid));
 		return true;
 	elseif type == "error" and is_kickable_error(stanza) then
 		log("debug", "%s kicked from %s for sending an error message", current_nick, self.jid);
@@ -778,14 +833,16 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 
 	local o_data = self:get_occupant_by_nick(to);
 	if not o_data then
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 		return true;
 	end
 	log("debug", "%s sent private message stanza to %s (%s)", from, to, o_data.jid);
 	stanza = muc_util.filter_muc_x(st.clone(stanza));
 	stanza:tag("x", { xmlns = "http://jabber.org/protocol/muc#user" }):up();
 	stanza.attr.from = current_nick;
-	self:route_to_occupant(o_data, stanza)
+	if module:fire_event("muc-private-message", { room = self, origin = origin, stanza = stanza }) ~= false then
+		self:route_to_occupant(o_data, stanza)
+	end
 	-- TODO: Remove x tag?
 	stanza.attr.from = from;
 	return true;
@@ -815,10 +872,12 @@ function room_mt:process_form(origin, stanza)
 	if form.attr.type == "cancel" then
 		origin.send(st.reply(stanza));
 	elseif form.attr.type == "submit" then
+		-- luacheck: ignore 231/errors
 		local fields, errors, present;
 		if form.tags[1] == nil then -- Instant room
 			fields, present = {}, {};
 		else
+			-- FIXME handle form errors
 			fields, errors, present = self:get_form_layout(stanza.attr.from):data(form);
 			if fields.FORM_TYPE ~= "http://jabber.org/protocol/muc#roomconfig" then
 				origin.send(st.error_reply(stanza, "cancel", "bad-request", "Form is not of type room configuration"));
@@ -873,13 +932,18 @@ function room_mt:clear(x)
 	x = x or st.stanza("x", {xmlns='http://jabber.org/protocol/muc#user'});
 	local occupants_updated = {};
 	for nick, occupant in self:each_occupant() do -- luacheck: ignore 213
+		local prev_role = occupant.role;
 		occupant.role = nil;
 		self:save_occupant(occupant);
-		occupants_updated[occupant] = true;
+		occupants_updated[occupant] = prev_role;
 	end
-	for occupant in pairs(occupants_updated) do
-		self:publicise_occupant_status(occupant, x);
-		module:fire_event("muc-occupant-left", { room = self; nick = occupant.nick; occupant = occupant;});
+	for occupant, prev_role in pairs(occupants_updated) do
+		self:publicise_occupant_status(occupant, x, nil, nil, nil, prev_role);
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 end
 
@@ -972,7 +1036,7 @@ function room_mt:handle_admin_query_get_command(origin, stanza)
 	local _aff_rank = valid_affiliations[_aff or "none"];
 	local _rol = item.attr.role;
 	if _aff and _aff_rank and not _rol then
-		-- You need to be at least an admin, and be requesting info about your affifiliation or lower
+		-- You need to be at least an admin, and be requesting info about your affiliation or lower
 		-- e.g. an admin can't ask for a list of owners
 		local affiliation_rank = valid_affiliations[affiliation or "none"];
 		if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank)
@@ -1049,6 +1113,9 @@ end
 function room_mt:handle_groupchat_to_room(origin, stanza)
 	local from = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(from);
+	if not stanza.attr.id then
+		stanza.attr.id = new_id()
+	end
 	if module:fire_event("muc-occupant-groupchat", {
 		room = self; origin = origin; stanza = stanza; from = from; occupant = occupant;
 	}) then return true; end
@@ -1218,7 +1285,7 @@ function room_mt:route_stanza(stanza) -- luacheck: ignore 212
 end
 
 function room_mt:get_affiliation(jid)
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	-- Affiliations are granted, revoked, and maintained based on the user's bare JID.
 	local bare = node and node.."@"..host or host;
 	local result = self._affiliations[bare];
@@ -1241,7 +1308,7 @@ end
 function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 	if not actor then return nil, "modify", "not-acceptable"; end;
 
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	if not host then return nil, "modify", "not-acceptable"; end
 	jid = jid_join(node, host); -- Bare
 	local is_host_only = node == nil;
@@ -1297,7 +1364,7 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 			-- Outcast can be by host.
 			is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host
 		) then
-			-- need to publcize in all cases; as affiliation in <item/> has changed.
+			-- need to publicize in all cases; as affiliation in <item/> has changed.
 			occupants_updated[occupant] = occupant.role;
 			if occupant.role ~= role and (
 				is_downgrade or
@@ -1322,16 +1389,20 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 
 	if next(occupants_updated) ~= nil then
 		for occupant, old_role in pairs(occupants_updated) do
-			self:publicise_occupant_status(occupant, x, nil, actor, reason);
+			self:publicise_occupant_status(occupant, x, nil, actor, reason, old_role);
 			if occupant.role == nil then
-				module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+				module:fire_event("muc-occupant-left", {
+						room = self;
+						nick = occupant.nick;
+						occupant = occupant;
+					});
 			elseif is_semi_anonymous and
 				(old_role == "moderator" and occupant.role ~= "moderator") or
 				(old_role ~= "moderator" and occupant.role == "moderator") then -- Has gained or lost moderator status
 				-- Send everyone else's presences (as jid visibility has changed)
 				for real_jid in occupant:each_session() do
 					self:send_occupant_list(real_jid, function(occupant_jid, occupant) --luacheck: ignore 212 433
-						return occupant.bare_jid ~= jid;
+						return (not occupant) or occupant.bare_jid ~= jid;
 					end);
 				end
 			end
@@ -1376,6 +1447,42 @@ function room_mt:get_role(nick)
 	return occupant and occupant.role or nil;
 end
 
+function room_mt:may_set_role(actor, occupant, role)
+	local event = {
+		room = self,
+		actor = actor,
+		occupant = occupant,
+		role = role,
+	};
+
+	module:fire_event("muc-pre-set-role", event);
+	if event.allowed ~= nil then
+		return event.allowed, event.error, event.condition;
+	end
+
+	-- Can't do anything to other owners or admins
+	local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
+	if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	-- If you are trying to give or take moderator role you need to be an owner or admin
+	if occupant.role == "moderator" or role == "moderator" then
+		local actor_affiliation = self:get_affiliation(actor);
+		if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
+			return nil, "cancel", "not-allowed";
+		end
+	end
+
+	-- Need to be in the room and a moderator
+	local actor_occupant = self:get_occupant_by_real_jid(actor);
+	if not actor_occupant or actor_occupant.role ~= "moderator" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	return true;
+end
+
 function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not actor then return nil, "modify", "not-acceptable"; end
 
@@ -1390,24 +1497,9 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if actor == true then
 		actor = nil -- So we can pass it safely to 'publicise_occupant_status' below
 	else
-		-- Can't do anything to other owners or admins
-		local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
-		if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
-			return nil, "cancel", "not-allowed";
-		end
-
-		-- If you are trying to give or take moderator role you need to be an owner or admin
-		if occupant.role == "moderator" or role == "moderator" then
-			local actor_affiliation = self:get_affiliation(actor);
-			if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
-				return nil, "cancel", "not-allowed";
-			end
-		end
-
-		-- Need to be in the room and a moderator
-		local actor_occupant = self:get_occupant_by_real_jid(actor);
-		if not actor_occupant or actor_occupant.role ~= "moderator" then
-			return nil, "cancel", "not-allowed";
+		local allowed, err, condition = self:may_set_role(actor, occupant, role)
+		if not allowed then
+			return allowed, err, condition;
 		end
 	end
 
@@ -1415,11 +1507,17 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not role then
 		x:tag("status", {code = "307"}):up();
 	end
+
+	local prev_role = occupant.role;
 	occupant.role = role;
 	self:save_occupant(occupant);
-	self:publicise_occupant_status(occupant, x, nil, actor, reason);
+	self:publicise_occupant_status(occupant, x, nil, actor, reason, prev_role);
 	if role == nil then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -1441,7 +1539,7 @@ function _M.new_room(jid, config)
 	}, room_mt);
 end
 
-local new_format = module:get_option_boolean("new_muc_storage_format", false);
+local new_format = module:get_option_boolean("new_muc_storage_format", true);
 
 function room_mt:freeze(live)
 	local frozen, state;
@@ -1505,7 +1603,7 @@ function _M.restore_room(frozen, state)
 	else
 		-- New storage format
 		for jid, data in pairs(frozen) do
-			local node, host, resource = jid_split(jid);
+			local _, host, resource = jid_split(jid);
 			if host:sub(1,1) ~= "_" and not resource and type(data) == "string" then
 				-- bare jid: affiliation
 				room._affiliations[jid] = data;
diff --git a/plugins/muc/name.lib.lua b/plugins/muc/name.lib.lua
index 37fe1259..5d73e74d 100644
--- a/plugins/muc/name.lib.lua
+++ b/plugins/muc/name.lib.lua
@@ -7,10 +7,8 @@
 -- COPYING file in the source package for more information.
 --
 
-local jid_split = require "util.jid".split;
-
 local function get_name(room)
-	return room._data.name or jid_split(room.jid);
+	return room._data.name;
 end
 
 local function set_name(room, name)
diff --git a/plugins/muc/password.lib.lua b/plugins/muc/password.lib.lua
index 1f4b2add..6695c0cf 100644
--- a/plugins/muc/password.lib.lua
+++ b/plugins/muc/password.lib.lua
@@ -50,7 +50,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	if get_password(room) ~= password then
 		local from, to = stanza.attr.from, stanza.attr.to;
 		module:log("debug", "%s couldn't join due to invalid password: %s", from, to);
-		local reply = st.error_reply(stanza, "auth", "not-authorized"):up();
+		local reply = st.error_reply(stanza, "auth", "not-authorized", nil, room.jid):up();
 		reply.tags[1].attr.code = "401";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
diff --git a/plugins/muc/presence_broadcast.lib.lua b/plugins/muc/presence_broadcast.lib.lua
new file mode 100644
index 00000000..82a89fee
--- /dev/null
+++ b/plugins/muc/presence_broadcast.lib.lua
@@ -0,0 +1,83 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+-- Copyright (C) 2014 Daurnimator
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local valid_roles = { "none", "visitor", "participant", "moderator" };
+local default_broadcast = {
+	visitor = true;
+	participant = true;
+	moderator = true;
+};
+
+local function get_presence_broadcast(room)
+	return room._data.presence_broadcast or default_broadcast;
+end
+
+local function set_presence_broadcast(room, broadcast_roles)
+	broadcast_roles = broadcast_roles or default_broadcast;
+
+	local changed = false;
+	local old_broadcast_roles = get_presence_broadcast(room);
+	for _, role in ipairs(valid_roles) do
+		if old_broadcast_roles[role] ~= broadcast_roles[role] then
+			changed = true;
+		end
+	end
+
+	if not changed then return false; end
+
+	room._data.presence_broadcast = broadcast_roles;
+
+	for _, occupant in room:each_occupant() do
+		local x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local role = occupant.role or "none";
+		if broadcast_roles[role] and not old_broadcast_roles[role] then
+			-- Presence broadcast is now enabled, so announce existing user
+			room:publicise_occupant_status(occupant, x);
+		elseif old_broadcast_roles[role] and not broadcast_roles[role] then
+			-- Presence broadcast is now disabled, so mark existing user as unavailable
+			room:publicise_occupant_status(occupant, x, nil, nil, nil, nil, true);
+		end
+	end
+
+	return true;
+end
+
+module:hook("muc-config-form", function(event)
+	local values = {};
+	for role, value in pairs(get_presence_broadcast(event.room)) do
+		if value then
+			values[#values + 1] = role;
+		end
+	end
+
+	table.insert(event.form, {
+		name = "muc#roomconfig_presencebroadcast";
+		type = "list-multi";
+		label = "Only show participants with roles:";
+		value = values;
+		options = valid_roles;
+	});
+end, 70-7);
+
+module:hook("muc-config-submitted/muc#roomconfig_presencebroadcast", function(event)
+	local broadcast_roles = {};
+	for _, role in ipairs(event.value) do
+		broadcast_roles[role] = true;
+	end
+	if set_presence_broadcast(event.room, broadcast_roles) then
+		event.status_codes["104"] = true;
+	end
+end);
+
+return {
+	get = get_presence_broadcast;
+	set = set_presence_broadcast;
+};
diff --git a/plugins/muc/register.lib.lua b/plugins/muc/register.lib.lua
index 95ed1a84..d917b396 100644
--- a/plugins/muc/register.lib.lua
+++ b/plugins/muc/register.lib.lua
@@ -15,8 +15,7 @@ local function get_reserved_nicks(room)
 	end
 	module:log("debug", "Refreshing reserved nicks...");
 	local reserved_nicks = {};
-	for jid in room:each_affiliation() do
-		local data = room._affiliation_data[jid];
+	for jid, _, data in room:each_affiliation() do
 		local nick = data and data.reserved_nickname;
 		module:log("debug", "Refreshed for %s: %s", jid, nick);
 		if nick then
@@ -54,7 +53,7 @@ end);
 
 local registration_form = dataforms.new {
 	{ name = "FORM_TYPE", type = "hidden", value = "http://jabber.org/protocol/muc#register" },
-	{ name = "muc#register_roomnick", type = "text-single", label = "Nickname"},
+	{ name = "muc#register_roomnick", type = "text-single", required = true, label = "Nickname"},
 };
 
 local function enforce_nick_policy(event)
@@ -67,7 +66,7 @@ local function enforce_nick_policy(event)
 	local reserved_by = get_registered_jid(room, requested_nick);
 	if reserved_by and reserved_by ~= jid_bare(stanza.attr.from) then
 		module:log("debug", "%s attempted to use nick %s reserved by %s", stanza.attr.from, requested_nick, reserved_by);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, room.jid):up();
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
 	end
@@ -80,7 +79,7 @@ local function enforce_nick_policy(event)
 				event.occupant.nick = jid_bare(event.occupant.nick) .. "/" .. nick;
 			elseif event.dest_occupant.nick ~= jid_bare(event.dest_occupant.nick) .. "/" .. nick then
 				module:log("debug", "Attempt by %s to join as %s, but their reserved nick is %s", stanza.attr.from, requested_nick, nick);
-				local reply = st.error_reply(stanza, "cancel", "not-acceptable"):up();
+				local reply = st.error_reply(stanza, "cancel", "not-acceptable", nil, room.jid):up();
 				origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 				return true;
 			end
@@ -104,7 +103,7 @@ local function handle_register_iq(room, origin, stanza)
 	local user_jid = jid_bare(stanza.attr.from)
 	local affiliation = room:get_affiliation(user_jid);
 	if affiliation == "outcast" then
-		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		origin.send(st.error_reply(stanza, "auth", "forbidden", room.jid));
 		return true;
 	elseif not (affiliation or allow_unaffiliated) then
 		origin.send(st.error_reply(stanza, "auth", "registration-required"));
@@ -135,13 +134,25 @@ local function handle_register_iq(room, origin, stanza)
 			return true;
 		end
 		local form_tag = query:get_child("x", "jabber:x:data");
-		local reg_data = form_tag and registration_form:data(form_tag);
+		if not form_tag then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing dataform"));
+			return true;
+		end
+		local form_type, err = dataforms.get_type(form_tag);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error with form: "..err));
+			return true;
+		elseif form_type ~= "http://jabber.org/protocol/muc#register" then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
+			return true;
+		end
+		local reg_data = registration_form:data(form_tag);
 		if not reg_data then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
 			return true;
 		end
 		-- Is the nickname valid?
-		local desired_nick = resourceprep(reg_data["muc#register_roomnick"]);
+		local desired_nick = resourceprep(reg_data["muc#register_roomnick"], true);
 		if not desired_nick then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid Nickname"));
 			return true;
diff --git a/plugins/muc/subject.lib.lua b/plugins/muc/subject.lib.lua
index 14256bbc..3230817c 100644
--- a/plugins/muc/subject.lib.lua
+++ b/plugins/muc/subject.lib.lua
@@ -94,6 +94,12 @@ module:hook("muc-occupant-groupchat", function(event)
 	local stanza = event.stanza;
 	local subject = stanza:get_child("subject");
 	if subject then
+		if stanza:get_child("body") or stanza:get_child("thread") then
+			-- Note: A message with a <subject/> and a <body/> or a <subject/> and
+			-- a <thread/> is a legitimate message, but it SHALL NOT be interpreted
+			-- as a subject change.
+			return;
+		end
 		local room = event.room;
 		local occupant = event.occupant;
 		-- Role check for subject changes
diff --git a/plugins/muc/util.lib.lua b/plugins/muc/util.lib.lua
index 53a83fae..0877150f 100644
--- a/plugins/muc/util.lib.lua
+++ b/plugins/muc/util.lib.lua
@@ -41,18 +41,22 @@ function _M.is_kickable_error(stanza)
 	return kickable_error_conditions[cond];
 end
 
-local muc_x_filters = {
-	["http://jabber.org/protocol/muc"] = true;
-	["http://jabber.org/protocol/muc#user"] = true;
-}
-local function muc_x_filter(tag)
-	if muc_x_filters[tag.attr.xmlns] then
+local filtered_namespaces = module:shared("filtered-namespaces");
+filtered_namespaces["http://jabber.org/protocol/muc"] = true;
+filtered_namespaces["http://jabber.org/protocol/muc#user"] = true;
+
+local function muc_ns_filter(tag)
+	if filtered_namespaces[tag.attr.xmlns] then
 		return nil;
 	end
 	return tag;
 end
 function _M.filter_muc_x(stanza)
-	return stanza:maptags(muc_x_filter);
+	return stanza:maptags(muc_ns_filter);
+end
+
+function _M.add_filtered_namespace(xmlns)
+	filtered_namespaces[xmlns] = true;
 end
 
 function _M.only_with_min_role(role)
diff --git a/prosody b/prosody
index 3659ca55..87f24291 100755
--- a/prosody
+++ b/prosody
@@ -90,4 +90,6 @@ cleanup();
 prosody.events.fire_event("server-stopped");
 prosody.log("info", "Shutdown complete");
 
+prosody.log("debug", "Shutdown reason was: %s", prosody.shutdown_reason or "not specified");
+prosody.log("debug", "Exiting with status code: %d", prosody.shutdown_code or 0);
 os.exit(prosody.shutdown_code);
diff --git a/prosody.cfg.lua.dist b/prosody.cfg.lua.dist
index f7f7b731..c6a210e5 100644
--- a/prosody.cfg.lua.dist
+++ b/prosody.cfg.lua.dist
@@ -32,6 +32,10 @@ admins = { }
 -- will look for modules first. For community modules, see https://modules.prosody.im/
 --plugin_paths = {}
 
+-- Single directory for custom prosody plugins and/or Lua libraries installation
+-- This path takes priority over plugin_paths, when prosody is searching for modules
+--installer_plugin_path = ""
+
 -- This is the list of modules Prosody will load on startup.
 -- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too.
 -- Documentation for bundled modules can be found at: https://prosody.im/doc/modules
@@ -88,7 +92,7 @@ modules_disabled = {
 	-- "offline"; -- Store offline messages
 	-- "c2s"; -- Handle client connections
 	-- "s2s"; -- Handle server-to-server connections
-	-- "posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
+	-- "posix"; -- POSIX functionality, sends server to background, etc.
 }
 
 -- Disable account creation by default, for security
diff --git a/prosodyctl b/prosodyctl
index 028e08d8..c74a04d1 100755
--- a/prosodyctl
+++ b/prosodyctl
@@ -10,7 +10,6 @@
 -- prosodyctl - command-line controller for Prosody XMPP server
 
 -- Will be modified by configure script if run --
-
 CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
 CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
 CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
@@ -50,20 +49,6 @@ startup.prosodyctl();
 
 -----------
 
-local error_messages = setmetatable({
-		["invalid-username"] = "The given username is invalid in a Jabber ID";
-		["invalid-hostname"] = "The given hostname is invalid";
-		["no-password"] = "No password was supplied";
-		["no-such-user"] = "The given user does not exist on the server";
-		["no-such-host"] = "The given hostname does not exist in the config";
-		["unable-to-save-data"] = "Unable to store, perhaps you don't have permission?";
-		["no-pidfile"] = "There is no 'pidfile' option in the configuration file, see https://prosody.im/doc/prosodyctl#pidfile for help";
-		["invalid-pidfile"] = "The 'pidfile' option in the configuration file is not a string, see https://prosody.im/doc/prosodyctl#pidfile for help";
-		["no-posix"] = "The mod_posix module is not enabled in the Prosody config file, see https://prosody.im/doc/prosodyctl for more info";
-		["no-such-method"] = "This module has no commands";
-		["not-running"] = "Prosody is not running";
-		}, { __index = function (_,k) return "Error: "..(tostring(k):gsub("%-", " "):gsub("^.", string.upper)); end });
-
 local configmanager = require "core.configmanager";
 local modulemanager = require "core.modulemanager"
 local prosodyctl = require "util.prosodyctl"
@@ -73,18 +58,45 @@ local lfs = dependencies.softreq "lfs";
 
 -----------------------
 
+local human_io = require "util.human.io";
+
 local show_message, show_warning = prosodyctl.show_message, prosodyctl.show_warning;
 local show_usage = prosodyctl.show_usage;
-local show_yesno = prosodyctl.show_yesno;
-local show_prompt = prosodyctl.show_prompt;
-local read_password = prosodyctl.read_password;
+local read_password = human_io.read_password;
+local call_luarocks = prosodyctl.call_luarocks;
+local error_messages = prosodyctl.error_messages;
 
 local jid_split = require "util.jid".prepped_split;
 
 local prosodyctl_timeout = (configmanager.get("*", "prosodyctl_timeout") or 5) * 2;
 -----------------------
 local commands = {};
-local command = arg[1];
+local command = table.remove(arg, 1);
+
+function commands.install(arg)
+	if arg[1] == "--help" then
+		show_usage([[install]], [[Installs a prosody/luarocks plugin]]);
+		return 1;
+	end
+	-- TODO finalize config option name
+	call_luarocks("install", arg[1], configmanager.get("*", "plugin_server") or "http://localhost/");
+end
+
+function commands.remove(arg)
+	if arg[1] == "--help" then
+		show_usage([[remove]], [[Removes a module installed in the working directory's plugins folder]]);
+		return 1;
+	end
+	call_luarocks("remove", arg[1])
+end
+
+function commands.list(arg)
+	if arg[1] == "--help" then
+		show_usage([[list]], [[Shows installed rocks]]);
+		return 1;
+	end
+	call_luarocks("list", arg[1])
+end
 
 function commands.adduser(arg)
 	if not arg[1] or arg[1] == "--help" then
@@ -121,7 +133,7 @@ function commands.adduser(arg)
 
 	if ok then return 0; end
 
-	show_message(msg)
+	show_message(error_messages[msg])
 	return 1;
 end
 
@@ -238,7 +250,15 @@ function commands.start(arg)
 	end
 
 	--luacheck: ignore 411/ret
-	local ok, ret = prosodyctl.start(prosody.paths.source);
+	local lua;
+	do
+		local i = 0;
+		repeat
+			i = i - 1;
+		until arg[i-1] == nil
+		lua = arg[i];
+	end
+	local ok, ret = prosodyctl.start(prosody.paths.source, lua);
 	if ok then
 		local daemonize = configmanager.get("*", "daemonize");
 		if daemonize == nil then
@@ -359,8 +379,7 @@ function commands.about(arg)
 	end
 
 	local pwd = ".";
-	local array = require "util.array";
-	local keys = require "util.iterators".keys;
+	local sorted_pairs = require "util.iterators".sorted_pairs;
 	local hg = require"util.mercurial";
 	local relpath = configmanager.resolve_relative_path;
 
@@ -383,6 +402,13 @@ function commands.about(arg)
 				.."\n  ";
 		end)));
 	print("");
+	local have_pposix, pposix = pcall(require, "util.pposix");
+	if have_pposix and pposix.uname then
+		print("# Operating system");
+		local uname, err = pposix.uname();
+		print(uname and uname.sysname .. " " .. uname.release or "Unknown POSIX", err or "");
+		print("");
+	end
 	print("# Lua environment");
 	print("Lua version:             ", _G._VERSION);
 	print("");
@@ -413,24 +439,48 @@ function commands.about(arg)
 	print("");
 	print("# Lua module versions");
 	local module_versions, longest_name = {}, 8;
-	local luaevent =dependencies.softreq"luaevent";
+	local library_versions = {};
 	dependencies.softreq"ssl";
 	dependencies.softreq"DBI";
+	local friendly_names = {
+		DBI = "LuaDBI";
+		lfs = "LuaFileSystem";
+		lunbound = "luaunbound";
+		lxp = "LuaExpat";
+		socket = "LuaSocket";
+		ssl = "LuaSec";
+	};
+	local lunbound = dependencies.softreq"lunbound";
 	for name, module in pairs(package.loaded) do
 		if type(module) == "table" and rawget(module, "_VERSION")
-		and name ~= "_G" and not name:match("%.") then
+			and name ~= "_G" and not name:match("%.") then
+			name = friendly_names[name] or name;
 			if #name > longest_name then
 				longest_name = #name;
 			end
-			module_versions[name] = module._VERSION;
+			local mod_version = module._VERSION;
+			if tostring(mod_version):sub(1, #name+1) == name .. " " then
+				mod_version = mod_version:sub(#name+2);
+			end
+			module_versions[name] = mod_version;
+		end
+	end
+	if lunbound then
+		if not module_versions["luaunbound"] then
+			module_versions["luaunbound"] = "0.5 (?)";
 		end
+		library_versions["libunbound"] = lunbound._LIBVER;
 	end
-	if luaevent then
-		module_versions["libevent"] = luaevent.core.libevent_version();
+	for name, version in sorted_pairs(module_versions) do
+		print(name..":"..string.rep(" ", longest_name-#name), version);
 	end
-	local sorted_keys = array.collect(keys(module_versions)):sort();
-	for _, name in ipairs(sorted_keys) do
-		print(name..":"..string.rep(" ", longest_name-#name), module_versions[name]);
+	print("");
+	print("# library versions");
+	if require "net.server".event_base then
+		library_versions["libevent"] = require"luaevent".core.libevent_version();
+	end
+	for name, version in sorted_pairs(library_versions) do
+		print(name..":"..string.rep(" ", longest_name-#name), version);
 	end
 	print("");
 end
@@ -513,810 +563,6 @@ function commands.unregister(arg)
 	return 1;
 end
 
-local openssl;
-
-local cert_commands = {};
-
--- If a file already exists, ask if the user wants to use it or replace it
--- Backups the old file if replaced
-local function use_existing(filename)
-	local attrs = lfs.attributes(filename);
-	if attrs then
-		if show_yesno(filename .. " exists, do you want to replace it? [y/n]") then
-			local backup = filename..".bkp~"..os.date("%FT%T", attrs.change);
-			os.rename(filename, backup);
-			show_message(filename.." backed up to "..backup);
-		else
-			-- Use the existing file
-			return true;
-		end
-	end
-end
-
-local have_pposix, pposix = pcall(require, "util.pposix");
-local cert_basedir = prosody.paths.data == "." and "./certs" or prosody.paths.data;
-if have_pposix and pposix.getuid() == 0 then
-	-- FIXME should be enough to check if this directory is writable
-	local cert_dir = configmanager.get("*", "certificates") or "certs";
-	cert_basedir = configmanager.resolve_relative_path(prosody.paths.config, cert_dir);
-end
-
-function cert_commands.config(arg)
-	if #arg >= 1 and arg[1] ~= "--help" then
-		local conf_filename = cert_basedir .. "/" .. arg[1] .. ".cnf";
-		if use_existing(conf_filename) then
-			return nil, conf_filename;
-		end
-		local distinguished_name;
-		if arg[#arg]:find("^/") then
-			distinguished_name = table.remove(arg);
-		end
-		local conf = openssl.config.new();
-		conf:from_prosody(prosody.hosts, configmanager, arg);
-		if distinguished_name then
-			local dn = {};
-			for k, v in distinguished_name:gmatch("/([^=/]+)=([^/]+)") do
-				table.insert(dn, k);
-				dn[k] = v;
-			end
-			conf.distinguished_name = dn;
-		else
-			show_message("Please provide details to include in the certificate config file.");
-			show_message("Leave the field empty to use the default value or '.' to exclude the field.")
-			for _, k in ipairs(openssl._DN_order) do
-				local v = conf.distinguished_name[k];
-				if v then
-					local nv = nil;
-					if k == "commonName" then
-						v = arg[1]
-					elseif k == "emailAddress" then
-						v = "xmpp@" .. arg[1];
-					elseif k == "countryName" then
-						local tld = arg[1]:match"%.([a-z]+)$";
-						if tld and #tld == 2 and tld ~= "uk" then
-							v = tld:upper();
-						end
-					end
-					nv = show_prompt(("%s (%s):"):format(k, nv or v));
-					nv = (not nv or nv == "") and v or nv;
-					if nv:find"[\192-\252][\128-\191]+" then
-						conf.req.string_mask = "utf8only"
-					end
-					conf.distinguished_name[k] = nv ~= "." and nv or nil;
-				end
-			end
-		end
-		local conf_file, err = io.open(conf_filename, "w");
-		if not conf_file then
-			show_warning("Could not open OpenSSL config file for writing");
-			show_warning(err);
-			os.exit(1);
-		end
-		conf_file:write(conf:serialize());
-		conf_file:close();
-		print("");
-		show_message("Config written to " .. conf_filename);
-		return nil, conf_filename;
-	else
-		show_usage("cert config HOSTNAME [HOSTNAME+]", "Builds a certificate config file covering the supplied hostname(s)")
-	end
-end
-
-function cert_commands.key(arg)
-	if #arg >= 1 and arg[1] ~= "--help" then
-		local key_filename = cert_basedir .. "/" .. arg[1] .. ".key";
-		if use_existing(key_filename) then
-			return nil, key_filename;
-		end
-		os.remove(key_filename); -- This file, if it exists is unlikely to have write permissions
-		local key_size = tonumber(arg[2] or show_prompt("Choose key size (2048):") or 2048);
-		local old_umask = pposix.umask("0377");
-		if openssl.genrsa{out=key_filename, key_size} then
-			os.execute(("chmod 400 '%s'"):format(key_filename));
-			show_message("Key written to ".. key_filename);
-			pposix.umask(old_umask);
-			return nil, key_filename;
-		end
-		show_message("There was a problem, see OpenSSL output");
-	else
-		show_usage("cert key HOSTNAME <bits>", "Generates a RSA key named HOSTNAME.key\n "
-		.."Prompts for a key size if none given")
-	end
-end
-
-function cert_commands.request(arg)
-	if #arg >= 1 and arg[1] ~= "--help" then
-		local req_filename = cert_basedir .. "/" .. arg[1] .. ".req";
-		if use_existing(req_filename) then
-			return nil, req_filename;
-		end
-		local _, key_filename = cert_commands.key({arg[1]});
-		local _, conf_filename = cert_commands.config(arg);
-		if openssl.req{new=true, key=key_filename, utf8=true, sha256=true, config=conf_filename, out=req_filename} then
-			show_message("Certificate request written to ".. req_filename);
-		else
-			show_message("There was a problem, see OpenSSL output");
-		end
-	else
-		show_usage("cert request HOSTNAME [HOSTNAME+]", "Generates a certificate request for the supplied hostname(s)")
-	end
-end
-
-function cert_commands.generate(arg)
-	if #arg >= 1 and arg[1] ~= "--help" then
-		local cert_filename = cert_basedir .. "/" .. arg[1] .. ".crt";
-		if use_existing(cert_filename) then
-			return nil, cert_filename;
-		end
-		local _, key_filename = cert_commands.key({arg[1]});
-		local _, conf_filename = cert_commands.config(arg);
-		if key_filename and conf_filename and cert_filename
-			and openssl.req{new=true, x509=true, nodes=true, key=key_filename,
-				days=365, sha256=true, utf8=true, config=conf_filename, out=cert_filename} then
-			show_message("Certificate written to ".. cert_filename);
-			print();
-		else
-			show_message("There was a problem, see OpenSSL output");
-		end
-	else
-		show_usage("cert generate HOSTNAME [HOSTNAME+]", "Generates a self-signed certificate for the current hostname(s)")
-	end
-end
-
-local function sh_esc(s)
-	return "'" .. s:gsub("'", "'\\''") .. "'";
-end
-
-local function copy(from, to, umask, owner, group)
-	local old_umask = umask and pposix.umask(umask);
-	local attrs = lfs.attributes(to);
-	if attrs then -- Move old file out of the way
-		local backup = to..".bkp~"..os.date("%FT%T", attrs.change);
-		os.rename(to, backup);
-	end
-	-- FIXME friendlier error handling, maybe move above backup back?
-	local input = assert(io.open(from));
-	local output = assert(io.open(to, "w"));
-	local data = input:read(2^11);
-	while data and output:write(data) do
-		data = input:read(2^11);
-	end
-	assert(input:close());
-	assert(output:close());
-	if not prosody.installed then
-		-- FIXME this is possibly specific to GNU chown
-		os.execute(("chown -c --reference=%s %s"):format(sh_esc(cert_basedir), sh_esc(to)));
-	elseif owner and group then
-		local ok = os.execute(("chown %s:%s %s"):format(sh_esc(owner), sh_esc(group), sh_esc(to)));
-		assert(ok == true or ok == 0, "Failed to change ownership of "..to);
-	end
-	if old_umask then pposix.umask(old_umask); end
-	return true;
-end
-
-function cert_commands.import(arg)
-	local hostnames = {};
-	-- Move hostname arguments out of arg, the rest should be a list of paths
-	while arg[1] and prosody.hosts[ arg[1] ] do
-		table.insert(hostnames, table.remove(arg, 1));
-	end
-	if hostnames[1] == nil then
-		local domains = os.getenv"RENEWED_DOMAINS"; -- Set if invoked via certbot
-		if domains then
-			for host in domains:gmatch("%S+") do
-				table.insert(hostnames, host);
-			end
-		else
-			for host in pairs(prosody.hosts) do
-				if host ~= "*" and configmanager.get(host, "enabled") ~= false then
-					table.insert(hostnames, host);
-				end
-			end
-		end
-	end
-	if not arg[1] or arg[1] == "--help" then -- Probably forgot the path
-		show_usage("cert import [HOSTNAME+] /path/to/certs [/other/paths/]+",
-			"Copies certificates to "..cert_basedir);
-		return 1;
-	end
-	local owner, group;
-	if pposix.getuid() == 0 then -- We need root to change ownership
-		owner = configmanager.get("*", "prosody_user") or "prosody";
-		group = configmanager.get("*", "prosody_group") or owner;
-	end
-	local cm = require "core.certmanager";
-	local imported = {};
-	for _, host in ipairs(hostnames) do
-		for _, dir in ipairs(arg) do
-			local paths = cm.find_cert(dir, host);
-			if paths then
-				copy(paths.certificate, cert_basedir .. "/" .. host .. ".crt", nil, owner, group);
-				copy(paths.key, cert_basedir .. "/" .. host .. ".key", "0377", owner, group);
-				table.insert(imported, host);
-			else
-				-- TODO Say where we looked
-				show_warning("No certificate for host "..host.." found :(");
-			end
-			-- TODO Additional checks
-			-- Certificate names matches the hostname
-			-- Private key matches public key in certificate
-		end
-	end
-	if imported[1] then
-		show_message("Imported certificate and key for hosts "..table.concat(imported, ", "));
-		local ok, err = prosodyctl.reload();
-		if not ok and err ~= "not-running" then
-			show_message(error_messages[err]);
-		end
-	else
-		show_warning("No certificates imported :(");
-		return 1;
-	end
-end
-
-function commands.cert(arg)
-	if #arg >= 1 and arg[1] ~= "--help" then
-		openssl = require "util.openssl";
-		lfs = require "lfs";
-		local cert_dir_attrs = lfs.attributes(cert_basedir);
-		if not cert_dir_attrs then
-			show_warning("The directory "..cert_basedir.." does not exist");
-			return 1; -- TODO Should we create it?
-		end
-		local uid = pposix.getuid();
-		if uid ~= 0 and uid ~= cert_dir_attrs.uid then
-			show_warning("The directory "..cert_basedir.." is not owned by the current user, won't be able to write files to it");
-			return 1;
-		elseif not cert_dir_attrs.permissions then -- COMPAT with LuaFilesystem < 1.6.2 (hey CentOS!)
-			show_message("Unable to check permissions on "..cert_basedir.." (LuaFilesystem 1.6.2+ required)");
-			show_message("Please confirm that Prosody (and only Prosody) can write to this directory)");
-		elseif cert_dir_attrs.permissions:match("^%.w..%-..%-.$") then
-			show_warning("The directory "..cert_basedir.." not only writable by its owner");
-			return 1;
-		end
-		local subcmd = table.remove(arg, 1);
-		if type(cert_commands[subcmd]) == "function" then
-			if subcmd ~= "import" then -- hostnames are optional for import
-				if not arg[1] then
-					show_message"You need to supply at least one hostname"
-					arg = { "--help" };
-				end
-				if arg[1] ~= "--help" and not prosody.hosts[arg[1]] then
-					show_message(error_messages["no-such-host"]);
-					return 1;
-				end
-			end
-			return cert_commands[subcmd](arg);
-		elseif subcmd == "check" then
-			return commands.check({"certs"});
-		end
-	end
-	show_usage("cert config|request|generate|key|import", "Helpers for generating X.509 certificates and keys.")
-	for _, cmd in pairs(cert_commands) do
-		print()
-		cmd{ "--help" }
-	end
-end
-
-function commands.check(arg)
-	if arg[1] == "--help" then
-		show_usage([[check]], [[Perform basic checks on your Prosody installation]]);
-		return 1;
-	end
-	local what = table.remove(arg, 1);
-	local set = require "util.set";
-	local it = require "util.iterators";
-	local ok = true;
-	local function disabled_hosts(host, conf) return host ~= "*" and conf.enabled ~= false; end
-	local function enabled_hosts() return it.filter(disabled_hosts, pairs(configmanager.getconfig())); end
-	if not (what == nil or what == "disabled" or what == "config" or what == "dns" or what == "certs") then
-		show_warning("Don't know how to check '%s'. Try one of 'config', 'dns', 'certs' or 'disabled'.", what);
-		return 1;
-	end
-	if not what or what == "disabled" then
-		local disabled_hosts_set = set.new();
-		for host, host_options in it.filter("*", pairs(configmanager.getconfig())) do
-			if host_options.enabled == false then
-				disabled_hosts_set:add(host);
-			end
-		end
-		if not disabled_hosts_set:empty() then
-			local msg = "Checks will be skipped for these disabled hosts: %s";
-			if what then msg = "These hosts are disabled: %s"; end
-			show_warning(msg, tostring(disabled_hosts_set));
-			if what then return 0; end
-			print""
-		end
-	end
-	if not what or what == "config" then
-		print("Checking config...");
-		local deprecated = set.new({
-			"bosh_ports", "disallow_s2s", "no_daemonize", "anonymous_login", "require_encryption",
-			"vcard_compatibility",
-		});
-		local known_global_options = set.new({
-			"pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize",
-			"umask", "prosodyctl_timeout", "use_ipv6", "use_libevent", "network_settings",
-			"network_backend", "http_default_host",
-			"statistics_interval", "statistics", "statistics_config",
-		});
-		local config = configmanager.getconfig();
-		-- Check that we have any global options (caused by putting a host at the top)
-		if it.count(it.filter("log", pairs(config["*"]))) == 0 then
-			ok = false;
-			print("");
-			print("    No global options defined. Perhaps you have put a host definition at the top")
-			print("    of the config file? They should be at the bottom, see https://prosody.im/doc/configure#overview");
-		end
-		if it.count(enabled_hosts()) == 0 then
-			ok = false;
-			print("");
-			if it.count(it.filter("*", pairs(config))) == 0 then
-				print("    No hosts are defined, please add at least one VirtualHost section")
-			elseif config["*"]["enabled"] == false then
-				print("    No hosts are enabled. Remove enabled = false from the global section or put enabled = true under at least one VirtualHost section")
-			else
-				print("    All hosts are disabled. Remove enabled = false from at least one VirtualHost section")
-			end
-		end
-		if not config["*"].modules_enabled then
-			print("    No global modules_enabled is set?");
-			local suggested_global_modules;
-			for host, options in enabled_hosts() do --luacheck: ignore 213/host
-				if not options.component_module and options.modules_enabled then
-					suggested_global_modules = set.intersection(suggested_global_modules or set.new(options.modules_enabled), set.new(options.modules_enabled));
-				end
-			end
-			if suggested_global_modules and not suggested_global_modules:empty() then
-				print("    Consider moving these modules into modules_enabled in the global section:")
-				print("    "..tostring(suggested_global_modules / function (x) return ("%q"):format(x) end));
-			end
-			print();
-		end
-
-		do -- Check for modules enabled both normally and as components
-			local modules = set.new(config["*"]["modules_enabled"]);
-			for host, options in enabled_hosts() do
-				local component_module = options.component_module;
-				if component_module and modules:contains(component_module) then
-					print(("    mod_%s is enabled both in modules_enabled and as Component %q %q"):format(component_module, host, component_module));
-					print("    This means the service is enabled on all VirtualHosts as well as the Component.");
-					print("    Are you sure this what you want? It may cause unexpected behaviour.");
-				end
-			end
-		end
-
-		-- Check for global options under hosts
-		local global_options = set.new(it.to_array(it.keys(config["*"])));
-		local deprecated_global_options = set.intersection(global_options, deprecated);
-		if not deprecated_global_options:empty() then
-			print("");
-			print("    You have some deprecated options in the global section:");
-			print("    "..tostring(deprecated_global_options))
-			ok = false;
-		end
-		for host, options in it.filter(function (h) return h ~= "*" end, pairs(configmanager.getconfig())) do
-			local host_options = set.new(it.to_array(it.keys(options)));
-			local misplaced_options = set.intersection(host_options, known_global_options);
-			for name in pairs(options) do
-				if name:match("^interfaces?")
-				or name:match("_ports?$") or name:match("_interfaces?$")
-				or (name:match("_ssl$") and not name:match("^[cs]2s_ssl$")) then
-					misplaced_options:add(name);
-				end
-			end
-			if not misplaced_options:empty() then
-				ok = false;
-				print("");
-				local n = it.count(misplaced_options);
-				print("    You have "..n.." option"..(n>1 and "s " or " ").."set under "..host.." that should be");
-				print("    in the global section of the config file, above any VirtualHost or Component definitions,")
-				print("    see https://prosody.im/doc/configure#overview for more information.")
-				print("");
-				print("    You need to move the following option"..(n>1 and "s" or "")..": "..table.concat(it.to_array(misplaced_options), ", "));
-			end
-		end
-		for host, options in enabled_hosts() do
-			local host_options = set.new(it.to_array(it.keys(options)));
-			local subdomain = host:match("^[^.]+");
-			if not(host_options:contains("component_module")) and (subdomain == "jabber" or subdomain == "xmpp"
-			   or subdomain == "chat" or subdomain == "im") then
-				print("");
-				print("    Suggestion: If "..host.. " is a new host with no real users yet, consider renaming it now to");
-				print("     "..host:gsub("^[^.]+%.", "")..". You can use SRV records to redirect XMPP clients and servers to "..host..".");
-				print("     For more information see: https://prosody.im/doc/dns");
-			end
-		end
-		local all_modules = set.new(config["*"].modules_enabled);
-		local all_options = set.new(it.to_array(it.keys(config["*"])));
-		for host in enabled_hosts() do
-			all_options:include(set.new(it.to_array(it.keys(config[host]))));
-			all_modules:include(set.new(config[host].modules_enabled));
-		end
-		for mod in all_modules do
-			if mod:match("^mod_") then
-				print("");
-				print("    Modules in modules_enabled should not have the 'mod_' prefix included.");
-				print("    Change '"..mod.."' to '"..mod:match("^mod_(.*)").."'.");
-			elseif mod:match("^auth_") then
-				print("");
-				print("    Authentication modules should not be added to modules_enabled,");
-				print("    but be specified in the 'authentication' option.");
-				print("    Remove '"..mod.."' from modules_enabled and instead add");
-				print("        authentication = '"..mod:match("^auth_(.*)").."'");
-				print("    For more information see https://prosody.im/doc/authentication");
-			elseif mod:match("^storage_") then
-				print("");
-				print("    storage modules should not be added to modules_enabled,");
-				print("    but be specified in the 'storage' option.");
-				print("    Remove '"..mod.."' from modules_enabled and instead add");
-				print("        storage = '"..mod:match("^storage_(.*)").."'");
-				print("    For more information see https://prosody.im/doc/storage");
-			end
-		end
-		if all_modules:contains("vcard") and all_modules:contains("vcard_legacy") then
-			print("");
-			print("    Both mod_vcard_legacy and mod_vcard are enabled but they conflict");
-			print("    with each other. Remove one.");
-		end
-		if all_modules:contains("pep") and all_modules:contains("pep_simple") then
-			print("");
-			print("    Both mod_pep_simple and mod_pep are enabled but they conflict");
-			print("    with each other. Remove one.");
-		end
-		for host, host_config in pairs(config) do --luacheck: ignore 213/host
-			if type(rawget(host_config, "storage")) == "string" and rawget(host_config, "default_storage") then
-				print("");
-				print("    The 'default_storage' option is not needed if 'storage' is set to a string.");
-				break;
-			end
-		end
-		local require_encryption = set.intersection(all_options, set.new({
-			"require_encryption", "c2s_require_encryption", "s2s_require_encryption"
-		})):empty();
-		local ssl = dependencies.softreq"ssl";
-		if not ssl then
-			if not require_encryption then
-				print("");
-				print("    You require encryption but LuaSec is not available.");
-				print("    Connections will fail.");
-				ok = false;
-			end
-		elseif not ssl.loadcertificate then
-			if all_options:contains("s2s_secure_auth") then
-				print("");
-				print("    You have set s2s_secure_auth but your version of LuaSec does ");
-				print("    not support certificate validation, so all s2s connections will");
-				print("    fail.");
-				ok = false;
-			elseif all_options:contains("s2s_secure_domains") then
-				local secure_domains = set.new();
-				for host in enabled_hosts() do
-					if config[host].s2s_secure_auth == true then
-						secure_domains:add("*");
-					else
-						secure_domains:include(set.new(config[host].s2s_secure_domains));
-					end
-				end
-				if not secure_domains:empty() then
-					print("");
-					print("    You have set s2s_secure_domains but your version of LuaSec does ");
-					print("    not support certificate validation, so s2s connections to/from ");
-					print("    these domains will fail.");
-					ok = false;
-				end
-			end
-		elseif require_encryption and not all_modules:contains("tls") then
-			print("");
-			print("    You require encryption but mod_tls is not enabled.");
-			print("    Connections will fail.");
-			ok = false;
-		end
-
-		print("Done.\n");
-	end
-	if not what or what == "dns" then
-		local dns = require "net.dns";
-		local idna = require "util.encodings".idna;
-		local ip = require "util.ip";
-		local c2s_ports = set.new(configmanager.get("*", "c2s_ports") or {5222});
-		local s2s_ports = set.new(configmanager.get("*", "s2s_ports") or {5269});
-
-		local c2s_srv_required, s2s_srv_required;
-		if not c2s_ports:contains(5222) then
-			c2s_srv_required = true;
-		end
-		if not s2s_ports:contains(5269) then
-			s2s_srv_required = true;
-		end
-
-		local problem_hosts = set.new();
-
-		local external_addresses, internal_addresses = set.new(), set.new();
-
-		local fqdn = socket.dns.tohostname(socket.dns.gethostname());
-		if fqdn then
-			do
-				local res = dns.lookup(idna.to_ascii(fqdn), "A");
-				if res then
-					for _, record in ipairs(res) do
-						external_addresses:add(record.a);
-					end
-				end
-			end
-			do
-				local res = dns.lookup(idna.to_ascii(fqdn), "AAAA");
-				if res then
-					for _, record in ipairs(res) do
-						external_addresses:add(record.aaaa);
-					end
-				end
-			end
-		end
-
-		local local_addresses = require"util.net".local_addresses() or {};
-
-		for addr in it.values(local_addresses) do
-			if not ip.new_ip(addr).private then
-				external_addresses:add(addr);
-			else
-				internal_addresses:add(addr);
-			end
-		end
-
-		if external_addresses:empty() then
-			print("");
-			print("   Failed to determine the external addresses of this server. Checks may be inaccurate.");
-			c2s_srv_required, s2s_srv_required = true, true;
-		end
-
-		local v6_supported = not not socket.tcp6;
-
-		for jid, host_options in enabled_hosts() do
-			local all_targets_ok, some_targets_ok = true, false;
-			local node, host = jid_split(jid);
-
-			local modules, component_module = modulemanager.get_modules_for_host(host);
-			if component_module then
-				modules:add(component_module);
-			end
-
-			local is_component = not not host_options.component_module;
-			print("Checking DNS for "..(is_component and "component" or "host").." "..jid.."...");
-			if node then
-				print("Only the domain part ("..host..") is used in DNS.")
-			end
-			local target_hosts = set.new();
-			if modules:contains("c2s") then
-				local res = dns.lookup("_xmpp-client._tcp."..idna.to_ascii(host)..".", "SRV");
-				if res then
-					for _, record in ipairs(res) do
-						target_hosts:add(record.srv.target);
-						if not c2s_ports:contains(record.srv.port) then
-							print("    SRV target "..record.srv.target.." contains unknown client port: "..record.srv.port);
-						end
-					end
-				else
-					if c2s_srv_required then
-						print("    No _xmpp-client SRV record found for "..host..", but it looks like you need one.");
-						all_targets_ok = false;
-					else
-						target_hosts:add(host);
-					end
-				end
-			end
-			if modules:contains("s2s") then
-				local res = dns.lookup("_xmpp-server._tcp."..idna.to_ascii(host)..".", "SRV");
-				if res then
-					for _, record in ipairs(res) do
-						target_hosts:add(record.srv.target);
-						if not s2s_ports:contains(record.srv.port) then
-							print("    SRV target "..record.srv.target.." contains unknown server port: "..record.srv.port);
-						end
-					end
-				else
-					if s2s_srv_required then
-						print("    No _xmpp-server SRV record found for "..host..", but it looks like you need one.");
-						all_targets_ok = false;
-					else
-						target_hosts:add(host);
-					end
-				end
-			end
-			if target_hosts:empty() then
-				target_hosts:add(host);
-			end
-
-			if target_hosts:contains("localhost") then
-				print("    Target 'localhost' cannot be accessed from other servers");
-				target_hosts:remove("localhost");
-			end
-
-			if modules:contains("proxy65") then
-				local proxy65_target = configmanager.get(host, "proxy65_address") or host;
-				if type(proxy65_target) == "string" then
-					local A, AAAA = dns.lookup(idna.to_ascii(proxy65_target), "A"), dns.lookup(idna.to_ascii(proxy65_target), "AAAA");
-					local prob = {};
-					if not A then
-						table.insert(prob, "A");
-					end
-					if v6_supported and not AAAA then
-						table.insert(prob, "AAAA");
-					end
-					if #prob > 0 then
-						print("    File transfer proxy "..proxy65_target.." has no "..table.concat(prob, "/")
-						.." record. Create one or set 'proxy65_address' to the correct host/IP.");
-					end
-				else
-					print("    proxy65_address for "..host.." should be set to a string, unable to perform DNS check");
-				end
-			end
-
-			for target_host in target_hosts do
-				local host_ok_v4, host_ok_v6;
-				do
-					local res = dns.lookup(idna.to_ascii(target_host), "A");
-					if res then
-						for _, record in ipairs(res) do
-							if external_addresses:contains(record.a) then
-								some_targets_ok = true;
-								host_ok_v4 = true;
-							elseif internal_addresses:contains(record.a) then
-								host_ok_v4 = true;
-								some_targets_ok = true;
-								print("    "..target_host.." A record points to internal address, external connections might fail");
-							else
-								print("    "..target_host.." A record points to unknown address "..record.a);
-								all_targets_ok = false;
-							end
-						end
-					end
-				end
-				do
-					local res = dns.lookup(idna.to_ascii(target_host), "AAAA");
-					if res then
-						for _, record in ipairs(res) do
-							if external_addresses:contains(record.aaaa) then
-								some_targets_ok = true;
-								host_ok_v6 = true;
-							elseif internal_addresses:contains(record.aaaa) then
-								host_ok_v6 = true;
-								some_targets_ok = true;
-								print("    "..target_host.." AAAA record points to internal address, external connections might fail");
-							else
-								print("    "..target_host.." AAAA record points to unknown address "..record.aaaa);
-								all_targets_ok = false;
-							end
-						end
-					end
-				end
-
-				local bad_protos = {}
-				if not host_ok_v4 then
-					table.insert(bad_protos, "IPv4");
-				end
-				if not host_ok_v6 then
-					table.insert(bad_protos, "IPv6");
-				end
-				if #bad_protos > 0 then
-					print("    Host "..target_host.." does not seem to resolve to this server ("..table.concat(bad_protos, "/")..")");
-				end
-				if host_ok_v6 and not v6_supported then
-					print("    Host "..target_host.." has AAAA records, but your version of LuaSocket does not support IPv6.");
-					print("      Please see https://prosody.im/doc/ipv6 for more information.");
-				end
-			end
-			if not all_targets_ok then
-				print("    "..(some_targets_ok and "Only some" or "No").." targets for "..host.." appear to resolve to this server.");
-				if is_component then
-					print("    DNS records are necessary if you want users on other servers to access this component.");
-				end
-				problem_hosts:add(host);
-			end
-			print("");
-		end
-		if not problem_hosts:empty() then
-			print("");
-			print("For more information about DNS configuration please see https://prosody.im/doc/dns");
-			print("");
-			ok = false;
-		end
-	end
-	if not what or what == "certs" then
-		local cert_ok;
-		print"Checking certificates..."
-		local x509_verify_identity = require"util.x509".verify_identity;
-		local create_context = require "core.certmanager".create_context;
-		local ssl = dependencies.softreq"ssl";
-		-- local datetime_parse = require"util.datetime".parse_x509;
-		local load_cert = ssl and ssl.loadcertificate;
-		-- or ssl.cert_from_pem
-		if not ssl then
-			print("LuaSec not available, can't perform certificate checks")
-			if what == "certs" then cert_ok = false end
-		elseif not load_cert then
-			print("This version of LuaSec (" .. ssl._VERSION .. ") does not support certificate checking");
-			cert_ok = false
-		else
-			local function skip_bare_jid_hosts(host)
-				if jid_split(host) then
-					-- See issue #779
-					return false;
-				end
-				return true;
-			end
-			for host in it.filter(skip_bare_jid_hosts, enabled_hosts()) do
-				print("Checking certificate for "..host);
-				-- First, let's find out what certificate this host uses.
-				local host_ssl_config = configmanager.rawget(host, "ssl")
-					or configmanager.rawget(host:match("%.(.*)"), "ssl");
-				local global_ssl_config = configmanager.rawget("*", "ssl");
-				local ok, err, ssl_config = create_context(host, "server", host_ssl_config, global_ssl_config);
-				if not ok then
-					print("  Error: "..err);
-					cert_ok = false
-				elseif not ssl_config.certificate then
-					print("  No 'certificate' found for "..host)
-					cert_ok = false
-				elseif not ssl_config.key then
-					print("  No 'key' found for "..host)
-					cert_ok = false
-				else
-					local key, err = io.open(ssl_config.key); -- Permissions check only
-					if not key then
-						print("    Could not open "..ssl_config.key..": "..err);
-						cert_ok = false
-					else
-						key:close();
-					end
-					local cert_fh, err = io.open(ssl_config.certificate); -- Load the file.
-					if not cert_fh then
-						print("    Could not open "..ssl_config.certificate..": "..err);
-						cert_ok = false
-					else
-						print("  Certificate: "..ssl_config.certificate)
-						local cert = load_cert(cert_fh:read"*a"); cert_fh:close();
-						if not cert:validat(os.time()) then
-							print("    Certificate has expired.")
-							cert_ok = false
-						elseif not cert:validat(os.time() + 86400) then
-							print("    Certificate expires within one day.")
-							cert_ok = false
-						elseif not cert:validat(os.time() + 86400*7) then
-							print("    Certificate expires within one week.")
-						elseif not cert:validat(os.time() + 86400*31) then
-							print("    Certificate expires within one month.")
-						end
-						if configmanager.get(host, "component_module") == nil
-							and not x509_verify_identity(host, "_xmpp-client", cert) then
-							print("    Not valid for client connections to "..host..".")
-							cert_ok = false
-						end
-						if (not (configmanager.get(host, "anonymous_login")
-							or configmanager.get(host, "authentication") == "anonymous"))
-							and not x509_verify_identity(host, "_xmpp-server", cert) then
-							print("    Not valid for server-to-server connections to "..host..".")
-							cert_ok = false
-						end
-					end
-				end
-			end
-		end
-		if cert_ok == false then
-			print("")
-			print("For more information about certificates please see https://prosody.im/doc/certificates");
-			ok = false
-		end
-		print("")
-	end
-	if not ok then
-		print("Problems found, see above.");
-	else
-		print("All checks passed, congratulations!");
-	end
-	return ok and 0 or 2;
-end
-
 ---------------------
 
 local async = require "util.async";
@@ -1340,8 +586,6 @@ local command_runner = async.runner(function ()
 			end
 		end
 
-		table.remove(arg, 1);
-
 		local module = modulemanager.get_module("*", module_name);
 		if not module then
 			show_message("Failed to load module '"..module_name.."': Unknown error");
@@ -1367,6 +611,13 @@ local command_runner = async.runner(function ()
 		end
 	end
 
+	if command and not commands[command] then
+		local ok, command_module = pcall(require, "util.prosodyctl."..command);
+		if ok and command_module[command] then
+			commands[command] = command_module[command];
+		end
+	end
+
 	if not commands[command] then -- Show help for all commands
 		function show_usage(usage, desc)
 			print(" "..usage);
@@ -1380,7 +631,8 @@ local command_runner = async.runner(function ()
 		print("Where COMMAND may be one of:\n");
 
 		local hidden_commands = require "util.set".new{ "register", "unregister", "addplugin" };
-		local commands_order = { "adduser", "passwd", "deluser", "start", "stop", "restart", "reload", "about" };
+		local commands_order = { "install", "remove", "list", "adduser", "passwd", "deluser", "start", "stop", "restart", "reload",
+			"about" };
 
 		local done = {};
 
@@ -1405,7 +657,7 @@ local command_runner = async.runner(function ()
 		os.exit(0);
 	end
 
-	os.exit(commands[command]({ select(2, unpack(arg)) }));
+	os.exit(commands[command](arg));
 end, watchers);
 
 command_runner:run(true);
diff --git a/spec/core_configmanager_spec.lua b/spec/core_configmanager_spec.lua
index afb7d492..7958ec6b 100644
--- a/spec/core_configmanager_spec.lua
+++ b/spec/core_configmanager_spec.lua
@@ -9,7 +9,9 @@ describe("core.configmanager", function()
 
 			configmanager.set("*", "testkey1", 321);
 			assert.are.equal(321, configmanager.get("*", "testkey1"), "Retrieving a set global key");
-			assert.are.equal(321, configmanager.get("example.com", "testkey1"), "Retrieving a set key of undefined host, of which only a globally set one exists");
+			assert.are.equal(321, configmanager.get("example.com", "testkey1"),
+				"Retrieving a set key of undefined host, of which only a globally set one exists"
+			);
 
 			configmanager.set("example.com", ""); -- Creates example.com host in config
 			assert.are.equal(321, configmanager.get("example.com", "testkey1"), "Retrieving a set key, of which only a globally set one exists");
diff --git a/spec/core_storagemanager_spec.lua b/spec/core_storagemanager_spec.lua
index a0a8b5ef..d07dc4ec 100644
--- a/spec/core_storagemanager_spec.lua
+++ b/spec/core_storagemanager_spec.lua
@@ -1,4 +1,4 @@
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local server = require "net.server_select";
 package.loaded["net.server"] = server;
 
@@ -90,6 +90,112 @@ describe("storagemanager", function ()
 				end);
 			end);
 
+			describe("map stores", function ()
+				-- These tests rely on being executed in order, disable any order
+				-- randomization for this block
+				randomize(false);
+
+				local store, kv_store;
+				it("may be opened", function ()
+					store = assert(sm.open(test_host, "test-map", "map"));
+				end);
+
+				it("may be opened as a keyval store", function ()
+					kv_store = assert(sm.open(test_host, "test-map", "keyval"));
+				end);
+
+				it("may set a specific key for a user", function ()
+					assert(store:set("user9999", "foo", "bar"));
+					assert.same(kv_store:get("user9999"), { foo = "bar" });
+				end);
+
+				it("may get a specific key for a user", function ()
+					assert.equal("bar", store:get("user9999", "foo"));
+				end);
+
+				it("may find all users with a specific key", function ()
+					assert.is_function(store.get_all);
+					assert(store:set("user9999b", "bar", "bar"));
+					assert(store:set("user9999c", "foo", "blah"));
+					local ret, err = store:get_all("foo");
+					assert.is_nil(err);
+					assert.same({ user9999 = "bar", user9999c = "blah" }, ret);
+				end);
+
+				it("rejects empty or non-string keys to get_all", function ()
+					assert.is_function(store.get_all);
+					do
+						local ret, err = store:get_all("");
+						assert.is_nil(ret);
+						assert.is_not_nil(err);
+					end
+					do
+						local ret, err = store:get_all(true);
+						assert.is_nil(ret);
+						assert.is_not_nil(err);
+					end
+				end);
+
+				it("rejects empty or non-string keys to delete_all", function ()
+					assert.is_function(store.delete_all);
+					do
+						local ret, err = store:delete_all("");
+						assert.is_nil(ret);
+						assert.is_not_nil(err);
+					end
+					do
+						local ret, err = store:delete_all(true);
+						assert.is_nil(ret);
+						assert.is_not_nil(err);
+					end
+				end);
+
+				it("may delete all instances of a specific key", function ()
+					assert.is_function(store.delete_all);
+					assert(store:set("user9999b", "foo", "hello"));
+
+					assert(store:delete_all("bar"));
+					-- Ensure key was deleted
+					do
+						local ret, err = store:get("user9999b", "bar");
+						assert.is_nil(ret);
+						assert.is_nil(err);
+					end
+					-- Ensure other users/keys are intact
+					do
+						local ret, err = store:get("user9999", "foo");
+						assert.equal("bar", ret);
+						assert.is_nil(err);
+					end
+					do
+						local ret, err = store:get("user9999b", "foo");
+						assert.equal("hello", ret);
+						assert.is_nil(err);
+					end
+					do
+						local ret, err = store:get("user9999c", "foo");
+						assert.equal("blah", ret);
+						assert.is_nil(err);
+					end
+				end);
+
+				it("may remove data for a specific key for a user", function ()
+					assert(store:set("user9999", "foo", nil));
+					do
+						local ret, err = store:get("user9999", "foo");
+						assert.is_nil(ret);
+						assert.is_nil(err);
+					end
+
+					assert(store:set("user9999b", "foo", nil));
+					do
+						local ret, err = store:get("user9999b", "foo");
+						assert.is_nil(ret);
+						assert.is_nil(err);
+					end
+				end);
+			end);
+
 			describe("archive stores", function ()
 				randomize(false);
 
@@ -100,7 +206,8 @@ describe("storagemanager", function ()
 
 				local test_stanza = st.stanza("test", { xmlns = "urn:example:foo" })
 					:tag("foo"):up()
-					:tag("foo"):up();
+					:tag("foo"):up()
+					:reset();
 				local test_time = 1539204123;
 
 				local test_data = {
@@ -108,17 +215,22 @@ describe("storagemanager", function ()
 					{ nil, test_stanza, test_time+1, "contact2@example.com" };
 					{ nil, test_stanza, test_time+2, "contact2@example.com" };
 					{ nil, test_stanza, test_time-1, "contact2@example.com" };
+					{ nil, test_stanza, test_time-1, "contact3@example.com" };
+					{ nil, test_stanza, test_time+0, "contact3@example.com" };
+					{ nil, test_stanza, test_time+1, "contact3@example.com" };
 				};
 
 				it("can be added to", function ()
 					for _, data_item in ipairs(test_data) do
-						local ok = archive:append("user", unpack(data_item, 1, 4));
-						assert.truthy(ok);
+						local id = archive:append("user", unpack(data_item, 1, 4));
+						assert.truthy(id);
+						data_item[1] = id;
 					end
 				end);
 
 				describe("can be queried", function ()
 					it("for all items", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {});
 						assert.truthy(data);
 						local count = 0;
@@ -135,6 +247,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by JID", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							with = "contact@example.com";
 						});
@@ -153,6 +266,7 @@ describe("storagemanager", function ()
 					end);
 
 					it("by time (end)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["end"] = test_time;
 						});
@@ -167,10 +281,11 @@ describe("storagemanager", function ()
 							assert.equal(2, #item.tags);
 							assert(test_time >= when);
 						end
-						assert.equal(2, count);
+						assert.equal(4, count);
 					end);
 
 					it("by time (start)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["start"] = test_time;
 						});
@@ -185,10 +300,11 @@ describe("storagemanager", function ()
 							assert.equal(2, #item.tags);
 							assert(test_time <= when);
 						end
-						assert.equal(#test_data -1, count);
+						assert.equal(#test_data - 2, count);
 					end);
 
 					it("by time (start+end)", function ()
+						-- luacheck: ignore 211/err
 						local data, err = archive:find("user", {
 							["start"] = test_time;
 							["end"] = test_time+1;
@@ -205,7 +321,45 @@ describe("storagemanager", function ()
 							assert(when >= test_time, ("%d >= %d"):format(when, test_time));
 							assert(when <= test_time+1, ("%d <= %d"):format(when, test_time+1));
 						end
-						assert.equal(2, count);
+						assert.equal(4, count);
+					end);
+
+					it("by id (after)", function ()
+						-- luacheck: ignore 211/err
+						local data, err = archive:find("user", {
+							["after"] = test_data[2][1];
+						});
+						assert.truthy(data);
+						local count = 0;
+						for id, item in data do
+							count = count + 1;
+							assert.truthy(id);
+							assert.equal(test_data[2+count][1], id);
+							assert(st.is_stanza(item));
+							assert.equal("test", item.name);
+							assert.equal("urn:example:foo", item.attr.xmlns);
+							assert.equal(2, #item.tags);
+						end
+						assert.equal(5, count);
+					end);
+
+					it("by id (before)", function ()
+						-- luacheck: ignore 211/err
+						local data, err = archive:find("user", {
+							["before"] = test_data[4][1];
+						});
+						assert.truthy(data);
+						local count = 0;
+						for id, item in data do
+							count = count + 1;
+							assert.truthy(id);
+							assert.equal(test_data[count][1], id);
+							assert(st.is_stanza(item));
+							assert.equal("test", item.name);
+							assert.equal("urn:example:foo", item.attr.xmlns);
+							assert.equal(2, #item.tags);
+						end
+						assert.equal(3, count);
 					end);
 				end);
 
@@ -239,6 +393,7 @@ describe("storagemanager", function ()
 				end);
 
 				it("can be purged", function ()
+					-- luacheck: ignore 211/err
 					local ok, err = archive:delete("user");
 					assert.truthy(ok);
 					local data, err = archive:find("user", {
@@ -326,6 +481,32 @@ describe("storagemanager", function ()
 					assert.equal(2, count);
 					assert(archive:delete("user-issue1073"));
 				end);
+
+				it("can be treated as a map store", function ()
+					assert.falsy(archive:get("mapuser", "no-such-id"));
+					assert.falsy(archive:set("mapuser", "no-such-id", test_stanza));
+
+					local id = archive:append("mapuser", nil, test_stanza, test_time, "contact@example.com");
+					do
+						local stanza_roundtrip, when, with = archive:get("mapuser", id);
+						assert.same(test_stanza, stanza_roundtrip, "same stanza is returned");
+						assert.equal(test_time, when, "same 'when' is returned");
+						assert.equal("contact@example.com", with, "same 'with' is returned");
+					end
+
+					local replacement_stanza = st.stanza("test", { xmlns = "urn:example:foo" })
+						:tag("bar"):up()
+						:reset();
+					assert(archive:set("mapuser", id, replacement_stanza, test_time+1));
+
+					do
+						local replaced, when, with = archive:get("mapuser", id);
+						assert.same(replacement_stanza, replaced, "replaced stanza is returned");
+						assert.equal(test_time+1, when, "modified 'when' is returned");
+						assert.equal("contact@example.com", with, "original 'with' is returned");
+					end
+				end);
+
 			end);
 		end);
 	end
diff --git a/spec/inputs/http/httpstream-chunked-test.txt b/spec/inputs/http/httpstream-chunked-test.txt
new file mode 100644
index 00000000..56efa067
--- /dev/null
+++ b/spec/inputs/http/httpstream-chunked-test.txt
@@ -0,0 +1,15 @@
+HTTP/1.1 200 OK

+Cache-Control: max-age=0, must-revalidate, private

+Content-Type: application/json

+Date: Fri, 21 Aug 2020 12:18:51 GMT

+Expires: Fri, 21 Aug 2020 12:18:51 GMT

+Server: Apache/2.4.38 (Debian)

+Set-Cookie: PHPSESSID=00000000000000000000000000; path=/; HttpOnly

+Strict-Transport-Security: max-age=29030400

+X-Powered-By: PHP/7.4.7

+Transfer-Encoding: chunked

+

+2b4d

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+0

+

diff --git a/spec/muc_util_spec.lua b/spec/muc_util_spec.lua
index cef68e80..3b2da4d0 100644
--- a/spec/muc_util_spec.lua
+++ b/spec/muc_util_spec.lua
@@ -3,11 +3,23 @@ local muc_util;
 local st = require "util.stanza";
 
 do
-	local old_pp = package.path;
-	package.path = "./?.lib.lua;"..package.path;
-	muc_util = require "plugins.muc.util";
-	package.path = old_pp;
-end
+	-- XXX Hack for lack of a mock moduleapi
+	local env = setmetatable({
+		module = {
+			_shared = {};
+			-- Close enough to the real module:shared() for our purposes here
+			shared = function (self, name)
+				local t = self._shared[name];
+				if t == nil then
+					t = {};
+					self._shared[name] = t;
+				end
+				return t;
+			end;
+		}
+	}, { __index = _ENV or _G });
+	muc_util = require "util.envload".envloadfile("plugins/muc/util.lib.lua", env)();
+	end
 
 describe("muc/util", function ()
 	describe("filter_muc_x()", function ()
diff --git a/spec/net_http_parser_spec.lua b/spec/net_http_parser_spec.lua
index 6bba087c..f71cad20 100644
--- a/spec/net_http_parser_spec.lua
+++ b/spec/net_http_parser_spec.lua
@@ -1,16 +1,76 @@
-local httpstreams = { [[
+local http_parser = require "net.http.parser";
+local sha1 = require "util.hashes".sha1;
+
+local parser_input_bytes = 3;
+
+local function CRLF(s)
+	return (s:gsub("\n", "\r\n"));
+end
+
+local function test_stream(stream, expect)
+	local success_cb = spy.new(function (packet)
+		assert.is_table(packet);
+		if packet.body ~= false then
+			assert.is_equal(expect.body, packet.body);
+		end
+	end);
+
+	local parser = http_parser.new(success_cb, error, stream:sub(1,4) == "HTTP" and "client" or "server")
+	for chunk in stream:gmatch("."..string.rep(".?", parser_input_bytes-1)) do
+		parser:feed(chunk);
+	end
+
+	assert.spy(success_cb).was_called(expect.count or 1);
+end
+
+
+describe("net.http.parser", function()
+	describe("parser", function()
+		it("should handle requests with no content-length or body", function ()
+			test_stream(
+CRLF[[
 GET / HTTP/1.1
 Host: example.com
 
-]], [[
+]],
+				{
+					body = "";
+				}
+			);
+		end);
+
+		it("should handle responses with empty body", function ()
+			test_stream(
+CRLF[[
 HTTP/1.1 200 OK
 Content-Length: 0
 
-]], [[
+]],
+				{
+					body = "";
+				}
+			);
+		end);
+
+		it("should handle simple responses", function ()
+			test_stream(
+
+CRLF[[
 HTTP/1.1 200 OK
 Content-Length: 7
 
 Hello
+]],
+				{
+					body = "Hello\r\n", count = 1;
+				}
+			);
+		end);
+
+		it("should handle chunked encoding in responses", function ()
+			test_stream(
+
+CRLF[[
 HTTP/1.1 200 OK
 Transfer-Encoding: chunked
 
@@ -25,28 +85,51 @@ o
 0
 
 
-]]
-}
+]],
+				{
+					body = "Hello", count = 2;
+				}
+			);
+		end);
 
+		it("should handle a stream of responses", function ()
+			test_stream(
 
-local http_parser = require "net.http.parser";
+CRLF[[
+HTTP/1.1 200 OK
+Content-Length: 5
 
-describe("net.http.parser", function()
-	describe("#new()", function()
-		it("should work", function()
-			for _, stream in ipairs(httpstreams) do
-				local success;
-				local function success_cb(packet)
-					success = true;
-				end
-				stream = stream:gsub("\n", "\r\n");
-				local parser = http_parser.new(success_cb, error, stream:sub(1,4) == "HTTP" and "client" or "server")
-				for chunk in stream:gmatch("..?.?") do
-					parser:feed(chunk);
-				end
-
-				assert.is_true(success);
-			end
+Hello
+HTTP/1.1 200 OK
+Transfer-Encoding: chunked
+
+1
+H
+1
+e
+2
+ll
+1
+o
+0
+
+
+]],
+				{
+					body = "Hello", count = 3;
+				}
+			);
 		end);
 	end);
+
+	it("should handle large chunked responses", function ()
+		local data = io.open("spec/inputs/http/httpstream-chunked-test.txt", "rb"):read("*a");
+
+		-- Just a sanity check... text editors and things may mess with line endings, etc.
+		assert.equal("25930f021785ae14053a322c2dbc1897c3769720", sha1(data, true), "test data malformed");
+
+		test_stream(data, {
+			body = string.rep("~", 11085), count = 2;
+		});
+	end);
 end);
diff --git a/spec/net_websocket_frames_spec.lua b/spec/net_websocket_frames_spec.lua
index 519be7b9..7c7d8f05 100644
--- a/spec/net_websocket_frames_spec.lua
+++ b/spec/net_websocket_frames_spec.lua
@@ -52,6 +52,26 @@ describe("net.websocket.frames", function ()
 			["RSV2"] = false;
 			["RSV3"] = false;
 		};
+		ping = {
+			["opcode"] = 0x9;
+			["length"] = 4;
+			["data"] = "ping";
+			["FIN"] = true;
+			["MASK"] = false;
+			["RSV1"] = false;
+			["RSV2"] = false;
+			["RSV3"] = false;
+		};
+		pong = {
+			["opcode"] = 0xa;
+			["length"] = 4;
+			["data"] = "pong";
+			["FIN"] = true;
+			["MASK"] = false;
+			["RSV1"] = false;
+			["RSV2"] = false;
+			["RSV3"] = false;
+		};
 	}
 
 	describe("build", function ()
@@ -62,6 +82,8 @@ describe("net.websocket.frames", function ()
 			assert.equal("\128\0", build(test_frames.simple_fin));
 			assert.equal("\128\133 \0 \0HeLlO", build(test_frames.with_mask))
 			assert.equal("\128\128 \0 \0", build(test_frames.empty_with_mask))
+			assert.equal("\137\4ping", build(test_frames.ping));
+			assert.equal("\138\4pong", build(test_frames.pong));
 		end);
 	end);
 
@@ -72,6 +94,8 @@ describe("net.websocket.frames", function ()
 			assert.same(test_frames.simple_data, parse("\0\5hello"));
 			assert.same(test_frames.simple_fin, parse("\128\0"));
 			assert.same(test_frames.with_mask, parse("\128\133 \0 \0HeLlO"));
+			assert.same(test_frames.ping, parse("\137\4ping"));
+			assert.same(test_frames.pong, parse("\138\4pong"));
 		end);
 	end);
 
diff --git a/spec/scansion/basic_message.scs b/spec/scansion/basic_message.scs
index fb21c465..1258dbf5 100644
--- a/spec/scansion/basic_message.scs
+++ b/spec/scansion/basic_message.scs
@@ -79,7 +79,7 @@ Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>Hello Juliet, are you there?</body>
 		<delay xmlns='urn:xmpp:delay' from='localhost' stamp='{scansion:any}' />
-	</message>	
+	</message>
 
 # Romeo sends another bare-JID message, it should be delivered
 # instantly to Juliet's phone
@@ -92,7 +92,7 @@ Romeo sends:
 Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>Oh, hi!</body>
-	</message>	
+	</message>
 
 # Juliet's laptop goes online, but with a negative priority
 
@@ -122,7 +122,7 @@ Romeo sends:
 Juliet's phone receives:
 	<message from="${Romeo's full JID}" type="chat">
 		<body>How are you?</body>
-	</message>	
+	</message>
 
 # Romeo sends direct to Juliet's full JID, and she should receive it
 
diff --git a/spec/scansion/blocking.scs b/spec/scansion/blocking.scs
index 6a9f199e..5bb5a41b 100644
--- a/spec/scansion/blocking.scs
+++ b/spec/scansion/blocking.scs
@@ -145,16 +145,16 @@ Juliet receives:
 	</message>
 
 # Bye
-Juliet disconnects
-
 Juliet sends:
 	<presence type="unavailable"/>
 
+Juliet disconnects
+
 Romeo receives:
 	<presence from="${Juliet's full JID}" to="${Romeo's JID}" type="unavailable"/>
 
-Romeo disconnects
-
 Romeo sends:
 	<presence type="unavailable"/>
 
+Romeo disconnects
+
diff --git a/spec/scansion/extdisco.scs b/spec/scansion/extdisco.scs
new file mode 100644
index 00000000..fd73c9da
--- /dev/null
+++ b/spec/scansion/extdisco.scs
@@ -0,0 +1,57 @@
+# XEP-0215: External Service Discovery
+
+[Client] Romeo
+	password: password
+	jid: user@localhost/mFquWxSr
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq type='get' xml:lang='sv' id='lx2' to='localhost'>
+		<services xmlns='urn:xmpp:extdisco:2'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' id='lx2' from='localhost'>
+		<services xmlns='urn:xmpp:extdisco:2'>
+			<service host='default.example' transport='udp' port='9876' type='stun'/>
+			<service port='9876' type='turn' restricted='1' password='yHYYBDN7M3mdlug0LTdJbW0GvvQ=' transport='udp' host='default.example' username='1219525744'/>
+			<service port='9876' type='turn' restricted='1' password='1Uc6QfrDhIlbK97rGCUQ/cUICxs=' transport='udp' host='default.example' username='1219525744'/>
+			<service port='2121' type='ftp' restricted='1' password='password' transport='tcp' host='default.example' username='john'/>
+			<service port='21' type='ftp' restricted='1' password='password' transport='tcp' host='ftp.example.com' username='john'/>
+		</services>
+	</iq>
+
+Romeo sends:
+	<iq type='get' xml:lang='sv' id='lx3' to='localhost'>
+		<services xmlns='urn:xmpp:extdisco:2' type='ftp'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' id='lx3' from='localhost'>
+		<services xmlns='urn:xmpp:extdisco:2'>
+			<service port='2121' type='ftp' restricted='1' password='password' transport='tcp' host='default.example' username='john'/>
+			<service port='21' type='ftp' restricted='1' password='password' transport='tcp' host='ftp.example.com' username='john'/>
+		</services>
+	</iq>
+
+Romeo sends:
+	<iq type='get' xml:lang='sv' id='lx4' to='localhost'>
+		<credentials xmlns='urn:xmpp:extdisco:2'>
+			<service host='default.example' type='turn'/>
+		</credentials>
+	</iq>
+
+Romeo receives:
+	<iq type='result' id='lx4' from='localhost'>
+		<credentials xmlns='urn:xmpp:extdisco:2'>
+			<service port='9876' type='turn' restricted='1' password='yHYYBDN7M3mdlug0LTdJbW0GvvQ=' transport='udp' host='default.example' username='1219525744'/>
+			<service port='9876' type='turn' restricted='1' password='1Uc6QfrDhIlbK97rGCUQ/cUICxs=' transport='udp' host='default.example' username='1219525744'/>
+		</credentials>
+	</iq>
+
+Romeo disconnects
+
+# recording ended on 2020-07-18T16:47:57Z
diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs
new file mode 100644
index 00000000..41ffec0d
--- /dev/null
+++ b/spec/scansion/keep_full_sub_req.scs
@@ -0,0 +1,58 @@
+# server MUST keep a record of the complete presence stanza comprising the subscription request (#689)
+
+[Client] Alice
+	jid: pars-a@localhost
+	password: password
+
+[Client] Bob
+	jid: pars-b@localhost
+	password: password
+
+[Client] Bob's phone
+	jid: pars-b@localhost/phone
+	password: password
+
+---------
+
+Alice connects
+
+Alice sends:
+	<presence to="${Bob's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Alice disconnects
+
+Bob connects
+
+Bob sends:
+	<presence/>
+
+Bob receives:
+	<presence from="${Bob's full JID}"/>
+
+Bob receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob disconnects
+
+# Works if they reconnect too
+
+Bob's phone connects
+
+Bob's phone sends:
+	<presence/>
+
+Bob's phone receives:
+	<presence from="${Bob's phone's full JID}"/>
+
+
+Bob's phone receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob's phone disconnects
+
diff --git a/spec/scansion/lastactivity.scs b/spec/scansion/lastactivity.scs
new file mode 100644
index 00000000..44f4e516
--- /dev/null
+++ b/spec/scansion/lastactivity.scs
@@ -0,0 +1,45 @@
+# XEP-0012: Last Activity / mod_lastactivity
+
+[Client] Romeo
+	jid: romeo@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence>
+		<status>Hello</status>
+	</presence>
+
+Romeo receives:
+	<presence from="${Romeo's full JID}">
+		<status>Hello</status>
+	</presence>
+
+Romeo sends:
+	<presence type="unavailable">
+		<status>Goodbye</status>
+	</presence>
+
+Romeo receives:
+	<presence from="${Romeo's full JID}" type="unavailable">
+		<status>Goodbye</status>
+	</presence>
+
+# mod_lastlog saves time + status message from the last unavailable presence
+
+Romeo sends:
+	<iq id='a' type='get'>
+		<query xmlns='jabber:iq:last'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' id='a'>
+		<query xmlns='jabber:iq:last' seconds='0'>Goodbye</query>
+	</iq>
+
+Romeo disconnects
+
+# recording ended on 2020-04-20T14:39:47Z
diff --git a/spec/scansion/muc_create_destroy.scs b/spec/scansion/muc_create_destroy.scs
new file mode 100644
index 00000000..789d4c41
--- /dev/null
+++ b/spec/scansion/muc_create_destroy.scs
@@ -0,0 +1,316 @@
+# MUC creation, basic messages and destruction
+
+[Client] Romeo
+	jid: romeo@localhost/mK0dD6Ha
+	password: password
+
+[Client] Juliet
+	jid: juliet@localhost/lVwkim_k
+	password: password
+
+[Client] Admin
+	jid: admin@localhost/DfNgg9VE
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence to="garden@conference.localhost/romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/romeo">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="moderator"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message from="garden@conference.localhost" type="groupchat">
+		<subject/>
+	</message>
+
+Romeo sends:
+	<iq to="garden@conference.localhost" id="lx3" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#owner">
+			<x type="submit" xmlns="jabber:x:data"/>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="lx3" type="result" from="garden@conference.localhost"/>
+
+Juliet connects
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm1">
+		<body>Where are thou my Juliet?</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="garden@conference.localhost/romeo" id="rm1">
+		<body>Where are thou my Juliet?</body>
+	</message>
+
+Juliet sends:
+	<presence to="garden@conference.localhost/juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/romeo">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator"/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/juliet">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="none" jid="${Juliet's full JID}" role="participant"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<message from="garden@conference.localhost/romeo" id="rm1" type="groupchat">
+		<body>Where are thou my Juliet?</body>
+		<delay stamp="{scansion:any}" xmlns="urn:xmpp:delay" from="garden@conference.localhost"/>
+	</message>
+
+Juliet receives:
+	<message from="garden@conference.localhost" type="groupchat">
+		<subject/>
+	</message>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/juliet">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="none" jid="${Juliet's full JID}" role="participant"/>
+		</x>
+	</presence>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm1">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm1" from="garden@conference.localhost/juliet">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm1" from="garden@conference.localhost/juliet">
+		<body>/me jumps out from behind a tree</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm2">
+		<body>Here I am!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm2" from="garden@conference.localhost/juliet">
+		<body>Here I am!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm2" from="garden@conference.localhost/juliet">
+		<body>Here I am!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm2">
+		<body>What is this place?</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm2" from="garden@conference.localhost/romeo">
+		<body>What is this place?</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm2" from="garden@conference.localhost/romeo">
+		<body>What is this place?</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm3">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm3" from="garden@conference.localhost/juliet">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm3" from="garden@conference.localhost/juliet">
+		<body>I think we&apos;re in a script!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm3">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm3" from="garden@conference.localhost/romeo">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm3" from="garden@conference.localhost/romeo">
+		<body>Oh no! Does that mean our love is not real?!</body>
+	</message>
+
+Juliet sends:
+	<message to="garden@conference.localhost" type="groupchat" id="jm4">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="jm4" from="garden@conference.localhost/juliet">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="jm4" from="garden@conference.localhost/juliet">
+		<body>I refuse to accept this! Let&apos;s burn this place to the ground!</body>
+	</message>
+
+Romeo sends:
+	<message to="garden@conference.localhost" type="groupchat" id="rm4">
+		<body>Yes!</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" id="rm4" from="garden@conference.localhost/romeo">
+		<body>Yes!</body>
+	</message>
+
+Juliet receives:
+	<message type="groupchat" id="rm4" from="garden@conference.localhost/romeo">
+		<body>Yes!</body>
+	</message>
+
+Romeo sends:
+	<iq to="garden@conference.localhost" id="lx4" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#owner">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+		</query>
+	</iq>
+
+Juliet receives:
+	<presence from="garden@conference.localhost/juliet" type="unavailable">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+			<item affiliation="none" jid="${Juliet's full JID}" role="none"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<presence from="garden@conference.localhost/romeo" type="unavailable">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<destroy>
+				<reason>We refuse to live in this fantasy!</reason>
+			</destroy>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="none"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<iq id="lx4" type="result" from="garden@conference.localhost"/>
+
+Juliet disconnects
+
+Romeo sends:
+	<presence to="elsewhere@conference.localhost/romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="elsewhere@conference.localhost/romeo">
+		<x xmlns="vcard-temp:x:update">
+			<photo/>
+		</x>
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="moderator"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message from="elsewhere@conference.localhost" type="groupchat">
+		<subject/>
+	</message>
+
+Romeo sends:
+	<iq to="elsewhere@conference.localhost" id="lx5" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#owner">
+			<x type="submit" xmlns="jabber:x:data"/>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="lx5" type="result" from="elsewhere@conference.localhost"/>
+
+Admin connects
+
+Admin sends:
+	<iq id="destroy" type="set" to="conference.localhost">
+		<command xmlns="http://jabber.org/protocol/commands" node="http://prosody.im/protocol/muc#destroy">
+			<x xmlns="jabber:x:data">
+				<field var="rooms">
+					<value>elsewhere@conference.localhost</value>
+				</field>
+			</x>
+		</command>
+	</iq>
+
+Romeo receives:
+	<presence from="elsewhere@conference.localhost/romeo" type="unavailable">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<destroy/>
+			<item affiliation="owner" jid="${Romeo's full JID}" role="none"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo disconnects
+
+Admin receives:
+	<iq id="destroy" type="result" from="conference.localhost">
+		<command xmlns="http://jabber.org/protocol/commands" node="http://prosody.im/protocol/muc#destroy" status="completed" sessionid="{scansion:any}">
+			<note type="info">The following rooms were destroyed:&#10;elsewhere@conference.localhost</note>
+		</command>
+	</iq>
+
+Admin disconnects
+
+# recording ended on 2019-08-31T13:45:32Z
diff --git a/spec/scansion/muc_members_only_change.scs b/spec/scansion/muc_members_only_change.scs
index dc40b5a0..a708dbfb 100644
--- a/spec/scansion/muc_members_only_change.scs
+++ b/spec/scansion/muc_members_only_change.scs
@@ -94,7 +94,7 @@ Romeo sends:
 			<item affiliation='none' jid="${Juliet's JID}" />
 		</query>
 	</iq>
-	
+
 # As a non-member, Juliet must now be removed from the room
 Romeo receives:
 	<presence type='unavailable' from='room@conference.localhost/Juliet'>
diff --git a/spec/scansion/muc_nickname_change.scs b/spec/scansion/muc_nickname_change.scs
new file mode 100644
index 00000000..73f81203
--- /dev/null
+++ b/spec/scansion/muc_nickname_change.scs
@@ -0,0 +1,127 @@
+# MUC: Change nickname
+# Make sure a role is not reset, see #1466
+
+[Client] Romeo
+	jid: user@localhost
+	password: password
+
+[Client] Juliet
+	jid: user2@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence to="room@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='201'/>
+			<item jid="${Romeo's full JID}" affiliation='owner' role='moderator'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo sends:
+	<iq id='config1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#owner'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#roomconfig</value>
+				</field>
+				<field var='muc#roomconfig_moderatedroom'>
+					<value>1</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="config1" from="room@conference.localhost" type="result"/>
+
+Juliet connects
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item role='moderator' xmlns='http://jabber.org/protocol/muc#user' affiliation='owner'/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation="none" role="visitor" jid="${Juliet's full JID}"/>
+		</x>
+	</presence>
+
+Romeo sends:
+	<iq id='config1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#owner'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#roomconfig</value>
+				</field>
+				<field var='muc#roomconfig_moderatedroom'>
+					<value>0</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="config1" from="room@conference.localhost" type="result"/>
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status xmlns='http://jabber.org/protocol/muc#user' code='104'/>
+		</x>
+	</message>
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet2">
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet' type='unavailable'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='303'/>
+			<item nick='Juliet2' jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet2'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='none' role='visitor'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
diff --git a/spec/scansion/muc_password.scs b/spec/scansion/muc_password.scs
index 82611183..63c821e0 100644
--- a/spec/scansion/muc_password.scs
+++ b/spec/scansion/muc_password.scs
@@ -58,7 +58,7 @@ Juliet sends:
 
 Juliet receives:
 	<presence from="room@conference.localhost/Juliet" type="error">
-		<error type="auth" code="401">
+		<error type="auth" code="401" by="room@conference.localhost">
 			<not-authorized xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 	</presence>
diff --git a/spec/scansion/muc_register.scs b/spec/scansion/muc_register.scs
index a7e57177..9fcce688 100644
--- a/spec/scansion/muc_register.scs
+++ b/spec/scansion/muc_register.scs
@@ -100,7 +100,9 @@ Juliet receives:
 				<field type='hidden' var='FORM_TYPE'>
 					<value>http://jabber.org/protocol/muc#register</value>
 				</field>
-				<field type='text-single' label='Nickname' var='muc#register_roomnick'/>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
 			</x>
 		</query>
 	</iq>
@@ -175,7 +177,7 @@ Rosaline sends:
 
 Rosaline receives:
 	<presence type='error' from='room@conference.localhost/Juliet'>
-		<error type='cancel'>
+		<error type='cancel' by='room@conference.localhost'>
 			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 		<x xmlns='http://jabber.org/protocol/muc'/>
@@ -286,7 +288,7 @@ Rosaline sends:
 
 Rosaline receives:
 	<presence type='error' from='room@conference.localhost/Juliet'>
-		<error type='cancel'>
+		<error type='cancel' by='room@conference.localhost'>
 			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
 		</error>
 		<x xmlns='http://jabber.org/protocol/muc'/>
@@ -326,7 +328,7 @@ Romeo receives:
 	</iq>
 
 # Romeo updates his own registration
-	
+
 Romeo sends:
 	<iq id='jw81b36f' to='room@conference.localhost' type='get'>
 		<query xmlns='jabber:iq:register'/>
@@ -339,7 +341,9 @@ Romeo receives:
 				<field type='hidden' var='FORM_TYPE'>
 					<value>http://jabber.org/protocol/muc#register</value>
 				</field>
-				<field type='text-single' label='Nickname' var='muc#register_roomnick'/>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
 			</x>
 		</query>
 	</iq>
diff --git a/spec/scansion/muc_show_offline.scs b/spec/scansion/muc_show_offline.scs
new file mode 100644
index 00000000..57b75ec7
--- /dev/null
+++ b/spec/scansion/muc_show_offline.scs
@@ -0,0 +1,544 @@
+# MUC: Room registration and presence broadcast of unavailable members
+
+[Client] Romeo
+	jid: user@localhost
+	password: password
+
+[Client] Juliet
+	jid: user2@localhost
+	password: password
+
+[Client] Rosaline
+	jid: user3@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<presence to="room@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='201'/>
+			<item jid="${Romeo's full JID}" affiliation='owner' role='moderator'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+# Submit config form
+Romeo sends:
+	<iq id='config1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#owner'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#roomconfig</value>
+				</field>
+				<field var='muc#roomconfig_presencebroadcast'>
+					<value>none</value>
+					<value>participant</value>
+					<value>moderator</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq id="config1" from="room@conference.localhost" type="result">
+	</iq>
+
+Romeo sends:
+	<iq id='member1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#admin'>
+			<item affiliation='member' jid="${Juliet's JID}" />
+		</query>
+	</iq>
+
+Romeo receives:
+	<message from='room@conference.localhost'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's JID}" affiliation='member' />
+		</x>
+	</message>
+
+Romeo receives:
+	<iq from='room@conference.localhost' id='member1' type='result'/>
+
+# Juliet connects, and joins the room
+Juliet connects
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from="room@conference.localhost/Romeo" />
+
+Juliet receives:
+	<presence from="room@conference.localhost/Juliet" />
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo receives:
+	<presence from="room@conference.localhost/Juliet" />
+
+# Juliet retrieves the registration form
+
+Juliet sends:
+	<iq id='jw81b36f' to='room@conference.localhost' type='get'>
+		<query xmlns='jabber:iq:register'/>
+	</iq>
+
+Juliet receives:
+	<iq type='result' from='room@conference.localhost' id='jw81b36f'>
+		<query xmlns='jabber:iq:register'>
+			<x type='form' xmlns='jabber:x:data'>
+				<field type='hidden' var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#register</value>
+				</field>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Juliet sends:
+	<iq id='nv71va54' to='room@conference.localhost' type='set'>
+		<query xmlns='jabber:iq:register'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#register</value>
+				</field>
+				<field var='muc#register_roomnick'>
+					<value>Juliet</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='member' jid="${Juliet's full JID}" role='participant'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Juliet receives:
+	<iq type='result' from='room@conference.localhost' id='nv71va54'/>
+
+# Juliet discovers her reserved nick
+
+Juliet sends:
+	<iq id='getnick1' to='room@conference.localhost' type='get'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item'/>
+	</iq>
+
+Juliet receives:
+	<iq type='result' from='room@conference.localhost' id='getnick1'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item'>
+			<identity category='conference' name='Juliet' type='text'/>
+		</query>
+	</iq>
+
+# Juliet leaves the room:
+
+Juliet sends:
+	<presence type="unavailable" to="room@conference.localhost/Juliet" />
+
+Juliet receives:
+	<presence type='unavailable' from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='member' role='none'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's full JID}" affiliation='member' role='participant'/>
+		</x>
+	</presence>
+
+# Rosaline connect and tries to join the room as Juliet
+
+Rosaline connects
+
+Rosaline sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Rosaline receives:
+	<presence type='error' from='room@conference.localhost/Juliet'>
+		<error type='cancel' by='room@conference.localhost'>
+			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+		</error>
+		<x xmlns='http://jabber.org/protocol/muc'/>
+	</presence>
+
+# In a heated moment, Juliet unregisters from the room
+
+Juliet sends:
+	<iq type='set' to='room@conference.localhost' id='unreg1'>
+		<query xmlns='jabber:iq:register'>
+			<remove/>
+		</query>
+	</iq>
+
+Juliet receives:
+	<iq type='result' from='room@conference.localhost' id='unreg1'/>
+
+# Romeo is notified of Juliet's sad decision
+
+Romeo receives:
+	<message from='room@conference.localhost'>
+		<x xmlns='http://jabber.org/protocol/muc#user' scansion:strict='true'>
+			<item jid="${Juliet's JID}" affiliation='none' />
+		</x>
+	</message>
+
+# Rosaline attempts once more to sneak into the room, disguised as Juliet
+
+Rosaline sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Rosaline receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='owner' role='moderator'/>
+		</x>
+	</presence>
+
+Rosaline receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='none' jid="${Rosaline's full JID}" role='participant'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='none' jid="${Rosaline's full JID}" role='participant'/>
+		</x>
+	</presence>
+
+# On discovering the ruse, Romeo restores Juliet's nick and status within the room
+
+Romeo sends:
+	<iq id='member1' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#admin'>
+			<item affiliation='member' jid="${Juliet's JID}" nick='Juliet' />
+		</query>
+	</iq>
+
+# Rosaline is evicted from the room
+
+Romeo receives:
+	<presence from='room@conference.localhost/Juliet' type='unavailable'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='307'/>
+			<item affiliation='none' role='none' jid="${Rosaline's full JID}">
+				<reason>This nickname is reserved</reason>
+			</item>
+		</x>
+	</presence>
+
+# An out-of-room affiliation change is received for Juliet
+
+Romeo receives:
+	<message from='room@conference.localhost'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Juliet's JID}" affiliation='member' />
+		</x>
+	</message>
+
+Romeo receives:
+	<iq type='result' id='member1' from='room@conference.localhost' />
+
+Rosaline receives:
+	<presence type='unavailable' from='room@conference.localhost/Juliet'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<status code='307'/>
+			<item affiliation='none' jid="${Rosaline's full JID}" role='none'>
+				<reason>This nickname is reserved</reason>
+			</item>
+			<status code='110'/>
+		</x>
+	</presence>
+
+# Rosaline, frustrated, attempts to get back into the room...
+
+Rosaline sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# ...but once again, is denied
+
+Rosaline receives:
+	<presence type='error' from='room@conference.localhost/Juliet'>
+		<error type='cancel' by='room@conference.localhost'>
+			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+		</error>
+		<x xmlns='http://jabber.org/protocol/muc'/>
+	</presence>
+
+# Juliet, however, quietly joins the room with success
+
+Juliet sends:
+	<presence to="room@conference.localhost/Juliet">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Juliet receives:
+	<presence from="room@conference.localhost/Romeo" />
+
+Juliet receives:
+	<presence from="room@conference.localhost/Juliet" />
+
+Juliet receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo receives:
+	<presence from="room@conference.localhost/Juliet" />
+
+# Romeo checks whether he has reserved his own nick yet
+
+Romeo sends:
+	<iq id='getnick1' to='room@conference.localhost' type='get'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item'/>
+	</iq>
+
+# But no nick is returned, as he hasn't registered yet!
+
+Romeo receives:
+	<iq type='result' from='room@conference.localhost' id='getnick1'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item' scansion:strict='true' />
+	</iq>
+
+# Romeo updates his own registration
+
+Romeo sends:
+	<iq id='jw81b36f' to='room@conference.localhost' type='get'>
+		<query xmlns='jabber:iq:register'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' from='room@conference.localhost' id='jw81b36f'>
+		<query xmlns='jabber:iq:register'>
+			<x type='form' xmlns='jabber:x:data'>
+				<field type='hidden' var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#register</value>
+				</field>
+				<field type='text-single' label='Nickname' var='muc#register_roomnick'>
+					<required/>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo sends:
+	<iq id='nv71va54' to='room@conference.localhost' type='set'>
+		<query xmlns='jabber:iq:register'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#register</value>
+				</field>
+				<field var='muc#register_roomnick'>
+					<value>Romeo</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='owner' jid="${Romeo's full JID}" role='moderator'/>
+			<status code='110'/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<iq type='result' from='room@conference.localhost' id='nv71va54'/>
+
+Juliet receives:
+	<presence from='room@conference.localhost/Romeo'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item role='moderator' xmlns='http://jabber.org/protocol/muc#user' affiliation='owner'/>
+		</x>
+	</presence>
+
+# Romeo discovers his reserved nick
+
+Romeo sends:
+	<iq id='getnick1' to='room@conference.localhost' type='get'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' from='room@conference.localhost' id='getnick1'>
+		<query xmlns='http://jabber.org/protocol/disco#info' node='x-roomuser-item'>
+			<identity category='conference' name='Romeo' type='text'/>
+		</query>
+	</iq>
+
+# To check the status of the room is as expected, Romeo requests the member list
+
+Romeo sends:
+	<iq id='member3' to='room@conference.localhost' type='get'>
+		<query xmlns='http://jabber.org/protocol/muc#admin'>
+			<item affiliation='member'/>
+		</query>
+	</iq>
+
+Romeo receives:
+	<iq from='room@conference.localhost' type='result' id='member3'>
+		<query xmlns='http://jabber.org/protocol/muc#admin'>
+			<item jid="${Juliet's JID}" affiliation='member' nick='Juliet'/>
+		</query>
+	</iq>
+
+Juliet sends:
+	<presence type="unavailable" to="room@conference.localhost/Juliet" />
+
+Juliet receives:
+	<presence from='room@conference.localhost/Juliet' type='unavailable' />
+
+Romeo receives:
+	<presence type='unavailable' from='room@conference.localhost/Juliet' />
+
+# Rosaline joins as herself
+
+Rosaline sends:
+	<presence to="room@conference.localhost/Rosaline">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Rosaline receives:
+	<presence from="room@conference.localhost/Romeo" />
+
+Rosaline receives:
+	<presence from='room@conference.localhost/Juliet' type='unavailable'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='member' role='none' nick='Juliet' xmlns='http://jabber.org/protocol/muc#user'/>
+		</x>
+	</presence>
+
+Rosaline receives:
+	<presence from="room@conference.localhost/Rosaline" />
+
+Rosaline receives:
+	<message type='groupchat' from='room@conference.localhost'><subject/></message>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Rosaline'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item jid="${Rosaline's full JID}" affiliation='none' role='participant'/>
+		</x>
+	</presence>
+
+# Rosaline tries to register her own nickname, but unaffiliated
+# registration is disabled by default
+
+Rosaline sends:
+	<iq id='reg990' to='room@conference.localhost' type='get'>
+		<query xmlns='jabber:iq:register'/>
+	</iq>
+
+Rosaline receives:
+	<iq type='error' from='room@conference.localhost' id='reg990'>
+		<error type='auth'>
+			<registration-required xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+		</error>
+	</iq>
+
+Rosaline sends:
+	<iq id='reg991' to='room@conference.localhost' type='set'>
+		<query xmlns='jabber:iq:register'>
+			<x xmlns='jabber:x:data' type='submit'>
+				<field var='FORM_TYPE'>
+					<value>http://jabber.org/protocol/muc#register</value>
+				</field>
+				<field var='muc#register_roomnick'>
+					<value>Romeo</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Rosaline receives:
+	<iq id='reg991' type='error'>
+		<error type='auth'>
+			<registration-required xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+		</error>
+	</iq>
+
+# Romeo reserves her nickname for her
+
+Romeo sends:
+	<iq id='member2' to='room@conference.localhost' type='set'>
+		<query xmlns='http://jabber.org/protocol/muc#admin'>
+			<item affiliation='member' jid="${Rosaline's JID}" nick='Rosaline' />
+		</query>
+	</iq>
+
+Romeo receives:
+	<presence from='room@conference.localhost/Rosaline'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='member' role='participant' jid="${Rosaline's full JID}">
+				<actor jid="${Romeo's full JID}" nick='Romeo'/>
+			</item>
+		</x>
+	</presence>
+
+Romeo receives:
+	<iq type='result' id='member2' from='room@conference.localhost' />
+
+Rosaline receives:
+	<presence from='room@conference.localhost/Rosaline'>
+		<x xmlns='http://jabber.org/protocol/muc#user'>
+			<item affiliation='member' role='participant' jid="${Rosaline's full JID}">
+				<actor nick='Romeo' />
+			</item>
+			<status xmlns='http://jabber.org/protocol/muc#user' code='110'/>
+		</x>
+	</presence>
+
+# Romeo sets their their own nickname via admin query (see #1273)
+Romeo sends:
+	<iq to="room@conference.localhost" id="reserve" type="set">
+		<query xmlns="http://jabber.org/protocol/muc#admin">
+			<item nick="Romeo" affiliation="owner" jid="${Romeo's JID}"/>
+		</query>
+	</iq>
+
+Romeo receives:
+	<presence from="room@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item xmlns="http://jabber.org/protocol/muc#user" role="moderator" jid="${Romeo's full JID}" affiliation="owner">
+				<actor xmlns="http://jabber.org/protocol/muc#user" nick="Romeo"/>
+			</item>
+			<status xmlns="http://jabber.org/protocol/muc#user" code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<iq from="room@conference.localhost" id="reserve" type="result"/>
+
diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs
new file mode 100644
index 00000000..74980073
--- /dev/null
+++ b/spec/scansion/muc_subject_issue_667.scs
@@ -0,0 +1,129 @@
+# #667 MUC message with subject and body SHALL NOT be interpreted as a subject change
+
+[Client] Romeo
+	password: password
+	jid: romeo@localhost
+
+-----
+
+Romeo connects
+
+# and creates a room
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# the default (empty) subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this should be treated as a normal message
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# the still empty subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this is a subject change
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+# a message without <subject>
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# History
+# These have delay tags but we ignore those for now
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Finally, the topic
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo disconnects
+
diff --git a/spec/scansion/presence_preapproval.scs b/spec/scansion/presence_preapproval.scs
new file mode 100644
index 00000000..e34ac7cf
--- /dev/null
+++ b/spec/scansion/presence_preapproval.scs
@@ -0,0 +1,74 @@
+# server supports contact subscription pre-approval (RFC 6121 3.4)
+
+[Client] Alice
+	jid: preappove-a@localhost
+	password: password
+
+[Client] Bob
+	jid: preapprove-b@localhost
+	password: password
+
+---------
+
+Alice connects
+
+Alice sends:
+	<presence/>
+
+Alice receives:
+	<presence/>
+
+Alice sends:
+	<presence to="${Bob's JID}" type="subscribed"/>
+
+Bob connects
+
+Bob sends:
+	<iq type="get" id="roster1">
+		<query xmlns="jabber:iq:roster"/>
+	</iq>
+
+Bob receives:
+	<iq type="result" id="roster1">
+		<query xmlns="jabber:iq:roster" ver="{scansion:any}">
+		</query>
+	</iq>
+
+Bob sends:
+	<presence/>
+
+Bob receives:
+	<presence from="${Bob's full JID}"/>
+
+Bob sends:
+	<presence to="${Alice's JID}" type="subscribe" />
+
+Bob receives:
+	<iq type='set' id='{scansion:any}'>
+		<query ver='1' xmlns='jabber:iq:roster'>
+			<item jid="${Alice's JID}" subscription='none' ask='subscribe' />
+		</query>
+	</iq>
+
+
+
+Bob receives:
+	<presence from="${Alice's JID}" type="subscribed" />
+
+Bob disconnects
+
+Alice sends:
+	<iq type="get" id="roster1">
+		<query xmlns="jabber:iq:roster"/>
+	</iq>
+
+Alice receives:
+	<iq type="result" id="roster1">
+		<query xmlns="jabber:iq:roster" ver="{scansion:any}">
+			<item jid="${Bob's JID}" subscription="from" />
+		</query>
+	</iq>
+
+Alice disconnects
+
+Bob disconnects
diff --git a/spec/scansion/prosody.cfg.lua b/spec/scansion/prosody.cfg.lua
index f95ea31b..4e43d312 100644
--- a/spec/scansion/prosody.cfg.lua
+++ b/spec/scansion/prosody.cfg.lua
@@ -1,23 +1,37 @@
 --luacheck: ignore
 
+-- Mock time functions to simplify tests
+function _G.os.time()
+	return 1219439344;
+end
+package.preload["util.time"] = function ()
+	return {
+		now = function () return 1219439344.1; end;
+		monotonic = function () return 0.1; end;
+	}
+end
+
 admins = { "admin@localhost" }
 
-use_libevent = true
+network_backend = "epoll"
+network_settings = {
+}
 
 modules_enabled = {
 	-- Generally required
 		"roster"; -- Allow users to have a roster. Recommended ;)
 		"saslauth"; -- Authentication for clients and servers. Recommended if you want to log in.
-		"tls"; -- Add support for secure TLS on c2s/s2s connections
-		"dialback"; -- s2s dialback support
+		--"tls"; -- Add support for secure TLS on c2s/s2s connections
+		--"dialback"; -- s2s dialback support
 		"disco"; -- Service discovery
 
 	-- Not essential, but recommended
 		"carbons"; -- Keep multiple clients in sync
-		"pep"; -- Enables users to publish their mood, activity, playing music and more
+		"pep"; -- Enables users to publish their avatar, mood, activity, playing music and more
 		"private"; -- Private XML storage (for room bookmarks, etc.)
 		"blocklist"; -- Allow users to block communications with other users
-		"vcard"; -- Allow users to set vCards
+		"vcard4"; -- User profiles (stored in PEP)
+		"vcard_legacy"; -- Conversion between legacy vCard and PEP Avatar, vcard
 
 	-- Nice to have
 		"version"; -- Replies to server version requests
@@ -26,6 +40,11 @@ modules_enabled = {
 		"ping"; -- Replies to XMPP pings with pongs
 		"register"; -- Allow users to register on this server using a client and change passwords
 		"mam"; -- Store messages in an archive and allow users to access it
+		--"csi_simple"; -- Simple Mobile optimizations
+
+	-- Admin interfaces
+		--"admin_adhoc"; -- Allows administration via an XMPP client that supports ad-hoc commands
+		--"admin_telnet"; -- Opens telnet console interface on localhost port 5582
 
 	-- HTTP modules
 		--"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
@@ -35,18 +54,44 @@ modules_enabled = {
 	-- Other specific functionality
 		--"limits"; -- Enable bandwidth limiting for XMPP connections
 		--"groups"; -- Shared roster support
-		--"server_contact_info"; -- Publish contact information for this service
+		"server_contact_info"; -- Publish contact information for this service
 		--"announce"; -- Send announcement to all online users
 		--"welcome"; -- Welcome users who register accounts
 		--"watchregistrations"; -- Alert admins of registrations
 		--"motd"; -- Send a message to users when they log in
 		--"legacyauth"; -- Legacy authentication. Only used by some old clients and bots.
 		--"proxy65"; -- Enables a file transfer proxy service which clients behind NAT can use
+		"lastactivity";
+		"external_services";
 
 	-- Useful for testing
 		--"scansion_record"; -- Records things that happen in scansion test case format
 }
 
+contact_info = {
+	abuse = { "mailto:abuse@localhost", "xmpp:abuse@localhost" };
+	admin = { "mailto:admin@localhost", "xmpp:admin@localhost" };
+	feedback = { "http://localhost/feedback.html", "mailto:feedback@localhost", "xmpp:feedback@localhost" };
+	sales = { "xmpp:sales@localhost" };
+	security = { "xmpp:security@localhost" };
+	status = { "gopher://status.localhost" };
+	support = { "https://localhost/support.html", "xmpp:support@localhost" };
+}
+
+external_service_host = "default.example"
+external_service_port = 9876
+external_service_secret = "<secret>"
+external_services = {
+	{type = "stun"; transport = "udp"};
+	{type = "turn"; transport = "udp"; secret = true};
+	{type = "turn"; transport = "udp"; secret = "foo"};
+	{type = "ftp"; transport = "tcp"; port = 2121; username = "john"; password = "password"};
+	{type = "ftp"; transport = "tcp"; host = "ftp.example.com"; port = 21; username = "john"; password = "password"};
+}
+
+modules_disabled = {
+	"s2s";
+}
 certificate = "certs"
 
 allow_registration = false
@@ -69,15 +114,20 @@ mam_smart_enable = true
 
 -- Logging configuration
 -- For advanced logging see https://prosody.im/doc/logging
-log = "*console"
+log = {debug = "*console" }
 
-daemonize = true
 pidfile = "prosody.pid"
 
 VirtualHost "localhost"
 
+hide_os_type = true -- absence tested for in version.scs
+
 Component "conference.localhost" "muc"
 	storage = "memory"
+	modules_enabled = {
+		"muc_mam";
+	}
+
 
 Component "pubsub.localhost" "pubsub"
 	storage = "memory"
diff --git a/spec/scansion/pubsub_basic.scs b/spec/scansion/pubsub_basic.scs
index d983ff66..0adf6857 100644
--- a/spec/scansion/pubsub_basic.scs
+++ b/spec/scansion/pubsub_basic.scs
@@ -32,7 +32,7 @@ Juliet connects
 -- 			<subscribe node="princely_musings" jid="${Romeo's full JID}"/>
 -- 		</pubsub>
 -- 	</iq>
--- 
+--
 -- Juliet receives:
 -- 	<iq type="error"/>
 
diff --git a/spec/scansion/pubsub_preconditions.scs b/spec/scansion/pubsub_preconditions.scs
new file mode 100644
index 00000000..25afaa8d
--- /dev/null
+++ b/spec/scansion/pubsub_preconditions.scs
@@ -0,0 +1,234 @@
+# Pubsub preconditions are enforced
+
+[Client] Romeo
+	password: password
+	jid: jqpcrbq2@localhost
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current">
+					<tune xmlns="http://jabber.org/protocol/tune"/>
+				</item>
+			</publish>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="result">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current"/>
+			</publish>
+		</pubsub>
+	</iq>
+
+Romeo sends:
+	<iq id="52d74a36-afb0-4028-87ed-b25b988b049e" type="get">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune"/>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="52d74a36-afb0-4028-87ed-b25b988b049e" type="result">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune">
+				<x xmlns="jabber:x:data" type="form">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#node_config</value>
+					</field>
+					<field var="pubsub#title" label="Title" type="text-single"/>
+					<field var="pubsub#description" label="Description" type="text-single"/>
+					<field var="pubsub#type" label="The type of node data, usually specified by the namespace of the payload (if any)" type="text-single"/>
+					<field var="pubsub#max_items" label="Max # of items to persist" type="text-single">
+						<validate xmlns="http://jabber.org/protocol/xdata-validate" datatype="xs:integer"/>
+						<value>1</value>
+					</field>
+					<field var="pubsub#persist_items" label="Persist items to storage" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#access_model" label="Specify the subscriber model" type="list-single">
+						<option label="authorize">
+							<value>authorize</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<option label="presence">
+							<value>presence</value>
+						</option>
+						<option label="roster">
+							<value>roster</value>
+						</option>
+						<option label="whitelist">
+							<value>whitelist</value>
+						</option>
+						<value>presence</value>
+					</field>
+					<field var="pubsub#publish_model" label="Specify the publisher model" type="list-single">
+						<option label="publishers">
+							<value>publishers</value>
+						</option>
+						<option label="subscribers">
+							<value>subscribers</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<value>publishers</value>
+					</field>
+					<field var="pubsub#deliver_notifications" label="Whether to deliver event notifications" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#deliver_payloads" label="Whether to deliver payloads with event notifications" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notification_type" label="Specify the delivery style for notifications" type="list-single">
+						<option label="Messages of type normal">
+							<value>normal</value>
+						</option>
+						<option label="Messages of type headline">
+							<value>headline</value>
+						</option>
+						<value>headline</value>
+					</field>
+					<field var="pubsub#notify_delete" label="Whether to notify subscribers when the node is deleted" type="boolean">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notify_retract" label="Whether to notify subscribers when items are removed from the node" type="boolean">
+						<value>1</value>
+					</field>
+				</x>
+			</configure>
+		</pubsub>
+	</iq>
+
+Romeo sends:
+	<iq id="a73aac09-74be-4ee2-97e5-571bbdbcd956" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub#owner">
+			<configure node="http://jabber.org/protocol/tune">
+				<x xmlns="jabber:x:data" type="submit">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#node_config</value>
+					</field>
+					<field var="pubsub#title" type="text-single" label="Title">
+						<value>Nice tunes</value>
+					</field>
+					<field var="pubsub#description" type="text-single" label="Description"/>
+					<field var="pubsub#type" type="text-single" label="The type of node data, usually specified by the namespace of the payload (if any)"/>
+					<field var="pubsub#max_items" type="text-single" label="Max # of items to persist">
+						<validate xmlns="http://jabber.org/protocol/xdata-validate" datatype="xs:integer"/>
+						<value>1</value>
+					</field>
+					<field var="pubsub#persist_items" type="boolean" label="Persist items to storage">
+						<value>1</value>
+					</field>
+					<field var="pubsub#access_model" type="list-single" label="Specify the subscriber model">
+						<option label="authorize">
+							<value>authorize</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<option label="presence">
+							<value>presence</value>
+						</option>
+						<option label="roster">
+							<value>roster</value>
+						</option>
+						<option label="whitelist">
+							<value>whitelist</value>
+						</option>
+						<value>presence</value>
+					</field>
+					<field var="pubsub#publish_model" type="list-single" label="Specify the publisher model">
+						<option label="publishers">
+							<value>publishers</value>
+						</option>
+						<option label="subscribers">
+							<value>subscribers</value>
+						</option>
+						<option label="open">
+							<value>open</value>
+						</option>
+						<value>publishers</value>
+					</field>
+					<field var="pubsub#deliver_notifications" type="boolean" label="Whether to deliver event notifications">
+						<value>1</value>
+					</field>
+					<field var="pubsub#deliver_payloads" type="boolean" label="Whether to deliver payloads with event notifications">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notification_type" type="list-single" label="Specify the delivery style for notifications">
+						<option label="Messages of type normal">
+							<value>normal</value>
+						</option>
+						<option label="Messages of type headline">
+							<value>headline</value>
+						</option>
+						<value>headline</value>
+					</field>
+					<field var="pubsub#notify_delete" type="boolean" label="Whether to notify subscribers when the node is deleted">
+						<value>1</value>
+					</field>
+					<field var="pubsub#notify_retract" type="boolean" label="Whether to notify subscribers when items are removed from the node">
+						<value>1</value>
+					</field>
+				</x>
+			</configure>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq id="a73aac09-74be-4ee2-97e5-571bbdbcd956" type="result"/>
+
+Romeo sends:
+	<iq id="ab0e92d2-c06b-4987-9d45-f9f9e7721709" type="get">
+		<query xmlns="http://jabber.org/protocol/disco#items"/>
+	</iq>
+
+Romeo receives:
+	<iq id="ab0e92d2-c06b-4987-9d45-f9f9e7721709" type="result">
+		<query xmlns="http://jabber.org/protocol/disco#items">
+			<item name="Nice tunes" node="http://jabber.org/protocol/tune" jid="${Romeo's JID}"/>
+		</query>
+	</iq>
+
+Romeo sends:
+	<iq id="67eb1f47-1e69-4cb3-91e2-4d5943e72d4c" type="set">
+		<pubsub xmlns="http://jabber.org/protocol/pubsub">
+			<publish node="http://jabber.org/protocol/tune">
+				<item id="current">
+					<tune xmlns="http://jabber.org/protocol/tune"/>
+				</item>
+			</publish>
+			<publish-options>
+				<x xmlns="jabber:x:data">
+					<field var="FORM_TYPE" type="hidden">
+						<value>http://jabber.org/protocol/pubsub#publish-options</value>
+					</field>
+					<field var="pubsub#access_model">
+						<value>whitelist</value>
+					</field>
+				</x>
+			</publish-options>
+		</pubsub>
+	</iq>
+
+Romeo receives:
+	<iq type='error' id='67eb1f47-1e69-4cb3-91e2-4d5943e72d4c'>
+		<error type='cancel'>
+			<conflict xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/>
+			<text xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'>Field does not match: access_model</text>
+			<precondition-not-met xmlns='http://jabber.org/protocol/pubsub#errors'/>
+		</error>
+	</iq>
+
+Romeo disconnects
+
diff --git a/spec/scansion/server_contact_info.scs b/spec/scansion/server_contact_info.scs
new file mode 100644
index 00000000..6efc7fd1
--- /dev/null
+++ b/spec/scansion/server_contact_info.scs
@@ -0,0 +1,56 @@
+# XEP-0157: Contact Addresses for XMPP Services
+# mod_server_contact_info
+
+[Client] Romeo
+	jid: romeo@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq type='get' id='lx2' to='localhost'>
+		<query xmlns='http://jabber.org/protocol/disco#info'/>
+	</iq>
+
+# Ignore other disco#info features, identities etc
+
+Romeo receives:
+	<iq from='localhost' id='lx2' type='result'>
+		<query xmlns='http://jabber.org/protocol/disco#info' scansion:strict='false'>
+			<x xmlns='jabber:x:data' type='result'>
+				<field type='hidden' var='FORM_TYPE'>
+					<value>http://jabber.org/network/serverinfo</value>
+				</field>
+				<field type='list-multi' var='abuse-addresses'>
+					<value>mailto:abuse@localhost</value>
+					<value>xmpp:abuse@localhost</value>
+				</field>
+				<field type='list-multi' var='admin-addresses'>
+					<value>mailto:admin@localhost</value>
+					<value>xmpp:admin@localhost</value>
+				</field>
+				<field type='list-multi' var='feedback-addresses'>
+					<value>http://localhost/feedback.html</value>
+					<value>mailto:feedback@localhost</value>
+					<value>xmpp:feedback@localhost</value>
+				</field>
+				<field type='list-multi' var='sales-addresses'>
+					<value>xmpp:sales@localhost</value>
+				</field>
+				<field type='list-multi' var='security-addresses'>
+					<value>xmpp:security@localhost</value>
+				</field>
+				<field type='list-multi' var='status-addresses'>
+					<value>gopher://status.localhost</value>
+				</field>
+				<field type='list-multi' var='support-addresses'>
+					<value>https://localhost/support.html</value>
+					<value>xmpp:support@localhost</value>
+				</field>
+			</x>
+		</query>
+	</iq>
+
+Romeo disconnects
diff --git a/spec/scansion/uptime.scs b/spec/scansion/uptime.scs
new file mode 100644
index 00000000..188b9eb5
--- /dev/null
+++ b/spec/scansion/uptime.scs
@@ -0,0 +1,21 @@
+# XEP-0012: Last Activity / mod_uptime
+
+[Client] Romeo
+	jid: romeo@localhost
+	password: password
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq id='a' type='get' to='localhost'>
+		<query xmlns='jabber:iq:last'/>
+	</iq>
+
+Romeo receives:
+	<iq type='result' id='a' from='localhost'>
+		<query xmlns='jabber:iq:last' seconds='0'/>
+	</iq>
+
+Romeo disconnects
diff --git a/spec/scansion/version.scs b/spec/scansion/version.scs
new file mode 100644
index 00000000..6c841dd9
--- /dev/null
+++ b/spec/scansion/version.scs
@@ -0,0 +1,27 @@
+# XEP-0092: Software Version / mod_version
+
+[Client] Romeo
+	password: password
+	jid: romeo@localhost/dfaZpuxV
+
+-----
+
+Romeo connects
+
+Romeo sends:
+	<iq id='lx2' to='localhost' type='get'>
+		<query xmlns='jabber:iq:version'/>
+	</iq>
+
+# Version string would vary so we can't do an exact match atm
+# Inclusion of <os/> is disabled in the config, it should be absent
+Romeo receives:
+	<iq id='lx2' from='localhost' type='result'>
+		<query xmlns='jabber:iq:version' scansion:strict='true'>
+			<name>Prosody</name>
+			<version scansion:strict='false'/>
+		</query>
+	</iq>
+
+
+Romeo disconnects
diff --git a/spec/util_array_spec.lua b/spec/util_array_spec.lua
new file mode 100644
index 00000000..1d9da947
--- /dev/null
+++ b/spec/util_array_spec.lua
@@ -0,0 +1,155 @@
+local array = require "util.array";
+describe("util.array", function ()
+	describe("creation", function ()
+		describe("from table", function ()
+			it("works", function ()
+				local a = array({"a", "b", "c"});
+				assert.same({"a", "b", "c"}, a);
+			end);
+		end);
+
+		describe("from iterator", function ()
+			it("works", function ()
+				-- collects the first value, ie the keys
+				local a = array(ipairs({true, true, true}));
+				assert.same({1, 2, 3}, a);
+			end);
+		end);
+
+		describe("collect", function ()
+			it("works", function ()
+				-- collects the first value, ie the keys
+				local a = array.collect(ipairs({true, true, true}));
+				assert.same({1, 2, 3}, a);
+			end);
+		end);
+
+	end);
+
+	describe("metatable", function ()
+		describe("operator", function ()
+			describe("addition", function ()
+				it("works", function ()
+					local a = array({ "a", "b" });
+					local b = array({ "c", "d" });
+					assert.same({"a", "b", "c", "d"}, a + b);
+				end);
+			end);
+
+			describe("equality", function ()
+				it("works", function ()
+					local a1 = array({ "a", "b" });
+					local a2 = array({ "a", "b" });
+					local b = array({ "c", "d" });
+					assert.truthy(a1 == a2);
+					assert.falsy(a1 == b);
+					assert.falsy(a1 == { "a", "b" }, "Behavior of metatables changed in Lua 5.3");
+				end);
+			end);
+
+			describe("division", function ()
+				it("works", function ()
+					local a = array({ "a", "b", "c" });
+					local b = a / function (i) if i ~= "b" then return i .. "x" end end;
+					assert.same({ "ax", "cx" }, b);
+				end);
+			end);
+
+		end);
+	end);
+
+	describe("methods", function ()
+		describe("map", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				local b = a:map(string.upper);
+				assert.same({ "A", "B", "C" }, b);
+			end);
+		end);
+
+		describe("filter", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:filter(function (i) return i ~= "b" end);
+				assert.same({ "a", "c" }, a);
+			end);
+		end);
+
+		describe("sort", function ()
+			it("works", function ()
+				local a = array({ 5, 4, 3, 1, 2, });
+				a:sort();
+				assert.same({ 1, 2, 3, 4, 5, }, a);
+			end);
+		end);
+
+		describe("unique", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c", "c", "a", "b" });
+				a:unique();
+				assert.same({ "a", "b", "c" }, a);
+			end);
+		end);
+
+		describe("pluck", function ()
+			it("works", function ()
+				local a = array({ { a = 1, b = -1 }, { a = 2, b = -2 }, });
+				a:pluck("a");
+				assert.same({ 1, 2 }, a);
+			end);
+		end);
+
+
+		describe("reverse", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:reverse();
+				assert.same({ "c", "b", "a" }, a);
+			end);
+		end);
+
+		-- TODO :shuffle
+
+		describe("append", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:append(array({ "d", "e", }));
+				assert.same({ "a", "b", "c", "d", "e" }, a);
+			end);
+		end);
+
+		describe("push", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				a:push("d"):push("e");
+				assert.same({ "a", "b", "c", "d", "e" }, a);
+			end);
+		end);
+
+		describe("pop", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal("c", a:pop());
+				assert.same({ "a", "b", }, a);
+			end);
+		end);
+
+		describe("concat", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal("a,b,c", a:concat(","));
+			end);
+		end);
+
+		describe("length", function ()
+			it("works", function ()
+				local a = array({ "a", "b", "c" });
+				assert.equal(3, a:length());
+			end);
+		end);
+
+	end);
+
+	-- TODO The various array.foo(array ina, array outa) functions
+end);
+
diff --git a/spec/util_async_spec.lua b/spec/util_async_spec.lua
index d2de8c94..8123503b 100644
--- a/spec/util_async_spec.lua
+++ b/spec/util_async_spec.lua
@@ -544,6 +544,8 @@ describe("util.async", function()
 			assert.equal(r1.state, "ready");
 		end);
 
+		-- luacheck: ignore 211/rf
+		-- FIXME what's rf?
 		it("should support multiple done() calls", function ()
 			local processed_item;
 			local wait, done;
diff --git a/spec/util_dataforms_spec.lua b/spec/util_dataforms_spec.lua
index 89759035..d2d1264a 100644
--- a/spec/util_dataforms_spec.lua
+++ b/spec/util_dataforms_spec.lua
@@ -106,11 +106,26 @@ describe("util.dataforms", function ()
 				name = "text-single-field",
 				value = "text-single-value",
 			},
+			{
+				-- XEP-0221
+				-- TODO Validate the XML produced by this.
+				type = "text-single",
+				label = "text-single-with-media-label",
+				name = "text-single-with-media-field",
+				media = {
+					height = 24,
+					width = 32,
+					{
+						type = "image/png",
+						uri = "data:",
+					},
+				},
+			},
 		});
 		xform = some_form:form();
 	end);
 
-	it("works", function ()
+	it("XML serialization looks like it should", function ()
 		assert.truthy(xform);
 		assert.truthy(st.is_stanza(xform));
 		assert.equal("x", xform.name);
@@ -316,7 +331,7 @@ describe("util.dataforms", function ()
 	end);
 
 	describe(":data", function ()
-		it("works", function ()
+		it("returns something", function ()
 			assert.truthy(some_form:data(xform));
 		end);
 	end);
@@ -402,26 +417,63 @@ describe("util.dataforms", function ()
 		end);
 	end);
 
-	describe("validation", function ()
+	describe("datatype validation", function ()
 		local f = dataforms.new {
 			{
 				name = "number",
 				type = "text-single",
 				datatype = "xs:integer",
+				range_min = -10,
+				range_max = 10,
 			},
 		};
 
-		it("works", function ()
+		it("integer roundtrip works", function ()
 			local d = f:data(f:form({number = 1}));
 			assert.equal(1, d.number);
 		end);
 
-		it("works", function ()
+		it("integer error handling works", function ()
 			local d,e = f:data(f:form({number = "nan"}));
 			assert.not_equal(1, d.number);
 			assert.table(e);
 			assert.string(e.number);
 		end);
+
+		it("works", function ()
+			local d,e = f:data(f:form({number = 100}));
+			assert.not_equal(100, d.number);
+			assert.table(e);
+			assert.string(e.number);
+		end);
+	end);
+	describe("media element", function ()
+		it("produced media element correctly", function ()
+			local f;
+			for field in xform:childtags("field") do
+				if field.attr.var == "text-single-with-media-field" then
+					f = field;
+					break;
+				end
+			end
+
+			assert.truthy(st.is_stanza(f));
+			assert.equal("text-single-with-media-field", f.attr.var);
+			assert.equal("text-single", f.attr.type);
+			assert.equal("text-single-with-media-label", f.attr.label);
+			assert.equal(0, iter.count(f:childtags("value")));
+
+			local m = f:get_child("media", "urn:xmpp:media-element");
+			assert.truthy(st.is_stanza(m));
+			assert.equal("24", m.attr.height);
+			assert.equal("32", m.attr.width);
+			assert.equal(1, iter.count(m:childtags("uri")));
+
+			local u = m:get_child("uri");
+			assert.truthy(st.is_stanza(u));
+			assert.equal("image/png", u.attr.type);
+			assert.equal("data:", u:get_text());
+		end);
 	end);
 end);
 
diff --git a/spec/util_error_spec.lua b/spec/util_error_spec.lua
new file mode 100644
index 00000000..34edd313
--- /dev/null
+++ b/spec/util_error_spec.lua
@@ -0,0 +1,146 @@
+local errors = require "util.error"
+
+describe("util.error", function ()
+	describe("new()", function ()
+		it("works", function ()
+			local err = errors.new("bork", "bork bork");
+			assert.not_nil(err);
+			assert.equal("cancel", err.type);
+			assert.equal("undefined-condition", err.condition);
+			assert.same("bork bork", err.context);
+		end);
+
+		describe("templates", function ()
+			it("works", function ()
+				local templates = {
+					["fail"] = {
+						type = "wait",
+						condition = "internal-server-error",
+						code = 555;
+					};
+				};
+				local err = errors.new("fail", { traceback = "in some file, somewhere" }, templates);
+				assert.equal("wait", err.type);
+				assert.equal("internal-server-error", err.condition);
+				assert.equal(555, err.code);
+				assert.same({ traceback = "in some file, somewhere" }, err.context);
+			end);
+		end);
+
+	end);
+
+	describe("is_err()", function ()
+		it("works", function ()
+			assert.truthy(errors.is_err(errors.new()));
+			assert.falsy(errors.is_err("not an error"));
+		end);
+	end);
+
+	describe("coerce", function ()
+		it("works", function ()
+			local ok, err = errors.coerce(nil, "it dun goofed");
+			assert.is_nil(ok);
+			assert.truthy(errors.is_err(err))
+		end);
+	end);
+
+	describe("from_stanza", function ()
+		it("works", function ()
+			local st = require "util.stanza";
+			local m = st.message({ type = "chat" });
+			local e = st.error_reply(m, "modify", "bad-request", nil, "error.example"):tag("extra", { xmlns = "xmpp:example.test" });
+			local err = errors.from_stanza(e);
+			assert.truthy(errors.is_err(err));
+			assert.equal("modify", err.type);
+			assert.equal("bad-request", err.condition);
+			assert.equal(e, err.context.stanza);
+			assert.equal("error.example", err.context.by);
+			assert.not_nil(err.extra.tag);
+		end);
+	end);
+
+	describe("__tostring", function ()
+		it("doesn't throw", function ()
+			assert.has_no.errors(function ()
+				-- See 6f317e51544d
+				tostring(errors.new());
+			end);
+		end);
+	end);
+
+	describe("extra", function ()
+		it("keeps some extra fields", function ()
+			local err = errors.new({condition="gone",text="Sorry mate, it's all gone",extra={uri="file:///dev/null"}});
+			assert.is_table(err.extra);
+			assert.equal("file:///dev/null", err.extra.uri);
+		end);
+	end)
+
+	describe("init", function()
+		it("basics works", function()
+			local reg = errors.init("test", {
+				broke = {type = "cancel"; condition = "internal-server-error"; text = "It broke :("};
+				nope = {type = "auth"; condition = "not-authorized"; text = "Can't let you do that Dave"};
+			});
+
+			local broke = reg.new("broke");
+			assert.equal("cancel", broke.type);
+			assert.equal("internal-server-error", broke.condition);
+			assert.equal("It broke :(", broke.text);
+			assert.equal("test", broke.source);
+
+			local nope = reg.new("nope");
+			assert.equal("auth", nope.type);
+			assert.equal("not-authorized", nope.condition);
+			assert.equal("Can't let you do that Dave", nope.text);
+		end);
+
+		it("compact mode works", function()
+			local reg = errors.init("test", "spec", {
+				broke = {"cancel"; "internal-server-error"; "It broke :("};
+				nope = {"auth"; "not-authorized"; "Can't let you do that Dave"; "sorry-dave"};
+			});
+
+			local broke = reg.new("broke");
+			assert.equal("cancel", broke.type);
+			assert.equal("internal-server-error", broke.condition);
+			assert.equal("It broke :(", broke.text);
+			assert.is_nil(broke.extra);
+
+			local nope = reg.new("nope");
+			assert.equal("auth", nope.type);
+			assert.equal("not-authorized", nope.condition);
+			assert.equal("Can't let you do that Dave", nope.text);
+			assert.equal("spec", nope.extra.namespace);
+			assert.equal("sorry-dave", nope.extra.condition);
+		end);
+
+		it("registry looks the same regardless of syntax", function()
+			local normal = errors.init("test", {
+				broke = {type = "cancel"; condition = "internal-server-error"; text = "It broke :("};
+				nope = {
+					type = "auth";
+					condition = "not-authorized";
+					text = "Can't let you do that Dave";
+					extra = {namespace = "spec"; condition = "sorry-dave"};
+				};
+			});
+			local compact1 = errors.init("test", "spec", {
+				broke = {"cancel"; "internal-server-error"; "It broke :("};
+				nope = {"auth"; "not-authorized"; "Can't let you do that Dave"; "sorry-dave"};
+			});
+			local compact2 = errors.init("test", {
+				broke = {"cancel"; "internal-server-error"; "It broke :("};
+				nope = {"auth"; "not-authorized"; "Can't let you do that Dave"};
+			});
+			assert.same(normal.registry, compact1.registry);
+
+			assert.same({
+				broke = {type = "cancel"; condition = "internal-server-error"; text = "It broke :("};
+				nope = {type = "auth"; condition = "not-authorized"; text = "Can't let you do that Dave"};
+			}, compact2.registry);
+		end);
+	end);
+
+end);
+
diff --git a/spec/util_events_spec.lua b/spec/util_events_spec.lua
index fee60f8f..fcfa6e53 100644
--- a/spec/util_events_spec.lua
+++ b/spec/util_events_spec.lua
@@ -208,5 +208,43 @@ describe("util.events", function ()
 				assert.spy(h).was_called(2);
 			end);
 		end);
+
+		describe("debug hooks", function ()
+			it("should get called", function ()
+				local d = spy.new(function (handler, event_name, event_data) --luacheck: ignore 212/event_name
+					return handler(event_data);
+				end);
+
+				e.add_handler("myevent", h);
+				e.fire_event("myevent");
+
+				assert.spy(h).was_called(1);
+				assert.spy(d).was_called(0);
+
+				assert.is_nil(e.set_debug_hook(d));
+
+				e.fire_event("myevent", { mydata = true });
+
+				assert.spy(h).was_called(2);
+				assert.spy(d).was_called(1);
+				assert.spy(d).was_called_with(h, "myevent", { mydata = true });
+
+				assert.equal(d, e.set_debug_hook(nil));
+
+				e.fire_event("myevent", { mydata = false });
+
+				assert.spy(h).was_called(3);
+				assert.spy(d).was_called(1);
+			end);
+			it("setting should return any existing debug hook", function ()
+				local function f() end
+				local function g() end
+				assert.is_nil(e.set_debug_hook(f));
+				assert.is_equal(f, e.set_debug_hook(g));
+				assert.is_equal(g, e.set_debug_hook(f));
+				assert.is_equal(f, e.set_debug_hook(nil));
+				assert.is_nil(e.set_debug_hook(f));
+			end);
+		end);
 	end);
 end);
diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua
index 7e6a0c6e..50509630 100644
--- a/spec/util_format_spec.lua
+++ b/spec/util_format_spec.lua
@@ -5,10 +5,15 @@ describe("util.format", function()
 		it("should work", function()
 			assert.equal("hello", format("%s", "hello"));
 			assert.equal("<nil>", format("%s"));
+			assert.equal("<nil>", format("%d"));
+			assert.equal("<nil>", format("%q"));
 			assert.equal(" [<nil>]", format("", nil));
 			assert.equal("true", format("%s", true));
 			assert.equal("[true]", format("%d", true));
 			assert.equal("% [true]", format("%%", true));
+			assert.equal("{ }", format("%q", { }));
+			assert.equal("[1.5]", format("%d", 1.5));
+			assert.equal("[7.3786976294838e+19]", format("%d", 73786976294838206464));
 		end);
 	end);
 end);
diff --git a/spec/util_hashes_spec.lua b/spec/util_hashes_spec.lua
new file mode 100644
index 00000000..3639dd4e
--- /dev/null
+++ b/spec/util_hashes_spec.lua
@@ -0,0 +1,55 @@
+-- Test vectors from RFC 6070
+local hashes = require "util.hashes";
+local hex = require "util.hex";
+
+-- Also see spec for util.hmac where HMAC test cases reside
+
+describe("PBKDF2-HMAC-SHA1", function ()
+	it("test vector 1", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 1
+		local DK = "0c60c80f961f0e71f3a9b524af6012062fe037a6";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha1(P, S, c)));
+	end);
+	it("test vector 2", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 2
+		local DK = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha1(P, S, c)));
+	end);
+	it("test vector 3", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 4096
+		local DK = "4b007901b765489abead49d926f721d065a429c1";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha1(P, S, c)));
+	end);
+	it("test vector 4 #SLOW", function ()
+		local P = "password"
+		local S = "salt"
+		local c = 16777216
+		local DK = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha1(P, S, c)));
+	end);
+end);
+
+describe("PBKDF2-HMAC-SHA256", function ()
+	it("test vector 1", function ()
+		local P = "password";
+		local S = "salt";
+		local c = 1
+		local DK = "120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha256(P, S, c)));
+	end);
+	it("test vector 2", function ()
+		local P = "password";
+		local S = "salt";
+		local c = 2
+		local DK = "ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43";
+		assert.equal(DK, hex.to(hashes.pbkdf2_hmac_sha256(P, S, c)));
+	end);
+end);
+
+
diff --git a/spec/util_hashring_spec.lua b/spec/util_hashring_spec.lua
new file mode 100644
index 00000000..d8801774
--- /dev/null
+++ b/spec/util_hashring_spec.lua
@@ -0,0 +1,85 @@
+local hashring = require "util.hashring";
+
+describe("util.hashring", function ()
+
+	local sha256 = require "util.hashes".sha256;
+
+	local ring = hashring.new(128, sha256);
+
+	it("should fail to get a node that does not exist", function ()
+		assert.is_nil(ring:get_node("foo"))
+	end);
+
+	it("should support adding nodes", function ()
+		ring:add_node("node1");
+	end);
+
+	it("should return a single node for all keys if only one node exists", function ()
+		for i = 1, 100 do
+			assert.is_equal("node1", ring:get_node(tostring(i)))
+		end
+	end);
+
+	it("should support adding a second node", function ()
+		ring:add_node("node2");
+	end);
+
+	it("should fail to remove a non-existent node", function ()
+		assert.is_falsy(ring:remove_node("node3"));
+	end);
+
+	it("should succeed to remove a node", function ()
+		assert.is_truthy(ring:remove_node("node1"));
+	end);
+
+	it("should return the only node for all keys", function ()
+		for i = 1, 100 do
+			assert.is_equal("node2", ring:get_node(tostring(i)))
+		end
+	end);
+
+	it("should support adding multiple nodes", function ()
+		ring:add_nodes({ "node1", "node3", "node4", "node5" });
+	end);
+
+	it("should disrupt a minimal number of keys on node removal", function ()
+		local orig_ring = ring:clone();
+		local node_tallies = {};
+
+		local n = 1000;
+
+		for i = 1, n do
+			local key = tostring(i);
+			local node = ring:get_node(key);
+			node_tallies[node] = (node_tallies[node] or 0) + 1;
+		end
+
+		--[[
+		for node, key_count in pairs(node_tallies) do
+			print(node, key_count, ("%.2f%%"):format((key_count/n)*100));
+		end
+		]]
+
+		ring:remove_node("node5");
+
+		local disrupted_keys = 0;
+		for i = 1, n do
+			local key = tostring(i);
+			if orig_ring:get_node(key) ~= ring:get_node(key) then
+				disrupted_keys = disrupted_keys + 1;
+			end
+		end
+		assert.is_equal(node_tallies["node5"], disrupted_keys);
+	end);
+
+	it("should support removing multiple nodes", function ()
+		ring:remove_nodes({"node2", "node3", "node4", "node5"});
+	end);
+
+	it("should return a single node for all keys if only one node remains", function ()
+		for i = 1, 100 do
+			assert.is_equal("node1", ring:get_node(tostring(i)))
+		end
+	end);
+
+end);
diff --git a/spec/util_hmac_spec.lua b/spec/util_hmac_spec.lua
new file mode 100644
index 00000000..a2125c3a
--- /dev/null
+++ b/spec/util_hmac_spec.lua
@@ -0,0 +1,106 @@
+-- Test cases from RFC 4231
+
+-- Yes, the lines are long, it's annoying to split the long hex things.
+-- luacheck: ignore 631
+
+local hmac = require "util.hmac";
+local hex = require "util.hex";
+
+describe("Test case 1", function ()
+	local Key  = hex.from("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b");
+	local Data = hex.from("4869205468657265");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 2", function ()
+	local Key  = hex.from("4a656665");
+	local Data = hex.from("7768617420646f2079612077616e7420666f72206e6f7468696e673f");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 3", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("fa73b0089d56a284efb0f0756c890be9b1b5dbdd8ee81a3655f83e33b2279d39bf3e848279a722c806b485a47e67c807b946a337bee8942674278859e13292fb", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 4", function ()
+	local Key  = hex.from("0102030405060708090a0b0c0d0e0f10111213141516171819");
+	local Data = hex.from("cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("b0ba465637458c6990e5a8c5f61d4af7e576d97ff94b872de76f8050361ee3dba91ca5c11aa25eb4d679275cc5788063a5f19741120c4f2de2adebeb10a298dd", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 5", function ()
+	local Key  = hex.from("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c");
+	local Data = hex.from("546573742057697468205472756e636174696f6e");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("a3b6167473100ee06e0c796c2955552b", hmac.sha256(Key, Data, true):sub(1,128/4))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("415fad6271580a531d4179bc891d87a6", hmac.sha512(Key, Data, true):sub(1,128/4))
+		end);
+	end);
+end);
+describe("Test case 6", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("54657374205573696e67204c6172676572205468616e20426c6f636b2d53697a65204b6579202d2048617368204b6579204669727374");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("80b24263c7c1a3ebb71493c1dd7be8b49b46d1f41b4aeec1121b013783f8f3526b56d037e05f2598bd0fd2215d6a1e5295e64f73f63f0aec8b915a985d786598", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
+describe("Test case 7", function ()
+	local Key  = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+	local Data = hex.from("5468697320697320612074657374207573696e672061206c6172676572207468616e20626c6f636b2d73697a65206b657920616e642061206c6172676572207468616e20626c6f636b2d73697a6520646174612e20546865206b6579206e6565647320746f20626520686173686564206265666f7265206265696e6720757365642062792074686520484d414320616c676f726974686d2e");
+	describe("HMAC-SHA-256", function ()
+		it("works", function()
+			assert.equal("9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", hmac.sha256(Key, Data, true))
+		end);
+	end);
+	describe("HMAC-SHA-512", function ()
+		it("works", function()
+			assert.equal("e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58", hmac.sha512(Key, Data, true))
+		end);
+	end);
+end);
diff --git a/spec/util_http_spec.lua b/spec/util_http_spec.lua
index 0f51a86c..c6087450 100644
--- a/spec/util_http_spec.lua
+++ b/spec/util_http_spec.lua
@@ -28,6 +28,11 @@ describe("util.http", function()
 		it("should decode important URL characters", function()
 			assert.are.equal("This & that = something", http.urldecode("This%20%26%20that%20%3d%20something"), "Important URL chars escaped");
 		end);
+
+		it("should decode both lower and uppercase", function ()
+			assert.are.equal("This & that = {something}.", http.urldecode("This%20%26%20that%20%3D%20%7Bsomething%7D%2E"), "Important URL chars escaped");
+		end);
+
 	end);
 
 	describe("#formencode()", function()
@@ -84,4 +89,23 @@ describe("util.http", function()
 			assert.equal("/foo/", http.normalize_path("/foo/", true));
 		end);
 	end);
+
+	describe("contains_token", function ()
+		it("is present in field", function ()
+			assert.is_true(http.contains_token("foo", "foo"));
+			assert.is_true(http.contains_token("foo, bar", "foo"));
+			assert.is_true(http.contains_token("foo,bar", "foo"));
+			assert.is_true(http.contains_token("bar,  foo,baz", "foo"));
+		end);
+
+		it("is absent from field", function ()
+			assert.is_false(http.contains_token("bar", "foo"));
+			assert.is_false(http.contains_token("fooo", "foo"));
+			assert.is_false(http.contains_token("foo o,bar", "foo"));
+		end);
+
+		it("is weird", function ()
+			assert.is_(http.contains_token("fo o", "foo"));
+		end);
+	end);
 end);
diff --git a/spec/util_human_io_spec.lua b/spec/util_human_io_spec.lua
new file mode 100644
index 00000000..ead117df
--- /dev/null
+++ b/spec/util_human_io_spec.lua
@@ -0,0 +1,29 @@
+describe("util.human.io", function ()
+	local human_io
+	setup(function ()
+		human_io = require "util.human.io";
+	end);
+	describe("table", function ()
+
+		it("alignment works", function ()
+			local row = human_io.table({
+					{
+						width = 3,
+						align = "right"
+					},
+					{
+						width = 3,
+					},
+				});
+
+			assert.equal("  1 | .  ", row({ 1, "." }));
+			assert.equal(" 10 | .. ", row({ 10, ".." }));
+			assert.equal("100 | ...", row({ 100, "..." }));
+			assert.equal("10… | ..…", row({ 1000, "...." }));
+
+		end);
+	end);
+end);
+
+
+
diff --git a/spec/util_human_units_spec.lua b/spec/util_human_units_spec.lua
new file mode 100644
index 00000000..4326cdd4
--- /dev/null
+++ b/spec/util_human_units_spec.lua
@@ -0,0 +1,15 @@
+local units = require "util.human.units";
+
+describe("util.human.units", function ()
+	describe("format", function ()
+		it("formats numbers with SI units", function ()
+			assert.equal("1 km", units.format(1000, "m"));
+			assert.equal("1 GJ", units.format(1000000000, "J"));
+			assert.equal("1 ms", units.format(1/1000, "s"));
+			assert.equal("10 ms", units.format(10/1000, "s"));
+			assert.equal("1 ns", units.format(1/1000000000, "s"));
+			assert.equal("1 KiB", units.format(1024, "B", 'b'));
+			assert.equal("1 MiB", units.format(1024*1024, "B", 'b'));
+		end);
+	end);
+end);
diff --git a/spec/util_interpolation_spec.lua b/spec/util_interpolation_spec.lua
new file mode 100644
index 00000000..614d1f54
--- /dev/null
+++ b/spec/util_interpolation_spec.lua
@@ -0,0 +1,61 @@
+local template = [[
+{greet!?Hi}, {name?world}!
+]];
+local expect1 = [[
+Hello, WORLD!
+]];
+local expect2 = [[
+Hello, world!
+]];
+local expect3 = [[
+Hi, YOU!
+]];
+local template_array = [[
+{foo#{idx}. {item}
+}]]
+local expect_array = [[
+1. HELLO
+2. WORLD
+]]
+local template_func_pipe = [[
+{foo|sort#{idx}. {item}
+}]]
+local expect_func_pipe = [[
+1. A
+2. B
+3. C
+4. D
+]]
+local template_map = [[
+{foo%{idx}: {item!}
+}]]
+local expect_map = [[
+FOO: bar
+]]
+local template_not = [[
+{thing~Thing is falsy}{thing&Thing is truthy}
+]]
+local expect_not_true = [[
+Thing is truthy
+]]
+local expect_not_nil = [[
+Thing is falsy
+]]
+local expect_not_false = [[
+Thing is falsy
+]]
+describe("util.interpolation", function ()
+	it("renders", function ()
+		local render = require "util.interpolation".new("%b{}", string.upper, { sort = function (t) table.sort(t) return t end });
+		assert.equal(expect1, render(template, { greet = "Hello", name = "world" }));
+		assert.equal(expect2, render(template, { greet = "Hello" }));
+		assert.equal(expect3, render(template, { name = "you" }));
+		assert.equal(expect_array, render(template_array, { foo = { "Hello", "World" } }));
+		assert.equal(expect_func_pipe, render(template_func_pipe, { foo = { "c", "a", "d", "b", } }));
+		-- assert.equal("", render(template_func_pipe, { foo = nil })); -- FIXME
+		assert.equal(expect_map, render(template_map, { foo = { foo = "bar" } }));
+		assert.equal(expect_not_true, render(template_not, { thing = true }));
+		assert.equal(expect_not_nil, render(template_not, { thing = nil }));
+		assert.equal(expect_not_false, render(template_not, { thing = false }));
+	end);
+end);
diff --git a/spec/util_jid_spec.lua b/spec/util_jid_spec.lua
index c075212f..17cadbee 100644
--- a/spec/util_jid_spec.lua
+++ b/spec/util_jid_spec.lua
@@ -75,6 +75,56 @@ describe("util.jid", function()
 		end);
 	end);
 
+	local jid_escaping_test_vectors = {
+		-- From https://xmpp.org/extensions/xep-0106.xml#examples sans @example.com
+		[[space cadet]], [[space\20cadet]],
+		[[call me "ishmael"]], [[call\20me\20\22ishmael\22]],
+		[[at&t guy]], [[at\26t\20guy]],
+		[[d'artagnan]], [[d\27artagnan]],
+		[[/.fanboy]], [[\2f.fanboy]],
+		[[::foo::]], [[\3a\3afoo\3a\3a]],
+		[[<foo>]], [[\3cfoo\3e]],
+		[[user@host]], [[user\40host]],
+		[[c:\net]], [[c\3a\net]],
+		[[c:\\net]], [[c\3a\\net]],
+		[[c:\cool stuff]], [[c\3a\cool\20stuff]],
+		[[c:\5commas]], [[c\3a\5c5commas]],
+
+		-- Section 4.2
+		[[\3and\2is\5cool]], [[\5c3and\2is\5c5cool]],
+
+		-- From aioxmpp
+		[[\5c]], [[\5c5c]],
+    -- [[\5C]], [[\5C]],
+    [[\2plus\2is\4]], [[\2plus\2is\4]],
+    [[foo\bar]], [[foo\bar]],
+    [[foo\41r]], [[foo\41r]],
+    -- additional test vectors
+    [[call\20me]], [[call\5c20me]],
+	};
+
+	describe("#escape()", function ()
+		it("should work", function ()
+			for i = 1, #jid_escaping_test_vectors, 2 do
+				local original = jid_escaping_test_vectors[i];
+				local escaped = jid_escaping_test_vectors[i+1];
+
+				assert.are.equal(escaped, jid.escape(original), ("Escapes '%s' -> '%s'"):format(original, escaped));
+			end
+		end);
+	end)
+
+	describe("#unescape()", function ()
+		it("should work", function ()
+			for i = 1, #jid_escaping_test_vectors, 2 do
+				local original = jid_escaping_test_vectors[i];
+				local escaped = jid_escaping_test_vectors[i+1];
+
+				assert.are.equal(original, jid.unescape(escaped), ("Unescapes '%s' -> '%s'"):format(escaped, original));
+			end
+		end);
+	end)
+
 	it("should work with nodes", function()
 		local function test(_jid, expected_node)
 			assert.are.equal(jid.node(_jid), expected_node, "Unexpected node for "..tostring(_jid));
diff --git a/spec/util_json_spec.lua b/spec/util_json_spec.lua
index 43360540..f07cd525 100644
--- a/spec/util_json_spec.lua
+++ b/spec/util_json_spec.lua
@@ -1,5 +1,6 @@
 
 local json = require "util.json";
+local array = require "util.array";
 
 describe("util.json", function()
 	describe("#encode()", function()
@@ -67,4 +68,13 @@ describe("util.json", function()
 			end
 		end);
 	end)
+
+	describe("util.array integration", function ()
+		it("works", function ()
+			assert.equal("[]", json.encode(array()));
+			assert.equal("[1,2,3]", json.encode(array({1,2,3})));
+			assert.equal(getmetatable(array()), getmetatable(json.decode("[]")));
+		end);
+	end);
+
 end);
diff --git a/spec/util_jwt_spec.lua b/spec/util_jwt_spec.lua
new file mode 100644
index 00000000..b391a870
--- /dev/null
+++ b/spec/util_jwt_spec.lua
@@ -0,0 +1,20 @@
+local jwt = require "util.jwt";
+
+describe("util.jwt", function ()
+	it("validates", function ()
+		local key = "secret";
+		local token = jwt.sign(key, { payload = "this" });
+		assert.string(token);
+		local ok, parsed = jwt.verify(key, token);
+		assert.truthy(ok)
+		assert.same({ payload = "this" }, parsed);
+	end);
+	it("rejects invalid", function ()
+		local key = "secret";
+		local token = jwt.sign("wrong", { payload = "this" });
+		assert.string(token);
+		local ok = jwt.verify(key, token);
+		assert.falsy(ok)
+	end);
+end);
+
diff --git a/spec/util_paths_spec.lua b/spec/util_paths_spec.lua
new file mode 100644
index 00000000..2e8a0c08
--- /dev/null
+++ b/spec/util_paths_spec.lua
@@ -0,0 +1,39 @@
+local sep = package.config:match("(.)\n");
+describe("util.paths", function ()
+	local paths = require "util.paths";
+	describe("#join()", function ()
+		it("returns single component as-is", function ()
+			assert.equal("foo", paths.join("foo"));
+		end);
+		it("joins paths", function ()
+			assert.equal("foo"..sep.."bar", paths.join("foo", "bar"))
+		end);
+		it("joins longer paths", function ()
+			assert.equal("foo"..sep.."bar"..sep.."baz", paths.join("foo", "bar", "baz"))
+		end);
+		it("joins even longer paths", function ()
+			assert.equal("foo"..sep.."bar"..sep.."baz"..sep.."moo", paths.join("foo", "bar", "baz", "moo"))
+		end);
+	end)
+
+	describe("#glob_to_pattern()", function ()
+		it("works", function ()
+			assert.equal("^thing.%..*$", paths.glob_to_pattern("thing?.*"))
+		end);
+	end)
+
+	describe("#resolve_relative_path()", function ()
+		it("returns absolute paths as-is", function ()
+			if sep == "/" then
+				assert.equal("/tmp/path", paths.resolve_relative_path("/run", "/tmp/path"));
+			elseif sep == "\\" then
+				assert.equal("C:\\Program Files", paths.resolve_relative_path("A:\\", "C:\\Program Files"));
+			end
+		end);
+		it("resolves relative paths", function ()
+			if sep == "/" then
+				assert.equal("/run/path", paths.resolve_relative_path("/run", "path"));
+			end
+		end);
+	end)
+end)
diff --git a/spec/util_promise_spec.lua b/spec/util_promise_spec.lua
index 65d252f6..6fb5d6bd 100644
--- a/spec/util_promise_spec.lua
+++ b/spec/util_promise_spec.lua
@@ -248,6 +248,30 @@ describe("util.promise", function ()
 			assert.spy(cb3).was_called(1);
 			assert.spy(cb3).was_called_with("goodbye");
 		end);
+
+		it("ordinary values", function ()
+			local p = promise.resolve()
+			local cb = spy.new(function ()
+				return "hello"
+			end);
+			local cb2 = spy.new(function () end);
+			p:next(cb):next(cb2);
+			assert.spy(cb).was_called(1);
+			assert.spy(cb2).was_called(1);
+			assert.spy(cb2).was_called_with("hello");
+		end);
+
+		it("nil", function ()
+			local p = promise.resolve()
+			local cb = spy.new(function ()
+				return
+			end);
+			local cb2 = spy.new(function () end);
+			p:next(cb):next(cb2);
+			assert.spy(cb).was_called(1);
+			assert.spy(cb2).was_called(1);
+			assert.spy(cb2).was_called_with(nil);
+		end);
 	end);
 
 	describe("race()", function ()
@@ -329,6 +353,60 @@ describe("util.promise", function ()
 			assert.equal("fail", result);
 		end);
 	end);
+	describe("all_settled()", function ()
+		it("works with fulfilled promises", function ()
+			local p1, p2 = promise.resolve("yep"), promise.resolve("nope");
+			local p = promise.all_settled({ p1, p2 });
+			local result;
+			p:next(function (v)
+				result = v;
+			end);
+			assert.same({
+				{ status = "fulfilled", value = "yep" };
+				{ status = "fulfilled", value = "nope" };
+			}, result);
+		end);
+		it("works with pending promises", function ()
+			local r1, r2;
+			local p1, p2 = promise.new(function (resolve) r1 = resolve end), promise.new(function (resolve) r2 = resolve end);
+			local p = promise.all_settled({ p1, p2 });
+
+			local result;
+			local cb = spy.new(function (v)
+				result = v;
+			end);
+			p:next(cb);
+			assert.spy(cb).was_called(0);
+			r2("yep");
+			assert.spy(cb).was_called(0);
+			r1("nope");
+			assert.spy(cb).was_called(1);
+			assert.same({
+				{ status = "fulfilled", value = "nope" };
+				{ status = "fulfilled", value = "yep" };
+			}, result);
+		end);
+		it("works when some promises reject", function ()
+			local r1, r2;
+			local p1, p2 = promise.new(function (resolve) r1 = resolve end), promise.new(function (_, reject) r2 = reject end);
+			local p = promise.all_settled({ p1, p2 });
+
+			local result;
+			local cb = spy.new(function (v)
+				result = v;
+			end);
+			p:next(cb);
+			assert.spy(cb).was_called(0);
+			r2("this fails");
+			assert.spy(cb).was_called(0);
+			r1("this succeeds");
+			assert.spy(cb).was_called(1);
+			assert.same({
+				{ status = "fulfilled", value = "this succeeds" };
+				{ status = "rejected", reason = "this fails" };
+			}, result);
+		end);
+	end);
 	describe("catch()", function ()
 		it("works", function ()
 			local result;
diff --git a/spec/util_pubsub_spec.lua b/spec/util_pubsub_spec.lua
index c982fb36..f876089e 100644
--- a/spec/util_pubsub_spec.lua
+++ b/spec/util_pubsub_spec.lua
@@ -101,13 +101,14 @@ describe("util.pubsub", function ()
 			assert(service:publish("node", true, "1", "item 1", { myoption = true }));
 
 			local ok, config = assert(service:get_node_config("node", true));
+			assert.truthy(ok);
 			assert.equals(true, config.myoption);
 		end);
 
 		it("fails to publish to a node with differing config", function ()
 			local ok, err = service:publish("node", true, "1", "item 2", { myoption = false });
 			assert.falsy(ok);
-			assert.equals("precondition-not-met", err);
+			assert.equals("precondition-not-met", err.pubsub_condition);
 		end);
 
 		it("allows to publish to a node with differing config when only defaults are suggested", function ()
@@ -229,6 +230,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be the default", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("open", config.access_model);
 			end);
 			it("should allow anyone to subscribe", function ()
@@ -250,6 +252,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be present in the configuration", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("whitelist", config.access_model);
 			end);
 			it("should not allow anyone to subscribe", function ()
@@ -294,6 +297,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should be the default", function ()
 				local ok, config = service:get_node_config("test", true);
+				assert.truthy(ok);
 				assert.equal("publishers", config.publish_model);
 			end);
 			it("should not allow anyone to publish", function ()
@@ -304,6 +308,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should allow publishers to publish", function ()
 				assert(service:set_affiliation("test", true, "mypublisher", "publisher"));
+				-- luacheck: ignore 211/err
 				local ok, err = service:publish("test", "mypublisher", "item1", "foo");
 				assert.is_true(ok);
 			end);
@@ -342,6 +347,7 @@ describe("util.pubsub", function ()
 			end);
 			it("should allow publishers to publish without a subscription", function ()
 				assert(service:set_affiliation("test", true, "mypublisher", "publisher"));
+				-- luacheck: ignore 211/err
 				local ok, err = service:publish("test", "mypublisher", "item1", "foo");
 				assert.is_true(ok);
 			end);
@@ -477,4 +483,30 @@ describe("util.pubsub", function ()
 
 	end);
 
+	describe("subscriber filter", function ()
+		it("works", function ()
+			local filter = spy.new(function (subs) -- luacheck: ignore 212/subs
+				return {["modified"] = true};
+			end);
+			local broadcaster = spy.new(function (notif_type, node_name, subscribers, item) -- luacheck: ignore 212
+			end);
+			local service = pubsub.new({
+					subscriber_filter = filter;
+					broadcaster = broadcaster;
+				});
+
+			local ok = service:create("node", true);
+			assert.truthy(ok);
+
+			local ok = service:add_subscription("node", true, "someone");
+			assert.truthy(ok);
+
+			local ok = service:publish("node", true, "1", "item");
+			assert.truthy(ok);
+			-- TODO how to match table arguments?
+			assert.spy(filter).was_called();
+			assert.spy(broadcaster).was_called();
+		end);
+	end);
+
 end);
diff --git a/spec/util_queue_spec.lua b/spec/util_queue_spec.lua
index 7cd3d695..d73f523d 100644
--- a/spec/util_queue_spec.lua
+++ b/spec/util_queue_spec.lua
@@ -100,4 +100,41 @@ describe("util.queue", function()
 
 		end);
 	end);
+	describe("consume()", function ()
+		it("should work", function ()
+			local q = queue.new(10);
+			for i = 1, 5 do
+				q:push(i);
+			end
+			local c = 0;
+			for i in q:consume() do
+				assert(i == c + 1);
+				assert(q:count() == (5-i));
+				c = i;
+			end
+		end);
+
+		it("should work even if items are pushed in the loop", function ()
+			local q = queue.new(10);
+			for i = 1, 5 do
+				q:push(i);
+			end
+			local c = 0;
+			for i in q:consume() do
+				assert(i == c + 1);
+				if c < 3 then
+					assert(q:count() == (5-i));
+				else
+					assert(q:count() == (6-i));
+				end
+
+				c = i;
+
+				if c == 3 then
+					q:push(6);
+				end
+			end
+			assert.equal(c, 6);
+		end);
+	end);
 end);
diff --git a/spec/util_ringbuffer_spec.lua b/spec/util_ringbuffer_spec.lua
new file mode 100644
index 00000000..633885a8
--- /dev/null
+++ b/spec/util_ringbuffer_spec.lua
@@ -0,0 +1,103 @@
+local rb = require "util.ringbuffer";
+describe("util.ringbuffer", function ()
+	describe("#new", function ()
+		it("has a constructor", function ()
+			assert.Function(rb.new);
+		end);
+		it("can be created", function ()
+			assert.truthy(rb.new());
+		end);
+		it("won't create an empty buffer", function ()
+			assert.has_error(function ()
+				rb.new(0);
+			end);
+		end);
+		it("won't create a negatively sized buffer", function ()
+			assert.has_error(function ()
+				rb.new(-1);
+			end);
+		end);
+	end);
+	describe(":write", function ()
+		local b = rb.new();
+		it("works", function ()
+			assert.truthy(b:write("hi"));
+		end);
+	end);
+
+	describe(":discard", function ()
+		local b = rb.new();
+		it("works", function ()
+			assert.truthy(b:write("hello world"));
+			assert.truthy(b:discard(6));
+			assert.equal(5, #b);
+			assert.equal("world", b:read(5));
+		end);
+	end);
+
+	describe(":sub", function ()
+		-- Helper function to compare buffer:sub() with string:sub()
+		local function test_sub(b, x, y)
+			local s = b:read(#b, true);
+			local string_result, buffer_result = s:sub(x, y), b:sub(x, y);
+			assert.equals(string_result, buffer_result, ("buffer:sub(%d, %s) does not match string:sub()"):format(x, y and ("%d"):format(y) or "nil"));
+		end
+
+		it("works", function ()
+			local b = rb.new();
+			assert.truthy(b:write("hello world"));
+			assert.equals("hello", b:sub(1, 5));
+		end);
+
+		it("supports optional end parameter", function ()
+			local b = rb.new();
+			assert.truthy(b:write("hello world"));
+			assert.equals("hello world", b:sub(1));
+			assert.equals("world", b:sub(-5));
+		end);
+
+		it("is equivalent to string:sub", function ()
+			local b = rb.new(6);
+			assert.truthy(b:write("foobar"));
+			b:read(3);
+			b:write("foo");
+			for i = -13, 13 do
+				for j = -13, 13 do
+					test_sub(b, i, j);
+				end
+			end
+		end);
+	end);
+
+	describe(":byte", function ()
+		-- Helper function to compare buffer:byte() with string:byte()
+		local function test_byte(b, x, y)
+			local s = b:read(#b, true);
+			local string_result, buffer_result = {s:byte(x, y)}, {b:byte(x, y)};
+			assert.same(string_result, buffer_result, ("buffer:byte(%d, %s) does not match string:byte()"):format(x, y and ("%d"):format(y) or "nil"));
+		end
+
+		it("is equivalent to string:byte", function ()
+			local b = rb.new(6);
+			assert.truthy(b:write("foobar"));
+			b:read(3);
+			b:write("foo");
+			test_byte(b, 1);
+			test_byte(b, 3);
+			test_byte(b, -1);
+			test_byte(b, -3);
+			for i = -13, 13 do
+				for j = -13, 13 do
+					test_byte(b, i, j);
+				end
+			end
+		end);
+
+		it("works with characters > 127", function ()
+			local b = rb.new();
+			b:write(string.char(0, 140));
+			local r = { b:byte(1, 2) };
+			assert.same({ 0, 140 }, r);
+		end);
+	end);
+end);
diff --git a/spec/util_rsm_spec.lua b/spec/util_rsm_spec.lua
new file mode 100644
index 00000000..08fb1b02
--- /dev/null
+++ b/spec/util_rsm_spec.lua
@@ -0,0 +1,89 @@
+local rsm = require "util.rsm";
+local xml = require "util.xml";
+
+local function strip(s)
+	return (s:gsub(">%s+<", "><"));
+end
+
+describe("util.rsm", function ()
+	describe("parse", function ()
+		it("works", function ()
+			local test = xml.parse(strip([[
+				<set xmlns='http://jabber.org/protocol/rsm'>
+					<max>10</max>
+				</set>
+				]]));
+			assert.same({ max = 10 }, rsm.parse(test));
+		end);
+
+		it("works", function ()
+			local test = xml.parse(strip([[
+				<set xmlns='http://jabber.org/protocol/rsm'>
+					<first index='0'>saint@example.org</first>
+					<last>peterpan@neverland.lit</last>
+					<count>800</count>
+				</set>
+				]]));
+			assert.same({ first = { index = 0, "saint@example.org" }, last = "peterpan@neverland.lit", count = 800 }, rsm.parse(test));
+		end);
+
+		it("works", function ()
+			local test = xml.parse(strip([[
+				<set xmlns='http://jabber.org/protocol/rsm'>
+					<max>10</max>
+					<before>peter@pixyland.org</before>
+				</set>
+				]]));
+			assert.same({ max = 10, before = "peter@pixyland.org" }, rsm.parse(test));
+		end);
+
+	end);
+
+	describe("generate", function ()
+		it("works", function ()
+			local test = xml.parse(strip([[
+				<set xmlns='http://jabber.org/protocol/rsm'>
+					<max>10</max>
+				</set>
+				]]));
+			local res = rsm.generate({ max = 10 });
+			assert.same(test:get_child_text("max"), res:get_child_text("max"));
+		end);
+
+		it("works", function ()
+			local test = xml.parse(strip([[
+				<set xmlns='http://jabber.org/protocol/rsm'>
+					<first index='0'>saint@example.org</first>
+					<last>peterpan@neverland.lit</last>
+					<count>800</count>
+				</set>
+				]]));
+			local res = rsm.generate({ first = { index = 0, "saint@example.org" }, last = "peterpan@neverland.lit", count = 800 });
+			assert.same(test:get_child("first").attr.index, res:get_child("first").attr.index);
+			assert.same(test:get_child_text("first"), res:get_child_text("first"));
+			assert.same(test:get_child_text("last"), res:get_child_text("last"));
+			assert.same(test:get_child_text("count"), res:get_child_text("count"));
+		end);
+
+		it("works", function ()
+			local test = xml.parse(strip([[
+			<set xmlns='http://jabber.org/protocol/rsm'>
+				<max>10</max>
+				<before>peter@pixyland.org</before>
+			</set>
+			]]));
+			local res = rsm.generate({ max = 10, before = "peter@pixyland.org" });
+			assert.same(test:get_child_text("max"), res:get_child_text("max"));
+			assert.same(test:get_child_text("before"), res:get_child_text("before"));
+		end);
+
+		it("handles floats", function ()
+			local r1 = rsm.generate({ max = 10.0, count = 100.0, first = { index = 1.0, "foo" } });
+			assert.equal("10", r1:get_child_text("max"));
+			assert.equal("100", r1:get_child_text("count"));
+			assert.equal("1", r1:get_child("first").attr.index);
+		end);
+
+	end);
+end);
+
diff --git a/spec/util_sasl_spec.lua b/spec/util_sasl_spec.lua
new file mode 100644
index 00000000..368291b3
--- /dev/null
+++ b/spec/util_sasl_spec.lua
@@ -0,0 +1,43 @@
+local sasl = require "util.sasl";
+
+-- profile * mechanism
+-- callbacks could use spies instead
+
+describe("util.sasl", function ()
+	describe("plain_test profile", function ()
+		local profile = {
+			plain_test = function (_, username, password, realm)
+				assert.equals("user", username)
+				assert.equals("pencil", password)
+				assert.equals("sasl.test", realm)
+				return true, true;
+			end;
+		};
+		it("works with PLAIN", function ()
+			local plain = sasl.new("sasl.test", profile);
+			assert.truthy(plain:select("PLAIN"));
+			assert.truthy(plain:process("\000user\000pencil"));
+			assert.equals("user", plain.username);
+		end);
+	end);
+
+	describe("plain profile", function ()
+		local profile = {
+			plain = function (_, username, realm)
+				assert.equals("user", username)
+				assert.equals("sasl.test", realm)
+				return "pencil", true;
+			end;
+		};
+
+		it("works with PLAIN", function ()
+			local plain = sasl.new("sasl.test", profile);
+			assert.truthy(plain:select("PLAIN"));
+			assert.truthy(plain:process("\000user\000pencil"));
+			assert.equals("user", plain.username);
+		end);
+
+		-- TODO SCRAM
+	end);
+end);
+
diff --git a/spec/util_stanza_spec.lua b/spec/util_stanza_spec.lua
index 6fbae41a..535f783b 100644
--- a/spec/util_stanza_spec.lua
+++ b/spec/util_stanza_spec.lua
@@ -1,5 +1,6 @@
 
 local st = require "util.stanza";
+local errors = require "util.error";
 
 describe("util.stanza", function()
 	describe("#preserialize()", function()
@@ -95,20 +96,31 @@ describe("util.stanza", function()
 
 	describe("#iq()", function()
 		it("should create an iq stanza", function()
-			local i = st.iq({ id = "foo" });
+			local i = st.iq({ type = "get", id = "foo" });
 			assert.are.equal("iq", i.name);
 			assert.are.equal("foo", i.attr.id);
+			assert.are.equal("get", i.attr.type);
 		end);
 
-		it("should reject stanzas with no id", function ()
+		it("should reject stanzas with no attributes", function ()
 			assert.has.error_match(function ()
 				st.iq();
-			end, "id attribute");
+			end, "attributes");
+		end);
 
+
+		it("should reject stanzas with no id", function ()
 			assert.has.error_match(function ()
-				st.iq({ foo = "bar" });
+				st.iq({ type = "get" });
 			end, "id attribute");
 		end);
+
+		it("should reject stanzas with no type", function ()
+			assert.has.error_match(function ()
+				st.iq({ id = "foo" });
+			end, "type attribute");
+
+		end);
 	end);
 
 	describe("#presence()", function ()
@@ -159,6 +171,19 @@ describe("util.stanza", function()
 			assert.are.equal(r.attr.type, "result");
 			assert.are.equal(#r.tags, 0, "A reply should not include children of the original stanza");
 		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.reply(not "a stanza");
+			end, "expected stanza");
+		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.reply({name="x"});
+			end, "expected stanza");
+		end);
+
 	end);
 
 	describe("#error_reply()", function()
@@ -167,13 +192,14 @@ describe("util.stanza", function()
 			local s = st.stanza("s", { to = "touser", from = "fromuser", id = "123" })
 				:tag("child1");
 			-- Make reply stanza
-			local r = st.error_reply(s, "cancel", "service-unavailable");
+			local r = st.error_reply(s, "cancel", "service-unavailable", nil, "host");
 			assert.are.equal(r.name, s.name);
 			assert.are.equal(r.id, s.id);
 			assert.are.equal(r.attr.to, s.attr.from);
 			assert.are.equal(r.attr.from, s.attr.to);
 			assert.are.equal(#r.tags, 1);
 			assert.are.equal(r.tags[1].tags[1].name, "service-unavailable");
+			assert.are.equal(r.tags[1].attr.by, "host");
 		end);
 
 		it("should work for <iq get>", function()
@@ -190,8 +216,79 @@ describe("util.stanza", function()
 			assert.are.equal(#r.tags, 1);
 			assert.are.equal(r.tags[1].tags[1].name, "service-unavailable");
 		end);
+
+		it("should reject not-stanzas", function ()
+			assert.has.error_match(function ()
+				st.error_reply(not "a stanza", "modify", "bad-request");
+			end, "expected stanza");
+		end);
+
+		it("should reject stanzas of type error", function ()
+			assert.has.error_match(function ()
+				st.error_reply(st.message({type="error"}), "cancel", "conflict");
+			end, "got stanza of type error");
+			assert.has.error_match(function ()
+				st.error_reply(st.error_reply(st.message({type="chat"}), "modify", "forbidden"), "cancel", "service-unavailable");
+			end, "got stanza of type error");
+		end);
+
+		describe("util.error integration", function ()
+		it("should accept util.error objects", function ()
+			local s = st.message({ to = "touser", from = "fromuser", id = "123", type = "chat" }, "Hello");
+			local e = errors.new({ type = "modify", condition = "not-acceptable", text = "Bork bork bork" }, { by = "this.test" });
+			local r = st.error_reply(s, e);
+
+			assert.are.equal(r.name, s.name);
+			assert.are.equal(r.id, s.id);
+			assert.are.equal(r.attr.to, s.attr.from);
+			assert.are.equal(r.attr.from, s.attr.to);
+			assert.are.equal(r.attr.type, "error");
+			assert.are.equal(r.tags[1].name, "error");
+			assert.are.equal(r.tags[1].attr.type, e.type);
+			assert.are.equal(r.tags[1].tags[1].name, e.condition);
+			assert.are.equal(r.tags[1].tags[2]:get_text(), e.text);
+			assert.are.equal("this.test", r.tags[1].attr.by);
+		end);
+
+		it("should accept util.error objects with an URI", function ()
+			local s = st.message({ to = "touser", from = "fromuser", id = "123", type = "chat" }, "Hello");
+			local gone = errors.new({ condition = "gone", extra = { uri = "file:///dev/null" } })
+			local gonner = st.error_reply(s, gone);
+			assert.are.equal("gone", gonner.tags[1].tags[1].name);
+			assert.are.equal("file:///dev/null", gonner.tags[1].tags[1][1]);
+		end);
+
+		it("should accept util.error objects with application specific error", function ()
+			local s = st.message({ to = "touser", from = "fromuser", id = "123", type = "chat" }, "Hello");
+			local e = errors.new({ condition = "internal-server-error", text = "Namespaced thing happened",
+				extra = {namespace="xmpp:example.test", condition="this-happened"} })
+			local r = st.error_reply(s, e);
+			assert.are.equal("xmpp:example.test", r.tags[1].tags[3].attr.xmlns);
+			assert.are.equal("this-happened", r.tags[1].tags[3].name);
+
+			local e2 = errors.new({ condition = "internal-server-error", text = "Namespaced thing happened",
+				extra = {tag=st.stanza("that-happened", { xmlns = "xmpp:example.test", ["another-attribute"] = "here" })} })
+			local r2 = st.error_reply(s, e2);
+			assert.are.equal("xmpp:example.test", r2.tags[1].tags[3].attr.xmlns);
+			assert.are.equal("that-happened", r2.tags[1].tags[3].name);
+			assert.are.equal("here", r2.tags[1].tags[3].attr["another-attribute"]);
+		end);
+		end);
 	end);
 
+	describe("#get_error()", function ()
+		describe("basics", function ()
+			local s = st.message();
+			local e = st.error_reply(s, "cancel", "not-acceptable", "UNACCEPTABLE!!!! ONE MILLION YEARS DUNGEON!")
+				:tag("dungeon", { xmlns = "urn:uuid:c9026187-5b05-4e70-b265-c3b6338a7d0f", period="1000000years"});
+			local typ, cond, text, extra = e:get_error();
+			assert.equal("cancel", typ);
+			assert.equal("not-acceptable", cond);
+			assert.equal("UNACCEPTABLE!!!! ONE MILLION YEARS DUNGEON!", text);
+			assert.not_nil(extra)
+		end)
+	end)
+
 	describe("should reject #invalid", function ()
 		local invalid_names = {
 			["empty string"] = "", ["characters"] = "<>";
@@ -370,4 +467,43 @@ describe("util.stanza", function()
 			end);
 		end);
 	end);
+
+	describe("top_tag", function ()
+		local xml_parse = require "util.xml".parse;
+		it("works", function ()
+			local s = st.message({type="chat"}, "Hello");
+			local top_tag = s:top_tag();
+			assert.is_string(top_tag);
+			assert.not_equal("/>", top_tag:sub(-2, -1));
+			assert.equal(">", top_tag:sub(-1, -1));
+			local s2 = xml_parse(top_tag.."</message>");
+			assert(st.is_stanza(s2));
+			assert.equal("message", s2.name);
+			assert.equal(0, #s2);
+			assert.equal(0, #s2.tags);
+			assert.equal("chat", s2.attr.type);
+		end);
+
+		it("works with namespaced attributes", function ()
+			local s = xml_parse[[<message foo:bar='true' xmlns:foo='my-awesome-ns'/>]];
+			local top_tag = s:top_tag();
+			assert.is_string(top_tag);
+			assert.not_equal("/>", top_tag:sub(-2, -1));
+			assert.equal(">", top_tag:sub(-1, -1));
+			local s2 = xml_parse(top_tag.."</message>");
+			assert(st.is_stanza(s2));
+			assert.equal("message", s2.name);
+			assert.equal(0, #s2);
+			assert.equal(0, #s2.tags);
+			assert.equal("true", s2.attr["my-awesome-ns\1bar"]);
+		end);
+	end);
+
+	describe("indent", function ()
+		local s = st.stanza("foo"):text("\n"):tag("bar"):tag("baz"):up():text_tag("cow", "moo");
+		assert.equal("<foo>\n\t<bar>\n\t\t<baz/>\n\t\t<cow>moo</cow>\n\t</bar>\n</foo>", tostring(s:indent()));
+		assert.equal("<foo>\n  <bar>\n    <baz/>\n    <cow>moo</cow>\n  </bar>\n</foo>", tostring(s:indent(1, "  ")));
+		assert.equal("<foo>\n\t\t<bar>\n\t\t\t<baz/>\n\t\t\t<cow>moo</cow>\n\t\t</bar>\n\t</foo>", tostring(s:indent(2, "\t")));
+	end);
+
 end);
diff --git a/spec/util_table_spec.lua b/spec/util_table_spec.lua
new file mode 100644
index 00000000..76f54b69
--- /dev/null
+++ b/spec/util_table_spec.lua
@@ -0,0 +1,17 @@
+local u_table = require "util.table";
+describe("util.table", function ()
+	describe("create()", function ()
+		it("works", function ()
+			-- Can't test the allocated sizes of the table, so what you gonna do?
+			assert.is.table(u_table.create(1,1));
+		end);
+	end);
+
+	describe("pack()", function ()
+		it("works", function ()
+			assert.same({ "lorem", "ipsum", "dolor", "sit", "amet", n = 5 }, u_table.pack("lorem", "ipsum", "dolor", "sit", "amet"));
+		end);
+	end);
+end);
+
+
diff --git a/spec/util_throttle_spec.lua b/spec/util_throttle_spec.lua
index 75daf1b9..985afae8 100644
--- a/spec/util_throttle_spec.lua
+++ b/spec/util_throttle_spec.lua
@@ -88,7 +88,7 @@ describe("util.throttle", function()
 				later(0.1);
 				a:update();
 			end
-			assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rouding errors
+			assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rounding errors
 		end);
 	end);
 
diff --git a/tools/cfgdump.lua b/tools/cfgdump.lua
new file mode 100755
index 00000000..b7ca0716
--- /dev/null
+++ b/tools/cfgdump.lua
@@ -0,0 +1,120 @@
+#!/usr/bin/env lua
+
+-- cfgdump.lua prosody.cfg.lua [[host] option]
+
+local s_format, print = string.format, print;
+local printf = function(fmt, ...) return print(s_format(fmt, ...)); end
+local serialization = require"util.serialization";
+local serialize = serialization.new and serialization.new({ unquoted = true }) or serialization.serialize;
+local configmanager = require"core.configmanager";
+local startup = require "util.startup";
+
+startup.set_function_metatable();
+local config_filename, onlyhost, onlyoption = ...;
+
+local ok, _, err = configmanager.load(config_filename or "./prosody.cfg.lua", "lua");
+assert(ok, err);
+
+if onlyhost then
+	if not onlyoption then
+		onlyhost, onlyoption = "*", onlyhost;
+	end
+	if onlyhost ~= "*" then
+		local component_module = configmanager.get(onlyhost, "component_module");
+
+		if component_module == "component" then
+			printf("Component %q", onlyhost);
+		elseif component_module then
+			printf("Component %q %q", onlyhost, component_module);
+		else
+			printf("VirtualHost %q", onlyhost);
+		end
+	end
+	printf("%s = %s", onlyoption or "?", serialize(configmanager.get(onlyhost, onlyoption)));
+	return;
+end
+
+local config = configmanager.getconfig();
+
+
+for host, hostcfg in pairs(config) do
+	local fixed = {};
+	for option, value in pairs(hostcfg) do
+		fixed[option] = value;
+		if option:match("ports?$") or option:match("interfaces?$") then
+			if option:match("s$") then
+				if type(value) ~= "table" then
+					fixed[option] = { value };
+				end
+			else
+				if type(value) == "table" and #value > 1 then
+					fixed[option] = nil;
+					fixed[option.."s"] = value;
+				end
+			end
+		end
+	end
+	config[host] = fixed;
+end
+
+local globals = config["*"]; config["*"] = nil;
+
+local function printsection(section)
+	local out, n = {}, 1;
+	for k,v in pairs(section) do
+		out[n], n = s_format("%s = %s", k, serialize(v)), n + 1;
+	end
+	table.sort(out);
+	print(table.concat(out, "\n"));
+end
+
+print("-------------- Prosody Exported Configuration File -------------");
+print();
+print("------------------------ Global section ------------------------");
+print();
+printsection(globals);
+print();
+
+local has_components = nil;
+
+print("------------------------ Virtual hosts -------------------------");
+
+for host, hostcfg in pairs(config) do
+	setmetatable(hostcfg, nil);
+	hostcfg.defined = nil;
+
+	if hostcfg.component_module == nil then
+		print();
+		printf("VirtualHost %q", host);
+		printsection(hostcfg);
+	else
+		has_components = true
+	end
+end
+
+print();
+
+if has_components then
+print("------------------------- Components ---------------------------");
+
+	for host, hostcfg in pairs(config) do
+		local component_module = hostcfg.component_module;
+		hostcfg.component_module = nil;
+
+		if component_module then
+			print();
+			if component_module == "component" then
+				printf("Component %q", host);
+			else
+				printf("Component %q %q", host, component_module);
+				hostcfg.component_module = nil;
+				hostcfg.load_global_modules = nil;
+			end
+			printsection(hostcfg);
+		end
+	end
+end
+
+print()
+print("------------------------- End of File --------------------------");
+
diff --git a/tools/form2table.lua b/tools/form2table.lua
new file mode 100644
index 00000000..49a6972b
--- /dev/null
+++ b/tools/form2table.lua
@@ -0,0 +1,48 @@
+-- Read an XML dataform and spit out a serialized Lua table of it
+
+local function from_stanza(stanza)
+	local layout = {
+		title = stanza:get_child_text("title");
+		instructions = stanza:get_child_text("instructions");
+	};
+	for tag in stanza:childtags("field") do
+		local field = {
+			name = tag.attr.var;
+			type = tag.attr.type;
+			label = tag.attr.label;
+			desc = tag:get_child_text("desc");
+			required = tag:get_child("required") and true or nil;
+			value = tag:get_child_text("value");
+			options = nil;
+		};
+
+		if field.type == "list-single" or field.type == "list-multi" then
+			local options = {};
+			for option in tag:childtags("option") do
+				options[#options+1] = { label = option.attr.label, value = option:get_child_text("value") };
+			end
+			field.options = options;
+		end
+
+		if field.type == "jid-multi" or field.type == "list-multi" or field.type == "text-multi" then
+			local values = {};
+			for value in tag:childtags("value") do
+				values[#values+1] = value:get_text();
+			end
+			if field.type == "text-multi" then
+				values = table.concat(values, "\n");
+			end
+			field.value = values;
+		end
+
+		if field.type == "boolean" then
+			field.value = field.value == "true" or field.value == "1";
+		end
+
+		layout[#layout+1] = field;
+
+	end
+	return layout;
+end
+
+print("dataforms.new " .. require "util.serialization".serialize(from_stanza(require "util.xml".parse(io.read("*a"))), { unquoted = true }))
diff --git a/tools/linedebug.lua b/tools/linedebug.lua
new file mode 100644
index 00000000..96214cc8
--- /dev/null
+++ b/tools/linedebug.lua
@@ -0,0 +1,18 @@
+local data = {}
+local getinfo = debug.getinfo;
+local function linehook(ev, li)
+	local S = getinfo(2, "S");
+	if S and S.source and S.source:match"^@" then
+		local file = S.source:sub(2);
+		local lines = data[file];
+		if not lines then
+			lines = {};
+			data[file] = lines;
+			for line in io.lines(file) do
+				lines[#lines+1] = line;
+			end
+		end
+		io.stderr:write(ev, " ", file, " ", li, " ", lines[li], "\n");
+	end
+end
+debug.sethook(linehook, "l");
diff --git a/tools/make_repo.lua b/tools/make_repo.lua
new file mode 100644
index 00000000..5b5e6234
--- /dev/null
+++ b/tools/make_repo.lua
@@ -0,0 +1,44 @@
+print("Getting all the available modules")
+if os.execute '[ -e "./downloaded_modules" ]' then
+	os.execute("rm -rf downloaded_modules")
+end
+os.execute("hg clone https://hg.prosody.im/prosody-modules/ downloaded_modules")
+local i, popen = 0, io.popen
+local flag = "mod_"
+if os.execute '[ -e "./repository" ]' then
+	os.execute("mkdir repository")
+end
+local pfile = popen('ls -a "downloaded_modules"')
+for filename in pfile:lines() do
+	i = i + 1
+	if filename:sub(1, #flag) == flag then
+		local file = io.open("repository/"..filename.."-scm-1.rockspec", "w")
+		file:write('package = "'..filename..'"', '\n')
+		file:write('version = "scm-1"', '\n')
+		file:write('source = {', '\n')
+		file:write('\turl = "hg+https://hg.prosody.im/prosody-modules",', '\n')
+		file:write('\tdir = "prosody-modules"', '\n')
+		file:write('}', '\n')
+		file:write('description = {', '\n')
+		file:write('\thomepage = "https://prosody.im/",', '\n')
+		file:write('\tlicense = "MIT"', '\n')
+		file:write('}', '\n')
+		file:write('dependencies = {', '\n')
+		file:write('\t"lua >= 5.1"', '\n')
+		file:write('}', '\n')
+		file:write('build = {', '\n')
+		file:write('\ttype = "builtin",', '\n')
+		file:write('\tmodules = {', '\n')
+		file:write('\t\t["'..filename..'.'..filename..'"] = "'..filename..'/'..filename..'.lua"', '\n')
+		file:write('\t}', '\n')
+		file:write('}', '\n')
+		file:close()
+	end
+end
+pfile:close()
+os.execute("cd repository/ && luarocks-admin make_manifest ./ && chmod -R 644 ./*")
+print("")
+print("Done!. Modules' sources are locally available at ./downloaded_modules")
+print("Repository is available at ./repository")
+print("The repository contains all of prosody modules' respective rockspecs, as well as manifest files and an html Index")
+print("You can now either point your server to this folder, or copy its contents to another configured folder.")
diff --git a/tools/migration/Makefile b/tools/migration/Makefile
index 713831d2..fce0f7d9 100644
--- a/tools/migration/Makefile
+++ b/tools/migration/Makefile
@@ -12,16 +12,12 @@ INSTALLEDCONFIG = $(SYSCONFDIR)
 INSTALLEDMODULES = $(LIBDIR)/prosody/modules
 INSTALLEDDATA = $(DATADIR)
 
-SOURCE_FILES = migrator/*.lua
-
-all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES)
+all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua
 
 install: prosody-migrator.install migrator.cfg.lua.install
-	install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator
+	install -d $(BIN) $(CONFIG) $(SOURCE)
 	install -d $(MAN)/man1
-	install -d $(SOURCE)/migrator
 	install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator
-	install -m644 $(SOURCE_FILES) $(SOURCE)/migrator
 	test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua
 
 clean:
@@ -31,7 +27,9 @@ clean:
 prosody-migrator.install: prosody-migrator.lua
 	sed "1s/\blua\b/$(RUNWITH)/; \
 		s|^CFG_SOURCEDIR=.*;$$|CFG_SOURCEDIR='$(INSTALLEDSOURCE)';|; \
-		s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|;" \
+		s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|; \
+		s|^CFG_DATADIR=.*;$$|CFG_DATADIR='$(INSTALLEDDATA)';|; \
+		s|^CFG_PLUGINDIR=.*;$$|CFG_PLUGINDIR='$(INSTALLEDMODULES)/';|;" \
 			< prosody-migrator.lua > prosody-migrator.install
 
 migrator.cfg.lua.install: migrator.cfg.lua
diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua
index fa37f2a3..c26fa869 100644
--- a/tools/migration/migrator.cfg.lua
+++ b/tools/migration/migrator.cfg.lua
@@ -1,12 +1,38 @@
 local data_path = "../../data";
 
+local vhost = {
+	"accounts",
+	"account_details",
+	"roster",
+	"vcard",
+	"private",
+	"blocklist",
+	"privacy",
+	"archive-archive",
+	"offline-archive",
+	"pubsub_nodes",
+	-- "pubsub_*-archive",
+	"pep",
+	-- "pep_*-archive",
+}
+local muc = {
+	"persistent",
+	"config",
+	"state",
+	"muc_log-archive",
+};
+
 input {
-	type = "prosody_files";
+	hosts = {
+		["example.com"] = vhost;
+		["conference.example.com"] = muc;
+	};
+	type = "internal";
 	path = data_path;
 }
 
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
@@ -14,11 +40,11 @@ output {
 --[[
 
 input {
-	type = "prosody_files";
+	type = "internal";
 	path = data_path;
 }
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua
deleted file mode 100644
index cfbfcce8..00000000
--- a/tools/migration/migrator/mtools.lua
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-local print = print;
-local t_insert = table.insert;
-local t_sort = table.sort;
-
-
-local function sorted(params)
-
-	local reader = params.reader; -- iterator to get items from
-	local sorter = params.sorter; -- sorting function
-	local filter = params.filter; -- filter function
-
-	local cache = {};
-	for item in reader do
-		if filter then item = filter(item); end
-		if item then t_insert(cache, item); end
-	end
-	if sorter then
-		t_sort(cache, sorter);
-	end
-	local i = 0;
-	return function()
-		i = i + 1;
-		return cache[i];
-	end;
-
-end
-
-local function merged(reader, merger)
-
-	local item1 = reader();
-	local merged = { item1 };
-	return function()
-		while true do
-			if not item1 then return nil; end
-			local item2 = reader();
-			if not item2 then item1 = nil; return merged; end
-			if merger(item1, item2) then
-			--print("merged")
-				item1 = item2;
-				t_insert(merged, item1);
-			else
-			--print("unmerged", merged)
-				item1 = item2;
-				local tmp = merged;
-				merged = { item1 };
-				return tmp;
-			end
-		end
-	end;
-
-end
-
-return {
-	sorted = sorted;
-	merged = merged;
-}
diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua
deleted file mode 100644
index 4de09273..00000000
--- a/tools/migration/migrator/prosody_files.lua
+++ /dev/null
@@ -1,144 +0,0 @@
-
-local print = print;
-local assert = assert;
-local setmetatable = setmetatable;
-local tonumber = tonumber;
-local char = string.char;
-local coroutine = coroutine;
-local lfs = require "lfs";
-local loadfile = loadfile;
-local pcall = pcall;
-local mtools = require "migrator.mtools";
-local next = next;
-local pairs = pairs;
-local json = require "util.json";
-local os_getenv = os.getenv;
-local error = error;
-
-prosody = {};
-local dm = require "util.datamanager"
-
-
-local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end
-local function is_file(path) return lfs.attributes(path, "mode") == "file"; end
-local function clean_path(path)
-	return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~");
-end
-local encode, decode; do
-	local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });
-	decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end
-	encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end
-end
-local function decode_dir(x)
-	if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x);
-	end
-end
-local function decode_file(x)
-	if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x:gsub("%.dat$", ""));
-	end
-end
-local function prosody_dir(path, ondir, onfile, ...)
-	for x in lfs.dir(path) do
-		local xpath = path.."/"..x;
-		if decode_dir(x) and is_dir(xpath) then
-			ondir(xpath, x, ...);
-		elseif decode_file(x) and is_file(xpath) then
-			onfile(xpath, x, ...);
-		end
-	end
-end
-
-local function handle_root_file(path, name)
-	--print("root file: ", decode_file(name))
-	coroutine.yield { user = nil, host = nil, store = decode_file(name) };
-end
-local function handle_host_file(path, name, host)
-	--print("host file: ", decode_dir(host).."/"..decode_file(name))
-	coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };
-end
-local function handle_store_file(path, name, store, host)
-	--print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))
-	coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };
-end
-local function handle_host_store(path, name, host)
-	prosody_dir(path, function() end, handle_store_file, name, host);
-end
-local function handle_host_dir(path, name)
-	prosody_dir(path, handle_host_store, handle_host_file, name);
-end
-local function handle_root_dir(path)
-	prosody_dir(path, handle_host_dir, handle_root_file);
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1].user;
-		host = item[1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		userdata.stores[store.store] = store.data;
-		store.user = nil; store.host = nil; store.store = nil;
-	end
-	return userdata;
-end
-
-local function reader(input)
-	local path = clean_path(assert(input.path, "no input.path specified"));
-	assert(is_dir(path), "input.path is not a directory");
-	local iter = coroutine.wrap(function()handle_root_dir(path);end);
-	-- get per-user stores, sorted
-	local iter = mtools.sorted {
-		reader = function()
-			local x = iter();
-			while x do
-				dm.set_data_path(path);
-				local err;
-				x.data, err = dm.load(x.user, x.host, x.store);
-				if x.data == nil and err then
-					local p = dm.getpath(x.user, x.host, x.store);
-					print(("Error loading data at path %s for %s@%s (%s store): %s")
-						:format(p, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>", err or "<nil>"));
-				else
-					return x;
-				end
-				x = iter();
-			end
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user);
-	end);
-
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end
-end
-
-local function writer(output)
-	local path = clean_path(assert(output.path, "no output.path specified"));
-	assert(is_dir(path), "output.path is not a directory");
-	return function(item)
-		if not item then return; end -- end of input
-		dm.set_data_path(path);
-		for store, data in pairs(item.stores) do
-			assert(dm.store(item.user, item.host, store, data));
-		end
-	end
-end
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua
deleted file mode 100644
index 6df2b025..00000000
--- a/tools/migration/migrator/prosody_sql.lua
+++ /dev/null
@@ -1,190 +0,0 @@
-
-local assert = assert;
-local have_DBI = pcall(require,"DBI");
-local print = print;
-local type = type;
-local next = next;
-local pairs = pairs;
-local t_sort = table.sort;
-local json = require "util.json";
-local mtools = require "migrator.mtools";
-local tostring = tostring;
-local tonumber = tonumber;
-
-if not have_DBI then
-	error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0);
-end
-
-local sql = require "util.sql";
-
-local function create_table(engine, name) -- luacheck: ignore 431/engine
-	local Table, Column, Index = sql.Table, sql.Column, sql.Index;
-
-	local ProsodyTable = Table {
-		name= name or "prosody";
-		Column { name="host", type="TEXT", nullable=false };
-		Column { name="user", type="TEXT", nullable=false };
-		Column { name="store", type="TEXT", nullable=false };
-		Column { name="key", type="TEXT", nullable=false };
-		Column { name="type", type="TEXT", nullable=false };
-		Column { name="value", type="MEDIUMTEXT", nullable=false };
-		Index { name="prosody_index", "host", "user", "store", "key" };
-	};
-	engine:transaction(function()
-		ProsodyTable:create(engine);
-	end);
-
-end
-
-local function serialize(value)
-	local t = type(value);
-	if t == "string" or t == "boolean" or t == "number" then
-		return t, tostring(value);
-	elseif t == "table" then
-		local value,err = json.encode(value);
-		if value then return "json", value; end
-		return nil, err;
-	end
-	return nil, "Unhandled value type: "..t;
-end
-local function deserialize(t, value)
-	if t == "string" then return value;
-	elseif t == "boolean" then
-		if value == "true" then return true;
-		elseif value == "false" then return false; end
-	elseif t == "number" then return tonumber(value);
-	elseif t == "json" then
-		return json.decode(value);
-	end
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1][1].user;
-		host = item[1][1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		for i=1,#store do -- loop over store data
-			local row = store[i];
-			local k = row.key;
-			local v = deserialize(row.type, row.value);
-			if k and v then
-				if k ~= "" then result[k] = v; elseif type(v) == "table" then
-					for a,b in pairs(v) do
-						result[a] = b;
-					end
-				end
-			end
-			userdata.stores[store[1].store] = result;
-		end
-	end
-	return userdata;
-end
-
-local function needs_upgrade(engine, params)
-	if params.driver == "MySQL" then
-		local success = engine:transaction(function()
-			local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'");
-			assert(result:rowcount() == 0);
-
-			-- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already
-			local check_encoding_query = [[
-			SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME"
-			FROM "information_schema"."columns"
-			WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' );
-			]];
-			check_encoding_query = check_encoding_query:format(engine.charset, engine.charset);
-			local result = engine:execute(check_encoding_query);
-			assert(result:rowcount() == 0)
-		end);
-		if not success then
-			-- Upgrade required
-			return true;
-		end
-	end
-	return false;
-end
-
-local function reader(input)
-	local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, input) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-	end));
-	local keys = {"host", "user", "store", "key", "type", "value"};
-	assert(engine:connect());
-	local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";"));
-	-- get SQL rows, sorted
-	local iter = mtools.sorted {
-		reader = function() val = f(s, val); return val; end;
-		filter = function(x)
-			for i=1,#keys do
-				x[ keys[i] ] = x[i];
-			end
-			if x.host  == "" then x.host  = nil; end
-			if x.user  == "" then x.user  = nil; end
-			if x.store == "" then x.store = nil; end
-			return x;
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge rows to get stores
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user and a.store == b.store);
-	end);
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a[1].host == b[1].host and a[1].user == b[1].user);
-	end);
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end;
-end
-
-local function writer(output, iter)
-	local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, output) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-		create_table(engine);
-	end));
-	assert(engine:connect());
-	assert(engine:delete("DELETE FROM \"prosody\""));
-	local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)";
-
-	return function(item)
-		if not item then assert(engine.conn:commit()) return end -- end of input
-		local host = item.host or "";
-		local user = item.user or "";
-		for store, data in pairs(item.stores) do
-			-- TODO transactions
-			local extradata = {};
-			for key, value in pairs(data) do
-				if type(key) == "string" and key ~= "" then
-					local t, value = assert(serialize(value));
-					local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value));
-				else
-					extradata[key] = value;
-				end
-			end
-			if next(extradata) ~= nil then
-				local t, extradata = assert(serialize(extradata));
-				local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata));
-			end
-		end
-	end;
-end
-
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua
index 1219d891..5c4800ad 100644
--- a/tools/migration/prosody-migrator.lua
+++ b/tools/migration/prosody-migrator.lua
@@ -1,19 +1,43 @@
 #!/usr/bin/env lua
 
-CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR");
-CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR");
+CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
+CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
+CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
+CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
 
--- Substitute ~ with path to home directory in paths
-if CFG_CONFIGDIR then
-	CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME"));
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+local function is_relative(path)
+	local path_sep = package.config:sub(1,1);
+        return ((path_sep == "/" and path:sub(1,1) ~= "/")
+	or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
 end
 
+-- Tell Lua where to find our libraries
 if CFG_SOURCEDIR then
-	CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME"));
+	local function filter_relative_paths(path)
+		if is_relative(path) then return ""; end
+	end
+	local function sanitise_paths(paths)
+		return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
+	end
+	package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
+	package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
+end
+
+-- Substitute ~ with path to home directory in data path
+if CFG_DATADIR then
+	if os.getenv("HOME") then
+		CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
+	end
 end
 
 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
 
+local startup = require "util.startup";
+startup.prosodyctl();
+-- TODO startup.migrator ?
+
 -- Command-line parsing
 local options = {};
 local i = 1;
@@ -29,13 +53,6 @@ while arg[i] do
 	end
 end
 
-if CFG_SOURCEDIR then
-	package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
-	package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
-else
-	package.path = "../../?.lua;"..package.path
-	package.cpath = "../../?.so;"..package.cpath
-end
 
 local envloadfile = require "util.envload".envloadfile;
 
@@ -69,24 +86,14 @@ if not config[to_store] then
 	print("Error: Output store '"..to_store.."' not found in the config file.");
 end
 
-function load_store_handler(name)
-	local store_type = config[name].type;
-	if not store_type then
-		print("Error: "..name.." store type not specified in the config file");
-		return false;
-	else
-		local ok, err = pcall(require, "migrator."..store_type);
-		if not ok then
-			print(("Error: Failed to initialize '%s' store:\n\t%s")
-				:format(name, err));
-			return false;
-		end
+for store, conf in pairs(config) do -- COMPAT
+	if conf.type == "prosody_files" then
+		conf.type = "internal";
+	elseif conf.type == "prosody_sql" then
+		conf.type = "sql";
 	end
-	return true;
 end
 
-have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
-
 if have_err then
 	print("");
 	print("Usage: "..arg[0].." FROM_STORE TO_STORE");
@@ -101,17 +108,88 @@ if have_err then
 	os.exit(1);
 end
 
-local itype = config[from_store].type;
-local otype = config[to_store].type;
-local reader = require("migrator."..itype).reader(config[from_store]);
-local writer = require("migrator."..otype).writer(config[to_store]);
+local async = require "util.async";
+local server = require "net.server";
+local watchers = {
+	error = function (_, err)
+		error(err);
+	end;
+	waiting = function ()
+		server.loop();
+	end;
+};
+
+local cm = require "core.configmanager";
+local hm = require "core.hostmanager";
+local sm = require "core.storagemanager";
+local um = require "core.usermanager";
+
+local function users(store, host)
+	if store.users then
+		return store:users();
+	else
+		return um.users(host);
+	end
+end
 
-local json = require "util.json";
+local function prepare_config(host, conf)
+	if conf.type == "internal" then
+		sm.olddm.set_data_path(conf.path or prosody.paths.data);
+	elseif conf.type == "sql" then
+		cm.set(host, "sql", conf);
+	end
+end
 
-io.stderr:write("Migrating...\n");
-for x in reader do
-	--print(json.encode(x))
-	writer(x);
+local function get_driver(host, conf)
+	prepare_config(host, conf);
+	return assert(sm.load_driver(host, conf.type));
 end
-writer(nil); -- close
+
+local migration_runner = async.runner(function (job)
+	for host, stores in pairs(job.input.hosts) do
+		prosody.hosts[host] = startup.make_host(host);
+		sm.initialize_host(host);
+		um.initialize_host(host);
+
+		local input_driver = get_driver(host, job.input);
+
+		local output_driver = get_driver(host, job.output);
+
+		for _, store in ipairs(stores) do
+			local p, typ = store:match("()%-(%w+)$");
+			if typ then store = store:sub(1, p-1); else typ = "keyval"; end
+			log("info", "Migrating host %s store %s (%s)", host, store, typ);
+
+			local origin = assert(input_driver:open(store, typ));
+			local destination = assert(output_driver:open(store, typ));
+
+			if typ == "keyval" then -- host data
+				local data, err = origin:get(nil);
+				assert(not err, err);
+				assert(destination:set(nil, data));
+			end
+
+			for user in users(origin, host) do
+				if typ == "keyval" then
+					local data, err = origin:get(user);
+					assert(not err, err);
+					assert(destination:set(user, data));
+				elseif typ == "archive" then
+					local iter, err = origin:find(user);
+					assert(iter, err);
+					for id, item, when, with in iter do
+						assert(destination:append(user, id, item, when, with));
+					end
+				else
+					error("Don't know how to migrate data of type '"..typ.."'.");
+				end
+			end
+		end
+	end
+end, watchers);
+
+io.stderr:write("Migrating...\n");
+
+migration_runner:run({ input = config[from_store], output = config[to_store] });
+
 io.stderr:write("Done!\n");
diff --git a/tools/modtrace.lua b/tools/modtrace.lua
new file mode 100644
index 00000000..45fa9f6a
--- /dev/null
+++ b/tools/modtrace.lua
@@ -0,0 +1,159 @@
+-- Trace module calls and method calls on created objects
+--
+-- Very rough and for debugging purposes only. It makes many
+-- assumptions and there are many ways it could fail.
+--
+-- Example use:
+--
+--   local dbuffer = require "tools.modtrace".trace("util.dbuffer");
+--
+
+local t_pack = require "util.table".pack;
+local serialize = require "util.serialization".serialize;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+local set = require "util.set";
+
+local serialize_cfg = {
+	preset = "oneline";
+	freeze = true;
+	fatal = false;
+	fallback = function (v) return "<"..tostring(v)..">" end;
+};
+
+local function stringify_value(v)
+	if type(v) == "string" and #v > 20 then
+		return ("<string(%d)>"):format(#v);
+	elseif type(v) == "function" then
+		return tostring(v);
+	end
+	return serialize(v, serialize_cfg);
+end
+
+local function stringify_params(...)
+	local n = select("#", ...);
+	local r = {};
+	for i = 1, n do
+		table.insert(r, stringify_value((select(i, ...))));
+	end
+	return table.concat(r, ", ");
+end
+
+local function stringify_result(ret)
+	local r = {};
+	for i = 1, ret.n do
+		table.insert(r, stringify_value(ret[i]));
+	end
+	return table.concat(r, ", ");
+end
+
+local function stringify_call(method_name, ...)
+	return ("%s(%s)"):format(method_name, stringify_params(...));
+end
+
+local function wrap_method(original_obj, original_method, method_name)
+	method_name = ("<%s>:%s"):format(getmetatable(original_obj).__name or "object", method_name);
+	return function (new_obj_self, ...)
+		local opts = new_obj_self._modtrace_opts;
+		local f = opts.output or io.stderr;
+		f:write(stringify_call(method_name, ...));
+		local ret = t_pack(original_method(original_obj, ...));
+		if ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function wrap_function(original_function, function_name, opts)
+	local f = opts.output or io.stderr;
+	return function (...)
+		f:write(stringify_call(function_name, ...));
+		local ret = t_pack(original_function(...));
+		if ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function wrap_metamethod(name, method)
+	if name == "__index" then
+		return function (new_obj, k)
+			local original_method;
+			if type(method) == "table" then
+				original_method = new_obj._modtrace_original_obj[k];
+			else
+				original_method = method(new_obj._modtrace_original_obj, k);
+			end
+			if original_method == nil then
+				return nil;
+			end
+			return wrap_method(new_obj._modtrace_original_obj, original_method, k);
+		end;
+	end
+	return function (new_obj, ...)
+		return method(new_obj._modtrace_original_obj, ...);
+	end;
+end
+
+local function wrap_mt(original_mt)
+	local new_mt = {};
+	for k, v in pairs(original_mt) do
+		new_mt[k] = wrap_metamethod(k, v);
+	end
+	return new_mt;
+end
+
+local function wrap_obj(original_obj, opts)
+	local new_mt = wrap_mt(getmetatable(original_obj));
+	return setmetatable({_modtrace_original_obj = original_obj, _modtrace_opts = opts}, new_mt);
+end
+
+local function wrap_new(original_new, function_name, opts)
+	local f = opts.output or io.stderr;
+	return function (...)
+		f:write(stringify_call(function_name, ...));
+		local ret = t_pack(original_new(...));
+		local obj = ret[1];
+
+		if ret.n == 1 and type(ret[1]) == "table" then
+			f:write(" = <", getmetatable(ret[1]).__name or "object", ">", "\n");
+		elseif ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+
+		if obj then
+			ret[1] = wrap_obj(obj, opts);
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function trace(module, opts)
+	if type(module) == "string" then
+		module = require(module);
+	end
+	opts = opts or {};
+	local new_methods = set.new(opts.new_methods or {"new"});
+	local fake_module = setmetatable({}, {
+		__index = function (_, k)
+			if new_methods:contains(k) then
+				return wrap_new(module[k], k, opts);
+			else
+				return wrap_function(module[k], k, opts);
+			end
+		end;
+	});
+	return fake_module;
+end
+
+return {
+	wrap = trace;
+	trace = trace;
+}
diff --git a/tools/tb2err b/tools/tb2err
new file mode 100755
index 00000000..7b676813
--- /dev/null
+++ b/tools/tb2err
@@ -0,0 +1,21 @@
+#!/usr/bin/env lua-any
+-- Lua-Versions: 5.3 5.2 5.1
+-- traceback to errors.err for vim -q
+local path_sep = package.config:sub(1,1);
+for line in io.lines() do
+	local src, err = line:match("%s*(%S+)(:%d+: .*)")
+	if src then
+		src = src:gsub("\\", path_sep);
+		local cut = src:match("/()core/")
+			or src:match("/()net/")
+			or src:match("/()util/")
+			or src:match("/()modules/")
+			or src:match("/()plugins/")
+			or src:match("/()prosody[ctl]*$") 
+		if cut then
+			src = src:sub(cut);
+		end
+		src = src:gsub("^modules/", "plugins/")
+		io.write(src, err, "\n");
+	end
+end
diff --git a/util-src/GNUmakefile b/util-src/GNUmakefile
index a8b3529f..0b96bf41 100644
--- a/util-src/GNUmakefile
+++ b/util-src/GNUmakefile
@@ -24,6 +24,7 @@ install: $(ALL)
 clean:
 	rm -f $(ALL) $(patsubst %.so,%.o,$(ALL))
 
+encodings.o: CFLAGS+=$(IDNA_FLAGS)
 encodings.so: LDLIBS+=$(IDNA_LIBS)
 
 hashes.so: LDLIBS+=$(OPENSSL_LIBS)
diff --git a/util-src/encodings.c b/util-src/encodings.c
index e55a3f44..72264da8 100644
--- a/util-src/encodings.c
+++ b/util-src/encodings.c
@@ -24,6 +24,9 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
 
 /***************** BASE64 *****************/
 
@@ -216,7 +219,7 @@ static const char *utf8_decode(const char *o, int *val) {
  * Check that a string is valid UTF-8
  * Returns NULL if not
  */
-const char *check_utf8(lua_State *L, int idx, size_t *l) {
+static const char *check_utf8(lua_State *L, int idx, size_t *l) {
 	size_t pos, len;
 	const char *s = luaL_checklstring(L, idx, &len);
 	pos = 0;
@@ -247,7 +250,7 @@ static int Lutf8_length(lua_State *L) {
 	size_t len;
 
 	if(!check_utf8(L, 1, &len)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushliteral(L, "invalid utf8");
 		return 2;
 	}
@@ -268,41 +271,47 @@ static const luaL_Reg Reg_utf8[] = {
 #include <unicode/usprep.h>
 #include <unicode/ustring.h>
 #include <unicode/utrace.h>
+#include <unicode/uspoof.h>
+#include <unicode/uidna.h>
 
 static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile) {
 	size_t input_len;
 	int32_t unprepped_len, prepped_len, output_len;
 	const char *input;
 	char output[1024];
+	int flags = USPREP_ALLOW_UNASSIGNED;
 
 	UChar unprepped[1024]; /* Temporary unicode buffer (1024 characters) */
 	UChar prepped[1024];
 
 	UErrorCode err = U_ZERO_ERROR;
 
-	if(!lua_isstring(L, 1)) {
-		lua_pushnil(L);
-		return 1;
-	}
-
-	input = lua_tolstring(L, 1, &input_len);
+	input = luaL_checklstring(L, 1, &input_len);
 
 	if(input_len >= 1024) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
+	/* strict */
+	if(!lua_isnoneornil(L, 2)) {
+		luaL_checktype(L, 2, LUA_TBOOLEAN);
+		if(lua_toboolean(L, 2)) {
+			flags = 0;
+		}
+	}
+
 	u_strFromUTF8(unprepped, 1024, &unprepped_len, input, input_len, &err);
 
 	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
-	prepped_len = usprep_prepare(profile, unprepped, unprepped_len, prepped, 1024, USPREP_ALLOW_UNASSIGNED, NULL, &err);
+	prepped_len = usprep_prepare(profile, unprepped, unprepped_len, prepped, 1024, flags, NULL, &err);
 
 	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	} else {
 		u_strToUTF8(output, 1024, &output_len, prepped, prepped_len, &err);
@@ -310,29 +319,62 @@ static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile)
 		if(U_SUCCESS(err) && output_len < 1024) {
 			lua_pushlstring(L, output, output_len);
 		} else {
-			lua_pushnil(L);
+			luaL_pushfail(L);
 		}
 
 		return 1;
 	}
 }
 
-UStringPrepProfile *icu_nameprep;
-UStringPrepProfile *icu_nodeprep;
-UStringPrepProfile *icu_resourceprep;
-UStringPrepProfile *icu_saslprep;
+static UStringPrepProfile *icu_nameprep;
+static UStringPrepProfile *icu_nodeprep;
+static UStringPrepProfile *icu_resourceprep;
+static UStringPrepProfile *icu_saslprep;
+static USpoofChecker *icu_spoofcheck;
+static UIDNA *icu_idna2008;
+
+#if (U_ICU_VERSION_MAJOR_NUM < 58)
+/* COMPAT */
+#define USPOOF_CONFUSABLE (USPOOF_SINGLE_SCRIPT_CONFUSABLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_WHOLE_SCRIPT_CONFUSABLE)
+#endif
 
 /* initialize global ICU stringprep profiles */
-void init_icu() {
+static void init_icu(void) {
 	UErrorCode err = U_ZERO_ERROR;
 	utrace_setLevel(UTRACE_VERBOSE);
 	icu_nameprep = usprep_openByType(USPREP_RFC3491_NAMEPREP, &err);
 	icu_nodeprep = usprep_openByType(USPREP_RFC3920_NODEPREP, &err);
 	icu_resourceprep = usprep_openByType(USPREP_RFC3920_RESOURCEPREP, &err);
 	icu_saslprep = usprep_openByType(USPREP_RFC4013_SASLPREP, &err);
+	icu_spoofcheck = uspoof_open(&err);
+	uspoof_setChecks(icu_spoofcheck, USPOOF_CONFUSABLE, &err);
+	int options = UIDNA_DEFAULT;
+#if 0
+	/* COMPAT with future Unicode versions */
+	options |= UIDNA_ALLOW_UNASSIGNED;
+#endif
+#if 1
+	/* Forbid eg labels starting with _ */
+	options |= UIDNA_USE_STD3_RULES;
+#endif
+#if 0
+	/* TODO determine if we need this */
+	options |= UIDNA_CHECK_BIDI;
+#endif
+#if 0
+	/* UTS46 makes it sound like these are the responsibility of registrars */
+	options |= UIDNA_CHECK_CONTEXTJ;
+	options |= UIDNA_CHECK_CONTEXTO;
+#endif
+#if 0
+	/* This disables COMPAT with IDNA 2003 */
+	options |= UIDNA_NONTRANSITIONAL_TO_ASCII;
+	options |= UIDNA_NONTRANSITIONAL_TO_UNICODE;
+#endif
+	icu_idna2008 = uidna_openUTS46(options, &err);
 
 	if(U_FAILURE(err)) {
-		fprintf(stderr, "[c] util.encodings: error: %s\n", u_errorName((UErrorCode)err));
+		fprintf(stderr, "[c] util.encodings: error: %s\n", u_errorName(err));
 	}
 }
 
@@ -362,27 +404,31 @@ static int stringprep_prep(lua_State *L, const Stringprep_profile *profile) {
 	const char *s;
 	char string[1024];
 	int ret;
-
-	if(!lua_isstring(L, 1)) {
-		lua_pushnil(L);
-		return 1;
-	}
+	Stringprep_profile_flags flags = 0;
 
 	s = check_utf8(L, 1, &len);
 
+	/* strict */
+	if(!lua_isnoneornil(L, 2)) {
+		luaL_checktype(L, 2, LUA_TBOOLEAN);
+		if(lua_toboolean(L, 2)) {
+			flags = STRINGPREP_NO_UNASSIGNED;
+		}
+	}
+
 	if(s == NULL || len >= 1024 || len != strlen(s)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1; /* TODO return error message */
 	}
 
 	strcpy(string, s);
-	ret = stringprep(string, 1024, (Stringprep_profile_flags)0, profile);
+	ret = stringprep(string, 1024, flags, profile);
 
 	if(ret == STRINGPREP_OK) {
 		lua_pushstring(L, string);
 		return 1;
 	} else {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1; /* TODO return error message */
 	}
 }
@@ -421,14 +467,15 @@ static int Lidna_to_ascii(lua_State *L) {	/** idna.to_ascii(s) */
 	u_strFromUTF8(ustr, 1024, &ulen, s, len, &err);
 
 	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
-	dest_len = uidna_IDNToASCII(ustr, ulen, dest, 1024, UIDNA_USE_STD3_RULES, NULL, &err);
+	UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+	dest_len = uidna_nameToASCII(icu_idna2008, ustr, ulen, dest, 256, &info, &err);
 
-	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+	if(U_FAILURE(err) || info.errors) {
+		luaL_pushfail(L);
 		return 1;
 	} else {
 		u_strToUTF8(output, 1024, &output_len, dest, dest_len, &err);
@@ -436,7 +483,7 @@ static int Lidna_to_ascii(lua_State *L) {	/** idna.to_ascii(s) */
 		if(U_SUCCESS(err) && output_len < 1024) {
 			lua_pushlstring(L, output, output_len);
 		} else {
-			lua_pushnil(L);
+			luaL_pushfail(L);
 		}
 
 		return 1;
@@ -455,14 +502,15 @@ static int Lidna_to_unicode(lua_State *L) {	/** idna.to_unicode(s) */
 	u_strFromUTF8(ustr, 1024, &ulen, s, len, &err);
 
 	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
-	dest_len = uidna_IDNToUnicode(ustr, ulen, dest, 1024, UIDNA_USE_STD3_RULES, NULL, &err);
+	UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+	dest_len = uidna_nameToUnicode(icu_idna2008, ustr, ulen, dest, 1024, &info, &err);
 
-	if(U_FAILURE(err)) {
-		lua_pushnil(L);
+	if(U_FAILURE(err) || info.errors) {
+		luaL_pushfail(L);
 		return 1;
 	} else {
 		u_strToUTF8(output, 1024, &output_len, dest, dest_len, &err);
@@ -470,13 +518,47 @@ static int Lidna_to_unicode(lua_State *L) {	/** idna.to_unicode(s) */
 		if(U_SUCCESS(err) && output_len < 1024) {
 			lua_pushlstring(L, output, output_len);
 		} else {
-			lua_pushnil(L);
+			luaL_pushfail(L);
 		}
 
 		return 1;
 	}
 }
 
+static int Lskeleton(lua_State *L) {
+	size_t len;
+	int32_t ulen, dest_len, output_len;
+	const char *s = luaL_checklstring(L, 1, &len);
+	UErrorCode err = U_ZERO_ERROR;
+	UChar ustr[1024];
+	UChar dest[1024];
+	char output[1024];
+
+	u_strFromUTF8(ustr, 1024, &ulen, s, len, &err);
+
+	if(U_FAILURE(err)) {
+		luaL_pushfail(L);
+		return 1;
+	}
+
+	dest_len = uspoof_getSkeleton(icu_spoofcheck, 0, ustr, ulen, dest, 1024, &err);
+
+	if(U_FAILURE(err)) {
+		luaL_pushfail(L);
+		return 1;
+	}
+
+	u_strToUTF8(output, 1024, &output_len, dest, dest_len, &err);
+
+	if(U_SUCCESS(err)) {
+		lua_pushlstring(L, output, output_len);
+		return 1;
+	}
+
+	luaL_pushfail(L);
+	return 1;
+}
+
 #else /* USE_STRINGPREP_ICU */
 /****************** libidn ********************/
 
@@ -490,7 +572,7 @@ static int Lidna_to_ascii(lua_State *L) {	/** idna.to_ascii(s) */
 	int ret;
 
 	if(s == NULL || len != strlen(s)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1; /* TODO return error message */
 	}
 
@@ -501,7 +583,7 @@ static int Lidna_to_ascii(lua_State *L) {	/** idna.to_ascii(s) */
 		idn_free(output);
 		return 1;
 	} else {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		idn_free(output);
 		return 1; /* TODO return error message */
 	}
@@ -518,7 +600,7 @@ static int Lidna_to_unicode(lua_State *L) {	/** idna.to_unicode(s) */
 		idn_free(output);
 		return 1;
 	} else {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		idn_free(output);
 		return 1; /* TODO return error message */
 	}
@@ -558,6 +640,13 @@ LUALIB_API int luaopen_util_encodings(lua_State *L) {
 	luaL_setfuncs(L, Reg_utf8, 0);
 	lua_setfield(L, -2, "utf8");
 
+#ifdef USE_STRINGPREP_ICU
+	lua_newtable(L);
+	lua_pushcfunction(L, Lskeleton);
+	lua_setfield(L, -2, "skeleton");
+	lua_setfield(L, -2, "confusable");
+#endif
+
 	lua_pushliteral(L, "-3.14");
 	lua_setfield(L, -2, "version");
 	return 1;
diff --git a/util-src/hashes.c b/util-src/hashes.c
index 903ecb6e..84a604ef 100644
--- a/util-src/hashes.c
+++ b/util-src/hashes.c
@@ -26,6 +26,7 @@ typedef unsigned __int32 uint32_t;
 #include <openssl/sha.h>
 #include <openssl/md5.h>
 #include <openssl/hmac.h>
+#include <openssl/evp.h>
 
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
@@ -34,8 +35,8 @@ typedef unsigned __int32 uint32_t;
 #define HMAC_IPAD 0x36363636
 #define HMAC_OPAD 0x5c5c5c5c
 
-const char *hex_tab = "0123456789abcdef";
-void toHex(const unsigned char *in, int length, unsigned char *out) {
+static const char *hex_tab = "0123456789abcdef";
+static void toHex(const unsigned char *in, int length, unsigned char *out) {
 	int i;
 
 	for(i = 0; i < length; i++) {
@@ -75,44 +76,6 @@ struct hash_desc {
 	void *ctx, *ctxo;
 };
 
-static void hmac(struct hash_desc *desc, const char *key, size_t key_len,
-                 const char *msg, size_t msg_len, unsigned char *result) {
-	union xory {
-		unsigned char bytes[64];
-		uint32_t quadbytes[16];
-	};
-
-	int i;
-	unsigned char hashedKey[64]; /* Maximum used digest length */
-	union xory k_ipad, k_opad;
-
-	if(key_len > 64) {
-		desc->Init(desc->ctx);
-		desc->Update(desc->ctx, key, key_len);
-		desc->Final(hashedKey, desc->ctx);
-		key = (const char *)hashedKey;
-		key_len = desc->digestLength;
-	}
-
-	memcpy(k_ipad.bytes, key, key_len);
-	memset(k_ipad.bytes + key_len, 0, 64 - key_len);
-	memcpy(k_opad.bytes, k_ipad.bytes, 64);
-
-	for(i = 0; i < 16; i++) {
-		k_ipad.quadbytes[i] ^= HMAC_IPAD;
-		k_opad.quadbytes[i] ^= HMAC_OPAD;
-	}
-
-	desc->Init(desc->ctx);
-	desc->Update(desc->ctx, k_ipad.bytes, 64);
-	desc->Init(desc->ctxo);
-	desc->Update(desc->ctxo, k_opad.bytes, 64);
-	desc->Update(desc->ctx, msg, msg_len);
-	desc->Final(result, desc->ctx);
-	desc->Update(desc->ctxo, result, desc->digestLength);
-	desc->Final(result, desc->ctxo);
-}
-
 #define MAKE_HMAC_FUNCTION(myFunc, evp, size, type) \
 static int myFunc(lua_State *L) { \
 	unsigned char hash[size], result[2*size]; \
@@ -136,56 +99,37 @@ MAKE_HMAC_FUNCTION(Lhmac_sha256, EVP_sha256, SHA256_DIGEST_LENGTH, SHA256_CTX)
 MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX)
 MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX)
 
-static int LscramHi(lua_State *L) {
-	union xory {
-		unsigned char bytes[SHA_DIGEST_LENGTH];
-		uint32_t quadbytes[SHA_DIGEST_LENGTH / 4];
-	};
-	int i;
-	SHA_CTX ctx, ctxo;
-	unsigned char Ust[SHA_DIGEST_LENGTH];
-	union xory Und;
-	union xory res;
-	size_t str_len, salt_len;
-	struct hash_desc desc;
-	const char *str = luaL_checklstring(L, 1, &str_len);
-	const char *salt = luaL_checklstring(L, 2, &salt_len);
-	char *salt2;
-	const int iter = luaL_checkinteger(L, 3);
-
-	desc.Init = (int (*)(void *))SHA1_Init;
-	desc.Update = (int (*)(void *, const void *, size_t))SHA1_Update;
-	desc.Final = (int (*)(unsigned char *, void *))SHA1_Final;
-	desc.digestLength = SHA_DIGEST_LENGTH;
-	desc.ctx = &ctx;
-	desc.ctxo = &ctxo;
+static int Lpbkdf2_sha1(lua_State *L) {
+	unsigned char out[SHA_DIGEST_LENGTH];
 
-	salt2 = malloc(salt_len + 4);
+	size_t pass_len, salt_len;
+	const char *pass = luaL_checklstring(L, 1, &pass_len);
+	const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len);
+	const int iter = luaL_checkinteger(L, 3);
 
-	if(salt2 == NULL) {
-		return luaL_error(L, "Out of memory in scramHi");
+	if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), SHA_DIGEST_LENGTH, out) == 0) {
+		return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed");
 	}
 
-	memcpy(salt2, salt, salt_len);
-	memcpy(salt2 + salt_len, "\0\0\0\1", 4);
-	hmac(&desc, str, str_len, salt2, salt_len + 4, Ust);
-	free(salt2);
+	lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH);
 
-	memcpy(res.bytes, Ust, sizeof(res));
+	return 1;
+}
 
-	for(i = 1; i < iter; i++) {
-		int j;
-		hmac(&desc, str, str_len, (char *)Ust, sizeof(Ust), Und.bytes);
 
-		for(j = 0; j < SHA_DIGEST_LENGTH / 4; j++) {
-			res.quadbytes[j] ^= Und.quadbytes[j];
-		}
+static int Lpbkdf2_sha256(lua_State *L) {
+	unsigned char out[SHA256_DIGEST_LENGTH];
 
-		memcpy(Ust, Und.bytes, sizeof(Ust));
-	}
+	size_t pass_len, salt_len;
+	const char *pass = luaL_checklstring(L, 1, &pass_len);
+	const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len);
+	const int iter = luaL_checkinteger(L, 3);
 
-	lua_pushlstring(L, (char *)res.bytes, SHA_DIGEST_LENGTH);
+	if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), SHA256_DIGEST_LENGTH, out) == 0) {
+		return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed");
+	}
 
+	lua_pushlstring(L, (char *)out, SHA256_DIGEST_LENGTH);
 	return 1;
 }
 
@@ -200,7 +144,9 @@ static const luaL_Reg Reg[] = {
 	{ "hmac_sha256",	Lhmac_sha256	},
 	{ "hmac_sha512",	Lhmac_sha512	},
 	{ "hmac_md5",		Lhmac_md5	},
-	{ "scram_Hi_sha1",	LscramHi	},
+	{ "scram_Hi_sha1",	Lpbkdf2_sha1	}, /* COMPAT */
+	{ "pbkdf2_hmac_sha1",	Lpbkdf2_sha1	},
+	{ "pbkdf2_hmac_sha256",	Lpbkdf2_sha256	},
 	{ NULL,			NULL		}
 };
 
@@ -209,7 +155,7 @@ LUALIB_API int luaopen_util_hashes(lua_State *L) {
 	luaL_checkversion(L);
 #endif
 	lua_newtable(L);
-	luaL_setfuncs(L, Reg, 0);;
+	luaL_setfuncs(L, Reg, 0);
 	lua_pushliteral(L, "-3.14");
 	lua_setfield(L, -2, "version");
 	return 1;
diff --git a/util-src/makefile b/util-src/makefile
index 02bad40a..1560fca4 100644
--- a/util-src/makefile
+++ b/util-src/makefile
@@ -23,6 +23,8 @@ install: $(ALL)
 clean:
 	rm -f $(ALL) $(patsubst %.so,%.o,$(ALL))
 
+encodings.o: encodings.c
+	$(CC) $(CFLAGS) $(IDNA_FLAGS) -c -o $@ $<
 encodings.so: encodings.o
 	$(LD) $(LDFLAGS) -o $@ $< $(LDLIBS) $(IDNA_LIBS)
 
diff --git a/util-src/net.c b/util-src/net.c
index 5f706d81..d786e885 100644
--- a/util-src/net.c
+++ b/util-src/net.c
@@ -33,10 +33,13 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
 
 /* Enumerate all locally configured IP addresses */
 
-const char *const type_strings[] = {
+static const char *const type_strings[] = {
 	"both",
 	"ipv4",
 	"ipv6",
@@ -46,8 +49,8 @@ const char *const type_strings[] = {
 static int lc_local_addresses(lua_State *L) {
 #ifndef _WIN32
 	/* Link-local IPv4 addresses; see RFC 3927 and RFC 5735 */
-	const long ip4_linklocal = htonl(0xa9fe0000); /* 169.254.0.0 */
-	const long ip4_mask      = htonl(0xffff0000);
+	const uint32_t ip4_linklocal = htonl(0xa9fe0000); /* 169.254.0.0 */
+	const uint32_t ip4_mask      = htonl(0xffff0000);
 	struct ifaddrs *addr = NULL, *a;
 #endif
 	int n = 1;
@@ -59,7 +62,7 @@ static int lc_local_addresses(lua_State *L) {
 #ifndef _WIN32
 
 	if(getifaddrs(&addr) < 0) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushfstring(L, "getifaddrs failed (%d): %s", errno,
 		                strerror(errno));
 		return 2;
@@ -141,14 +144,14 @@ static int lc_pton(lua_State *L) {
 
 		case -1:
 			errno_ = errno;
-			lua_pushnil(L);
+			luaL_pushfail(L);
 			lua_pushstring(L, strerror(errno_));
 			lua_pushinteger(L, errno_);
 			return 3;
 
 		default:
 		case 0:
-			lua_pushnil(L);
+			luaL_pushfail(L);
 			lua_pushstring(L, strerror(EINVAL));
 			lua_pushinteger(L, EINVAL);
 			return 3;
@@ -170,7 +173,7 @@ static int lc_ntop(lua_State *L) {
 		family = AF_INET;
 	}
 	else {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(EAFNOSUPPORT));
 		lua_pushinteger(L, EAFNOSUPPORT);
 		return 3;
@@ -179,7 +182,7 @@ static int lc_ntop(lua_State *L) {
 	if(!inet_ntop(family, ipaddr, buf, INET6_ADDRSTRLEN))
 	{
 		errno_ = errno;
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(errno_));
 		lua_pushinteger(L, errno_);
 		return 3;
diff --git a/util-src/poll.c b/util-src/poll.c
index 0ca0cf28..6c6a4e63 100644
--- a/util-src/poll.c
+++ b/util-src/poll.c
@@ -37,6 +37,9 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setmetatable(L, tname) luaL_getmetatable(L, tname); lua_setmetatable(L, -2)
 #endif
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
 
 /*
  * Structure to keep state for each type of API
@@ -59,7 +62,7 @@ typedef struct Lpoll_state {
 /*
  * Add an FD to be watched
  */
-int Ladd(lua_State *L) {
+static int Ladd(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -67,7 +70,7 @@ int Ladd(lua_State *L) {
 	int wantwrite = lua_toboolean(L, 4);
 
 	if(fd < 0) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(EBADF));
 		lua_pushinteger(L, EBADF);
 		return 3;
@@ -84,7 +87,7 @@ int Ladd(lua_State *L) {
 
 	if(ret < 0) {
 		ret = errno;
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ret));
 		lua_pushinteger(L, ret);
 		return 3;
@@ -96,14 +99,14 @@ int Ladd(lua_State *L) {
 #else
 
 	if(fd > FD_SETSIZE) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(EBADF));
 		lua_pushinteger(L, EBADF);
 		return 3;
 	}
 
 	if(FD_ISSET(fd, &state->all)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(EEXIST));
 		lua_pushinteger(L, EEXIST);
 		return 3;
@@ -137,7 +140,7 @@ int Ladd(lua_State *L) {
 /*
  * Set events to watch for, readable and/or writable
  */
-int Lset(lua_State *L) {
+static int Lset(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -160,7 +163,7 @@ int Lset(lua_State *L) {
 	}
 	else {
 		ret = errno;
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ret));
 		lua_pushinteger(L, ret);
 		return 3;
@@ -169,9 +172,10 @@ int Lset(lua_State *L) {
 #else
 
 	if(!FD_ISSET(fd, &state->all)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ENOENT));
 		lua_pushinteger(L, ENOENT);
+		return 3;
 	}
 
 	if(!lua_isnoneornil(L, 3)) {
@@ -200,7 +204,7 @@ int Lset(lua_State *L) {
 /*
  * Remove FDs
  */
-int Ldel(lua_State *L) {
+static int Ldel(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	int fd = luaL_checkinteger(L, 2);
 
@@ -217,7 +221,7 @@ int Ldel(lua_State *L) {
 	}
 	else {
 		ret = errno;
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ret));
 		lua_pushinteger(L, ret);
 		return 3;
@@ -226,9 +230,10 @@ int Ldel(lua_State *L) {
 #else
 
 	if(!FD_ISSET(fd, &state->all)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ENOENT));
 		lua_pushinteger(L, ENOENT);
+		return 3;
 	}
 
 	FD_CLR(fd, &state->wantread);
@@ -247,7 +252,7 @@ int Ldel(lua_State *L) {
 /*
  * Check previously manipulated event state for FDs ready for reading or writing
  */
-int Lpushevent(lua_State *L, struct Lpoll_state *state) {
+static int Lpushevent(lua_State *L, struct Lpoll_state *state) {
 #ifdef USE_EPOLL
 
 	if(state->processed > 0) {
@@ -281,7 +286,7 @@ int Lpushevent(lua_State *L, struct Lpoll_state *state) {
 /*
  * Wait for event
  */
-int Lwait(lua_State *L) {
+static int Lwait(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 
 	int ret = Lpushevent(L, state);
@@ -312,18 +317,20 @@ int Lwait(lua_State *L) {
 #endif
 
 	if(ret == 0) {
+		/* Is this an error? */
 		lua_pushnil(L);
 		lua_pushstring(L, "timeout");
 		return 2;
 	}
 	else if(ret < 0 && errno == EINTR) {
+		/* Is this an error? */
 		lua_pushnil(L);
 		lua_pushstring(L, "signal");
 		return 2;
 	}
 	else if(ret < 0) {
 		ret = errno;
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(ret));
 		lua_pushinteger(L, ret);
 		return 3;
@@ -344,7 +351,7 @@ int Lwait(lua_State *L) {
 /*
  * Return Epoll FD
  */
-int Lgetfd(lua_State *L) {
+static int Lgetfd(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	lua_pushinteger(L, state->epoll_fd);
 	return 1;
@@ -353,7 +360,7 @@ int Lgetfd(lua_State *L) {
 /*
  * Close epoll FD
  */
-int Lgc(lua_State *L) {
+static int Lgc(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 
 	if(state->epoll_fd == -1) {
@@ -375,7 +382,7 @@ int Lgc(lua_State *L) {
 /*
  * String representation
  */
-int Ltos(lua_State *L) {
+static int Ltos(lua_State *L) {
 	struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT);
 	lua_pushfstring(L, "%s: %p", STATE_MT, state);
 	return 1;
@@ -384,7 +391,7 @@ int Ltos(lua_State *L) {
 /*
  * Create a new context
  */
-int Lnew(lua_State *L) {
+static int Lnew(lua_State *L) {
 	/* Allocate state */
 	Lpoll_state *state = lua_newuserdata(L, sizeof(Lpoll_state));
 	luaL_setmetatable(L, STATE_MT);
@@ -397,7 +404,7 @@ int Lnew(lua_State *L) {
 	int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
 
 	if(epoll_fd <= 0) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(errno));
 		lua_pushinteger(L, errno);
 		return 3;
diff --git a/util-src/pposix.c b/util-src/pposix.c
index 004f61a6..856905b0 100644
--- a/util-src/pposix.c
+++ b/util-src/pposix.c
@@ -25,14 +25,18 @@
 #define _DEFAULT_SOURCE
 #endif
 #endif
+
 #if defined(__APPLE__)
 #ifndef _DARWIN_C_SOURCE
 #define _DARWIN_C_SOURCE
 #endif
 #endif
+
+#if ! defined(__FreeBSD__)
 #ifndef _POSIX_C_SOURCE
 #define _POSIX_C_SOURCE 200809L
 #endif
+#endif
 
 #include <stdlib.h>
 #include <math.h>
@@ -57,6 +61,12 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
+#if (LUA_VERSION_NUM < 503)
+#define lua_isinteger(L, n) lua_isnumber(L, n)
+#endif
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
 
 #include <fcntl.h>
 #if defined(__linux__)
@@ -102,7 +112,7 @@ static int lc_daemonize(lua_State *L) {
 	} else if(pid != 0) {
 		/* We are the parent process */
 		lua_pushboolean(L, 1);
-		lua_pushnumber(L, pid);
+		lua_pushinteger(L, pid);
 		return 2;
 	}
 
@@ -133,7 +143,7 @@ static int lc_daemonize(lua_State *L) {
 
 /* Syslog support */
 
-const char *const facility_strings[] = {
+static const char *const facility_strings[] = {
 	"auth",
 #if !(defined(sun) || defined(__sun))
 	"authpriv",
@@ -159,7 +169,7 @@ const char *const facility_strings[] = {
 	"uucp",
 	NULL
 };
-int facility_constants[] =	{
+static int facility_constants[] =	{
 	LOG_AUTH,
 #if !(defined(sun) || defined(__sun))
 	LOG_AUTHPRIV,
@@ -195,9 +205,9 @@ int facility_constants[] =	{
        constant.
    " -- syslog manpage
 */
-char *syslog_ident = NULL;
+static char *syslog_ident = NULL;
 
-int lc_syslog_open(lua_State *L) {
+static int lc_syslog_open(lua_State *L) {
 	int facility = luaL_checkoption(L, 2, "daemon", facility_strings);
 	facility = facility_constants[facility];
 
@@ -213,7 +223,7 @@ int lc_syslog_open(lua_State *L) {
 	return 0;
 }
 
-const char *const level_strings[] = {
+static const char *const level_strings[] = {
 	"debug",
 	"info",
 	"notice",
@@ -221,7 +231,7 @@ const char *const level_strings[] = {
 	"error",
 	NULL
 };
-int level_constants[] = 	{
+static int level_constants[] = 	{
 	LOG_DEBUG,
 	LOG_INFO,
 	LOG_NOTICE,
@@ -229,7 +239,7 @@ int level_constants[] = 	{
 	LOG_CRIT,
 	-1
 };
-int lc_syslog_log(lua_State *L) {
+static int lc_syslog_log(lua_State *L) {
 	int level = level_constants[luaL_checkoption(L, 1, "notice", level_strings)];
 
 	if(lua_gettop(L) == 3) {
@@ -241,7 +251,7 @@ int lc_syslog_log(lua_State *L) {
 	return 0;
 }
 
-int lc_syslog_close(lua_State *L) {
+static int lc_syslog_close(lua_State *L) {
 	(void)L;
 	closelog();
 
@@ -253,7 +263,7 @@ int lc_syslog_close(lua_State *L) {
 	return 0;
 }
 
-int lc_syslog_setmask(lua_State *L) {
+static int lc_syslog_setmask(lua_State *L) {
 	int level_idx = luaL_checkoption(L, 1, "notice", level_strings);
 	int mask = 0;
 
@@ -267,31 +277,31 @@ int lc_syslog_setmask(lua_State *L) {
 
 /* getpid */
 
-int lc_getpid(lua_State *L) {
+static int lc_getpid(lua_State *L) {
 	lua_pushinteger(L, getpid());
 	return 1;
 }
 
 /* UID/GID functions */
 
-int lc_getuid(lua_State *L) {
+static int lc_getuid(lua_State *L) {
 	lua_pushinteger(L, getuid());
 	return 1;
 }
 
-int lc_getgid(lua_State *L) {
+static int lc_getgid(lua_State *L) {
 	lua_pushinteger(L, getgid());
 	return 1;
 }
 
-int lc_setuid(lua_State *L) {
+static int lc_setuid(lua_State *L) {
 	int uid = -1;
 
 	if(lua_gettop(L) < 1) {
 		return 0;
 	}
 
-	if(!lua_isnumber(L, 1) && lua_tostring(L, 1)) {
+	if(!lua_isinteger(L, 1) && lua_tostring(L, 1)) {
 		/* Passed UID is actually a string, so look up the UID */
 		struct passwd *p;
 		p = getpwnam(lua_tostring(L, 1));
@@ -304,7 +314,7 @@ int lc_setuid(lua_State *L) {
 
 		uid = p->pw_uid;
 	} else {
-		uid = lua_tonumber(L, 1);
+		uid = lua_tointeger(L, 1);
 	}
 
 	if(uid > -1) {
@@ -342,14 +352,14 @@ int lc_setuid(lua_State *L) {
 	return 2;
 }
 
-int lc_setgid(lua_State *L) {
+static int lc_setgid(lua_State *L) {
 	int gid = -1;
 
 	if(lua_gettop(L) < 1) {
 		return 0;
 	}
 
-	if(!lua_isnumber(L, 1) && lua_tostring(L, 1)) {
+	if(!lua_isinteger(L, 1) && lua_tostring(L, 1)) {
 		/* Passed GID is actually a string, so look up the GID */
 		struct group *g;
 		g = getgrnam(lua_tostring(L, 1));
@@ -362,7 +372,7 @@ int lc_setgid(lua_State *L) {
 
 		gid = g->gr_gid;
 	} else {
-		gid = lua_tonumber(L, 1);
+		gid = lua_tointeger(L, 1);
 	}
 
 	if(gid > -1) {
@@ -400,13 +410,13 @@ int lc_setgid(lua_State *L) {
 	return 2;
 }
 
-int lc_initgroups(lua_State *L) {
+static int lc_initgroups(lua_State *L) {
 	int ret;
 	gid_t gid;
 	struct passwd *p;
 
 	if(!lua_isstring(L, 1)) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, "invalid-username");
 		return 2;
 	}
@@ -414,7 +424,7 @@ int lc_initgroups(lua_State *L) {
 	p = getpwnam(lua_tostring(L, 1));
 
 	if(!p) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, "no-such-user");
 		return 2;
 	}
@@ -433,7 +443,7 @@ int lc_initgroups(lua_State *L) {
 			break;
 
 		default:
-			lua_pushnil(L);
+			luaL_pushfail(L);
 			lua_pushstring(L, "invalid-gid");
 			return 2;
 	}
@@ -443,17 +453,17 @@ int lc_initgroups(lua_State *L) {
 	if(ret) {
 		switch(errno) {
 			case ENOMEM:
-				lua_pushnil(L);
+				luaL_pushfail(L);
 				lua_pushstring(L, "no-memory");
 				break;
 
 			case EPERM:
-				lua_pushnil(L);
+				luaL_pushfail(L);
 				lua_pushstring(L, "permission-denied");
 				break;
 
 			default:
-				lua_pushnil(L);
+				luaL_pushfail(L);
 				lua_pushstring(L, "unknown-error");
 		}
 	} else {
@@ -464,7 +474,7 @@ int lc_initgroups(lua_State *L) {
 	return 2;
 }
 
-int lc_umask(lua_State *L) {
+static int lc_umask(lua_State *L) {
 	char old_mode_string[7];
 	mode_t old_mode = umask(strtoul(luaL_checkstring(L, 1), NULL, 8));
 
@@ -475,7 +485,7 @@ int lc_umask(lua_State *L) {
 	return 1;
 }
 
-int lc_mkdir(lua_State *L) {
+static int lc_mkdir(lua_State *L) {
 	int ret = mkdir(luaL_checkstring(L, 1), S_IRUSR | S_IWUSR | S_IXUSR
 	                | S_IRGRP | S_IWGRP | S_IXGRP
 	                | S_IROTH | S_IXOTH); /* mode 775 */
@@ -500,7 +510,7 @@ int lc_mkdir(lua_State *L) {
  *	Example usage:
  *	pposix.setrlimit("NOFILE", 1000, 2000)
  */
-int string2resource(const char *s) {
+static int string2resource(const char *s) {
 	if(!strcmp(s, "CORE")) {
 		return RLIMIT_CORE;
 	}
@@ -550,7 +560,7 @@ int string2resource(const char *s) {
 	return -1;
 }
 
-rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
+static rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
 	switch(lua_type(L, idx)) {
 		case LUA_TSTRING:
 
@@ -571,7 +581,7 @@ rlim_t arg_to_rlimit(lua_State *L, int idx, rlim_t current) {
 	}
 }
 
-int lc_setrlimit(lua_State *L) {
+static int lc_setrlimit(lua_State *L) {
 	struct rlimit lim;
 	int arguments = lua_gettop(L);
 	int rid = -1;
@@ -610,7 +620,7 @@ int lc_setrlimit(lua_State *L) {
 	return 1;
 }
 
-int lc_getrlimit(lua_State *L) {
+static int lc_getrlimit(lua_State *L) {
 	int arguments = lua_gettop(L);
 	const char *resource = NULL;
 	int rid = -1;
@@ -643,29 +653,29 @@ int lc_getrlimit(lua_State *L) {
 	if(lim.rlim_cur == RLIM_INFINITY) {
 		lua_pushstring(L, "unlimited");
 	} else {
-		lua_pushnumber(L, lim.rlim_cur);
+		lua_pushinteger(L, lim.rlim_cur);
 	}
 
 	if(lim.rlim_max == RLIM_INFINITY) {
 		lua_pushstring(L, "unlimited");
 	} else {
-		lua_pushnumber(L, lim.rlim_max);
+		lua_pushinteger(L, lim.rlim_max);
 	}
 
 	return 3;
 }
 
-int lc_abort(lua_State *L) {
+static int lc_abort(lua_State *L) {
 	(void)L;
 	abort();
 	return 0;
 }
 
-int lc_uname(lua_State *L) {
+static int lc_uname(lua_State *L) {
 	struct utsname uname_info;
 
 	if(uname(&uname_info) != 0) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(errno));
 		return 2;
 	}
@@ -688,14 +698,14 @@ int lc_uname(lua_State *L) {
 	return 1;
 }
 
-int lc_setenv(lua_State *L) {
+static int lc_setenv(lua_State *L) {
 	const char *var = luaL_checkstring(L, 1);
 	const char *value;
 
 	/* If the second argument is nil or nothing, unset the var */
 	if(lua_isnoneornil(L, 2)) {
 		if(unsetenv(var) != 0) {
-			lua_pushnil(L);
+			luaL_pushfail(L);
 			lua_pushstring(L, strerror(errno));
 			return 2;
 		}
@@ -707,7 +717,7 @@ int lc_setenv(lua_State *L) {
 	value = luaL_checkstring(L, 2);
 
 	if(setenv(var, value, 1) != 0) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushstring(L, strerror(errno));
 		return 2;
 	}
@@ -717,7 +727,7 @@ int lc_setenv(lua_State *L) {
 }
 
 #ifdef WITH_MALLINFO
-int lc_meminfo(lua_State *L) {
+static int lc_meminfo(lua_State *L) {
 	struct mallinfo info = mallinfo();
 	lua_createtable(L, 0, 5);
 	/* This is the total size of memory allocated with sbrk by malloc, in bytes. */
@@ -745,7 +755,7 @@ int lc_meminfo(lua_State *L) {
  * Attempt to allocate space first
  * Truncate to original size on failure
  */
-int lc_atomic_append(lua_State *L) {
+static int lc_atomic_append(lua_State *L) {
 	int err;
 	size_t len;
 
@@ -769,7 +779,7 @@ int lc_atomic_append(lua_State *L) {
 
 			case ENOSPC: /* No space left */
 			default: /* Other issues */
-				lua_pushnil(L);
+				luaL_pushfail(L);
 				lua_pushstring(L, strerror(err));
 				lua_pushinteger(L, err);
 				return 3;
@@ -796,7 +806,7 @@ int lc_atomic_append(lua_State *L) {
 		return luaL_error(L, "atomic_append() failed in ftruncate(): %s", strerror(errno));
 	}
 
-	lua_pushnil(L);
+	luaL_pushfail(L);
 	lua_pushstring(L, strerror(err));
 	lua_pushinteger(L, err);
 	return 3;
diff --git a/util-src/ringbuffer.c b/util-src/ringbuffer.c
index 8f9013f7..0f250c12 100644
--- a/util-src/ringbuffer.c
+++ b/util-src/ringbuffer.c
@@ -2,11 +2,14 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
-#include <stdio.h>
 
 #include <lua.h>
 #include <lauxlib.h>
 
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
+
 typedef struct {
 	size_t rpos; /* read position */
 	size_t wpos; /* write position */
@@ -15,23 +18,67 @@ typedef struct {
 	char buffer[];
 } ringbuffer;
 
-char readchar(ringbuffer *b) {
-	b->blen--;
-	return b->buffer[(b->rpos++) % b->alen];
+/* Translate absolute idx to a wrapped index within the buffer,
+   based on current read position */
+static int wrap_pos(const ringbuffer *b, const long idx, long *pos) {
+	if(idx > (long)b->blen) {
+		return 0;
+	}
+	if(idx + (long)b->rpos > (long)b->alen) {
+		*pos = idx - (b->alen - b->rpos);
+	} else {
+		*pos = b->rpos + idx;
+	}
+	return 1;
+}
+
+static int calc_splice_positions(const ringbuffer *b, long start, long end, long *out_start, long *out_end) {
+	if(start < 0) {
+		start = 1 + start + b->blen;
+	}
+	if(start <= 0) {
+		start = 1;
+	}
+
+	if(end < 0) {
+		end = 1 + end + b->blen;
+	}
+
+	if(end > (long)b->blen) {
+		end = b->blen;
+	}
+	if(start < 1) {
+		start = 1;
+	}
+
+	if(start > end) {
+		return 0;
+	}
+
+	start = start - 1;
+
+	if(!wrap_pos(b, start, out_start)) {
+		return 0;
+	}
+	if(!wrap_pos(b, end, out_end)) {
+		return 0;
+	}
+
+	return 1;
 }
 
-void writechar(ringbuffer *b, char c) {
+static void writechar(ringbuffer *b, char c) {
 	b->blen++;
 	b->buffer[(b->wpos++) % b->alen] = c;
 }
 
 /* make sure position counters stay within the allocation */
-void modpos(ringbuffer *b) {
+static void modpos(ringbuffer *b) {
 	b->rpos = b->rpos % b->alen;
 	b->wpos = b->wpos % b->alen;
 }
 
-int find(ringbuffer *b, const char *s, size_t l) {
+static int find(ringbuffer *b, const char *s, size_t l) {
 	size_t i, j;
 	int m;
 
@@ -64,7 +111,7 @@ int find(ringbuffer *b, const char *s, size_t l) {
  * Find first position of a substring in buffer
  * (buffer, string) -> number
  */
-int rb_find(lua_State *L) {
+static int rb_find(lua_State *L) {
 	size_t l, m;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
@@ -82,7 +129,7 @@ int rb_find(lua_State *L) {
  * Move read position forward without returning the data
  * (buffer, number) -> boolean
  */
-int rb_discard(lua_State *L) {
+static int rb_discard(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	size_t r = luaL_checkinteger(L, 2);
 
@@ -103,13 +150,13 @@ int rb_discard(lua_State *L) {
  * Read bytes from buffer
  * (buffer, number, boolean?) -> string
  */
-int rb_read(lua_State *L) {
+static int rb_read(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	size_t r = luaL_checkinteger(L, 2);
 	int peek = lua_toboolean(L, 3);
 
 	if(r > b->blen) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
@@ -135,7 +182,7 @@ int rb_read(lua_State *L) {
  * Read buffer until first occurrence of a substring
  * (buffer, string) -> string
  */
-int rb_readuntil(lua_State *L) {
+static int rb_readuntil(lua_State *L) {
 	size_t l, m;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
@@ -154,14 +201,14 @@ int rb_readuntil(lua_State *L) {
  * Write bytes into the buffer
  * (buffer, string) -> integer
  */
-int rb_write(lua_State *L) {
+static int rb_write(lua_State *L) {
 	size_t l, w = 0;
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	const char *s = luaL_checklstring(L, 2, &l);
 
 	/* Does `l` bytes fit? */
 	if((l + b->blen) > b->alen) {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		return 1;
 	}
 
@@ -177,32 +224,82 @@ int rb_write(lua_State *L) {
 	return 1;
 }
 
-int rb_tostring(lua_State *L) {
+static int rb_tostring(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushfstring(L, "ringbuffer: %p %d/%d", b, b->blen, b->alen);
 	return 1;
 }
 
-int rb_length(lua_State *L) {
+static int rb_sub(lua_State *L) {
+	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
+
+	long start = luaL_checkinteger(L, 2);
+	long end = luaL_optinteger(L, 3, -1);
+
+	long wrapped_start, wrapped_end;
+	if(!calc_splice_positions(b, start, end, &wrapped_start, &wrapped_end)) {
+		lua_pushstring(L, "");
+	} else if(wrapped_end <= wrapped_start) {
+		lua_pushlstring(L, &b->buffer[wrapped_start], b->alen - wrapped_start);
+		lua_pushlstring(L, b->buffer, wrapped_end);
+		lua_concat(L, 2);
+	} else {
+		lua_pushlstring(L, &b->buffer[wrapped_start], (wrapped_end - wrapped_start));
+	}
+
+	return 1;
+}
+
+static int rb_byte(lua_State *L) {
+	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
+
+	long start = luaL_optinteger(L, 2, 1);
+	long end = luaL_optinteger(L, 3, start);
+
+	long i;
+
+	long wrapped_start, wrapped_end;
+	if(calc_splice_positions(b, start, end, &wrapped_start, &wrapped_end)) {
+		if(wrapped_end <= wrapped_start) {
+			for(i = wrapped_start; i < (long)b->alen; i++) {
+				lua_pushinteger(L, (unsigned char)b->buffer[i]);
+			}
+			for(i = 0; i < wrapped_end; i++) {
+				lua_pushinteger(L, (unsigned char)b->buffer[i]);
+			}
+			return wrapped_end + (b->alen - wrapped_start);
+		} else {
+			for(i = wrapped_start; i < wrapped_end; i++) {
+				lua_pushinteger(L, (unsigned char)b->buffer[i]);
+			}
+			return wrapped_end - wrapped_start;
+		}
+	}
+
+	return 0;
+}
+
+static int rb_length(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->blen);
 	return 1;
 }
 
-int rb_size(lua_State *L) {
+static int rb_size(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->alen);
 	return 1;
 }
 
-int rb_free(lua_State *L) {
+static int rb_free(lua_State *L) {
 	ringbuffer *b = luaL_checkudata(L, 1, "ringbuffer_mt");
 	lua_pushinteger(L, b->alen - b->blen);
 	return 1;
 }
 
-int rb_new(lua_State *L) {
-	size_t size = luaL_optinteger(L, 1, sysconf(_SC_PAGESIZE));
+static int rb_new(lua_State *L) {
+	lua_Integer size = luaL_optinteger(L, 1, sysconf(_SC_PAGESIZE));
+	luaL_argcheck(L, size > 0, 1, "positive integer expected");
 	ringbuffer *b = lua_newuserdata(L, sizeof(ringbuffer) + size);
 
 	b->rpos = 0;
@@ -243,6 +340,10 @@ int luaopen_util_ringbuffer(lua_State *L) {
 			lua_setfield(L, -2, "size");
 			lua_pushcfunction(L, rb_length);
 			lua_setfield(L, -2, "length");
+			lua_pushcfunction(L, rb_sub);
+			lua_setfield(L, -2, "sub");
+			lua_pushcfunction(L, rb_byte);
+			lua_setfield(L, -2, "byte");
 			lua_pushcfunction(L, rb_free);
 			lua_setfield(L, -2, "free");
 		}
diff --git a/util-src/signal.c b/util-src/signal.c
index c696a3a2..1a398fa0 100644
--- a/util-src/signal.c
+++ b/util-src/signal.c
@@ -39,6 +39,9 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
+#if (LUA_VERSION_NUM < 503)
+#define lua_isinteger(L, n) lua_isnumber(L, n)
+#endif
 
 #ifndef lsig
 
@@ -164,8 +167,8 @@ static lua_Hook Hsig = NULL;
 static int Hmask = 0;
 static int Hcount = 0;
 
-int signals[MAX_PENDING_SIGNALS];
-int nsig = 0;
+static int signals[MAX_PENDING_SIGNALS];
+static int nsig = 0;
 
 static void sighook(lua_State *L, lua_Debug *ar) {
 	(void)ar;
@@ -176,7 +179,7 @@ static void sighook(lua_State *L, lua_Debug *ar) {
 	lua_gettable(L, LUA_REGISTRYINDEX);
 
 	for(int i = 0; i < nsig; i++) {
-		lua_pushnumber(L, signals[i]);
+		lua_pushinteger(L, signals[i]);
 		lua_gettable(L, -2);
 		lua_call(L, 0, 0);
 	};
@@ -223,18 +226,18 @@ static int l_signal(lua_State *L) {
 	t = lua_type(L, 1);
 
 	if(t == LUA_TNUMBER) {
-		sig = (int) lua_tonumber(L, 1);
+		sig = (int) lua_tointeger(L, 1);
 	} else if(t == LUA_TSTRING) {
 		lua_pushstring(L, LUA_SIGNAL);
 		lua_gettable(L, LUA_REGISTRYINDEX);
 		lua_pushvalue(L, 1);
 		lua_gettable(L, -2);
 
-		if(!lua_isnumber(L, -1)) {
+		if(!lua_isinteger(L, -1)) {
 			return luaL_error(L, "invalid signal string");
 		}
 
-		sig = (int) lua_tonumber(L, -1);
+		sig = (int) lua_tointeger(L, -1);
 		lua_pop(L, 1); /* get rid of number we pushed */
 	} else {
 		luaL_checknumber(L, 1);    /* will always error, with good error msg */
@@ -245,9 +248,9 @@ static int l_signal(lua_State *L) {
 	if(args == 1 || lua_isnil(L, 2)) { /* clear handler */
 		lua_pushstring(L, LUA_SIGNAL);
 		lua_gettable(L, LUA_REGISTRYINDEX);
-		lua_pushnumber(L, sig);
+		lua_pushinteger(L, sig);
 		lua_gettable(L, -2); /* return old handler */
-		lua_pushnumber(L, sig);
+		lua_pushinteger(L, sig);
 		lua_pushnil(L);
 		lua_settable(L, -4);
 		lua_remove(L, -2); /* remove LUA_SIGNAL table */
@@ -258,7 +261,7 @@ static int l_signal(lua_State *L) {
 		lua_pushstring(L, LUA_SIGNAL);
 		lua_gettable(L, LUA_REGISTRYINDEX);
 
-		lua_pushnumber(L, sig);
+		lua_pushinteger(L, sig);
 		lua_pushvalue(L, 2);
 		lua_settable(L, -3);
 
@@ -292,15 +295,15 @@ static int l_signal(lua_State *L) {
 static int l_raise(lua_State *L) {
 	/* int args = lua_gettop(L); */
 	int t = 0; /* type */
-	lua_Number ret;
+	lua_Integer ret;
 
 	luaL_checkany(L, 1);
 
 	t = lua_type(L, 1);
 
 	if(t == LUA_TNUMBER) {
-		ret = (lua_Number) raise((int) lua_tonumber(L, 1));
-		lua_pushnumber(L, ret);
+		ret = (lua_Integer) raise((int) lua_tointeger(L, 1));
+		lua_pushinteger(L, ret);
 	} else if(t == LUA_TSTRING) {
 		lua_pushstring(L, LUA_SIGNAL);
 		lua_gettable(L, LUA_REGISTRYINDEX);
@@ -311,9 +314,9 @@ static int l_raise(lua_State *L) {
 			return luaL_error(L, "invalid signal string");
 		}
 
-		ret = (lua_Number) raise((int) lua_tonumber(L, -1));
+		ret = (lua_Integer) raise((int) lua_tointeger(L, -1));
 		lua_pop(L, 1); /* get rid of number we pushed */
-		lua_pushnumber(L, ret);
+		lua_pushinteger(L, ret);
 	} else {
 		luaL_checknumber(L, 1);    /* will always error, with good error msg */
 	}
@@ -334,7 +337,7 @@ static int l_raise(lua_State *L) {
 
 static int l_kill(lua_State *L) {
 	int t; /* type */
-	lua_Number ret; /* return value */
+	lua_Integer ret; /* return value */
 
 	luaL_checknumber(L, 1); /* must be int for pid */
 	luaL_checkany(L, 2); /* check for a second arg */
@@ -342,9 +345,9 @@ static int l_kill(lua_State *L) {
 	t = lua_type(L, 2);
 
 	if(t == LUA_TNUMBER) {
-		ret = (lua_Number) kill((int) lua_tonumber(L, 1),
-		                        (int) lua_tonumber(L, 2));
-		lua_pushnumber(L, ret);
+		ret = (lua_Integer) kill((int) lua_tointeger(L, 1),
+		                         (int) lua_tointeger(L, 2));
+		lua_pushinteger(L, ret);
 	} else if(t == LUA_TSTRING) {
 		lua_pushstring(L, LUA_SIGNAL);
 		lua_gettable(L, LUA_REGISTRYINDEX);
@@ -355,10 +358,10 @@ static int l_kill(lua_State *L) {
 			return luaL_error(L, "invalid signal string");
 		}
 
-		ret = (lua_Number) kill((int) lua_tonumber(L, 1),
-		                        (int) lua_tonumber(L, -1));
+		ret = (lua_Integer) kill((int) lua_tointeger(L, 1),
+		                         (int) lua_tointeger(L, -1));
 		lua_pop(L, 1); /* get rid of number we pushed */
-		lua_pushnumber(L, ret);
+		lua_pushinteger(L, ret);
 	} else {
 		luaL_checknumber(L, 2);    /* will always error, with good error msg */
 	}
@@ -396,11 +399,11 @@ int luaopen_util_signal(lua_State *L) {
 	while(lua_signals[i].name != NULL) {
 		/* registry table */
 		lua_pushstring(L, lua_signals[i].name);
-		lua_pushnumber(L, lua_signals[i].sig);
+		lua_pushinteger(L, lua_signals[i].sig);
 		lua_settable(L, -3);
 		/* signal table */
 		lua_pushstring(L, lua_signals[i].name);
-		lua_pushnumber(L, lua_signals[i].sig);
+		lua_pushinteger(L, lua_signals[i].sig);
 		lua_settable(L, -5);
 		i++;
 	}
diff --git a/util-src/time.c b/util-src/time.c
index bfad52ee..afef3df5 100644
--- a/util-src/time.c
+++ b/util-src/time.c
@@ -1,22 +1,22 @@
 #ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 199309L
+#define _POSIX_C_SOURCE 200809L
 #endif
 
 #include <time.h>
 #include <lua.h>
 
-lua_Number tv2number(struct timespec *tv) {
+static lua_Number tv2number(struct timespec *tv) {
 	return tv->tv_sec + tv->tv_nsec * 1e-9;
 }
 
-int lc_time_realtime(lua_State *L) {
+static int lc_time_realtime(lua_State *L) {
 	struct timespec t;
 	clock_gettime(CLOCK_REALTIME, &t);
 	lua_pushnumber(L, tv2number(&t));
 	return 1;
 }
 
-int lc_time_monotonic(lua_State *L) {
+static int lc_time_monotonic(lua_State *L) {
 	struct timespec t;
 	clock_gettime(CLOCK_MONOTONIC, &t);
 	lua_pushnumber(L, tv2number(&t));
diff --git a/util-src/windows.c b/util-src/windows.c
index 89bec57b..57af79d5 100644
--- a/util-src/windows.c
+++ b/util-src/windows.c
@@ -22,6 +22,9 @@
 #if (LUA_VERSION_NUM == 501)
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
+#if (LUA_VERSION_NUM < 504)
+#define luaL_pushfail lua_pushnil
+#endif
 
 static int Lget_nameservers(lua_State *L) {
 	char stack_buffer[1024]; // stack allocated buffer
@@ -45,14 +48,14 @@ static int Lget_nameservers(lua_State *L) {
 
 		return 1;
 	} else {
-		lua_pushnil(L);
+		luaL_pushfail(L);
 		lua_pushfstring(L, "DnsQueryConfig returned %d", status);
 		return 2;
 	}
 }
 
 static int lerror(lua_State *L, char *string) {
-	lua_pushnil(L);
+	luaL_pushfail(L);
 	lua_pushfstring(L, "%s: %d", string, GetLastError());
 	return 2;
 }
diff --git a/util/adhoc.lua b/util/adhoc.lua
index d81b8242..19c94c34 100644
--- a/util/adhoc.lua
+++ b/util/adhoc.lua
@@ -2,7 +2,7 @@
 
 local function new_simple_form(form, result_handler)
 	return function(self, data, state)
-		if state then
+		if state or data.form then
 			if data.action == "cancel" then
 				return { status = "canceled" };
 			end
@@ -16,7 +16,7 @@ end
 
 local function new_initial_data_form(form, initial_data, result_handler)
 	return function(self, data, state)
-		if state then
+		if state or data.form then
 			if data.action == "cancel" then
 				return { status = "canceled" };
 			end
diff --git a/util/adminstream.lua b/util/adminstream.lua
new file mode 100644
index 00000000..5b592e76
--- /dev/null
+++ b/util/adminstream.lua
@@ -0,0 +1,336 @@
+local st = require "util.stanza";
+local new_xmpp_stream = require "util.xmppstream".new;
+local sessionlib = require "util.session";
+local gettime = require "util.time".now;
+local runner = require "util.async".runner;
+local add_task = require "util.timer".add_task;
+local events = require "util.events";
+local server = require "net.server";
+
+local stream_close_timeout = 5;
+
+local log = require "util.logger".init("adminstream");
+
+local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
+
+local stream_callbacks = { default_ns = "xmpp:prosody.im/admin" };
+
+function stream_callbacks.streamopened(session, attr)
+	-- run _streamopened in async context
+	session.thread:run({ stream = "opened", attr = attr });
+end
+
+function stream_callbacks._streamopened(session, attr) --luacheck: ignore 212/attr
+	if session.type ~= "client" then
+		session:open_stream();
+	end
+	session.notopen = nil;
+end
+
+function stream_callbacks.streamclosed(session, attr)
+	-- run _streamclosed in async context
+	session.thread:run({ stream = "closed", attr = attr });
+end
+
+function stream_callbacks._streamclosed(session)
+	session.log("debug", "Received </stream:stream>");
+	session:close(false);
+end
+
+function stream_callbacks.error(session, error, data)
+	if error == "no-stream" then
+		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
+		session:close("invalid-namespace");
+	elseif error == "parse-error" then
+		session.log("debug", "Client XML parse error: %s", data);
+		session:close("not-well-formed");
+	elseif error == "stream-error" then
+		local condition, text = "undefined-condition";
+		for child in data:childtags(nil, xmlns_xmpp_streams) do
+			if child.name ~= "text" then
+				condition = child.name;
+			else
+				text = child:get_text();
+			end
+			if condition ~= "undefined-condition" and text then
+				break;
+			end
+		end
+		text = condition .. (text and (" ("..text..")") or "");
+		session.log("info", "Session closed by remote with error: %s", text);
+		session:close(nil, text);
+	end
+end
+
+function stream_callbacks.handlestanza(session, stanza)
+	session.thread:run(stanza);
+end
+
+local runner_callbacks = {};
+
+function runner_callbacks:error(err)
+	self.data.log("error", "Traceback[c2s]: %s", err);
+end
+
+local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
+
+local function destroy_session(session, reason)
+	if session.destroyed then return; end
+	session.destroyed = true;
+	session.log("debug", "Destroying session: %s", reason or "unknown reason");
+end
+
+local function session_close(session, reason)
+	local log = session.log or log;
+	if session.conn then
+		if session.notopen then
+			session:open_stream();
+		end
+		if reason then -- nil == no err, initiated by us, false == initiated by client
+			local stream_error = st.stanza("stream:error");
+			if type(reason) == "string" then -- assume stream error
+				stream_error:tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' });
+			elseif type(reason) == "table" then
+				if reason.condition then
+					stream_error:tag(reason.condition, stream_xmlns_attr):up();
+					if reason.text then
+						stream_error:tag("text", stream_xmlns_attr):text(reason.text):up();
+					end
+					if reason.extra then
+						stream_error:add_child(reason.extra);
+					end
+				elseif reason.name then -- a stanza
+					stream_error = reason;
+				end
+			end
+			stream_error = tostring(stream_error);
+			log("debug", "Disconnecting client, <stream:error> is: %s", stream_error);
+			session.send(stream_error);
+		end
+
+		session.send("</stream:stream>");
+		function session.send() return false; end
+
+		local reason_text = (reason and (reason.name or reason.text or reason.condition)) or reason;
+		session.log("debug", "c2s stream for %s closed: %s", session.full_jid or session.ip or "<unknown>", reason_text or "session closed");
+
+		-- Authenticated incoming stream may still be sending us stanzas, so wait for </stream:stream> from remote
+		local conn = session.conn;
+		if reason_text == nil and not session.notopen and session.type == "c2s" then
+			-- Grace time to process data from authenticated cleanly-closed stream
+			add_task(stream_close_timeout, function ()
+				if not session.destroyed then
+					session.log("warn", "Failed to receive a stream close response, closing connection anyway...");
+					destroy_session(session);
+					conn:close();
+				end
+			end);
+		else
+			destroy_session(session, reason_text);
+			conn:close();
+		end
+	else
+		local reason_text = (reason and (reason.name or reason.text or reason.condition)) or reason;
+		destroy_session(session, reason_text);
+	end
+end
+
+--- Public methods
+
+local function new_connection(socket_path, listeners)
+	local have_unix, unix = pcall(require, "socket.unix");
+	if type(unix) ~= "table" then
+		have_unix = false;
+	end
+	local conn, sock;
+
+	return {
+		connect = function ()
+			if not have_unix then
+				return nil, "no unix socket support";
+			end
+			if sock or conn then
+				return nil, "already connected";
+			end
+			sock = unix.stream();
+			sock:settimeout(0);
+			local ok, err = sock:connect(socket_path);
+			if not ok then
+				return nil, err;
+			end
+			conn = server.wrapclient(sock, nil, nil, listeners, "*a");
+			return true;
+		end;
+		disconnect = function ()
+			if conn then
+				conn:close();
+				conn = nil;
+			end
+			if sock then
+				sock:close();
+				sock = nil;
+			end
+			return true;
+		end;
+	};
+end
+
+local function new_server(sessions, stanza_handler)
+	local listeners = {};
+
+	function listeners.onconnect(conn)
+		log("debug", "New connection");
+		local session = sessionlib.new("admin");
+		sessionlib.set_id(session);
+		sessionlib.set_logger(session);
+		sessionlib.set_conn(session, conn);
+
+		session.conntime = gettime();
+		session.type = "admin";
+
+		local stream = new_xmpp_stream(session, stream_callbacks);
+		session.stream = stream;
+		session.notopen = true;
+
+		session.thread = runner(function (stanza)
+			if st.is_stanza(stanza) then
+				stanza_handler(session, stanza);
+			elseif stanza.stream == "opened" then
+				stream_callbacks._streamopened(session, stanza.attr);
+			elseif stanza.stream == "closed" then
+				stream_callbacks._streamclosed(session, stanza.attr);
+			end
+		end, runner_callbacks, session);
+
+		function session.data(data)
+			-- Parse the data, which will store stanzas in session.pending_stanzas
+			if data then
+				local ok, err = stream:feed(data);
+				if not ok then
+					session.log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+					session:close("not-well-formed");
+				end
+			end
+		end
+
+		session.close = session_close;
+
+		session.send = function (t)
+			session.log("debug", "Sending[%s]: %s", session.type, t.top_tag and t:top_tag() or t:match("^[^>]*>?"));
+			return session.rawsend(tostring(t));
+		end
+
+		function session.rawsend(t)
+			local ret, err = conn:write(t);
+			if not ret then
+				session.log("debug", "Error writing to connection: %s", err);
+				return false, err;
+			end
+			return true;
+		end
+
+		sessions[conn] = session;
+	end
+
+	function listeners.onincoming(conn, data)
+		local session = sessions[conn];
+		if session then
+			session.data(data);
+		end
+	end
+
+	function listeners.ondisconnect(conn, err)
+		local session = sessions[conn];
+		if session then
+			session.log("info", "Admin client disconnected: %s", err or "connection closed");
+			session.conn = nil;
+			sessions[conn]  = nil;
+		end
+	end
+
+	function listeners.onreadtimeout(conn)
+		return conn:send(" ");
+	end
+
+	return {
+		listeners = listeners;
+	};
+end
+
+local function new_client()
+	local client = {
+		type = "client";
+		events = events.new();
+		log = log;
+	};
+
+	local listeners = {};
+
+	function listeners.onconnect(conn)
+		log("debug", "Connected");
+		client.conn = conn;
+
+		local stream = new_xmpp_stream(client, stream_callbacks);
+		client.stream = stream;
+		client.notopen = true;
+
+		client.thread = runner(function (stanza)
+			if st.is_stanza(stanza) then
+				if not client.events.fire_event("received", stanza) and not stanza.attr.xmlns then
+					client.events.fire_event("received/"..stanza.name, stanza);
+				end
+			elseif stanza.stream == "opened" then
+				stream_callbacks._streamopened(client, stanza.attr);
+				client.events.fire_event("connected");
+			elseif stanza.stream == "closed" then
+				client.events.fire_event("disconnected");
+				stream_callbacks._streamclosed(client, stanza.attr);
+			end
+		end, runner_callbacks, client);
+
+		client.close = session_close;
+
+		function client.send(t)
+			client.log("debug", "Sending: %s", t.top_tag and t:top_tag() or t:match("^[^>]*>?"));
+			return client.rawsend(tostring(t));
+		end
+
+		function client.rawsend(t)
+			local ret, err = conn:write(t);
+			if not ret then
+				client.log("debug", "Error writing to connection: %s", err);
+				return false, err;
+			end
+			return true;
+		end
+		client.log("debug", "Opening stream...");
+		client:open_stream();
+	end
+
+	function listeners.onincoming(conn, data) --luacheck: ignore 212/conn
+		local ok, err = client.stream:feed(data);
+		if not ok then
+			client.log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+			client:close("not-well-formed");
+		end
+	end
+
+	function listeners.ondisconnect(conn, err) --luacheck: ignore 212/conn
+		client.log("info", "Admin client disconnected: %s", err or "connection closed");
+		client.conn = nil;
+	end
+
+	function listeners.onreadtimeout(conn)
+		conn:send(" ");
+	end
+
+	client.listeners = listeners;
+
+	return client;
+end
+
+return {
+	connection = new_connection;
+	server = new_server;
+	client = new_client;
+};
diff --git a/util/argparse.lua b/util/argparse.lua
new file mode 100644
index 00000000..dde4fcc3
--- /dev/null
+++ b/util/argparse.lua
@@ -0,0 +1,55 @@
+local function parse(arg, config)
+	local short_params = config and config.short_params or {};
+	local value_params = config and config.value_params or {};
+
+	local parsed_opts = {};
+
+	if #arg == 0 then
+		return parsed_opts;
+	end
+	while true do
+		local raw_param = arg[1];
+		if not raw_param then
+			break;
+		end
+
+		local prefix = raw_param:match("^%-%-?");
+		if not prefix then
+			break;
+		elseif prefix == "--" and raw_param == "--" then
+			table.remove(arg, 1);
+			break;
+		end
+		local param = table.remove(arg, 1):sub(#prefix+1);
+		if #param == 1 and short_params then
+			param = short_params[param];
+		end
+
+		if not param then
+			return nil, "param-not-found", param;
+		end
+
+		local param_k, param_v;
+		if value_params[param] then
+			param_k, param_v = param, table.remove(arg, 1);
+			if not param_v then
+				return nil, "missing-value", raw_param;
+			end
+		else
+			param_k, param_v = param:match("^([^=]+)=(.+)$");
+			if not param_k then
+				if param:match("^no%-") then
+					param_k, param_v = param:sub(4), false;
+				else
+					param_k, param_v = param, true;
+				end
+			end
+		end
+		parsed_opts[param_k] = param_v;
+	end
+	return parsed_opts;
+end
+
+return {
+	parse = parse;
+}
diff --git a/util/array.lua b/util/array.lua
index 0b60a4fd..6e5c8383 100644
--- a/util/array.lua
+++ b/util/array.lua
@@ -10,6 +10,7 @@ local t_insert, t_sort, t_remove, t_concat
     = table.insert, table.sort, table.remove, table.concat;
 
 local setmetatable = setmetatable;
+local getmetatable = getmetatable;
 local math_random = math.random;
 local math_floor = math.floor;
 local pairs, ipairs = pairs, ipairs;
@@ -40,6 +41,10 @@ function array_mt.__add(a1, a2)
 end
 
 function array_mt.__eq(a, b)
+	if getmetatable(a) ~= array_mt or getmetatable(b) ~= array_mt then
+		-- Lua 5.3+ calls this if both operands are tables, even if metatables differ
+		return false;
+	end
 	if #a == #b then
 		for i = 1, #a do
 			if a[i] ~= b[i] then
@@ -129,9 +134,13 @@ function array_base.unique(outa, ina)
 	end);
 end
 
-function array_base.pluck(outa, ina, key)
+function array_base.pluck(outa, ina, key, default)
 	for i = 1, #ina do
-		outa[i] = ina[i][key];
+		local v = ina[i][key];
+		if v == nil then
+			v = default;
+		end
+		outa[i] = v;
 	end
 	return outa;
 end
diff --git a/util/async.lua b/util/async.lua
index 20397785..341128d2 100644
--- a/util/async.lua
+++ b/util/async.lua
@@ -53,7 +53,7 @@ local function runner_continue(thread)
 			return false;
 		end
 		call_watcher(runner, "error", debug.traceback(thread, err));
-		runner.state, runner.thread = "ready", nil;
+		runner.state = "ready";
 		return runner:run();
 	elseif state == "ready" then
 		-- If state is 'ready', it is our responsibility to update runner.state from 'waiting'.
@@ -159,6 +159,10 @@ function runner_mt:run(input)
 
 	local q, thread = self.queue, self.thread;
 	if not thread or coroutine.status(thread) == "dead" then
+		--luacheck: ignore 143/coroutine
+		if thread and coroutine.close then
+			coroutine.close(thread);
+		end
 		self:log("debug", "creating new coroutine");
 		-- Create a new coroutine for this runner
 		thread = runner_create_thread(self.func, self);
@@ -246,9 +250,26 @@ local function ready()
 	return pcall(checkthread);
 end
 
+local function wait_for(promise)
+	local async_wait, async_done = waiter();
+	local ret, err = nil, nil;
+	promise:next(
+		function (r) ret = r; end,
+		function (e) err = e; end)
+		:finally(async_done);
+	async_wait();
+	if ret then
+		return ret;
+	else
+		return nil, err;
+	end
+end
+
 return {
 	ready = ready;
 	waiter = waiter;
 	guarder = guarder;
 	runner = runner;
+	wait = wait_for; -- COMPAT w/trunk pre-0.12
+	wait_for = wait_for;
 };
diff --git a/util/bit53.lua b/util/bit53.lua
new file mode 100644
index 00000000..4b5c2e9c
--- /dev/null
+++ b/util/bit53.lua
@@ -0,0 +1,7 @@
+-- Only the operators needed by net.websocket.frames are provided at this point
+return {
+	band   = function (a, b) return a & b end;
+	bor    = function (a, b) return a | b end;
+	bxor   = function (a, b) return a ~ b end;
+};
+
diff --git a/util/bitcompat.lua b/util/bitcompat.lua
new file mode 100644
index 00000000..454181af
--- /dev/null
+++ b/util/bitcompat.lua
@@ -0,0 +1,32 @@
+-- Compatibility layer for bitwise operations
+
+-- First try the bit32 lib
+-- Lua 5.3 has it with compat enabled
+-- Lua 5.2 has it by default
+if _G.bit32 then
+	return _G.bit32;
+else
+	-- Lua 5.1 may have it as a standalone module that can be installed
+	local ok, bitop = pcall(require, "bit32")
+	if ok then
+		return bitop;
+	end
+end
+
+do
+	-- Lua 5.3 and 5.4 would be able to use native infix operators
+	local ok, bitop = pcall(require, "util.bit53")
+	if ok then
+		return bitop;
+	end
+end
+
+do
+	-- Lastly, try the LuaJIT bitop library
+	local ok, bitop = pcall(require, "bit")
+	if ok then
+		return bitop;
+	end
+end
+
+error "No bit module found. See https://prosody.im/doc/depends#bitop";
diff --git a/util/cache.lua b/util/cache.lua
index a5fd5e6d..cd1b4544 100644
--- a/util/cache.lua
+++ b/util/cache.lua
@@ -28,7 +28,7 @@ local function _insert(list, m)
 end
 
 local cache_methods = {};
-local cache_mt = { __index = cache_methods };
+local cache_mt = { __name = "cache", __index = cache_methods };
 
 function cache_methods:set(k, v)
 	local m = self._data[k];
diff --git a/util/dataforms.lua b/util/dataforms.lua
index 052d6a55..efb13ac9 100644
--- a/util/dataforms.lua
+++ b/util/dataforms.lua
@@ -10,6 +10,7 @@ local setmetatable = setmetatable;
 local ipairs = ipairs;
 local type, next = type, next;
 local tonumber = tonumber;
+local tostring = tostring;
 local t_concat = table.concat;
 local st = require "util.stanza";
 local jid_prep = require "util.jid".prep;
@@ -54,6 +55,12 @@ function form_t.form(layout, data, formtype)
 
 		if formtype == "form" and field.datatype then
 			form:tag("validate", { xmlns = xmlns_validate, datatype = field.datatype });
+			if field.range_min or field.range_max then
+				form:tag("range", {
+						min = field.range_min and tostring(field.range_min),
+						max = field.range_max and tostring(field.range_max),
+					}):up();
+			end
 			-- <basic/> assumed
 			form:up();
 		end
@@ -136,7 +143,7 @@ function form_t.form(layout, data, formtype)
 
 		local media = field.media;
 		if media then
-			form:tag("media", { xmlns = "urn:xmpp:media-element", height = media.height, width = media.width });
+			form:tag("media", { xmlns = "urn:xmpp:media-element", height = ("%g"):format(media.height), width = ("%g"):format(media.width) });
 			for _, val in ipairs(media) do
 				form:tag("uri", { type = val.type }):text(val.uri):up()
 			end
@@ -290,13 +297,18 @@ field_readers["hidden"] =
 	end
 
 data_validators["xs:integer"] =
-	function (data)
+	function (data, field)
 		local n = tonumber(data);
 		if not n then
 			return false, "not a number";
 		elseif n % 1 ~= 0 then
 			return false, "not an integer";
 		end
+		if field.range_max and n > field.range_max then
+			return false, "out of bounds";
+		elseif field.range_min and n < field.range_min then
+			return false, "out of bounds";
+		end
 		return true, n;
 	end
 
diff --git a/util/datamanager.lua b/util/datamanager.lua
index 0d7060b7..26dede08 100644
--- a/util/datamanager.lua
+++ b/util/datamanager.lua
@@ -24,7 +24,7 @@ local t_concat = table.concat;
 local envloadfile = require"util.envload".envloadfile;
 local serialize = require "util.serialization".serialize;
 local lfs = require "lfs";
--- Extract directory seperator from package.config (an undocumented string that comes with lua)
+-- Extract directory separator from package.config (an undocumented string that comes with lua)
 local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" )
 
 local prosody = prosody;
@@ -157,7 +157,8 @@ end
 
 local function atomic_store(filename, data)
 	local scratch = filename.."~";
-	local f, ok, msg, errno;
+	local f, ok, msg, errno; -- luacheck: ignore errno
+	-- TODO return util.error with code=errno?
 
 	f, msg, errno = io_open(scratch, "w");
 	if not f then
diff --git a/util/dbuffer.lua b/util/dbuffer.lua
index 640c1449..54e12266 100644
--- a/util/dbuffer.lua
+++ b/util/dbuffer.lua
@@ -2,7 +2,7 @@ local queue = require "util.queue";
 
 local s_byte, s_sub = string.byte, string.sub;
 local dbuffer_methods = {};
-local dynamic_buffer_mt = { __index = dbuffer_methods };
+local dynamic_buffer_mt = { __name = "dbuffer", __index = dbuffer_methods };
 
 function dbuffer_methods:write(data)
 	if self.max_size and #data + self._length > self.max_size then
diff --git a/util/dependencies.lua b/util/dependencies.lua
index 24975567..c117bfc2 100644
--- a/util/dependencies.lua
+++ b/util/dependencies.lua
@@ -7,24 +7,22 @@
 --
 
 local function softreq(...) local ok, lib =  pcall(require, ...); if ok then return lib; else return nil, lib; end end
+local platform_table = require "util.human.io".table({ { width = 15, align = "right" }, { width = "100%" } });
 
 -- Required to be able to find packages installed with luarocks
 if not softreq "luarocks.loader" then -- LuaRocks 2.x
 	softreq "luarocks.require"; -- LuaRocks <1.x
 end
 
-local function missingdep(name, sources, msg)
+local function missingdep(name, sources, msg, err) -- luacheck: ignore err
+	-- TODO print something about the underlying error, useful for debugging
 	print("");
 	print("**************************");
 	print("Prosody was unable to find "..tostring(name));
 	print("This package can be obtained in the following ways:");
 	print("");
-	local longest_platform = 0;
-	for platform in pairs(sources) do
-		longest_platform = math.max(longest_platform, #platform);
-	end
-	for platform, source in pairs(sources) do
-		print("", platform..":"..(" "):rep(4+longest_platform-#platform)..source);
+	for _, row in ipairs(sources) do
+		print(platform_table(row));
 	end
 	print("");
 	print(msg or (name.." is required for Prosody to run, so we will now exit."));
@@ -44,25 +42,25 @@ local function check_dependencies()
 
 	local fatal;
 
-	local lxp = softreq "lxp"
+	local lxp, err = softreq "lxp"
 
 	if not lxp then
 		missingdep("luaexpat", {
-				["Debian/Ubuntu"] = "sudo apt-get install lua-expat";
-				["luarocks"] = "luarocks install luaexpat";
-				["Source"] = "http://matthewwild.co.uk/projects/luaexpat/";
-			});
+				{ "Debian/Ubuntu", "sudo apt-get install lua-expat" };
+				{ "luarocks", "luarocks install luaexpat" };
+				{ "Source", "http://matthewwild.co.uk/projects/luaexpat/" };
+			}, nil, err);
 		fatal = true;
 	end
 
-	local socket = softreq "socket"
+	local socket, err = softreq "socket"
 
 	if not socket then
 		missingdep("luasocket", {
-				["Debian/Ubuntu"] = "sudo apt-get install lua-socket";
-				["luarocks"] = "luarocks install luasocket";
-				["Source"] = "http://www.tecgraf.puc-rio.br/~diego/professional/luasocket/";
-			});
+				{ "Debian/Ubuntu", "sudo apt-get install lua-socket" };
+				{ "luarocks", "luarocks install luasocket" };
+				{ "Source", "http://www.tecgraf.puc-rio.br/~diego/professional/luasocket/" };
+			}, nil, err);
 		fatal = true;
 	elseif not socket.tcp4 then
 		-- COMPAT LuaSocket before being IP-version agnostic
@@ -73,39 +71,54 @@ local function check_dependencies()
 	local lfs, err = softreq "lfs"
 	if not lfs then
 		missingdep("luafilesystem", {
-			["luarocks"] = "luarocks install luafilesystem";
-			["Debian/Ubuntu"] = "sudo apt-get install lua-filesystem";
-			["Source"] = "http://www.keplerproject.org/luafilesystem/";
-		});
+			{ "luarocks", "luarocks install luafilesystem" };
+			{ "Debian/Ubuntu", "sudo apt-get install lua-filesystem" };
+			{ "Source", "http://www.keplerproject.org/luafilesystem/" };
+		}, nil, err);
 		fatal = true;
 	end
 
-	local ssl = softreq "ssl"
+	local ssl, err = softreq "ssl"
 
 	if not ssl then
 		missingdep("LuaSec", {
-				["Debian/Ubuntu"] = "sudo apt-get install lua-sec";
-				["luarocks"] = "luarocks install luasec";
-				["Source"] = "https://github.com/brunoos/luasec";
-			}, "SSL/TLS support will not be available");
+				{ "Debian/Ubuntu", "sudo apt-get install lua-sec" };
+				{ "luarocks", "luarocks install luasec" };
+				{ "Source", "https://github.com/brunoos/luasec" };
+			}, "SSL/TLS support will not be available", err);
 	end
 
-	local bit = softreq"bit" or softreq"bit32";
+	local bit, err = softreq"util.bitcompat";
 
 	if not bit then
 		missingdep("lua-bitops", {
-			["Debian/Ubuntu"] = "sudo apt-get install lua-bitop";
-			["luarocks"] = "luarocks install luabitop";
-			["Source"] = "http://bitop.luajit.org/";
-		}, "WebSocket support will not be available");
+			{ "Debian/Ubuntu", "sudo apt-get install lua-bitop" };
+			{ "luarocks", "luarocks install luabitop" };
+			{ "Source", "http://bitop.luajit.org/" };
+		}, "WebSocket support will not be available", err);
+	end
+
+	local unbound, err = softreq"lunbound"; -- luacheck: ignore 211/err
+	if not unbound then -- luacheck: ignore 542
+		--[[ TODO Re-enable once packages are available
+		missingdep("lua-unbound", {
+				{ "luarocks", "luarocks install luaunbound" };
+				{ "Source", "https://www.zash.se/luaunbound.html" };
+			}, "Old DNS resolver library will be used", err);
+		--]]
+	else
+		package.preload["net.adns"] = function ()
+			local ub = require "net.unbound";
+			return ub;
+		end
 	end
 
 	local encodings, err = softreq "util.encodings"
 	if not encodings then
 		if err:match("module '[^']*' not found") then
 			missingdep("util.encodings", {
-				["Windows"] = "Make sure you have encodings.dll from the Prosody distribution in util/";
-				["GNU/Linux"] = "Run './configure' and 'make' in the Prosody source directory to build util/encodings.so";
+				{ "Windows", "Make sure you have encodings.dll from the Prosody distribution in util/" };
+				{ "GNU/Linux", "Run './configure' and 'make' in the Prosody source directory to build util/encodings.so" };
 			});
 		else
 			print "***********************************"
@@ -122,8 +135,8 @@ local function check_dependencies()
 	if not hashes then
 		if err:match("module '[^']*' not found") then
 			missingdep("util.hashes", {
-				["Windows"] = "Make sure you have hashes.dll from the Prosody distribution in util/";
-				["GNU/Linux"] = "Run './configure' and 'make' in the Prosody source directory to build util/hashes.so";
+				{ "Windows", "Make sure you have hashes.dll from the Prosody distribution in util/" };
+				{ "GNU/Linux", "Run './configure' and 'make' in the Prosody source directory to build util/hashes.so" };
 			});
 		else
 			print "***********************************"
@@ -140,7 +153,7 @@ local function check_dependencies()
 end
 
 local function log_warnings()
-	if _VERSION > "Lua 5.2" then
+	if _VERSION > "Lua 5.3" then
 		prosody.log("warn", "Support for %s is experimental, please report any issues", _VERSION);
 	end
 	local ssl = softreq"ssl";
diff --git a/util/dns.lua b/util/dns.lua
new file mode 100644
index 00000000..282c4d42
--- /dev/null
+++ b/util/dns.lua
@@ -0,0 +1,277 @@
+-- libunbound based net.adns replacement for Prosody IM
+-- Copyright (C) 2012-2015 Kim Alvefur
+-- Copyright (C) 2012 Waqas Hussain
+--
+-- This file is MIT licensed.
+
+local setmetatable = setmetatable;
+local table = table;
+local t_concat = table.concat;
+local t_insert = table.insert;
+local s_byte = string.byte;
+local s_char = string.char;
+local s_format = string.format;
+local s_gsub = string.gsub;
+local s_sub = string.sub;
+local s_match = string.match;
+local s_gmatch = string.gmatch;
+
+local have_net, net_util = pcall(require, "util.net");
+
+if have_net and not net_util.ntop then -- Added in Prosody 0.11
+	have_net = false;
+end
+
+local chartohex = {};
+
+for c = 0, 255 do
+	chartohex[s_char(c)] = s_format("%02X", c);
+end
+
+local function tohex(s)
+	return (s_gsub(s, ".", chartohex));
+end
+
+-- Converted from
+-- http://www.iana.org/assignments/dns-parameters
+-- 2020-06-25
+
+local classes = {
+	IN = 1; "IN";
+	nil;
+	CH = 3; "CH";
+	HS = 4; "HS";
+};
+
+local types = {
+"A";"NS";"MD";"MF";"CNAME";"SOA";"MB";"MG";"MR";"NULL";"WKS";"PTR";"HINFO";
+"MINFO";"MX";"TXT";"RP";"AFSDB";"X25";"ISDN";"RT";"NSAP";"NSAP-PTR";"SIG";
+"KEY";"PX";"GPOS";"AAAA";"LOC";"NXT";"EID";"NIMLOC";"SRV";"ATMA";"NAPTR";
+"KX";"CERT";"A6";"DNAME";"SINK";"OPT";"APL";"DS";"SSHFP";"IPSECKEY";"RRSIG";
+"NSEC";"DNSKEY";"DHCID";"NSEC3";"NSEC3PARAM";"TLSA";"SMIMEA";[55]="HIP";
+[56]="NINFO";[57]="RKEY";[58]="TALINK";[59]="CDS";[60]="CDNSKEY";[61]="OPENPGPKEY";
+[62]="CSYNC";[63]="ZONEMD";[99]="SPF";[100]="UINFO";[101]="UID";[102]="GID";
+[103]="UNSPEC";[104]="NID";[105]="L32";[106]="L64";[107]="LP";[108]="EUI48";
+[109]="EUI64";["CSYNC"]=62;["TXT"]=16;["NAPTR"]=35;["A6"]=38;["RP"]=17;
+["TALINK"]=58;["NXT"]=30;["MR"]=9;["UINFO"]=100;["X25"]=19;["TKEY"]=249;
+["CERT"]=37;["SMIMEA"]=53;[252]="AXFR";[253]="MAILB";["CDS"]=59;[32769]="DLV";
+["RT"]=21;["WKS"]=11;[249]="TKEY";["LP"]=107;[250]="TSIG";["SSHFP"]=44;["DS"]=43;
+["ISDN"]=20;["ATMA"]=34;["NS"]=2;[257]="CAA";["PX"]=26;["MX"]=15;["TSIG"]=250;
+["EID"]=31;["TLSA"]=52;["GID"]=102;["KX"]=36;["SPF"]=99;["DOA"]=259;["GPOS"]=27;
+["IPSECKEY"]=45;["NIMLOC"]=32;["RRSIG"]=46;["UID"]=101;["DNAME"]=39;["NSAP"]=22;
+["DNSKEY"]=48;["SINK"]=40;["DHCID"]=49;[32768]="TA";["NSAP-PTR"]=23;["AAAA"]=28;
+["PTR"]=12;["MINFO"]=14;["TA"]=32768;["EUI64"]=109;[260]="AMTRELAY";
+["AMTRELAY"]=260;["CDNSKEY"]=60;[259]="DOA";["LOC"]=29;[258]="AVC";["AVC"]=258;
+["CAA"]=257;["MB"]=7;["*"]=255;[256]="URI";["URI"]=256;["SRV"]=33;["EUI48"]=108;
+[255]="*";[254]="MAILA";["MAILA"]=254;["MAILB"]=253;["CNAME"]=5;[251]="IXFR";
+["APL"]=42;["OPENPGPKEY"]=61;["MD"]=3;["NINFO"]=56;["ZONEMD"]=63;["RKEY"]=57;
+["L32"]=105;["NID"]=104;["HIP"]=55;["NSEC"]=47;["DLV"]=32769;["UNSPEC"]=103;
+["NSEC3PARAM"]=51;["MF"]=4;["MG"]=8;["AFSDB"]=18;["A"]=1;["SIG"]=24;["NSEC3"]=50;
+["HINFO"]=13;["IXFR"]=251;["NULL"]=10;["AXFR"]=252;["KEY"]=25;["OPT"]=41;
+["SOA"]=6;["L64"]=106;
+}
+
+local errors = {
+	NoError = "No Error"; [0] = "NoError";
+	FormErr = "Format Error"; "FormErr";
+	ServFail = "Server Failure"; "ServFail";
+	NXDomain = "Non-Existent Domain"; "NXDomain";
+	NotImp = "Not Implemented"; "NotImp";
+	Refused = "Query Refused"; "Refused";
+	YXDomain = "Name Exists when it should not"; "YXDomain";
+	YXRRSet = "RR Set Exists when it should not"; "YXRRSet";
+	NXRRSet = "RR Set that should exist does not"; "NXRRSet";
+	NotAuth = "Server Not Authoritative for zone"; "NotAuth";
+	NotZone = "Name not contained in zone"; "NotZone";
+};
+
+-- Simplified versions of Waqas DNS parsers
+-- Only the per RR parsers are needed and only feed a single RR
+
+local parsers = {};
+
+-- No support for pointers, but libunbound appears to take care of that.
+local function readDnsName(packet, pos)
+	if s_byte(packet, pos) == 0 then return "."; end
+	local pack_len, r, len = #packet, {};
+	pos = pos or 1;
+	repeat
+		len = s_byte(packet, pos) or 0;
+		t_insert(r, s_sub(packet, pos + 1, pos + len));
+		pos = pos + len + 1;
+	until len == 0 or pos >= pack_len;
+	return t_concat(r, "."), pos;
+end
+
+-- These are just simple names.
+parsers.CNAME = readDnsName;
+parsers.NS = readDnsName
+parsers.PTR = readDnsName;
+
+local soa_mt = {
+	__tostring = function(rr)
+		return s_format("%s %s %d %d %d %d %d", rr.mname, rr.rname, rr.serial, rr.refresh, rr.retry, rr.expire, rr.minimum);
+	end;
+};
+function parsers.SOA(packet)
+	local mname, rname, offset;
+
+	mname, offset = readDnsName(packet, 1);
+	rname, offset = readDnsName(packet, offset);
+
+	-- Extract all the bytes of these fields in one call
+	local
+		s1, s2, s3, s4, -- serial
+		r1, r2, r3, r4, -- refresh
+		t1, t2, t3, t4, -- retry
+		e1, e2, e3, e4, -- expire
+		m1, m2, m3, m4  -- minimum
+			= s_byte(packet, offset, offset + 19);
+
+	return setmetatable({
+		mname = mname;
+		rname = rname;
+		serial  = s1*0x1000000 + s2*0x10000 + s3*0x100 + s4;
+		refresh = r1*0x1000000 + r2*0x10000 + r3*0x100 + r4;
+		retry   = t1*0x1000000 + t2*0x10000 + t3*0x100 + t4;
+		expire  = e1*0x1000000 + e2*0x10000 + e3*0x100 + e4;
+		minimum = m1*0x1000000 + m2*0x10000 + m3*0x100 + m4;
+	}, soa_mt);
+end
+
+function parsers.A(packet)
+	return s_format("%d.%d.%d.%d", s_byte(packet, 1, 4));
+end
+
+local aaaa = { nil, nil, nil, nil, nil, nil, nil, nil, };
+function parsers.AAAA(packet)
+	local hi, lo, ip, len, token;
+	for i = 1, 8 do
+		hi, lo = s_byte(packet, i * 2 - 1, i * 2);
+		aaaa[i] = s_format("%x", hi * 256 + lo); -- skips leading zeros
+	end
+	ip = t_concat(aaaa, ":", 1, 8);
+	len = (s_match(ip, "^0:[0:]+()") or 1) - 1;
+	for s in s_gmatch(ip, ":0:[0:]+") do
+		if len < #s then len, token = #s, s; end -- find longest sequence of zeros
+	end
+	return (s_gsub(ip, token or "^0:[0:]+", "::", 1));
+end
+
+if have_net then
+	parsers.A = net_util.ntop;
+	parsers.AAAA = net_util.ntop;
+end
+
+local mx_mt = {
+	__tostring = function(rr)
+		return s_format("%d %s", rr.pref, rr.mx)
+	end
+};
+function parsers.MX(packet)
+	local name = readDnsName(packet, 3);
+	local b1,b2 = s_byte(packet, 1, 2);
+	return setmetatable({
+		pref = b1*256+b2;
+		mx = name;
+	}, mx_mt);
+end
+
+local srv_mt = {
+	__tostring = function(rr)
+		return s_format("%d %d %d %s", rr.priority, rr.weight, rr.port, rr.target);
+	end
+};
+function parsers.SRV(packet)
+	local name = readDnsName(packet, 7);
+	local b1, b2, b3, b4, b5, b6 = s_byte(packet, 1, 6);
+	return setmetatable({
+		priority = b1*256+b2;
+		weight   = b3*256+b4;
+		port     = b5*256+b6;
+		target   = name;
+	}, srv_mt);
+end
+
+local txt_mt = { __tostring = t_concat };
+function parsers.TXT(packet)
+	local pack_len = #packet;
+	local r, pos, len = {}, 1;
+	repeat
+		len = s_byte(packet, pos) or 0;
+		t_insert(r, s_sub(packet, pos + 1, pos + len));
+		pos = pos + len + 1;
+	until pos >= pack_len;
+	return setmetatable(r, txt_mt);
+end
+
+parsers.SPF = parsers.TXT;
+
+-- Acronyms from RFC 7218
+local tlsa_usages = {
+	[0] = "PKIX-CA";
+	[1] = "PKIX-EE";
+	[2] = "DANE-TA";
+	[3] = "DANE-EE";
+	[255] = "PrivCert";
+};
+local tlsa_selectors = {
+	[0] = "Cert",
+	[1] = "SPKI",
+	[255] = "PrivSel",
+};
+local tlsa_match_types = {
+	[0] = "Full",
+	[1] = "SHA2-256",
+	[2] = "SHA2-512",
+	[255] = "PrivMatch",
+};
+local tlsa_mt = {
+	__tostring = function(rr)
+		return s_format("%s %s %s %s",
+			tlsa_usages[rr.use] or rr.use,
+			tlsa_selectors[rr.select] or rr.select,
+			tlsa_match_types[rr.match] or rr.match,
+			tohex(rr.data));
+	end;
+	__index = {
+		getUsage = function(rr) return tlsa_usages[rr.use] end;
+		getSelector = function(rr) return tlsa_selectors[rr.select] end;
+		getMatchType = function(rr) return tlsa_match_types[rr.match] end;
+	}
+};
+function parsers.TLSA(packet)
+	local use, select, match = s_byte(packet, 1,3);
+	return setmetatable({
+		use = use;
+		select = select;
+		match = match;
+		data = s_sub(packet, 4);
+	}, tlsa_mt);
+end
+
+local params = {
+	TLSA = {
+		use = tlsa_usages;
+		select = tlsa_selectors;
+		match = tlsa_match_types;
+	};
+};
+
+local fallback_mt = {
+	__tostring = function(rr)
+		return s_format([[\# %d %s]], #rr.raw, tohex(rr.raw));
+	end;
+};
+local function fallback_parser(packet)
+	return setmetatable({ raw = packet },fallback_mt);
+end
+setmetatable(parsers, { __index = function() return fallback_parser end });
+
+return {
+	parsers = parsers;
+	classes = classes;
+	types = types;
+	errors = errors;
+	params = params;
+};
diff --git a/util/error.lua b/util/error.lua
new file mode 100644
index 00000000..85265e47
--- /dev/null
+++ b/util/error.lua
@@ -0,0 +1,152 @@
+local id = require "util.id";
+
+-- Library configuration (see configure())
+local auto_inject_traceback = false;
+local display_tracebacks = false;
+
+
+local error_mt = { __name = "error" };
+
+function error_mt:__tostring()
+	if display_tracebacks and self.context.traceback then
+		return ("error<%s:%s:%s:%s>"):format(self.type, self.condition, self.text or "", self.context.traceback);
+	end
+	return ("error<%s:%s:%s>"):format(self.type, self.condition, self.text or "");
+end
+
+local function is_err(e)
+	return getmetatable(e) == error_mt;
+end
+
+local function configure(opt)
+	if opt.display_tracebacks ~= nil then
+		display_tracebacks = opt.display_tracebacks;
+	end
+	if opt.auto_inject_traceback ~= nil then
+		auto_inject_traceback = opt.auto_inject_traceback;
+	end
+end
+
+-- Do we want any more well-known fields?
+-- Or could we just copy all fields from `e`?
+-- Sometimes you want variable details in the `text`, how to handle that?
+-- Translations?
+-- Should the `type` be restricted to the stanza error types or free-form?
+-- What to set `type` to for stream errors or SASL errors? Those don't have a 'type' attr.
+
+local function new(e, context, registry, source)
+	if is_err(e) then return e; end
+	local template = registry and registry[e];
+	if not template then
+		if type(e) == "table" then
+			template = {
+				code = e.code;
+				type = e.type;
+				condition = e.condition;
+				text = e.text;
+				extra = e.extra;
+			};
+		else
+			template = {};
+		end
+	end
+	context = context or {};
+
+	if auto_inject_traceback then
+		context.traceback = debug.traceback("error stack", 2);
+	end
+
+	local error_instance = setmetatable({
+		instance_id = id.short();
+
+		type = template.type or "cancel";
+		condition = template.condition or "undefined-condition";
+		text = template.text;
+		code = template.code;
+		extra = template.extra;
+
+		context = context;
+		source = source;
+	}, error_mt);
+
+	return error_instance;
+end
+
+-- compact --> normal form
+local function expand_registry(namespace, registry)
+	local mapped = {}
+	for err,template in pairs(registry) do
+		local e = {
+			type = template[1];
+			condition = template[2];
+			text = template[3];
+		};
+		if namespace and template[4] then
+			e.extra = { namespace = namespace, condition = template[4] };
+		end
+		mapped[err] = e;
+	end
+	return mapped;
+end
+
+local function init(source, namespace, registry)
+	if type(namespace) == "table" then
+		-- registry can be given as second argument if namespace is either not used
+		registry, namespace = namespace, nil;
+	end
+	local _, protoerr = next(registry, nil);
+	if protoerr and type(next(protoerr)) == "number" then
+		registry = expand_registry(namespace, registry);
+	end
+	return {
+		source = source;
+		registry = registry;
+		new = function (e, context)
+			return new(e, context, registry, source);
+		end;
+	};
+end
+
+local function coerce(ok, err, ...)
+	if ok or is_err(err) then
+		return ok, err, ...;
+	end
+
+	local new_err = new({
+		type = "cancel", condition = "undefined-condition"
+	}, { wrapped_error = err });
+
+	return ok, new_err, ...;
+end
+
+local function from_stanza(stanza, context, source)
+	local error_type, condition, text, extra_tag = stanza:get_error();
+	local error_tag = stanza:get_child("error");
+	context = context or {};
+	context.stanza = stanza;
+	context.by = error_tag.attr.by or stanza.attr.from;
+
+	local uri;
+	if condition == "gone" or condition == "redirect" then
+		uri = error_tag:get_child_text(condition, "urn:ietf:params:xml:ns:xmpp-stanzas");
+	end
+
+	return new({
+		type = error_type or "cancel";
+		condition = condition or "undefined-condition";
+		text = text;
+		extra = (extra_tag or uri) and {
+			uri = uri;
+			tag = extra_tag;
+		} or nil;
+	}, context, nil, source);
+end
+
+return {
+	new = new;
+	init = init;
+	coerce = coerce;
+	is_err = is_err;
+	from_stanza = from_stanza;
+	configure = configure;
+}
diff --git a/util/events.lua b/util/events.lua
index 0bf0ddcb..5205a457 100644
--- a/util/events.lua
+++ b/util/events.lua
@@ -26,6 +26,8 @@ local function new()
 	local wrappers = {};
 	-- Event map: event_map[handler_function] = priority_number
 	local event_map = {};
+	-- Debug hook, if any
+	local active_debug_hook = nil;
 	-- Called on-demand to build handlers entries
 	local function _rebuild_index(self, event)
 		local _handlers = event_map[event];
@@ -74,11 +76,16 @@ local function new()
 	end;
 	local function _fire_event(event_name, event_data)
 		local h = handlers[event_name];
-		if h then
+		if h and not active_debug_hook then
 			for i=1,#h do
 				local ret = h[i](event_data);
 				if ret ~= nil then return ret; end
 			end
+		elseif h and active_debug_hook then
+			for i=1,#h do
+				local ret = active_debug_hook(h[i], event_name, event_data);
+				if ret ~= nil then return ret; end
+			end
 		end
 	end;
 	local function fire_event(event_name, event_data)
@@ -140,6 +147,13 @@ local function new()
 			end
 		end
 	end
+
+	local function set_debug_hook(new_hook)
+		local old_hook = active_debug_hook;
+		active_debug_hook = new_hook;
+		return old_hook;
+	end
+
 	return {
 		add_handler = add_handler;
 		remove_handler = remove_handler;
@@ -150,8 +164,12 @@ local function new()
 			add_handler = add_wrapper;
 			remove_handler = remove_wrapper;
 		};
+
 		add_wrapper = add_wrapper;
 		remove_wrapper = remove_wrapper;
+
+		set_debug_hook = set_debug_hook;
+
 		fire_event = fire_event;
 		_handlers = handlers;
 		_event_map = event_map;
diff --git a/util/format.lua b/util/format.lua
index c5e513fa..1ce670f3 100644
--- a/util/format.lua
+++ b/util/format.lua
@@ -3,12 +3,20 @@
 --
 
 local tostring = tostring;
-local select = select;
 local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack
+local pack = require "util.table".pack; -- TODO table.pack in 5.2+
 local type = type;
+local dump = require "util.serialization".new("debug");
+local num_type = math.type or function (n)
+	return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float";
+end
+
+-- In Lua 5.3+ these formats throw an error if given a float
+local expects_integer = { c = true, d = true, i = true, o = true, u = true, X = true, x = true, };
 
 local function format(formatstring, ...)
-	local args, args_length = { ... }, select('#', ...);
+	local args = pack(...);
+	local args_length = args.n;
 
 	-- format specifier spec:
 	-- 1. Start: '%%'
@@ -28,17 +36,22 @@ local function format(formatstring, ...)
 		if spec ~= "%%" then
 			i = i + 1;
 			local arg = args[i];
-			if arg == nil then -- special handling for nil
-				arg = "<nil>"
-				args[i] = "<nil>";
-			end
 
 			local option = spec:sub(-1);
-			if option == "q" or option == "s" then -- arg should be string
+			if arg == nil then
+				args[i] = "nil";
+				spec = "<%s>";
+			elseif option == "q" then
+				args[i] = dump(arg);
+				spec = "%s";
+			elseif option == "s" then
 				args[i] = tostring(arg);
 			elseif type(arg) ~= "number" then -- arg isn't number as expected?
 				args[i] = tostring(arg);
 				spec = "[%s]";
+			elseif expects_integer[option] and num_type(arg) ~= "integer" then
+				args[i] = tostring(arg);
+				spec = "[%s]";
 			end
 		end
 		return spec;
diff --git a/util/hashring.lua b/util/hashring.lua
new file mode 100644
index 00000000..322bc005
--- /dev/null
+++ b/util/hashring.lua
@@ -0,0 +1,88 @@
+local function generate_ring(nodes, num_replicas, hash)
+	local new_ring = {};
+        for _, node_name in ipairs(nodes) do
+                for replica = 1, num_replicas do
+                        local replica_hash = hash(node_name..":"..replica);
+                        new_ring[replica_hash] = node_name;
+                        table.insert(new_ring, replica_hash);
+                end
+        end
+        table.sort(new_ring);
+	return new_ring;
+end
+
+local hashring_methods = {};
+local hashring_mt = {
+	__index = function (self, k)
+		-- Automatically build self.ring if it's missing
+		if k == "ring" then
+			local ring = generate_ring(self.nodes, self.num_replicas, self.hash);
+			rawset(self, "ring", ring);
+			return ring;
+		end
+		return rawget(hashring_methods, k);
+	end
+};
+
+local function new(num_replicas, hash_function)
+	return setmetatable({ nodes = {}, num_replicas = num_replicas, hash = hash_function }, hashring_mt);
+end;
+
+function hashring_methods:add_node(name)
+	self.ring = nil;
+	self.nodes[name] = true;
+	table.insert(self.nodes, name);
+	return true;
+end
+
+function hashring_methods:add_nodes(nodes)
+	self.ring = nil;
+	for _, node_name in ipairs(nodes) do
+		if not self.nodes[node_name] then
+			self.nodes[node_name] = true;
+			table.insert(self.nodes, node_name);
+		end
+	end
+	return true;
+end
+
+function hashring_methods:remove_node(node_name)
+	self.ring = nil;
+	if self.nodes[node_name] then
+		for i, stored_node_name in ipairs(self.nodes) do
+			if node_name == stored_node_name then
+				self.nodes[node_name] = nil;
+				table.remove(self.nodes, i);
+				return true;
+			end
+		end
+	end
+	return false;
+end
+
+function hashring_methods:remove_nodes(nodes)
+	self.ring = nil;
+	for _, node_name in ipairs(nodes) do
+		self:remove_node(node_name);
+	end
+end
+
+function hashring_methods:clone()
+	local clone_hashring = new(self.num_replicas, self.hash);
+	clone_hashring:add_nodes(self.nodes);
+	return clone_hashring;
+end
+
+function hashring_methods:get_node(key)
+	local key_hash = self.hash(key);
+	for _, replica_hash in ipairs(self.ring) do
+		if key_hash < replica_hash then
+			return self.ring[replica_hash];
+		end
+	end
+	return self.ring[self.ring[1]];
+end
+
+return {
+	new = new;
+}
diff --git a/util/helpers.lua b/util/helpers.lua
index 02257ffa..139b62ec 100644
--- a/util/helpers.lua
+++ b/util/helpers.lua
@@ -23,12 +23,27 @@ local function log_events(events, name, logger)
 		logger("debug", "%s firing event: %s", name, event);
 		return f(event, ...);
 	end
+
+	local function event_handler_hook(handler, event_name, event_data)
+		logger("debug", "calling handler for %s: %s", event_name, handler);
+		local ok, ret = pcall(handler, event_data);
+		if not ok then
+			logger("error", "error in event handler %s: %s", handler, ret);
+			error(ret);
+		end
+		if ret ~= nil then
+			logger("debug", "event chain ended for %s by %s with result: %s", event_name, handler, ret);
+		end
+		return ret;
+	end
+	events.set_debug_hook(event_handler_hook);
 	events[events.fire_event] = f;
 	return events;
 end
 
 local function revert_log_events(events)
 	events.fire_event, events[events.fire_event] = events[events.fire_event], nil; -- :))
+	events.set_debug_hook(nil);
 end
 
 local function log_host_events(host)
diff --git a/util/hmac.lua b/util/hmac.lua
index 2c4cc6ef..4cad17cc 100644
--- a/util/hmac.lua
+++ b/util/hmac.lua
@@ -10,6 +10,9 @@
 
 local hashes = require "util.hashes"
 
-return { md5 = hashes.hmac_md5,
-	 sha1 = hashes.hmac_sha1,
-	 sha256 = hashes.hmac_sha256 };
+return {
+	md5 = hashes.hmac_md5,
+	sha1 = hashes.hmac_sha1,
+	sha256 = hashes.hmac_sha256,
+	sha512 = hashes.hmac_sha512,
+};
diff --git a/util/http.lua b/util/http.lua
index cfb89193..3852f91c 100644
--- a/util/http.lua
+++ b/util/http.lua
@@ -6,24 +6,26 @@
 --
 
 local format, char = string.format, string.char;
-local pairs, ipairs, tonumber = pairs, ipairs, tonumber;
+local pairs, ipairs = pairs, ipairs;
 local t_insert, t_concat = table.insert, table.concat;
 
+local url_codes = {};
+for i = 0, 255 do
+	local c = char(i);
+	local u = format("%%%02x", i);
+	url_codes[c] = u;
+	url_codes[u] = c;
+	url_codes[u:upper()] = c;
+end
 local function urlencode(s)
-	return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end));
+	return s and (s:gsub("[^a-zA-Z0-9.~_-]", url_codes));
 end
 local function urldecode(s)
-	return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end));
+	return s and (s:gsub("%%%x%x", url_codes));
 end
 
 local function _formencodepart(s)
-	return s and (s:gsub("%W", function (c)
-		if c ~= " " then
-			return format("%%%02x", c:byte());
-		else
-			return "+";
-		end
-	end));
+	return s and (urlencode(s):gsub("%%20", "+"));
 end
 
 local function formencode(form)
diff --git a/util/human/io.lua b/util/human/io.lua
new file mode 100644
index 00000000..a38ab5dd
--- /dev/null
+++ b/util/human/io.lua
@@ -0,0 +1,161 @@
+local array = require "util.array";
+
+local function getchar(n)
+	local stty_ret = os.execute("stty raw -echo 2>/dev/null");
+	local ok, char;
+	if stty_ret == true or stty_ret == 0 then
+		ok, char = pcall(io.read, n or 1);
+		os.execute("stty sane");
+	else
+		ok, char = pcall(io.read, "*l");
+		if ok then
+			char = char:sub(1, n or 1);
+		end
+	end
+	if ok then
+		return char;
+	end
+end
+
+local function getline()
+	local ok, line = pcall(io.read, "*l");
+	if ok then
+		return line;
+	end
+end
+
+local function getpass()
+	local stty_ret, _, status_code = os.execute("stty -echo 2>/dev/null");
+	if status_code then -- COMPAT w/ Lua 5.1
+		stty_ret = status_code;
+	end
+	if stty_ret ~= 0 then
+		io.write("\027[08m"); -- ANSI 'hidden' text attribute
+	end
+	local ok, pass = pcall(io.read, "*l");
+	if stty_ret == 0 then
+		os.execute("stty sane");
+	else
+		io.write("\027[00m");
+	end
+	io.write("\n");
+	if ok then
+		return pass;
+	end
+end
+
+local function show_yesno(prompt)
+	io.write(prompt, " ");
+	local choice = getchar():lower();
+	io.write("\n");
+	if not choice:match("%a") then
+		choice = prompt:match("%[.-(%U).-%]$");
+		if not choice then return nil; end
+	end
+	return (choice == "y");
+end
+
+local function read_password()
+	local password;
+	while true do
+		io.write("Enter new password: ");
+		password = getpass();
+		if not password then
+			print("No password - cancelled");
+			return;
+		end
+		io.write("Retype new password: ");
+		if getpass() ~= password then
+			if not show_yesno [=[Passwords did not match, try again? [Y/n]]=] then
+				return;
+			end
+		else
+			break;
+		end
+	end
+	return password;
+end
+
+local function show_prompt(prompt)
+	io.write(prompt, " ");
+	local line = getline();
+	line = line and line:gsub("\n$","");
+	return (line and #line > 0) and line or nil;
+end
+
+local function printf(fmt, ...)
+	print(fmt:format(...));
+end
+
+local function padright(s, width)
+	return s..string.rep(" ", width-#s);
+end
+
+local function padleft(s, width)
+	return string.rep(" ", width-#s)..s;
+end
+
+local function new_table(col_specs, max_width)
+	max_width = max_width or tonumber(os.getenv("COLUMNS")) or 80;
+	local separator = " | ";
+
+	local widths = {};
+	local total_width = max_width - #separator * (#col_specs-1);
+	local free_width = total_width;
+	-- Calculate width of fixed-size columns
+	for i = 1, #col_specs do
+		local width = col_specs[i].width or "0";
+		if not(type(width) == "string" and width:sub(-1) == "%") then
+			local title = col_specs[i].title;
+			width = math.max(tonumber(width), title and (#title+1) or 0);
+			widths[i] = width;
+			free_width = free_width - width;
+			if i > 1 then
+				free_width = free_width - #separator;
+			end
+		end
+	end
+	-- Calculate width of %-based columns
+	for i = 1, #col_specs do
+		if not widths[i] then
+			local pc_width = tonumber((col_specs[i].width:gsub("%%$", "")));
+			widths[i] = math.floor(free_width*(pc_width/100));
+		end
+	end
+
+	return function (row)
+		local titles;
+		if not row then
+			titles, row = true, array.pluck(col_specs, "title", "");
+		end
+		local output = {};
+		for i, column in ipairs(col_specs) do
+			local width = widths[i];
+			local v = (not titles and column.mapper or tostring)(row[not titles and column.key or i] or "", row);
+			if #v < width then
+				if column.align == "right" then
+					v = padleft(v, width);
+				else
+					v = padright(v, width);
+				end
+			elseif #v > width then
+				v = v:sub(1, width-1) .. "…";
+			end
+			table.insert(output, v);
+		end
+		return table.concat(output, separator);
+	end;
+end
+
+return {
+	getchar = getchar;
+	getline = getline;
+	getpass = getpass;
+	show_yesno = show_yesno;
+	read_password = read_password;
+	show_prompt = show_prompt;
+	printf = printf;
+	padleft = padleft;
+	padright = padright;
+	table = new_table;
+};
diff --git a/util/human/units.lua b/util/human/units.lua
new file mode 100644
index 00000000..af233e98
--- /dev/null
+++ b/util/human/units.lua
@@ -0,0 +1,80 @@
+local math_abs = math.abs;
+local math_ceil = math.ceil;
+local math_floor = math.floor;
+local math_log = math.log;
+local math_max = math.max;
+local math_min = math.min;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+
+if math_log(10, 10) ~= 1 then
+	-- Lua 5.1 COMPAT
+	local log10 = math.log10;
+	function math_log(n, base)
+		return log10(n) / log10(base);
+	end
+end
+
+local large = {
+	"k", 1000,
+	"M", 1000000,
+	"G", 1000000000,
+	"T", 1000000000000,
+	"P", 1000000000000000,
+	"E", 1000000000000000000,
+	"Z", 1000000000000000000000,
+	"Y", 1000000000000000000000000,
+}
+local small = {
+	"m", 0.001,
+	"μ", 0.000001,
+	"n", 0.000000001,
+	"p", 0.000000000001,
+	"f", 0.000000000000001,
+	"a", 0.000000000000000001,
+	"z", 0.000000000000000000001,
+	"y", 0.000000000000000000000001,
+}
+
+local binary = {
+	"Ki", 2^10,
+	"Mi", 2^20,
+	"Gi", 2^30,
+	"Ti", 2^40,
+	"Pi", 2^50,
+	"Ei", 2^60,
+	"Zi", 2^70,
+	"Yi", 2^80,
+}
+
+local function adjusted_unit(n, b)
+	local round = math_floor;
+	local prefixes = large;
+	local logbase = 1000;
+	if b == 'b' then
+		prefixes = binary;
+		logbase = 1024;
+	elseif n < 1 then
+		prefixes = small;
+		round = math_ceil;
+	end
+	local m = math_max(0, math_min(8, round(math_abs(math_log(math_abs(n), logbase)))));
+	local prefix, multiplier = unpack(prefixes, m * 2-1, m*2);
+	return multiplier or 1, prefix;
+end
+
+-- n: number, the number to format
+-- unit: string, the base unit
+-- b: optional enum 'b', thousands base
+local function format(n, unit, b) --> string
+	local fmt = "%.3g %s%s";
+	if n == 0 then
+		return fmt:format(n, "", unit);
+	end
+	local multiplier, prefix = adjusted_unit(n, b);
+	return fmt:format(n / multiplier, prefix or "", unit);
+end
+
+return {
+	adjust = adjusted_unit;
+	format = format;
+};
diff --git a/util/import.lua b/util/import.lua
index 8ecfe43c..1007bc0a 100644
--- a/util/import.lua
+++ b/util/import.lua
@@ -8,7 +8,7 @@
 
 
 
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 local t_insert = table.insert;
 function _G.import(module, ...)
 	local m = package.loaded[module] or require(module);
diff --git a/util/interpolation.lua b/util/interpolation.lua
index e0ccf47b..808a45f2 100644
--- a/util/interpolation.lua
+++ b/util/interpolation.lua
@@ -64,6 +64,9 @@ local function new_render(pat, escape, funcs)
 			elseif opt == '&' then
 				if not value then return ""; end
 				return render(s_sub(block, e), values);
+			elseif opt == '~' then
+				if value then return ""; end
+				return render(s_sub(block, e), values);
 			elseif opt == '?' and not value then
 				return render(s_sub(block, e), values);
 			elseif value ~= nil then
diff --git a/util/ip.lua b/util/ip.lua
index d1808225..d039e475 100644
--- a/util/ip.lua
+++ b/util/ip.lua
@@ -19,8 +19,14 @@ local ip_mt = {
 		return ret;
 	end,
 	__tostring = function (ip) return ip.addr; end,
-	__eq = function (ipA, ipB) return ipA.packed == ipB.packed; end
 };
+ip_mt.__eq = function (ipA, ipB)
+	if getmetatable(ipA) ~= ip_mt or getmetatable(ipB) ~= ip_mt then
+		-- Lua 5.3+ calls this if both operands are tables, even if metatables differ
+		return false;
+	end
+	return ipA.packed == ipB.packed;
+end
 
 local hex2bits = {
 	["0"] = "0000", ["1"] = "0001", ["2"] = "0010", ["3"] = "0011",
diff --git a/util/iterators.lua b/util/iterators.lua
index 302cca36..c03c2fd6 100644
--- a/util/iterators.lua
+++ b/util/iterators.lua
@@ -11,9 +11,9 @@
 local it = {};
 
 local t_insert = table.insert;
-local select, next = select, next;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
-local pack = table.pack or function (...) return { n = select("#", ...), ... }; end -- luacheck: ignore 143
+local next = next;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+local pack = table.pack or require "util.table".pack;
 local type = type;
 local table, setmetatable = table, setmetatable;
 
diff --git a/util/jid.lua b/util/jid.lua
index ec31f180..a1180534 100644
--- a/util/jid.lua
+++ b/util/jid.lua
@@ -22,7 +22,11 @@ local escapes = {
 	["@"] = "\\40"; ["\\"] = "\\5c";
 };
 local unescapes = {};
-for k,v in pairs(escapes) do unescapes[v] = k; end
+local backslash_escapes = {};
+for k,v in pairs(escapes) do
+	unescapes[v] = k;
+	backslash_escapes[v] = v:gsub("\\", escapes)
+end
 
 local _ENV = nil;
 -- luacheck: std none
@@ -45,20 +49,20 @@ local function bare(jid)
 	return host;
 end
 
-local function prepped_split(jid)
+local function prepped_split(jid, strict)
 	local node, host, resource = split(jid);
 	if host and host ~= "." then
 		if sub(host, -1, -1) == "." then -- Strip empty root label
 			host = sub(host, 1, -2);
 		end
-		host = nameprep(host);
+		host = nameprep(host, strict);
 		if not host then return; end
 		if node then
-			node = nodeprep(node);
+			node = nodeprep(node, strict);
 			if not node then return; end
 		end
 		if resource then
-			resource = resourceprep(resource);
+			resource = resourceprep(resource, strict);
 			if not resource then return; end
 		end
 		return node, host, resource;
@@ -77,8 +81,8 @@ local function join(node, host, resource)
 	return host;
 end
 
-local function prep(jid)
-	local node, host, resource = prepped_split(jid);
+local function prep(jid, strict)
+	local node, host, resource = prepped_split(jid, strict);
 	return join(node, host, resource);
 end
 
@@ -107,7 +111,7 @@ local function resource(jid)
 	return (select(3, split(jid)));
 end
 
-local function escape(s) return s and (s:gsub(".", escapes)); end
+local function escape(s) return s and (s:gsub("\\%x%x", backslash_escapes):gsub("[\"&'/:<>@ ]", escapes)); end
 local function unescape(s) return s and (s:gsub("\\%x%x", unescapes)); end
 
 return {
diff --git a/util/jwt.lua b/util/jwt.lua
new file mode 100644
index 00000000..2b172d38
--- /dev/null
+++ b/util/jwt.lua
@@ -0,0 +1,50 @@
+local s_gsub = string.gsub;
+local json = require "util.json";
+local hashes = require "util.hashes";
+local base64_encode = require "util.encodings".base64.encode;
+local base64_decode = require "util.encodings".base64.decode;
+
+local b64url_rep = { ["+"] = "-", ["/"] = "_", ["="] = "", ["-"] = "+", ["_"] = "/" };
+local function b64url(data)
+	return (s_gsub(base64_encode(data), "[+/=]", b64url_rep));
+end
+local function unb64url(data)
+	return base64_decode(s_gsub(data, "[-_]", b64url_rep).."==");
+end
+
+local static_header = b64url('{"alg":"HS256","typ":"JWT"}') .. '.';
+
+local function sign(key, payload)
+	local encoded_payload = json.encode(payload);
+	local signed = static_header .. b64url(encoded_payload);
+	local signature = hashes.hmac_sha256(key, signed);
+	return signed .. "." .. b64url(signature);
+end
+
+local jwt_pattern = "^(([A-Za-z0-9-_]+)%.([A-Za-z0-9-_]+))%.([A-Za-z0-9-_]+)$"
+local function verify(key, blob)
+	local signed, bheader, bpayload, signature = string.match(blob, jwt_pattern);
+	if not signed then
+		return nil, "invalid-encoding";
+	end
+	local header = json.decode(unb64url(bheader));
+	if not header or type(header) ~= "table" then
+		return nil, "invalid-header";
+	elseif header.alg ~= "HS256" then
+		return nil, "unsupported-algorithm";
+	end
+	if b64url(hashes.hmac_sha256(key, signed)) ~= signature then
+		return false, "signature-mismatch";
+	end
+	local payload, err = json.decode(unb64url(bpayload));
+	if err ~= nil then
+		return nil, "json-decode-error";
+	end
+	return true, payload;
+end
+
+return {
+	sign = sign;
+	verify = verify;
+};
+
diff --git a/util/mercurial.lua b/util/mercurial.lua
index 3f75c4c1..0f2b1d04 100644
--- a/util/mercurial.lua
+++ b/util/mercurial.lua
@@ -19,7 +19,7 @@ function hg.check_id(path)
 			hg_changelog:close();
 		end
 	else
-		local hg_archival,e = io.open(path.."/.hg_archival.txt");
+		local hg_archival,e = io.open(path.."/.hg_archival.txt"); -- luacheck: ignore 211/e
 		if hg_archival then
 			local repo = hg_archival:read("*l");
 			local node = hg_archival:read("*l");
diff --git a/util/multitable.lua b/util/multitable.lua
index 8d32ed8a..4f2cd972 100644
--- a/util/multitable.lua
+++ b/util/multitable.lua
@@ -9,7 +9,7 @@
 local select = select;
 local t_insert = table.insert;
 local pairs, next, type = pairs, next, type;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 
 local _ENV = nil;
 -- luacheck: std none
diff --git a/util/paths.lua b/util/paths.lua
index 89f4cad9..b75c35e5 100644
--- a/util/paths.lua
+++ b/util/paths.lua
@@ -37,8 +37,34 @@ function path_util.glob_to_pattern(glob)
 	end).."$";
 end
 
-function path_util.join(...)
-	return t_concat({...}, path_sep);
+function path_util.join(a, b, c, ...) -- (... : string) --> string
+	-- Optimization: Avoid creating table for most uses
+	if b then
+		if c then
+			if ... then
+				return t_concat({a,b,c,...}, path_sep);
+			end
+			return a..path_sep..b..path_sep..c;
+		end
+		return a..path_sep..b;
+	end
+	return a;
+end
+
+function path_util.complement_lua_path(installer_plugin_path)
+	-- Checking for duplicates
+	-- The commands using luarocks need the path to the directory that has the /share and /lib folders.
+	local lua_version = _VERSION:match(" (.+)$");
+	local lua_path_sep = package.config:sub(3,3);
+	local dir_sep = package.config:sub(1,1);
+	local sub_path = dir_sep.."lua"..dir_sep..lua_version..dir_sep;
+	if not string.find(package.path, installer_plugin_path, 1, true) then
+		package.path = package.path..lua_path_sep..installer_plugin_path..dir_sep.."share"..sub_path.."?.lua";
+		package.path = package.path..lua_path_sep..installer_plugin_path..dir_sep.."share"..sub_path.."?"..dir_sep.."init.lua";
+	end
+	if not string.find(package.path, installer_plugin_path, 1, true) then
+		package.cpath = package.cpath..lua_path_sep..installer_plugin_path..dir_sep.."lib"..sub_path.."?.so";
+	end
 end
 
 return path_util;
diff --git a/util/pluginloader.lua b/util/pluginloader.lua
index 9ab8f245..8e37ffc4 100644
--- a/util/pluginloader.lua
+++ b/util/pluginloader.lua
@@ -8,6 +8,7 @@
 -- luacheck: ignore 113/CFG_PLUGINDIR
 
 local dir_sep, path_sep = package.config:match("^(%S+)%s(%S+)");
+local lua_version = _VERSION:match(" (.+)$");
 local plugin_dir = {};
 for path in (CFG_PLUGINDIR or "./plugins/"):gsub("[/\\]", dir_sep):gmatch("[^"..path_sep.."]+") do
 	path = path..dir_sep; -- add path separator to path end
@@ -36,12 +37,13 @@ end
 
 local function load_resource(plugin, resource)
 	resource = resource or "mod_"..plugin..".lua";
-
 	local names = {
 		"mod_"..plugin..dir_sep..plugin..dir_sep..resource; -- mod_hello/hello/mod_hello.lua
 		"mod_"..plugin..dir_sep..resource;                  -- mod_hello/mod_hello.lua
 		plugin..dir_sep..resource;                          -- hello/mod_hello.lua
 		resource;                                           -- mod_hello.lua
+		"share"..dir_sep.."lua"..dir_sep..lua_version..dir_sep..resource;
+		"share"..dir_sep.."lua"..dir_sep..lua_version..dir_sep.."mod_"..plugin..dir_sep..resource;
 	};
 
 	return load_file(names);
@@ -58,6 +60,9 @@ end
 
 local function load_code_ext(plugin, resource, extension, env)
 	local content, err = load_resource(plugin, resource.."."..extension);
+	if not content and extension == "lib.lua" then
+		content, err = load_resource(plugin, resource..".lua");
+	end
 	if not content then
 		content, err = load_resource(resource, resource.."."..extension);
 		if not content then
diff --git a/util/promise.lua b/util/promise.lua
index 07c9c4dc..6ebb0dd6 100644
--- a/util/promise.lua
+++ b/util/promise.lua
@@ -49,6 +49,9 @@ local function promise_settle(promise, new_state, new_next, cbs, value)
 	for _, cb in ipairs(cbs) do
 		cb(value);
 	end
+	-- No need to keep references to callbacks
+	promise._pending_on_fulfilled = nil;
+	promise._pending_on_rejected = nil;
 	return true;
 end
 
@@ -101,6 +104,27 @@ local function all(promises)
 	end);
 end
 
+local function all_settled(promises)
+	return new(function (resolve)
+		local count, total, results = 0, #promises, {};
+		for i = 1, total do
+			promises[i]:next(function (v)
+				results[i] = { status = "fulfilled", value = v };
+				count = count + 1;
+				if count == total then
+					resolve(results);
+				end
+			end, function (e)
+				results[i] = { status = "rejected", reason = e };
+				count = count + 1;
+				if count == total then
+					resolve(results);
+				end
+			end);
+		end
+	end);
+end
+
 local function race(promises)
 	return new(function (resolve, reject)
 		for i = 1, #promises do
@@ -146,6 +170,7 @@ return {
 	resolve = resolve;
 	reject = reject;
 	all = all;
+	all_settled = all_settled;
 	race = race;
 	try = try;
 	is_promise = is_promise;
diff --git a/util/prosodyctl.lua b/util/prosodyctl.lua
index 6c84ab6e..80052163 100644
--- a/util/prosodyctl.lua
+++ b/util/prosodyctl.lua
@@ -12,10 +12,10 @@ local encodings = require "util.encodings";
 local stringprep = encodings.stringprep;
 local storagemanager = require "core.storagemanager";
 local usermanager = require "core.usermanager";
+local interpolation = require "util.interpolation";
 local signal = require "util.signal";
 local set = require "util.set";
 local lfs = require "lfs";
-local pcall = pcall;
 local type = type;
 
 local nodeprep, nameprep = stringprep.nodeprep, stringprep.nameprep;
@@ -27,10 +27,22 @@ local tonumber = tonumber;
 local _G = _G;
 local prosody = prosody;
 
+local error_messages = setmetatable({
+		["invalid-username"] = "The given username is invalid in a Jabber ID";
+		["invalid-hostname"] = "The given hostname is invalid";
+		["no-password"] = "No password was supplied";
+		["no-such-user"] = "The given user does not exist on the server";
+		["no-such-host"] = "The given hostname does not exist in the config";
+		["unable-to-save-data"] = "Unable to store, perhaps you don't have permission?";
+		["no-pidfile"] = "There is no 'pidfile' option in the configuration file, see https://prosody.im/doc/prosodyctl#pidfile for help";
+		["invalid-pidfile"] = "The 'pidfile' option in the configuration file is not a string, see https://prosody.im/doc/prosodyctl#pidfile for help";
+		["no-posix"] = "The mod_posix module is not enabled in the Prosody config file, see https://prosody.im/doc/prosodyctl for more info";
+		["no-such-method"] = "This module has no commands";
+		["not-running"] = "Prosody is not running";
+		}, { __index = function (_,k) return "Error: "..(tostring(k):gsub("%-", " "):gsub("^.", string.upper)); end });
+
 -- UI helpers
-local function show_message(msg, ...)
-	print(msg:format(...));
-end
+local show_message = require "util.human.io".printf;
 
 local function show_usage(usage, desc)
 	print("Usage: ".._G.arg[0].." "..usage);
@@ -39,92 +51,19 @@ local function show_usage(usage, desc)
 	end
 end
 
-local function getchar(n)
-	local stty_ret = os.execute("stty raw -echo 2>/dev/null");
-	local ok, char;
-	if stty_ret == true or stty_ret == 0 then
-		ok, char = pcall(io.read, n or 1);
-		os.execute("stty sane");
-	else
-		ok, char = pcall(io.read, "*l");
-		if ok then
-			char = char:sub(1, n or 1);
-		end
-	end
-	if ok then
-		return char;
-	end
-end
-
-local function getline()
-	local ok, line = pcall(io.read, "*l");
-	if ok then
-		return line;
-	end
-end
-
-local function getpass()
-	local stty_ret, _, status_code = os.execute("stty -echo 2>/dev/null");
-	if status_code then -- COMPAT w/ Lua 5.1
-		stty_ret = status_code;
-	end
-	if stty_ret ~= 0 then
-		io.write("\027[08m"); -- ANSI 'hidden' text attribute
-	end
-	local ok, pass = pcall(io.read, "*l");
-	if stty_ret == 0 then
-		os.execute("stty sane");
-	else
-		io.write("\027[00m");
-	end
-	io.write("\n");
-	if ok then
-		return pass;
-	end
-end
-
-local function show_yesno(prompt)
-	io.write(prompt, " ");
-	local choice = getchar():lower();
-	io.write("\n");
-	if not choice:match("%a") then
-		choice = prompt:match("%[.-(%U).-%]$");
-		if not choice then return nil; end
-	end
-	return (choice == "y");
-end
-
-local function read_password()
-	local password;
-	while true do
-		io.write("Enter new password: ");
-		password = getpass();
-		if not password then
-			show_message("No password - cancelled");
-			return;
-		end
-		io.write("Retype new password: ");
-		if getpass() ~= password then
-			if not show_yesno [=[Passwords did not match, try again? [Y/n]]=] then
-				return;
-			end
-		else
-			break;
-		end
-	end
-	return password;
-end
-
-local function show_prompt(prompt)
-	io.write(prompt, " ");
-	local line = getline();
-	line = line and line:gsub("\n$","");
-	return (line and #line > 0) and line or nil;
+local function show_module_configuration_help(mod_name)
+	print("Done.")
+	print("If you installed a prosody plugin, don't forget to add its name under the 'modules_enabled' section inside your configuration file.")
+	print("Depending on the module, there might be further configuration steps required.")
+	print("")
+	print("More info about: ")
+	print("	modules_enabled: https://prosody.im/doc/modules_enabled")
+	print("	"..mod_name..": https://modules.prosody.im/"..mod_name..".html")
 end
 
 -- Server control
 local function adduser(params)
-	local user, host, password = nodeprep(params.user), nameprep(params.host), params.password;
+	local user, host, password = nodeprep(params.user, true), nameprep(params.host), params.password;
 	if not user then
 		return false, "invalid-username";
 	elseif not host then
@@ -200,7 +139,7 @@ local function getpid()
 		return false, "pidfile-read-failed", err;
 	end
 
-	local locked, err = lfs.lock(file, "w");
+	local locked, err = lfs.lock(file, "w"); -- luacheck: ignore 211/err
 	if locked then
 		file:close();
 		return false, "pidfile-not-locked";
@@ -217,7 +156,7 @@ local function getpid()
 end
 
 local function isrunning()
-	local ok, pid, err = getpid();
+	local ok, pid, err = getpid(); -- luacheck: ignore 211/err
 	if not ok then
 		if pid == "pidfile-read-failed" or pid == "pidfile-not-locked" then
 			-- Report as not running, since we can't open the pidfile
@@ -229,7 +168,8 @@ local function isrunning()
 	return true, signal.kill(pid, 0) == 0;
 end
 
-local function start(source_dir)
+local function start(source_dir, lua)
+	lua = lua and lua .. " " or "";
 	local ok, ret = isrunning();
 	if not ok then
 		return ok, ret;
@@ -238,9 +178,9 @@ local function start(source_dir)
 		return false, "already-running";
 	end
 	if not source_dir then
-		os.execute("./prosody -D");
+		os.execute(lua .. "./prosody -D");
 	else
-		os.execute(source_dir.."/../../bin/prosody -D");
+		os.execute(lua .. source_dir.."/../../bin/prosody -D");
 	end
 	return true;
 end
@@ -277,16 +217,36 @@ local function reload()
 	return true;
 end
 
+local function get_path_custom_plugins(plugin_paths)
+		-- I'm considering that the custom plugins' path is the first one at prosody.paths.plugins
+	-- luacheck: ignore 512
+	for path in plugin_paths:gmatch("[^;]+") do
+		return path;
+	end
+end
+
+local render_cli = interpolation.new("%b{}", function (s) return "'"..s:gsub("'","'\\''").."'" end)
+
+local function call_luarocks(operation, mod, server)
+	local dir = get_path_custom_plugins(prosody.paths.plugins);
+	if operation == "install" then
+		show_message("Installing %s at %s", mod, dir);
+	elseif operation == "remove" then
+		show_message("Removing %s from %s", mod, dir);
+	end
+	os.execute(render_cli("luarocks {op} --tree={dir} {server&--server={server}} {mod?}", {
+				dir = dir; op = operation; mod = mod; server = server;
+		}));
+	if operation == "install" then
+		show_module_configuration_help(mod);
+	end
+end
+
 return {
 	show_message = show_message;
 	show_warning = show_message;
 	show_usage = show_usage;
-	getchar = getchar;
-	getline = getline;
-	getpass = getpass;
-	show_yesno = show_yesno;
-	read_password = read_password;
-	show_prompt = show_prompt;
+	show_module_configuration_help = show_module_configuration_help;
 	adduser = adduser;
 	user_exists = user_exists;
 	passwd = passwd;
@@ -296,4 +256,7 @@ return {
 	start = start;
 	stop = stop;
 	reload = reload;
+	get_path_custom_plugins = get_path_custom_plugins;
+	call_luarocks = call_luarocks;
+	error_messages = error_messages;
 };
diff --git a/util/prosodyctl/cert.lua b/util/prosodyctl/cert.lua
new file mode 100644
index 00000000..29e26ed8
--- /dev/null
+++ b/util/prosodyctl/cert.lua
@@ -0,0 +1,293 @@
+local lfs = require "lfs";
+
+local pctl = require "util.prosodyctl";
+local configmanager = require "core.configmanager";
+
+local openssl;
+
+local cert_commands = {};
+
+-- If a file already exists, ask if the user wants to use it or replace it
+-- Backups the old file if replaced
+local function use_existing(filename)
+	local attrs = lfs.attributes(filename);
+	if attrs then
+		if pctl.show_yesno(filename .. " exists, do you want to replace it? [y/n]") then
+			local backup = filename..".bkp~"..os.date("%FT%T", attrs.change);
+			os.rename(filename, backup);
+			pctl.show_message("%s backed up to %s", filename, backup);
+		else
+			-- Use the existing file
+			return true;
+		end
+	end
+end
+
+local have_pposix, pposix = pcall(require, "util.pposix");
+local cert_basedir = prosody.paths.data == "." and "./certs" or prosody.paths.data;
+if have_pposix and pposix.getuid() == 0 then
+	-- FIXME should be enough to check if this directory is writable
+	local cert_dir = configmanager.get("*", "certificates") or "certs";
+	cert_basedir = configmanager.resolve_relative_path(prosody.paths.config, cert_dir);
+end
+
+function cert_commands.config(arg)
+	if #arg >= 1 and arg[1] ~= "--help" then
+		local conf_filename = cert_basedir .. "/" .. arg[1] .. ".cnf";
+		if use_existing(conf_filename) then
+			return nil, conf_filename;
+		end
+		local distinguished_name;
+		if arg[#arg]:find("^/") then
+			distinguished_name = table.remove(arg);
+		end
+		local conf = openssl.config.new();
+		conf:from_prosody(prosody.hosts, configmanager, arg);
+		if distinguished_name then
+			local dn = {};
+			for k, v in distinguished_name:gmatch("/([^=/]+)=([^/]+)") do
+				table.insert(dn, k);
+				dn[k] = v;
+			end
+			conf.distinguished_name = dn;
+		else
+			pctl.show_message("Please provide details to include in the certificate config file.");
+			pctl.show_message("Leave the field empty to use the default value or '.' to exclude the field.")
+			for _, k in ipairs(openssl._DN_order) do
+				local v = conf.distinguished_name[k];
+				if v then
+					local nv = nil;
+					if k == "commonName" then
+						v = arg[1]
+					elseif k == "emailAddress" then
+						v = "xmpp@" .. arg[1];
+					elseif k == "countryName" then
+						local tld = arg[1]:match"%.([a-z]+)$";
+						if tld and #tld == 2 and tld ~= "uk" then
+							v = tld:upper();
+						end
+					end
+					nv = pctl.show_prompt(("%s (%s):"):format(k, nv or v));
+					nv = (not nv or nv == "") and v or nv;
+					if nv:find"[\192-\252][\128-\191]+" then
+						conf.req.string_mask = "utf8only"
+					end
+					conf.distinguished_name[k] = nv ~= "." and nv or nil;
+				end
+			end
+		end
+		local conf_file, err = io.open(conf_filename, "w");
+		if not conf_file then
+			pctl.show_warning("Could not open OpenSSL config file for writing");
+			pctl.show_warning(err);
+			os.exit(1);
+		end
+		conf_file:write(conf:serialize());
+		conf_file:close();
+		print("");
+		pctl.show_message("Config written to %s", conf_filename);
+		return nil, conf_filename;
+	else
+		pctl.show_usage("cert config HOSTNAME [HOSTNAME+]", "Builds a certificate config file covering the supplied hostname(s)")
+	end
+end
+
+function cert_commands.key(arg)
+	if #arg >= 1 and arg[1] ~= "--help" then
+		local key_filename = cert_basedir .. "/" .. arg[1] .. ".key";
+		if use_existing(key_filename) then
+			return nil, key_filename;
+		end
+		os.remove(key_filename); -- This file, if it exists is unlikely to have write permissions
+		local key_size = tonumber(arg[2] or pctl.show_prompt("Choose key size (2048):") or 2048);
+		local old_umask = pposix.umask("0377");
+		if openssl.genrsa{out=key_filename, key_size} then
+			os.execute(("chmod 400 '%s'"):format(key_filename));
+			pctl.show_message("Key written to %s", key_filename);
+			pposix.umask(old_umask);
+			return nil, key_filename;
+		end
+		pctl.show_message("There was a problem, see OpenSSL output");
+	else
+		pctl.show_usage("cert key HOSTNAME <bits>", "Generates a RSA key named HOSTNAME.key\n "
+		.."Prompts for a key size if none given")
+	end
+end
+
+function cert_commands.request(arg)
+	if #arg >= 1 and arg[1] ~= "--help" then
+		local req_filename = cert_basedir .. "/" .. arg[1] .. ".req";
+		if use_existing(req_filename) then
+			return nil, req_filename;
+		end
+		local _, key_filename = cert_commands.key({arg[1]});
+		local _, conf_filename = cert_commands.config(arg);
+		if openssl.req{new=true, key=key_filename, utf8=true, sha256=true, config=conf_filename, out=req_filename} then
+			pctl.show_message("Certificate request written to %s", req_filename);
+		else
+			pctl.show_message("There was a problem, see OpenSSL output");
+		end
+	else
+		pctl.show_usage("cert request HOSTNAME [HOSTNAME+]", "Generates a certificate request for the supplied hostname(s)")
+	end
+end
+
+function cert_commands.generate(arg)
+	if #arg >= 1 and arg[1] ~= "--help" then
+		local cert_filename = cert_basedir .. "/" .. arg[1] .. ".crt";
+		if use_existing(cert_filename) then
+			return nil, cert_filename;
+		end
+		local _, key_filename = cert_commands.key({arg[1]});
+		local _, conf_filename = cert_commands.config(arg);
+		if key_filename and conf_filename and cert_filename
+			and openssl.req{new=true, x509=true, nodes=true, key=key_filename,
+				days=365, sha256=true, utf8=true, config=conf_filename, out=cert_filename} then
+			pctl.show_message("Certificate written to %s", cert_filename);
+			print();
+		else
+			pctl.show_message("There was a problem, see OpenSSL output");
+		end
+	else
+		pctl.show_usage("cert generate HOSTNAME [HOSTNAME+]", "Generates a self-signed certificate for the current hostname(s)")
+	end
+end
+
+local function sh_esc(s)
+	return "'" .. s:gsub("'", "'\\''") .. "'";
+end
+
+local function copy(from, to, umask, owner, group)
+	local old_umask = umask and pposix.umask(umask);
+	local attrs = lfs.attributes(to);
+	if attrs then -- Move old file out of the way
+		local backup = to..".bkp~"..os.date("%FT%T", attrs.change);
+		os.rename(to, backup);
+	end
+	-- FIXME friendlier error handling, maybe move above backup back?
+	local input = assert(io.open(from));
+	local output = assert(io.open(to, "w"));
+	local data = input:read(2^11);
+	while data and output:write(data) do
+		data = input:read(2^11);
+	end
+	assert(input:close());
+	assert(output:close());
+	if not prosody.installed then
+		-- FIXME this is possibly specific to GNU chown
+		os.execute(("chown -c --reference=%s %s"):format(sh_esc(cert_basedir), sh_esc(to)));
+	elseif owner and group then
+		local ok = os.execute(("chown %s:%s %s"):format(sh_esc(owner), sh_esc(group), sh_esc(to)));
+		assert(ok == true or ok == 0, "Failed to change ownership of "..to);
+	end
+	if old_umask then pposix.umask(old_umask); end
+	return true;
+end
+
+function cert_commands.import(arg)
+	local hostnames = {};
+	-- Move hostname arguments out of arg, the rest should be a list of paths
+	while arg[1] and prosody.hosts[ arg[1] ] do
+		table.insert(hostnames, table.remove(arg, 1));
+	end
+	if hostnames[1] == nil then
+		local domains = os.getenv"RENEWED_DOMAINS"; -- Set if invoked via certbot
+		if domains then
+			for host in domains:gmatch("%S+") do
+				table.insert(hostnames, host);
+			end
+		else
+			for host in pairs(prosody.hosts) do
+				if host ~= "*" and configmanager.get(host, "enabled") ~= false then
+					table.insert(hostnames, host);
+				end
+			end
+		end
+	end
+	if not arg[1] or arg[1] == "--help" then -- Probably forgot the path
+		pctl.show_usage("cert import [HOSTNAME+] /path/to/certs [/other/paths/]+",
+			"Copies certificates to "..cert_basedir);
+		return 1;
+	end
+	local owner, group;
+	if pposix.getuid() == 0 then -- We need root to change ownership
+		owner = configmanager.get("*", "prosody_user") or "prosody";
+		group = configmanager.get("*", "prosody_group") or owner;
+	end
+	local cm = require "core.certmanager";
+	local imported = {};
+	for _, host in ipairs(hostnames) do
+		for _, dir in ipairs(arg) do
+			local paths = cm.find_cert(dir, host);
+			if paths then
+				copy(paths.certificate, cert_basedir .. "/" .. host .. ".crt", nil, owner, group);
+				copy(paths.key, cert_basedir .. "/" .. host .. ".key", "0377", owner, group);
+				table.insert(imported, host);
+			else
+				-- TODO Say where we looked
+				pctl.show_warning("No certificate for host "..host.." found :(");
+			end
+			-- TODO Additional checks
+			-- Certificate names matches the hostname
+			-- Private key matches public key in certificate
+		end
+	end
+	if imported[1] then
+		pctl.show_message("Imported certificate and key for hosts %s", table.concat(imported, ", "));
+		local ok, err = pctl.reload();
+		if not ok and err ~= "not-running" then
+			pctl.show_message(pctl.error_messages[err]);
+		end
+	else
+		pctl.show_warning("No certificates imported :(");
+		return 1;
+	end
+end
+
+local function cert(arg)
+	if #arg >= 1 and arg[1] ~= "--help" then
+		openssl = require "util.openssl";
+		lfs = require "lfs";
+		local cert_dir_attrs = lfs.attributes(cert_basedir);
+		if not cert_dir_attrs then
+			pctl.show_warning("The directory "..cert_basedir.." does not exist");
+			return 1; -- TODO Should we create it?
+		end
+		local uid = pposix.getuid();
+		if uid ~= 0 and uid ~= cert_dir_attrs.uid then
+			pctl.show_warning("The directory "..cert_basedir.." is not owned by the current user, won't be able to write files to it");
+			return 1;
+		elseif not cert_dir_attrs.permissions then -- COMPAT with LuaFilesystem < 1.6.2 (hey CentOS!)
+			pctl.show_message("Unable to check permissions on %s (LuaFilesystem 1.6.2+ required)", cert_basedir);
+			pctl.show_message("Please confirm that Prosody (and only Prosody) can write to this directory)");
+		elseif cert_dir_attrs.permissions:match("^%.w..%-..%-.$") then
+			pctl.show_warning("The directory "..cert_basedir.." not only writable by its owner");
+			return 1;
+		end
+		local subcmd = table.remove(arg, 1);
+		if type(cert_commands[subcmd]) == "function" then
+			if subcmd ~= "import" then -- hostnames are optional for import
+				if not arg[1] then
+					pctl.show_message"You need to supply at least one hostname"
+					arg = { "--help" };
+				end
+				if arg[1] ~= "--help" and not prosody.hosts[arg[1]] then
+					pctl.show_message(pctl.error_messages["no-such-host"]);
+					return 1;
+				end
+			end
+			return cert_commands[subcmd](arg);
+		elseif subcmd == "check" then
+			return require "util.prosodyctl.check".check({"certs"});
+		end
+	end
+	pctl.show_usage("cert config|request|generate|key|import", "Helpers for generating X.509 certificates and keys.")
+	for _, cmd in pairs(cert_commands) do
+		print()
+		cmd{ "--help" }
+	end
+end
+
+return {
+	cert = cert;
+};
diff --git a/util/prosodyctl/check.lua b/util/prosodyctl/check.lua
new file mode 100644
index 00000000..86394a18
--- /dev/null
+++ b/util/prosodyctl/check.lua
@@ -0,0 +1,541 @@
+local configmanager = require "core.configmanager";
+local show_usage = require "util.prosodyctl".show_usage;
+local show_warning = require "util.prosodyctl".show_warning;
+local dependencies = require "util.dependencies";
+local socket = require "socket";
+local jid_split = require "util.jid".prepped_split;
+local modulemanager = require "core.modulemanager";
+
+local function check(arg)
+	if arg[1] == "--help" then
+		show_usage([[check]], [[Perform basic checks on your Prosody installation]]);
+		return 1;
+	end
+	local what = table.remove(arg, 1);
+	local set = require "util.set";
+	local it = require "util.iterators";
+	local ok = true;
+	local function disabled_hosts(host, conf) return host ~= "*" and conf.enabled ~= false; end
+	local function enabled_hosts() return it.filter(disabled_hosts, pairs(configmanager.getconfig())); end
+	if not (what == nil or what == "disabled" or what == "config" or what == "dns" or what == "certs") then
+		show_warning("Don't know how to check '%s'. Try one of 'config', 'dns', 'certs' or 'disabled'.", what);
+		return 1;
+	end
+	if not what or what == "disabled" then
+		local disabled_hosts_set = set.new();
+		for host, host_options in it.filter("*", pairs(configmanager.getconfig())) do
+			if host_options.enabled == false then
+				disabled_hosts_set:add(host);
+			end
+		end
+		if not disabled_hosts_set:empty() then
+			local msg = "Checks will be skipped for these disabled hosts: %s";
+			if what then msg = "These hosts are disabled: %s"; end
+			show_warning(msg, tostring(disabled_hosts_set));
+			if what then return 0; end
+			print""
+		end
+	end
+	if not what or what == "config" then
+		print("Checking config...");
+		local deprecated = set.new({
+			"bosh_ports", "disallow_s2s", "no_daemonize", "anonymous_login", "require_encryption",
+			"vcard_compatibility", "cross_domain_bosh", "cross_domain_websocket", "daemonize",
+		});
+		local known_global_options = set.new({
+			"pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize",
+			"umask", "prosodyctl_timeout", "use_ipv6", "use_libevent", "network_settings",
+			"network_backend", "http_default_host",
+			"statistics_interval", "statistics", "statistics_config",
+		});
+		local config = configmanager.getconfig();
+		-- Check that we have any global options (caused by putting a host at the top)
+		if it.count(it.filter("log", pairs(config["*"]))) == 0 then
+			ok = false;
+			print("");
+			print("    No global options defined. Perhaps you have put a host definition at the top")
+			print("    of the config file? They should be at the bottom, see https://prosody.im/doc/configure#overview");
+		end
+		if it.count(enabled_hosts()) == 0 then
+			ok = false;
+			print("");
+			if it.count(it.filter("*", pairs(config))) == 0 then
+				print("    No hosts are defined, please add at least one VirtualHost section")
+			elseif config["*"]["enabled"] == false then
+				print("    No hosts are enabled. Remove enabled = false from the global section or put enabled = true under at least one VirtualHost section")
+			else
+				print("    All hosts are disabled. Remove enabled = false from at least one VirtualHost section")
+			end
+		end
+		if not config["*"].modules_enabled then
+			print("    No global modules_enabled is set?");
+			local suggested_global_modules;
+			for host, options in enabled_hosts() do --luacheck: ignore 213/host
+				if not options.component_module and options.modules_enabled then
+					suggested_global_modules = set.intersection(suggested_global_modules or set.new(options.modules_enabled), set.new(options.modules_enabled));
+				end
+			end
+			if suggested_global_modules and not suggested_global_modules:empty() then
+				print("    Consider moving these modules into modules_enabled in the global section:")
+				print("    "..tostring(suggested_global_modules / function (x) return ("%q"):format(x) end));
+			end
+			print();
+		end
+
+		do -- Check for modules enabled both normally and as components
+			local modules = set.new(config["*"]["modules_enabled"]);
+			for host, options in enabled_hosts() do
+				local component_module = options.component_module;
+				if component_module and modules:contains(component_module) then
+					print(("    mod_%s is enabled both in modules_enabled and as Component %q %q"):format(component_module, host, component_module));
+					print("    This means the service is enabled on all VirtualHosts as well as the Component.");
+					print("    Are you sure this what you want? It may cause unexpected behaviour.");
+				end
+			end
+		end
+
+		-- Check for global options under hosts
+		local global_options = set.new(it.to_array(it.keys(config["*"])));
+		local deprecated_global_options = set.intersection(global_options, deprecated);
+		if not deprecated_global_options:empty() then
+			print("");
+			print("    You have some deprecated options in the global section:");
+			print("    "..tostring(deprecated_global_options))
+			ok = false;
+		end
+		for host, options in it.filter(function (h) return h ~= "*" end, pairs(configmanager.getconfig())) do
+			local host_options = set.new(it.to_array(it.keys(options)));
+			local misplaced_options = set.intersection(host_options, known_global_options);
+			for name in pairs(options) do
+				if name:match("^interfaces?")
+				or name:match("_ports?$") or name:match("_interfaces?$")
+				or (name:match("_ssl$") and not name:match("^[cs]2s_ssl$")) then
+					misplaced_options:add(name);
+				end
+			end
+			if not misplaced_options:empty() then
+				ok = false;
+				print("");
+				local n = it.count(misplaced_options);
+				print("    You have "..n.." option"..(n>1 and "s " or " ").."set under "..host.." that should be");
+				print("    in the global section of the config file, above any VirtualHost or Component definitions,")
+				print("    see https://prosody.im/doc/configure#overview for more information.")
+				print("");
+				print("    You need to move the following option"..(n>1 and "s" or "")..": "..table.concat(it.to_array(misplaced_options), ", "));
+			end
+		end
+		for host, options in enabled_hosts() do
+			local host_options = set.new(it.to_array(it.keys(options)));
+			local subdomain = host:match("^[^.]+");
+			if not(host_options:contains("component_module")) and (subdomain == "jabber" or subdomain == "xmpp"
+			   or subdomain == "chat" or subdomain == "im") then
+				print("");
+				print("    Suggestion: If "..host.. " is a new host with no real users yet, consider renaming it now to");
+				print("     "..host:gsub("^[^.]+%.", "")..". You can use SRV records to redirect XMPP clients and servers to "..host..".");
+				print("     For more information see: https://prosody.im/doc/dns");
+			end
+		end
+		local all_modules = set.new(config["*"].modules_enabled);
+		local all_options = set.new(it.to_array(it.keys(config["*"])));
+		for host in enabled_hosts() do
+			all_options:include(set.new(it.to_array(it.keys(config[host]))));
+			all_modules:include(set.new(config[host].modules_enabled));
+		end
+		for mod in all_modules do
+			if mod:match("^mod_") then
+				print("");
+				print("    Modules in modules_enabled should not have the 'mod_' prefix included.");
+				print("    Change '"..mod.."' to '"..mod:match("^mod_(.*)").."'.");
+			elseif mod:match("^auth_") then
+				print("");
+				print("    Authentication modules should not be added to modules_enabled,");
+				print("    but be specified in the 'authentication' option.");
+				print("    Remove '"..mod.."' from modules_enabled and instead add");
+				print("        authentication = '"..mod:match("^auth_(.*)").."'");
+				print("    For more information see https://prosody.im/doc/authentication");
+			elseif mod:match("^storage_") then
+				print("");
+				print("    storage modules should not be added to modules_enabled,");
+				print("    but be specified in the 'storage' option.");
+				print("    Remove '"..mod.."' from modules_enabled and instead add");
+				print("        storage = '"..mod:match("^storage_(.*)").."'");
+				print("    For more information see https://prosody.im/doc/storage");
+			end
+		end
+		if all_modules:contains("vcard") and all_modules:contains("vcard_legacy") then
+			print("");
+			print("    Both mod_vcard_legacy and mod_vcard are enabled but they conflict");
+			print("    with each other. Remove one.");
+		end
+		if all_modules:contains("pep") and all_modules:contains("pep_simple") then
+			print("");
+			print("    Both mod_pep_simple and mod_pep are enabled but they conflict");
+			print("    with each other. Remove one.");
+		end
+		for host, host_config in pairs(config) do --luacheck: ignore 213/host
+			if type(rawget(host_config, "storage")) == "string" and rawget(host_config, "default_storage") then
+				print("");
+				print("    The 'default_storage' option is not needed if 'storage' is set to a string.");
+				break;
+			end
+		end
+		local require_encryption = set.intersection(all_options, set.new({
+			"require_encryption", "c2s_require_encryption", "s2s_require_encryption"
+		})):empty();
+		local ssl = dependencies.softreq"ssl";
+		if not ssl then
+			if not require_encryption then
+				print("");
+				print("    You require encryption but LuaSec is not available.");
+				print("    Connections will fail.");
+				ok = false;
+			end
+		elseif not ssl.loadcertificate then
+			if all_options:contains("s2s_secure_auth") then
+				print("");
+				print("    You have set s2s_secure_auth but your version of LuaSec does ");
+				print("    not support certificate validation, so all s2s connections will");
+				print("    fail.");
+				ok = false;
+			elseif all_options:contains("s2s_secure_domains") then
+				local secure_domains = set.new();
+				for host in enabled_hosts() do
+					if config[host].s2s_secure_auth == true then
+						secure_domains:add("*");
+					else
+						secure_domains:include(set.new(config[host].s2s_secure_domains));
+					end
+				end
+				if not secure_domains:empty() then
+					print("");
+					print("    You have set s2s_secure_domains but your version of LuaSec does ");
+					print("    not support certificate validation, so s2s connections to/from ");
+					print("    these domains will fail.");
+					ok = false;
+				end
+			end
+		elseif require_encryption and not all_modules:contains("tls") then
+			print("");
+			print("    You require encryption but mod_tls is not enabled.");
+			print("    Connections will fail.");
+			ok = false;
+		end
+
+		print("Done.\n");
+	end
+	if not what or what == "dns" then
+		local dns = require "net.dns";
+		pcall(function ()
+			dns = require"net.unbound".dns;
+		end)
+		local idna = require "util.encodings".idna;
+		local ip = require "util.ip";
+		local c2s_ports = set.new(configmanager.get("*", "c2s_ports") or {5222});
+		local s2s_ports = set.new(configmanager.get("*", "s2s_ports") or {5269});
+
+		local c2s_srv_required, s2s_srv_required;
+		if not c2s_ports:contains(5222) then
+			c2s_srv_required = true;
+		end
+		if not s2s_ports:contains(5269) then
+			s2s_srv_required = true;
+		end
+
+		local problem_hosts = set.new();
+
+		local external_addresses, internal_addresses = set.new(), set.new();
+
+		local fqdn = socket.dns.tohostname(socket.dns.gethostname());
+		if fqdn then
+			do
+				local res = dns.lookup(idna.to_ascii(fqdn), "A");
+				if res then
+					for _, record in ipairs(res) do
+						external_addresses:add(record.a);
+					end
+				end
+			end
+			do
+				local res = dns.lookup(idna.to_ascii(fqdn), "AAAA");
+				if res then
+					for _, record in ipairs(res) do
+						external_addresses:add(record.aaaa);
+					end
+				end
+			end
+		end
+
+		local local_addresses = require"util.net".local_addresses() or {};
+
+		for addr in it.values(local_addresses) do
+			if not ip.new_ip(addr).private then
+				external_addresses:add(addr);
+			else
+				internal_addresses:add(addr);
+			end
+		end
+
+		if external_addresses:empty() then
+			print("");
+			print("   Failed to determine the external addresses of this server. Checks may be inaccurate.");
+			c2s_srv_required, s2s_srv_required = true, true;
+		end
+
+		local v6_supported = not not socket.tcp6;
+
+		for jid, host_options in enabled_hosts() do
+			local all_targets_ok, some_targets_ok = true, false;
+			local node, host = jid_split(jid);
+
+			local modules, component_module = modulemanager.get_modules_for_host(host);
+			if component_module then
+				modules:add(component_module);
+			end
+
+			local is_component = not not host_options.component_module;
+			print("Checking DNS for "..(is_component and "component" or "host").." "..jid.."...");
+			if node then
+				print("Only the domain part ("..host..") is used in DNS.")
+			end
+			local target_hosts = set.new();
+			if modules:contains("c2s") then
+				local res = dns.lookup("_xmpp-client._tcp."..idna.to_ascii(host)..".", "SRV");
+				if res then
+					for _, record in ipairs(res) do
+						if record.srv.target == "." then -- TODO is this an error if mod_c2s is enabled?
+							print("    'xmpp-client' service disabled by pointing to '.'"); -- FIXME Explain better what this is
+							break;
+						end
+						target_hosts:add(record.srv.target);
+						if not c2s_ports:contains(record.srv.port) then
+							print("    SRV target "..record.srv.target.." contains unknown client port: "..record.srv.port);
+						end
+					end
+				else
+					if c2s_srv_required then
+						print("    No _xmpp-client SRV record found for "..host..", but it looks like you need one.");
+						all_targets_ok = false;
+					else
+						target_hosts:add(host);
+					end
+				end
+			end
+			if modules:contains("s2s") then
+				local res = dns.lookup("_xmpp-server._tcp."..idna.to_ascii(host)..".", "SRV");
+				if res then
+					for _, record in ipairs(res) do
+						if record.srv.target == "." then -- TODO Is this an error if mod_s2s is enabled?
+							print("    'xmpp-server' service disabled by pointing to '.'"); -- FIXME Explain better what this is
+							break;
+						end
+						target_hosts:add(record.srv.target);
+						if not s2s_ports:contains(record.srv.port) then
+							print("    SRV target "..record.srv.target.." contains unknown server port: "..record.srv.port);
+						end
+					end
+				else
+					if s2s_srv_required then
+						print("    No _xmpp-server SRV record found for "..host..", but it looks like you need one.");
+						all_targets_ok = false;
+					else
+						target_hosts:add(host);
+					end
+				end
+			end
+			if target_hosts:empty() then
+				target_hosts:add(host);
+			end
+
+			if target_hosts:contains("localhost") then
+				print("    Target 'localhost' cannot be accessed from other servers");
+				target_hosts:remove("localhost");
+			end
+
+			if modules:contains("proxy65") then
+				local proxy65_target = configmanager.get(host, "proxy65_address") or host;
+				if type(proxy65_target) == "string" then
+					local A, AAAA = dns.lookup(idna.to_ascii(proxy65_target), "A"), dns.lookup(idna.to_ascii(proxy65_target), "AAAA");
+					local prob = {};
+					if not A then
+						table.insert(prob, "A");
+					end
+					if v6_supported and not AAAA then
+						table.insert(prob, "AAAA");
+					end
+					if #prob > 0 then
+						print("    File transfer proxy "..proxy65_target.." has no "..table.concat(prob, "/")
+						.." record. Create one or set 'proxy65_address' to the correct host/IP.");
+					end
+				else
+					print("    proxy65_address for "..host.." should be set to a string, unable to perform DNS check");
+				end
+			end
+
+			for target_host in target_hosts do
+				local host_ok_v4, host_ok_v6;
+				do
+					local res = dns.lookup(idna.to_ascii(target_host), "A");
+					if res then
+						for _, record in ipairs(res) do
+							if external_addresses:contains(record.a) then
+								some_targets_ok = true;
+								host_ok_v4 = true;
+							elseif internal_addresses:contains(record.a) then
+								host_ok_v4 = true;
+								some_targets_ok = true;
+								print("    "..target_host.." A record points to internal address, external connections might fail");
+							else
+								print("    "..target_host.." A record points to unknown address "..record.a);
+								all_targets_ok = false;
+							end
+						end
+					end
+				end
+				do
+					local res = dns.lookup(idna.to_ascii(target_host), "AAAA");
+					if res then
+						for _, record in ipairs(res) do
+							if external_addresses:contains(record.aaaa) then
+								some_targets_ok = true;
+								host_ok_v6 = true;
+							elseif internal_addresses:contains(record.aaaa) then
+								host_ok_v6 = true;
+								some_targets_ok = true;
+								print("    "..target_host.." AAAA record points to internal address, external connections might fail");
+							else
+								print("    "..target_host.." AAAA record points to unknown address "..record.aaaa);
+								all_targets_ok = false;
+							end
+						end
+					end
+				end
+
+				local bad_protos = {}
+				if not host_ok_v4 then
+					table.insert(bad_protos, "IPv4");
+				end
+				if not host_ok_v6 then
+					table.insert(bad_protos, "IPv6");
+				end
+				if #bad_protos > 0 then
+					print("    Host "..target_host.." does not seem to resolve to this server ("..table.concat(bad_protos, "/")..")");
+				end
+				if host_ok_v6 and not v6_supported then
+					print("    Host "..target_host.." has AAAA records, but your version of LuaSocket does not support IPv6.");
+					print("      Please see https://prosody.im/doc/ipv6 for more information.");
+				end
+			end
+			if not all_targets_ok then
+				print("    "..(some_targets_ok and "Only some" or "No").." targets for "..host.." appear to resolve to this server.");
+				if is_component then
+					print("    DNS records are necessary if you want users on other servers to access this component.");
+				end
+				problem_hosts:add(host);
+			end
+			print("");
+		end
+		if not problem_hosts:empty() then
+			print("");
+			print("For more information about DNS configuration please see https://prosody.im/doc/dns");
+			print("");
+			ok = false;
+		end
+	end
+	if not what or what == "certs" then
+		local cert_ok;
+		print"Checking certificates..."
+		local x509_verify_identity = require"util.x509".verify_identity;
+		local create_context = require "core.certmanager".create_context;
+		local ssl = dependencies.softreq"ssl";
+		-- local datetime_parse = require"util.datetime".parse_x509;
+		local load_cert = ssl and ssl.loadcertificate;
+		-- or ssl.cert_from_pem
+		if not ssl then
+			print("LuaSec not available, can't perform certificate checks")
+			if what == "certs" then cert_ok = false end
+		elseif not load_cert then
+			print("This version of LuaSec (" .. ssl._VERSION .. ") does not support certificate checking");
+			cert_ok = false
+		else
+			local function skip_bare_jid_hosts(host)
+				if jid_split(host) then
+					-- See issue #779
+					return false;
+				end
+				return true;
+			end
+			for host in it.filter(skip_bare_jid_hosts, enabled_hosts()) do
+				print("Checking certificate for "..host);
+				-- First, let's find out what certificate this host uses.
+				local host_ssl_config = configmanager.rawget(host, "ssl")
+					or configmanager.rawget(host:match("%.(.*)"), "ssl");
+				local global_ssl_config = configmanager.rawget("*", "ssl");
+				local ok, err, ssl_config = create_context(host, "server", host_ssl_config, global_ssl_config);
+				if not ok then
+					print("  Error: "..err);
+					cert_ok = false
+				elseif not ssl_config.certificate then
+					print("  No 'certificate' found for "..host)
+					cert_ok = false
+				elseif not ssl_config.key then
+					print("  No 'key' found for "..host)
+					cert_ok = false
+				else
+					local key, err = io.open(ssl_config.key); -- Permissions check only
+					if not key then
+						print("    Could not open "..ssl_config.key..": "..err);
+						cert_ok = false
+					else
+						key:close();
+					end
+					local cert_fh, err = io.open(ssl_config.certificate); -- Load the file.
+					if not cert_fh then
+						print("    Could not open "..ssl_config.certificate..": "..err);
+						cert_ok = false
+					else
+						print("  Certificate: "..ssl_config.certificate)
+						local cert = load_cert(cert_fh:read"*a"); cert_fh:close();
+						if not cert:validat(os.time()) then
+							print("    Certificate has expired.")
+							cert_ok = false
+						elseif not cert:validat(os.time() + 86400) then
+							print("    Certificate expires within one day.")
+							cert_ok = false
+						elseif not cert:validat(os.time() + 86400*7) then
+							print("    Certificate expires within one week.")
+						elseif not cert:validat(os.time() + 86400*31) then
+							print("    Certificate expires within one month.")
+						end
+						if configmanager.get(host, "component_module") == nil
+							and not x509_verify_identity(host, "_xmpp-client", cert) then
+							print("    Not valid for client connections to "..host..".")
+							cert_ok = false
+						end
+						if (not (configmanager.get(host, "anonymous_login")
+							or configmanager.get(host, "authentication") == "anonymous"))
+							and not x509_verify_identity(host, "_xmpp-server", cert) then
+							print("    Not valid for server-to-server connections to "..host..".")
+							cert_ok = false
+						end
+					end
+				end
+			end
+		end
+		if cert_ok == false then
+			print("")
+			print("For more information about certificates please see https://prosody.im/doc/certificates");
+			ok = false
+		end
+		print("")
+	end
+	if not ok then
+		print("Problems found, see above.");
+	else
+		print("All checks passed, congratulations!");
+	end
+	return ok and 0 or 2;
+end
+
+return {
+	check = check;
+};
diff --git a/util/prosodyctl/shell.lua b/util/prosodyctl/shell.lua
new file mode 100644
index 00000000..9ac982f7
--- /dev/null
+++ b/util/prosodyctl/shell.lua
@@ -0,0 +1,114 @@
+local config = require "core.configmanager";
+local server = require "net.server";
+local st = require "util.stanza";
+local path = require "util.paths";
+local parse_args = require "util.argparse".parse;
+
+local have_readline, readline = pcall(require, "readline");
+
+local adminstream = require "util.adminstream";
+
+if have_readline then
+	readline.set_readline_name("prosody");
+	readline.set_options({
+			histfile = path.join(prosody.paths.data, ".shell_history");
+			ignoredups = true;
+		});
+end
+
+local function read_line()
+	if have_readline then
+		return readline.readline("prosody> ");
+	else
+		io.write("prosody> ");
+		return io.read("*line");
+	end
+end
+
+local function send_line(client, line)
+	client.send(st.stanza("repl-input"):text(line));
+end
+
+local function repl(client)
+	local line = read_line();
+	if not line or line == "quit" or line == "exit" or line == "bye" then
+		if not line then
+			print("");
+		end
+		if have_readline then
+			readline.save_history();
+		end
+		os.exit();
+	end
+	send_line(client, line);
+end
+
+local function printbanner()
+	print([[
+                     ____                \   /     _
+                    |  _ \ _ __ ___  ___  _-_   __| |_   _
+                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
+                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
+                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
+                    A study in simplicity            |___/
+
+]]);
+	print("Welcome to the Prosody administration console. For a list of commands, type: help");
+	print("You may find more help on using this console in our online documentation at ");
+	print("https://prosody.im/doc/console\n");
+end
+
+local function start(arg) --luacheck: ignore 212/arg
+	local client = adminstream.client();
+	local opts, err, where = parse_args(arg);
+
+	if not opts then
+		if err == "param-not-found" then
+			print("Unknown command-line option: "..tostring(where));
+		elseif err == "missing-value" then
+			print("Expected a value to follow command-line option: "..where);
+		end
+		os.exit(1);
+	end
+
+	client.events.add_handler("connected", function ()
+		if not arg.quiet then
+			printbanner();
+		end
+		repl(client);
+	end);
+
+	client.events.add_handler("disconnected", function ()
+		print("--- session closed ---");
+		os.exit();
+	end);
+
+	client.events.add_handler("received", function (stanza)
+		if stanza.name == "repl-output" or stanza.name == "repl-result" then
+			local result_prefix = stanza.attr.type == "error" and "!" or "|";
+			print(result_prefix.." "..stanza:get_text());
+		end
+		if stanza.name == "repl-result" then
+			repl(client);
+		end
+	end);
+
+	local socket_path = path.resolve_relative_path(prosody.paths.data, opts.socket or config.get("*", "admin_socket") or "prosody.sock");
+	local conn = adminstream.connection(socket_path, client.listeners);
+	local ok, err = conn:connect();
+	if not ok then
+		if err == "no unix socket support" then
+			print("** LuaSocket unix socket support not available or incompatible, ensure your");
+			print("** version is up to date.");
+		else
+			print("** Unable to connect to server - is it running? Is mod_admin_shell enabled?");
+			print("** Connection error: "..err);
+		end
+		os.exit(1);
+	end
+	server.loop();
+end
+
+return {
+	shell = start;
+};
diff --git a/util/pubsub.lua b/util/pubsub.lua
index 1674b9a7..cfac7a68 100644
--- a/util/pubsub.lua
+++ b/util/pubsub.lua
@@ -1,5 +1,6 @@
 local events = require "util.events";
 local cache = require "util.cache";
+local errors = require "util.error";
 
 local service_mt = {};
 
@@ -280,7 +281,8 @@ function service:set_affiliation(node, actor, jid, affiliation) --> ok, err
 	node_obj.affiliations[jid] = affiliation;
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.affiliations[jid] = old_affiliation;
 			return ok, "internal-server-error";
@@ -344,7 +346,8 @@ function service:add_subscription(node, actor, jid, options) --> ok, err
 	end
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.subscribers[jid] = old_subscription;
 			self.subscriptions[normal_jid][jid][node] = old_subscription and true or nil;
@@ -396,7 +399,8 @@ function service:remove_subscription(node, actor, jid) --> ok, err
 	end
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.subscribers[jid] = old_subscription;
 			self.subscriptions[normal_jid][jid][node] = old_subscription and true or nil;
@@ -454,7 +458,8 @@ function service:create(node, actor, options) --> ok, err
 	};
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, self.nodes[node]);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, self.nodes[node]); -- luacheck: ignore 211/err
 		if not ok then
 			self.nodes[node] = nil;
 			return ok, "internal-server-error";
@@ -511,7 +516,7 @@ local function check_preconditions(node_config, required_config)
 	end
 	for config_field, value in pairs(required_config) do
 		if node_config[config_field] ~= value then
-			return false;
+			return false, config_field;
 		end
 	end
 	return true;
@@ -547,8 +552,13 @@ function service:publish(node, actor, id, item, requested_config) --> ok, err
 		node_obj = self.nodes[node];
 	elseif requested_config and not requested_config._defaults_only then
 		-- Check that node has the requested config before we publish
-		if not check_preconditions(node_obj.config, requested_config) then
-			return false, "precondition-not-met";
+		local ok, field = check_preconditions(node_obj.config, requested_config);
+		if not ok then
+			local err = errors.new({
+				type = "cancel", condition = "conflict", text = "Field does not match: "..field;
+			});
+			err.pubsub_condition = "precondition-not-met";
+			return false, err;
 		end
 	end
 	if not self.config.itemcheck(item) then
@@ -768,7 +778,8 @@ function service:set_node_config(node, actor, new_config) --> ok, err
 	node_obj.config = new_config;
 
 	if self.config.nodestore then
-		local ok, err = save_node_to_store(self, node_obj);
+		-- TODO pass the error from storage to caller eg wrapped in an util.error
+		local ok, err = save_node_to_store(self, node_obj); -- luacheck: ignore 211/err
 		if not ok then
 			node_obj.config = old_config;
 			return ok, "internal-server-error";
diff --git a/util/queue.lua b/util/queue.lua
index c8e71514..c94c62ae 100644
--- a/util/queue.lua
+++ b/util/queue.lua
@@ -59,18 +59,20 @@ local function new(size, allow_wrapping)
 			return true;
 		end;
 		items = function (self)
-			--luacheck: ignore 431/t
-			return function (t, pos)
-				if pos >= t:count() then
+			return function (_, pos)
+				if pos >= items then
 					return nil;
 				end
 				local read_pos = tail + pos;
-				if read_pos > t.size then
+				if read_pos > self.size then
 					read_pos = (read_pos%size);
 				end
-				return pos+1, t._items[read_pos];
+				return pos+1, t[read_pos];
 			end, self, 0;
 		end;
+		consume = function (self)
+			return self.pop, self;
+		end;
 	};
 end
 
diff --git a/util/rsm.lua b/util/rsm.lua
index 40a78fb5..ad725d76 100644
--- a/util/rsm.lua
+++ b/util/rsm.lua
@@ -10,10 +10,15 @@
 --
 
 local stanza = require"util.stanza".stanza;
-local tostring, tonumber = tostring, tonumber;
+local tonumber = tonumber;
+local s_format = string.format;
 local type = type;
 local pairs = pairs;
 
+local function inttostr(n)
+	return s_format("%d", n);
+end
+
 local xmlns_rsm = 'http://jabber.org/protocol/rsm';
 
 local element_parsers = {};
@@ -45,22 +50,28 @@ end
 local element_generators = setmetatable({
 	first = function(st, data)
 		if type(data) == "table" then
-			st:tag("first", { index = data.index }):text(data[1]):up();
+			st:tag("first", { index = inttostr(data.index) }):text(data[1]):up();
 		else
-			st:tag("first"):text(tostring(data)):up();
+			st:tag("first"):text(data):up();
 		end
 	end;
 	before = function(st, data)
 		if data == true then
 			st:tag("before"):up();
 		else
-			st:tag("before"):text(tostring(data)):up();
+			st:tag("before"):text(data):up();
 		end
-	end
+	end;
+	max = function (st, data)
+		st:tag("max"):text(inttostr(data)):up();
+	end;
+	count = function (st, data)
+		st:tag("count"):text(inttostr(data)):up();
+	end;
 }, {
 	__index = function(_, name)
 		return function(st, data)
-			st:tag(name):text(tostring(data)):up();
+			st:tag(name):text(data):up();
 		end
 	end;
 });
diff --git a/util/sasl.lua b/util/sasl.lua
index 50851405..fc2abdf3 100644
--- a/util/sasl.lua
+++ b/util/sasl.lua
@@ -134,7 +134,6 @@ end
 
 -- load the mechanisms
 require "util.sasl.plain"     .init(registerMechanism);
-require "util.sasl.digest-md5".init(registerMechanism);
 require "util.sasl.anonymous" .init(registerMechanism);
 require "util.sasl.scram"     .init(registerMechanism);
 require "util.sasl.external"  .init(registerMechanism);
diff --git a/util/sasl/digest-md5.lua b/util/sasl/digest-md5.lua
deleted file mode 100644
index 7542a037..00000000
--- a/util/sasl/digest-md5.lua
+++ /dev/null
@@ -1,251 +0,0 @@
--- sasl.lua v0.4
--- Copyright (C) 2008-2010 Tobias Markmann
---
---    All rights reserved.
---
---    Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
---
---        * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
---        * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
---        * Neither the name of Tobias Markmann nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
---
---    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-local tostring = tostring;
-local type = type;
-
-local s_gmatch = string.gmatch;
-local s_match = string.match;
-local t_concat = table.concat;
-local t_insert = table.insert;
-local to_byte, to_char = string.byte, string.char;
-
-local md5 = require "util.hashes".md5;
-local log = require "util.logger".init("sasl");
-local generate_uuid = require "util.uuid".generate;
-local nodeprep = require "util.encodings".stringprep.nodeprep;
-
-local _ENV = nil;
--- luacheck: std none
-
---=========================
---SASL DIGEST-MD5 according to RFC 2831
-
---[[
-Supported Authentication Backends
-
-digest_md5:
-	function(username, domain, realm, encoding) -- domain and realm are usually the same; for some broken
-												-- implementations it's not
-		return digesthash, state;
-	end
-
-digest_md5_test:
-	function(username, domain, realm, encoding, digesthash)
-		return true or false, state;
-	end
-]]
-
-local function digest(self, message)
-	--TODO complete support for authzid
-
-	local function serialize(message)
-		local data = ""
-
-		-- testing all possible values
-		if message["realm"] then data = data..[[realm="]]..message.realm..[[",]] end
-		if message["nonce"] then data = data..[[nonce="]]..message.nonce..[[",]] end
-		if message["qop"] then data = data..[[qop="]]..message.qop..[[",]] end
-		if message["charset"] then data = data..[[charset=]]..message.charset.."," end
-		if message["algorithm"] then data = data..[[algorithm=]]..message.algorithm.."," end
-		if message["rspauth"] then data = data..[[rspauth=]]..message.rspauth.."," end
-		data = data:gsub(",$", "")
-		return data
-	end
-
-	local function utf8tolatin1ifpossible(passwd)
-		local i = 1;
-		while i <= #passwd do
-			local passwd_i = to_byte(passwd:sub(i, i));
-			if passwd_i > 0x7F then
-				if passwd_i < 0xC0 or passwd_i > 0xC3 then
-					return passwd;
-				end
-				i = i + 1;
-				passwd_i = to_byte(passwd:sub(i, i));
-				if passwd_i < 0x80 or passwd_i > 0xBF then
-					return passwd;
-				end
-			end
-			i = i + 1;
-		end
-
-		local p = {};
-		local j = 0;
-		i = 1;
-		while (i <= #passwd) do
-			local passwd_i = to_byte(passwd:sub(i, i));
-			if passwd_i > 0x7F then
-				i = i + 1;
-				local passwd_i_1 = to_byte(passwd:sub(i, i));
-				t_insert(p, to_char(passwd_i%4*64 + passwd_i_1%64)); -- I'm so clever
-			else
-				t_insert(p, to_char(passwd_i));
-			end
-			i = i + 1;
-		end
-		return t_concat(p);
-	end
-	local function latin1toutf8(str)
-		local p = {};
-		for ch in s_gmatch(str, ".") do
-			ch = to_byte(ch);
-			if (ch < 0x80) then
-				t_insert(p, to_char(ch));
-			elseif (ch < 0xC0) then
-				t_insert(p, to_char(0xC2, ch));
-			else
-				t_insert(p, to_char(0xC3, ch - 64));
-			end
-		end
-		return t_concat(p);
-	end
-	local function parse(data)
-		local message = {}
-		-- COMPAT: %z in the pattern to work around jwchat bug (sends "charset=utf-8\0")
-		for k, v in s_gmatch(data, [[([%w%-]+)="?([^",%z]*)"?,?]]) do -- FIXME The hacky regex makes me shudder
-			message[k] = v;
-		end
-		return message;
-	end
-
-	if not self.nonce then
-		self.nonce = generate_uuid();
-		self.step = 0;
-		self.nonce_count = {};
-	end
-
-	self.step = self.step + 1;
-	if (self.step == 1) then
-		local challenge = serialize({	nonce = self.nonce,
-										qop = "auth",
-										charset = "utf-8",
-										algorithm = "md5-sess",
-										realm = self.realm});
-		return "challenge", challenge;
-	elseif (self.step == 2) then
-		local response = parse(message);
-		-- check for replay attack
-		if response["nc"] then
-			if self.nonce_count[response["nc"]] then return "failure", "not-authorized" end
-		end
-
-		-- check for username, it's REQUIRED by RFC 2831
-		local username = response["username"];
-		local _nodeprep = self.profile.nodeprep;
-		if username and _nodeprep ~= false then
-			username = (_nodeprep or nodeprep)(username); -- FIXME charset
-		end
-		if not username or username == "" then
-			return "failure", "malformed-request";
-		end
-		self.username = username;
-
-		-- check for nonce, ...
-		if not response["nonce"] then
-			return "failure", "malformed-request";
-		else
-			-- check if it's the right nonce
-			if response["nonce"] ~= tostring(self.nonce) then return "failure", "malformed-request" end
-		end
-
-		if not response["cnonce"] then return "failure", "malformed-request", "Missing entry for cnonce in SASL message." end
-		if not response["qop"] then response["qop"] = "auth" end
-
-		if response["realm"] == nil or response["realm"] == "" then
-			response["realm"] = "";
-		elseif response["realm"] ~= self.realm then
-			return "failure", "not-authorized", "Incorrect realm value";
-		end
-
-		local decoder;
-		if response["charset"] == nil then
-			decoder = utf8tolatin1ifpossible;
-		elseif response["charset"] ~= "utf-8" then
-			return "failure", "incorrect-encoding", "The client's response uses "..response["charset"].." for encoding with isn't supported by sasl.lua. Supported encodings are latin or utf-8.";
-		end
-
-		local domain = "";
-		local protocol = "";
-		if response["digest-uri"] then
-			protocol, domain = response["digest-uri"]:match("(%w+)/(.*)$");
-			if protocol == nil or domain == nil then return "failure", "malformed-request" end
-		else
-			return "failure", "malformed-request", "Missing entry for digest-uri in SASL message."
-		end
-
-		--TODO maybe realm support
-		local Y, state;
-		if self.profile.plain then
-			local password, state = self.profile.plain(self, response["username"], self.realm)
-			if state == nil then return "failure", "not-authorized"
-			elseif state == false then return "failure", "account-disabled" end
-			Y = md5(response["username"]..":"..response["realm"]..":"..password);
-		elseif self.profile["digest-md5"] then
-			Y, state = self.profile["digest-md5"](self, response["username"], self.realm, response["realm"], response["charset"])
-			if state == nil then return "failure", "not-authorized"
-			elseif state == false then return "failure", "account-disabled" end
-		elseif self.profile["digest-md5-test"] then
-			-- TODO
-		end
-		--local password_encoding, Y = self.credentials_handler("DIGEST-MD5", response["username"], self.realm, response["realm"], decoder);
-		--if Y == nil then return "failure", "not-authorized"
-		--elseif Y == false then return "failure", "account-disabled" end
-		local A1 = "";
-		if response.authzid then
-			if response.authzid == self.username or response.authzid == self.username.."@"..self.realm then
-				-- COMPAT
-				log("warn", "Client is violating RFC 3920 (section 6.1, point 7).");
-				A1 = Y..":"..response["nonce"]..":"..response["cnonce"]..":"..response.authzid;
-			else
-				return "failure", "invalid-authzid";
-			end
-		else
-			A1 = Y..":"..response["nonce"]..":"..response["cnonce"];
-		end
-		local A2 = "AUTHENTICATE:"..protocol.."/"..domain;
-
-		local HA1 = md5(A1, true);
-		local HA2 = md5(A2, true);
-
-		local KD = HA1..":"..response["nonce"]..":"..response["nc"]..":"..response["cnonce"]..":"..response["qop"]..":"..HA2;
-		local response_value = md5(KD, true);
-
-		if response_value == response["response"] then
-			-- calculate rspauth
-			A2 = ":"..protocol.."/"..domain;
-
-			HA1 = md5(A1, true);
-			HA2 = md5(A2, true);
-
-			KD = HA1..":"..response["nonce"]..":"..response["nc"]..":"..response["cnonce"]..":"..response["qop"]..":"..HA2
-			local rspauth = md5(KD, true);
-			self.authenticated = true;
-			--TODO: considering sending the rspauth in a success node for saving one roundtrip; allowed according to http://tools.ietf.org/html/draft-saintandre-rfc3920bis-09#section-7.3.6
-			return "challenge", serialize({rspauth = rspauth});
-		else
-			return "failure", "not-authorized", "The response provided by the client doesn't match the one we calculated."
-		end
-	elseif self.step == 3 then
-		if self.authenticated ~= nil then return "success"
-		else return "failure", "malformed-request" end
-	end
-end
-
-local function init(registerMechanism)
-	registerMechanism("DIGEST-MD5", {"plain"}, digest);
-end
-
-return {
-	init = init;
-}
diff --git a/util/sasl/scram.lua b/util/sasl/scram.lua
index f64feb8b..865f8cf7 100644
--- a/util/sasl/scram.lua
+++ b/util/sasl/scram.lua
@@ -14,16 +14,12 @@
 local s_match = string.match;
 local type = type
 local base64 = require "util.encodings".base64;
-local hmac_sha1 = require "util.hashes".hmac_sha1;
-local sha1 = require "util.hashes".sha1;
-local Hi = require "util.hashes".scram_Hi_sha1;
+local hashes = require "util.hashes";
 local generate_uuid = require "util.uuid".generate;
 local saslprep = require "util.encodings".stringprep.saslprep;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
 local log = require "util.logger".init("sasl");
-local t_concat = table.concat;
-local char = string.char;
-local byte = string.byte;
+local	binaryXOR = require "util.strbitop".sxor;
 
 local _ENV = nil;
 -- luacheck: std none
@@ -47,32 +43,6 @@ Supported Channel Binding Backends
 
 local default_i = 4096
 
-local xor_map = {
-	0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,1,0,3,2,5,4,7,6,9,8,11,10,
-	13,12,15,14,2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13,3,2,1,0,7,6,5,
-	4,11,10,9,8,15,14,13,12,4,5,6,7,0,1,2,3,12,13,14,15,8,9,10,11,5,
-	4,7,6,1,0,3,2,13,12,15,14,9,8,11,10,6,7,4,5,2,3,0,1,14,15,12,13,
-	10,11,8,9,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,8,9,10,11,12,13,
-	14,15,0,1,2,3,4,5,6,7,9,8,11,10,13,12,15,14,1,0,3,2,5,4,7,6,10,
-	11,8,9,14,15,12,13,2,3,0,1,6,7,4,5,11,10,9,8,15,14,13,12,3,2,1,
-	0,7,6,5,4,12,13,14,15,8,9,10,11,4,5,6,7,0,1,2,3,13,12,15,14,9,8,
-	11,10,5,4,7,6,1,0,3,2,14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1,15,
-	14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,
-};
-
-local result = {};
-local function binaryXOR( a, b )
-	for i=1, #a do
-		local x, y = byte(a, i), byte(b, i);
-		local lowx, lowy = x % 16, y % 16;
-		local hix, hiy = (x - lowx) / 16, (y - lowy) / 16;
-		local lowr, hir = xor_map[lowx * 16 + lowy + 1], xor_map[hix * 16 + hiy + 1];
-		local r = hir * 16 + lowr;
-		result[i] = char(r)
-	end
-	return t_concat(result);
-end
-
 local function validate_username(username, _nodeprep)
 	-- check for forbidden char sequences
 	for eq in username:gmatch("=(.?.?)") do
@@ -99,24 +69,26 @@ local function hashprep(hashname)
 	return hashname:lower():gsub("-", "_");
 end
 
-local function getAuthenticationDatabaseSHA1(password, salt, iteration_count)
-	if type(password) ~= "string" or type(salt) ~= "string" or type(iteration_count) ~= "number" then
-		return false, "inappropriate argument types"
-	end
-	if iteration_count < 4096 then
-		log("warn", "Iteration count < 4096 which is the suggested minimum according to RFC 5802.")
-	end
-	password = saslprep(password);
-	if not password then
-		return false, "password fails SASLprep";
+local function get_scram_hasher(H, HMAC, Hi)
+	return function (password, salt, iteration_count)
+		if type(password) ~= "string" or type(salt) ~= "string" or type(iteration_count) ~= "number" then
+			return false, "inappropriate argument types"
+		end
+		if iteration_count < 4096 then
+			log("warn", "Iteration count < 4096 which is the suggested minimum according to RFC 5802.")
+		end
+		password = saslprep(password);
+		if not password then
+			return false, "password fails SASLprep";
+		end
+		local salted_password = Hi(password, salt, iteration_count);
+		local stored_key = H(HMAC(salted_password, "Client Key"))
+		local server_key = HMAC(salted_password, "Server Key");
+		return true, stored_key, server_key
 	end
-	local salted_password = Hi(password, salt, iteration_count);
-	local stored_key = sha1(hmac_sha1(salted_password, "Client Key"))
-	local server_key = hmac_sha1(salted_password, "Server Key");
-	return true, stored_key, server_key
 end
 
-local function scram_gen(hash_name, H_f, HMAC_f)
+local function scram_gen(hash_name, H_f, HMAC_f, get_auth_db, expect_cb)
 	local profile_name = "scram_" .. hashprep(hash_name);
 	local function scram_hash(self, message)
 		local support_channel_binding = false;
@@ -129,6 +101,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 			local client_first_message = message;
 
 			-- TODO: fail if authzid is provided, since we don't support them yet
+			-- luacheck: ignore 211/authzid
 			local gs2_header, gs2_cbind_flag, gs2_cbind_name, authzid, client_first_message_bare, username, clientnonce
 				= s_match(client_first_message, "^(([pny])=?([^,]*),([^,]*),)(m?=?[^,]*,?n=([^,]*),r=([^,]*),?.*)$");
 
@@ -144,6 +117,10 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 
 			if gs2_cbind_flag == "n" then
 				-- "n" -> client doesn't support channel binding.
+				if expect_cb then
+					log("debug", "Client unexpectedly doesn't support channel binding");
+					-- XXX Is it sensible to abort if the client starts -PLUS but doesn't use channel binding?
+				end
 				support_channel_binding = false;
 			end
 
@@ -181,7 +158,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 				iteration_count = default_i;
 
 				local succ;
-				succ, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, iteration_count);
+				succ, stored_key, server_key = get_auth_db(password, salt, iteration_count);
 				if not succ then
 					log("error", "Generating authentication database failed. Reason: %s", stored_key);
 					return "failure", "temporary-auth-failure";
@@ -194,7 +171,7 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 			end
 
 			local nonce = clientnonce .. generate_uuid();
-			local server_first_message = "r="..nonce..",s="..base64.encode(salt)..",i="..iteration_count;
+			local server_first_message = ("r=%s,s=%s,i=%d"):format(nonce, base64.encode(salt), iteration_count);
 			self.state = {
 				gs2_header = gs2_header;
 				gs2_cbind_name = gs2_cbind_name;
@@ -251,22 +228,28 @@ local function scram_gen(hash_name, H_f, HMAC_f)
 	return scram_hash;
 end
 
+local auth_db_getters = {}
 local function init(registerMechanism)
-	local function registerSCRAMMechanism(hash_name, hash, hmac_hash)
+	local function registerSCRAMMechanism(hash_name, hash, hmac_hash, pbkdf2)
+		local get_auth_db = get_scram_hasher(hash, hmac_hash, pbkdf2);
+		auth_db_getters[hash_name] = get_auth_db;
 		registerMechanism("SCRAM-"..hash_name,
 			{"plain", "scram_"..(hashprep(hash_name))},
-			scram_gen(hash_name:lower(), hash, hmac_hash));
+			scram_gen(hash_name:lower(), hash, hmac_hash, get_auth_db));
 
 		-- register channel binding equivalent
 		registerMechanism("SCRAM-"..hash_name.."-PLUS",
 			{"plain", "scram_"..(hashprep(hash_name))},
-			scram_gen(hash_name:lower(), hash, hmac_hash), {"tls-unique"});
+			scram_gen(hash_name:lower(), hash, hmac_hash, get_auth_db, true), {"tls-unique"});
 	end
 
-	registerSCRAMMechanism("SHA-1", sha1, hmac_sha1);
+	registerSCRAMMechanism("SHA-1", hashes.sha1, hashes.hmac_sha1, hashes.pbkdf2_hmac_sha1);
+	registerSCRAMMechanism("SHA-256", hashes.sha256, hashes.hmac_sha256, hashes.pbkdf2_hmac_sha256);
 end
 
 return {
-	getAuthenticationDatabaseSHA1 = getAuthenticationDatabaseSHA1;
+	get_hash = get_scram_hasher;
+	hashers = auth_db_getters;
+	getAuthenticationDatabaseSHA1 = get_scram_hasher(hashes.sha1, hashes.hmac_sha1, hashes.pbkdf2_hmac_sha1); -- COMPAT
 	init = init;
 }
diff --git a/util/serialization.lua b/util/serialization.lua
index 5121a9f9..d310a3e8 100644
--- a/util/serialization.lua
+++ b/util/serialization.lua
@@ -16,22 +16,18 @@ local s_char = string.char;
 local s_match = string.match;
 local t_concat = table.concat;
 
+local to_hex = require "util.hex".to;
+
 local pcall = pcall;
 local envload = require"util.envload".envload;
 
 local pos_inf, neg_inf = math.huge, -math.huge;
--- luacheck: ignore 143/math
 local m_type = math.type or function (n)
 	return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float";
 end;
 
-local char_to_hex = {};
-for i = 0,255 do
-	char_to_hex[s_char(i)] = s_format("%02x", i);
-end
-
-local function to_hex(s)
-	return (s_gsub(s, ".", char_to_hex));
+local function rawpairs(t)
+	return next, t, nil;
 end
 
 local function fatal_error(obj, why)
@@ -123,6 +119,7 @@ local function new(opt)
 	local freeze = opt.freeze;
 	local maxdepth = opt.maxdepth or 127;
 	local multirefs = opt.multiref;
+	local table_pairs = opt.table_iterator or rawpairs;
 
 	-- serialize one table, recursively
 	-- t - table being serialized
@@ -153,6 +150,10 @@ local function new(opt)
 
 				if type(fr) == "function" then
 					t = fr(t);
+					if type(t) == "string" then
+						o[l], l = t, l + 1;
+						return l;
+					end
 					if type(tag) == "string" then
 						o[l], l = tag, l + 1;
 					end
@@ -164,7 +165,9 @@ local function new(opt)
 		local indent = s_rep(indentwith, d);
 		local numkey = 1;
 		local ktyp, vtyp;
-		for k,v in next,t do
+		local had_items = false;
+		for k,v in table_pairs(t) do
+			had_items = true;
 			o[l], l = itemstart, l + 1;
 			o[l], l = indent, l + 1;
 			ktyp, vtyp = type(k), type(v);
@@ -195,14 +198,10 @@ local function new(opt)
 			else
 				o[l], l = ser(v), l + 1;
 			end
-			-- last item?
-			if next(t, k) ~= nil then
-				o[l], l = itemsep, l + 1;
-			else
-				o[l], l = itemlast, l + 1;
-			end
+			o[l], l = itemsep, l + 1;
 		end
-		if next(t) ~= nil then
+		if had_items then
+			o[l - 1] = itemlast;
 			o[l], l = s_rep(indentwith, d-1), l + 1;
 		end
 		o[l], l = tend, l +1;
diff --git a/util/session.lua b/util/session.lua
index b2a726ce..25b22faf 100644
--- a/util/session.lua
+++ b/util/session.lua
@@ -4,12 +4,13 @@ local logger = require "util.logger";
 local function new_session(typ)
 	local session = {
 		type = typ .. "_unauthed";
+		base_type = typ;
 	};
 	return session;
 end
 
 local function set_id(session)
-	local id = session.type .. tostring(session):match("%x+$"):lower();
+	local id = session.base_type .. tostring(session):match("%x+$"):lower();
 	session.id = id;
 	return session;
 end
@@ -30,7 +31,7 @@ local function set_send(session)
 	local conn = session.conn;
 	if not conn then
 		function session.send(data)
-			session.log("debug", "Discarding data sent to unconnected session: %s", tostring(data));
+			session.log("debug", "Discarding data sent to unconnected session: %s", data);
 			return false;
 		end
 		return session;
@@ -46,7 +47,7 @@ local function set_send(session)
 			if t then
 				local ret, err = w(conn, t);
 				if not ret then
-					session.log("debug", "Error writing to connection: %s", tostring(err));
+					session.log("debug", "Error writing to connection: %s", err);
 					return false, err;
 				end
 			end
diff --git a/util/set.lua b/util/set.lua
index 02fabc6a..827a9158 100644
--- a/util/set.lua
+++ b/util/set.lua
@@ -8,6 +8,7 @@
 
 local ipairs, pairs, setmetatable, next, tostring =
       ipairs, pairs, setmetatable, next, tostring;
+local getmetatable = getmetatable;
 local t_concat = table.concat;
 
 local _ENV = nil;
@@ -146,6 +147,11 @@ function set_mt.__div(set, func)
 	return new_set;
 end
 function set_mt.__eq(set1, set2)
+	if getmetatable(set1) ~= set_mt or getmetatable(set2) ~= set_mt then
+		-- Lua 5.3+ calls this if both operands are tables, even if metatables differ
+		return false;
+	end
+
 	set1, set2 = set1._items, set2._items;
 	for item in pairs(set1) do
 		if not set2[item] then
diff --git a/util/sql.lua b/util/sql.lua
index 00c7b57f..9d1c86ca 100644
--- a/util/sql.lua
+++ b/util/sql.lua
@@ -201,31 +201,31 @@ function engine:_transaction(func, ...)
 		if not ok then return ok, err; end
 	end
 	--assert(not self.__transaction, "Recursive transactions not allowed");
-	log("debug", "SQL transaction begin [%s]", tostring(func));
+	log("debug", "SQL transaction begin [%s]", func);
 	self.__transaction = true;
 	local success, a, b, c = xpcall(func, handleerr, ...);
 	self.__transaction = nil;
 	if success then
-		log("debug", "SQL transaction success [%s]", tostring(func));
+		log("debug", "SQL transaction success [%s]", func);
 		local ok, err = self.conn:commit();
 		-- LuaDBI doesn't actually return an error message here, just a boolean
 		if not ok then return ok, err or "commit failed"; end
 		return success, a, b, c;
 	else
-		log("debug", "SQL transaction failure [%s]: %s", tostring(func), a.err);
+		log("debug", "SQL transaction failure [%s]: %s", func, a.err);
 		if self.conn then self.conn:rollback(); end
 		return success, a.err;
 	end
 end
 function engine:transaction(...)
-	local ok, ret = self:_transaction(...);
+	local ok, ret, b, c = self:_transaction(...);
 	if not ok then
 		local conn = self.conn;
 		if not conn or not conn:ping() then
 			log("debug", "Database connection was closed. Will reconnect and retry.");
 			self.conn = nil;
-			log("debug", "Retrying SQL transaction [%s]", tostring((...)));
-			ok, ret = self:_transaction(...);
+			log("debug", "Retrying SQL transaction [%s]", (...));
+			ok, ret, b, c = self:_transaction(...);
 			log("debug", "SQL transaction retry %s", ok and "succeeded" or "failed");
 		else
 			log("debug", "SQL connection is up, so not retrying");
@@ -234,7 +234,7 @@ function engine:transaction(...)
 			log("error", "Error in SQL transaction: %s", ret);
 		end
 	end
-	return ok, ret;
+	return ok, ret, b, c;
 end
 function engine:_create_index(index)
 	local sql = "CREATE INDEX \""..index.name.."\" ON \""..index.table.."\" (";
@@ -335,6 +335,9 @@ function engine:set_encoding() -- to UTF-8
 		local ok, actual_charset = self:transaction(function ()
 			return self:select"SHOW SESSION VARIABLES LIKE 'character_set_client'";
 		end);
+		if not ok then
+			return false, "Failed to detect connection encoding";
+		end
 		local charset_ok = true;
 		for row in actual_charset do
 			if row[2] ~= charset then
diff --git a/util/sslconfig.lua b/util/sslconfig.lua
index a5827a76..6074a1fb 100644
--- a/util/sslconfig.lua
+++ b/util/sslconfig.lua
@@ -67,6 +67,9 @@ end
 -- Curve list too
 finalisers.curveslist = finalisers.ciphers;
 
+-- TLS 1.3 ciphers
+finalisers.ciphersuites = finalisers.ciphers;
+
 -- protocol = "x" should enable only that protocol
 -- protocol = "x+" should enable x and later versions
 
diff --git a/util/stanza.lua b/util/stanza.lua
index a90d56b3..94815346 100644
--- a/util/stanza.lua
+++ b/util/stanza.lua
@@ -98,7 +98,7 @@ function stanza_mt:query(xmlns)
 end
 
 function stanza_mt:body(text, attr)
-	return self:tag("body", attr):text(text);
+	return self:text_tag("body", text, attr);
 end
 
 function stanza_mt:text_tag(name, text, attr, namespaces)
@@ -270,6 +270,34 @@ function stanza_mt:find(path)
 	until not self
 end
 
+local function _clone(stanza, only_top)
+	local attr, tags = {}, {};
+	for k,v in pairs(stanza.attr) do attr[k] = v; end
+	local old_namespaces, namespaces = stanza.namespaces;
+	if old_namespaces then
+		namespaces = {};
+		for k,v in pairs(old_namespaces) do namespaces[k] = v; end
+	end
+	local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags };
+	if not only_top then
+		for i=1,#stanza do
+			local child = stanza[i];
+			if child.name then
+				child = _clone(child);
+				t_insert(tags, child);
+			end
+			t_insert(new, child);
+		end
+	end
+	return setmetatable(new, stanza_mt);
+end
+
+local function clone(stanza, only_top)
+	if not is_stanza(stanza) then
+		error("bad argument to clone: expected stanza, got "..type(stanza));
+	end
+	return _clone(stanza, only_top);
+end
 
 local escape_table = { ["'"] = "&apos;", ["\""] = "&quot;", ["<"] = "&lt;", [">"] = "&gt;", ["&"] = "&amp;" };
 local function xml_escape(str) return (s_gsub(str, "['&<>\"]", escape_table)); end
@@ -310,11 +338,8 @@ function stanza_mt.__tostring(t)
 end
 
 function stanza_mt.top_tag(t)
-	local attr_string = "";
-	if t.attr then
-		for k, v in pairs(t.attr) do if type(k) == "string" then attr_string = attr_string .. s_format(" %s='%s'", k, xml_escape(tostring(v))); end end
-	end
-	return s_format("<%s%s>", t.name, attr_string);
+	local top_tag_clone = clone(t, true);
+	return tostring(top_tag_clone):sub(1,-3)..">";
 end
 
 function stanza_mt.get_text(t)
@@ -324,11 +349,11 @@ function stanza_mt.get_text(t)
 end
 
 function stanza_mt.get_error(stanza)
-	local error_type, condition, text;
+	local error_type, condition, text, extra_tag;
 
 	local error_tag = stanza:get_child("error");
 	if not error_tag then
-		return nil, nil, nil;
+		return nil, nil, nil, nil;
 	end
 	error_type = error_tag.attr.type;
 
@@ -339,12 +364,14 @@ function stanza_mt.get_error(stanza)
 			elseif not condition then
 				condition = child.name;
 			end
-			if condition and text then
-				break;
-			end
+		else
+			extra_tag = child;
+		end
+		if condition and text and extra_tag then
+			break;
 		end
 	end
-	return error_type, condition or "undefined-condition", text;
+	return error_type, condition or "undefined-condition", text, extra_tag;
 end
 
 local function preserialize(stanza)
@@ -388,50 +415,32 @@ local function deserialize(serialized)
 	end
 end
 
-local function _clone(stanza)
-	local attr, tags = {}, {};
-	for k,v in pairs(stanza.attr) do attr[k] = v; end
-	local old_namespaces, namespaces = stanza.namespaces;
-	if old_namespaces then
-		namespaces = {};
-		for k,v in pairs(old_namespaces) do namespaces[k] = v; end
-	end
-	local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags };
-	for i=1,#stanza do
-		local child = stanza[i];
-		if child.name then
-			child = _clone(child);
-			t_insert(tags, child);
-		end
-		t_insert(new, child);
-	end
-	return setmetatable(new, stanza_mt);
-end
-
-local function clone(stanza)
-	if not is_stanza(stanza) then
-		error("bad argument to clone: expected stanza, got "..type(stanza));
-	end
-	return _clone(stanza);
-end
-
 local function message(attr, body)
 	if not body then
 		return new_stanza("message", attr);
 	else
-		return new_stanza("message", attr):tag("body"):text(body):up();
+		return new_stanza("message", attr):text_tag("body", body);
 	end
 end
 local function iq(attr)
-	if not (attr and attr.id) then
+	if not attr then
+		error("iq stanzas require id and type attributes");
+	end
+	if not attr.id then
 		error("iq stanzas require an id attribute");
 	end
+	if not attr.type then
+		error("iq stanzas require a type attribute");
+	end
 	return new_stanza("iq", attr);
 end
 
 local function reply(orig)
+	if not is_stanza(orig) then
+		error("bad argument to reply: expected stanza, got "..type(orig));
+	end
 	return new_stanza(orig.name,
-		orig.attr and {
+		{
 			to = orig.attr.from,
 			from = orig.attr.to,
 			id = orig.attr.id,
@@ -440,12 +449,37 @@ local function reply(orig)
 end
 
 local xmpp_stanzas_attr = { xmlns = xmlns_stanzas };
-local function error_reply(orig, error_type, condition, error_message)
+local function error_reply(orig, error_type, condition, error_message, error_by)
+	if not is_stanza(orig) then
+		error("bad argument to error_reply: expected stanza, got "..type(orig));
+	elseif orig.attr.type == "error" then
+		error("bad argument to error_reply: got stanza of type error which must not be replied to");
+	end
 	local t = reply(orig);
 	t.attr.type = "error";
-	t:tag("error", {type = error_type}) --COMPAT: Some day xmlns:stanzas goes here
-	:tag(condition, xmpp_stanzas_attr):up();
-	if error_message then t:tag("text", xmpp_stanzas_attr):text(error_message):up(); end
+	local extra;
+	if type(error_type) == "table" then -- an util.error or similar object
+		if type(error_type.extra) == "table" then
+			extra = error_type.extra;
+		end
+		if type(error_type.context) == "table" and type(error_type.context.by) == "string" then error_by = error_type.context.by; end
+		error_type, condition, error_message = error_type.type, error_type.condition, error_type.text;
+	end
+	if t.attr.from == error_by then
+		error_by = nil;
+	end
+	t:tag("error", {type = error_type, by = error_by}) --COMPAT: Some day xmlns:stanzas goes here
+	:tag(condition, xmpp_stanzas_attr);
+	if extra and condition == "gone" and type(extra.uri) == "string" then
+		t:text(extra.uri);
+	end
+	t:up();
+	if error_message then t:text_tag("text", error_message, xmpp_stanzas_attr); end
+	if extra and is_stanza(extra.tag) then
+		t:add_child(extra.tag);
+	elseif extra and extra.namespace and extra.condition then
+		t:tag(extra.condition, { xmlns = extra.namespace }):up();
+	end
 	return t; -- stanza ready for adding app-specific errors
 end
 
@@ -493,6 +527,36 @@ else
 	stanza_mt.pretty_top_tag = stanza_mt.top_tag;
 end
 
+function stanza_mt.indent(t, level, indent)
+	if #t == 0 or (#t == 1 and type(t[1]) == "string") then
+		-- Empty nodes wouldn't have any indentation
+		-- Text-only nodes are preserved as to not alter the text content
+		-- Optimization: Skip clone of these since we don't alter them
+		return t;
+	end
+
+	indent = indent or "\t";
+	level = level or 1;
+	local tag = clone(t, true);
+
+	for child in t:children() do
+		if type(child) == "string" then
+			-- Already indented text would look weird but let's ignore that for now.
+			if child:find("%S") then
+				tag:text("\n" .. indent:rep(level));
+				tag:text(child);
+			end
+		elseif is_stanza(child) then
+			tag:text("\n" .. indent:rep(level));
+			tag:add_direct_child(child:indent(level+1, indent));
+		end
+	end
+	-- before the closing tag
+	tag:text("\n" .. indent:rep((level-1)));
+
+	return tag;
+end
+
 return {
 	stanza_mt = stanza_mt;
 	stanza = new_stanza;
diff --git a/util/startup.lua b/util/startup.lua
index c1f4ec8b..27420c57 100644
--- a/util/startup.lua
+++ b/util/startup.lua
@@ -5,8 +5,10 @@ local startup = {};
 local prosody = { events = require "util.events".new() };
 local logger = require "util.logger";
 local log = logger.init("startup");
+local parse_args = require "util.argparse".parse;
 
 local config = require "core.configmanager";
+local config_warnings;
 
 local dependencies = require "util.dependencies";
 
@@ -18,55 +20,20 @@ local short_params = { D = "daemonize", F = "no-daemonize" };
 local value_params = { config = true };
 
 function startup.parse_args()
-	local parsed_opts = {};
-	prosody.opts = parsed_opts;
-
-	if #arg == 0 then
-		return;
-	end
-	while true do
-		local raw_param = arg[1];
-		if not raw_param then
-			break;
-		end
-
-		local prefix = raw_param:match("^%-%-?");
-		if not prefix then
-			break;
-		elseif prefix == "--" and raw_param == "--" then
-			table.remove(arg, 1);
-			break;
-		end
-		local param = table.remove(arg, 1):sub(#prefix+1);
-		if #param == 1 then
-			param = short_params[param];
-		end
-
-		if not param then
-			print("Unknown command-line option: "..tostring(param));
+	local opts, err, where = parse_args(arg, {
+			short_params = short_params,
+			value_params = value_params,
+		});
+	if not opts then
+		if err == "param-not-found" then
+			print("Unknown command-line option: "..tostring(where));
 			print("Perhaps you meant to use prosodyctl instead?");
-			os.exit(1);
-		end
-
-		local param_k, param_v;
-		if value_params[param] then
-			param_k, param_v = param, table.remove(arg, 1);
-			if not param_v then
-				print("Expected a value to follow command-line option: "..raw_param);
-				os.exit(1);
-			end
-		else
-			param_k, param_v = param:match("^([^=]+)=(.+)$");
-			if not param_k then
-				if param:match("^no%-") then
-					param_k, param_v = param:sub(4), false;
-				else
-					param_k, param_v = param, true;
-				end
-			end
+		elseif err == "missing-value" then
+			print("Expected a value to follow command-line option: "..where);
 		end
-		parsed_opts[param_k] = param_v;
+		os.exit(1);
 	end
+	prosody.opts = opts;
 end
 
 function startup.read_config()
@@ -121,6 +88,8 @@ function startup.read_config()
 		print("**************************");
 		print("");
 		os.exit(1);
+	elseif err and #err > 0 then
+		config_warnings = err;
 	end
 	prosody.config_loaded = true;
 end
@@ -153,8 +122,13 @@ function startup.init_logging()
 	end);
 end
 
-function startup.log_dependency_warnings()
+function startup.log_startup_warnings()
 	dependencies.log_warnings();
+	if config_warnings then
+		for _, warning in ipairs(config_warnings) do
+			log("warn", "Configuration warning: %s", warning);
+		end
+	end
 end
 
 function startup.sanity_check()
@@ -223,8 +197,15 @@ function startup.set_function_metatable()
 		end
 	end
 	function mt.__tostring(f)
-		local info = debug.getinfo(f);
-		return ("function(%s:%d)"):format(info.short_src:match("[^\\/]*$"), info.linedefined);
+		local info = debug.getinfo(f, "Su");
+		local n_params = info.nparams or 0;
+		for i = 1, n_params do
+			info[i] = debug.getlocal(f, i);
+		end
+		if info.isvararg then
+			info[n_params+1] = "...";
+		end
+		return ("function<%s:%d>(%s)"):format(info.short_src:match("[^\\/]*$"), info.linedefined, table.concat(info, ", "));
 	end
 	debug.setmetatable(function() end, mt);
 end
@@ -276,8 +257,8 @@ end
 
 function startup.setup_plugindir()
 	local custom_plugin_paths = config.get("*", "plugin_paths");
+	local path_sep = package.config:sub(3,3);
 	if custom_plugin_paths then
-		local path_sep = package.config:sub(3,3);
 		-- path1;path2;path3;defaultpath...
 		-- luacheck: ignore 111
 		CFG_PLUGINDIR = table.concat(custom_plugin_paths, path_sep)..path_sep..(CFG_PLUGINDIR or "plugins");
@@ -285,6 +266,20 @@ function startup.setup_plugindir()
 	end
 end
 
+function startup.setup_plugin_install_path()
+	local installer_plugin_path = config.get("*", "installer_plugin_path") or (CFG_DATADIR or "data").."/custom_plugins";
+	local path_sep = package.config:sub(3,3);
+	-- TODO Figure out what this should be relative to, because CWD could be anywhere
+	installer_plugin_path = config.resolve_relative_path(require "lfs".currentdir(), installer_plugin_path);
+	-- TODO Can probably move directory creation to the install command
+	require "lfs".mkdir(installer_plugin_path);
+	require"util.paths".complement_lua_path(installer_plugin_path);
+	-- luacheck: ignore 111
+	CFG_PLUGINDIR = installer_plugin_path..path_sep..(CFG_PLUGINDIR or "plugins");
+	prosody.paths.installer = installer_plugin_path;
+	prosody.paths.plugins = CFG_PLUGINDIR;
+end
+
 function startup.chdir()
 	if prosody.installed then
 		local lfs = require "lfs";
@@ -306,9 +301,9 @@ function startup.add_global_prosody_functions()
 		local ok, level, err = config.load(prosody.config_file);
 		if not ok then
 			if level == "parser" then
-				log("error", "There was an error parsing the configuration file: %s", tostring(err));
+				log("error", "There was an error parsing the configuration file: %s", err);
 			elseif level == "file" then
-				log("error", "Couldn't read the config file when trying to reload: %s", tostring(err));
+				log("error", "Couldn't read the config file when trying to reload: %s", err);
 			end
 		else
 			prosody.events.fire_event("config-reloaded", {
@@ -482,7 +477,7 @@ function startup.switch_user()
 				print("Warning: Couldn't switch to Prosody user/group '"..tostring(desired_user).."'/'"..tostring(desired_group).."': "..tostring(err));
 			else
 				-- Make sure the Prosody user can read the config
-				local conf, err, errno = io.open(prosody.config_file);
+				local conf, err, errno = io.open(prosody.config_file); --luacheck: ignore 211/errno
 				if conf then
 					conf:close();
 				else
@@ -559,6 +554,10 @@ function startup.init_gc()
 	return true;
 end
 
+function startup.init_errors()
+	require "util.error".configure(config.get("*", "error_library") or {});
+end
+
 function startup.make_host(hostname)
 	return {
 		type = "local",
@@ -583,19 +582,22 @@ end
 
 -- prosodyctl only
 function startup.prosodyctl()
+	prosody.process_type = "prosodyctl";
 	startup.parse_args();
 	startup.init_global_state();
 	startup.read_config();
 	startup.force_console_logging();
 	startup.init_logging();
 	startup.init_gc();
+	startup.init_errors();
 	startup.setup_plugindir();
+	startup.setup_plugin_install_path();
 	startup.setup_datadir();
 	startup.chdir();
 	startup.read_version();
 	startup.switch_user();
 	startup.check_dependencies();
-	startup.log_dependency_warnings();
+	startup.log_startup_warnings();
 	startup.check_unwriteable();
 	startup.load_libraries();
 	startup.init_http_client();
@@ -605,23 +607,26 @@ end
 function startup.prosody()
 	-- These actions are in a strict order, as many depend on
 	-- previous steps to have already been performed
+	prosody.process_type = "prosody";
 	startup.parse_args();
 	startup.init_global_state();
 	startup.read_config();
 	startup.init_logging();
 	startup.init_gc();
+	startup.init_errors();
 	startup.sanity_check();
 	startup.sandbox_require();
 	startup.set_function_metatable();
 	startup.check_dependencies();
 	startup.load_libraries();
 	startup.setup_plugindir();
+	startup.setup_plugin_install_path();
 	startup.setup_datadir();
 	startup.chdir();
 	startup.add_global_prosody_functions();
 	startup.read_version();
 	startup.log_greeting();
-	startup.log_dependency_warnings();
+	startup.log_startup_warnings();
 	startup.load_secondary_libraries();
 	startup.init_http_client();
 	startup.init_data_store();
diff --git a/util/statistics.lua b/util/statistics.lua
index 39954652..db608217 100644
--- a/util/statistics.lua
+++ b/util/statistics.lua
@@ -44,25 +44,33 @@ local function new_registry(config)
 	local registry = {};
 	local methods;
 	methods = {
-		amount = function (name, initial)
-			local v = initial or 0;
-			registry[name..":amount"] = function () return "amount", v; end
+		amount = function (name, conf)
+			local v = conf and conf.initial or 0;
+			registry[name..":amount"] = function ()
+				return "amount", v, conf;
+			end
 			return function (new_v) v = new_v; end
 		end;
-		counter = function (name, initial)
-			local v = initial or 0;
-			registry[name..":amount"] = function () return "amount", v; end
+		counter = function (name, conf)
+			local v = conf and conf.initial or 0;
+			registry[name..":amount"] = function ()
+				return "amount", v, conf;
+			end
 			return function (delta)
 				v = v + delta;
 			end;
 		end;
-		rate = function (name)
-			local since, n = time(), 0;
+		rate = function (name, conf)
+			local since, n, total = time(), 0, 0;
 			registry[name..":rate"] = function ()
+				total = total + n;
 				local t = time();
 				local stats = {
 					rate = n/(t-since);
 					count = n;
+					total = total;
+					units = conf and conf.units;
+					type = conf and conf.type;
 				};
 				since, n = t, 0;
 				return "rate", stats.rate, stats;
@@ -71,15 +79,16 @@ local function new_registry(config)
 				n = n + 1;
 			end;
 		end;
-		distribution = function (name, unit, type)
-			type = type or "distribution";
+		distribution = function (name, conf)
+			local units = conf and conf.units;
+			local type = conf and conf.type or "distribution";
 			local events, last_event = {}, 0;
 			local n_actual_events = 0;
 			local since = time();
 
 			registry[name..":"..type] = function ()
 				local new_time = time();
-				local stats = get_distribution_stats(events, n_actual_events, since, new_time, unit);
+				local stats = get_distribution_stats(events, n_actual_events, since, new_time, units);
 				events, last_event = {}, 0;
 				n_actual_events = 0;
 				since = new_time;
@@ -94,17 +103,19 @@ local function new_registry(config)
 				end
 			end;
 		end;
-		sizes = function (name)
-			return methods.distribution(name, "bytes", "size");
+		sizes = function (name, conf)
+			conf = conf or { units = "bytes", type = "size" }
+			return methods.distribution(name, conf);
 		end;
-		times = function (name)
+		times = function (name, conf)
+			local units = conf and conf.units or "seconds";
 			local events, last_event = {}, 0;
 			local n_actual_events = 0;
 			local since = time();
 
 			registry[name..":duration"] = function ()
 				local new_time = time();
-				local stats = get_distribution_stats(events, n_actual_events, since, new_time, "seconds");
+				local stats = get_distribution_stats(events, n_actual_events, since, new_time, units);
 				events, last_event = {}, 0;
 				n_actual_events = 0;
 				since = new_time;
diff --git a/util/statsd.lua b/util/statsd.lua
index 67481c36..8f6151c6 100644
--- a/util/statsd.lua
+++ b/util/statsd.lua
@@ -38,13 +38,13 @@ local function new(config)
 
 	local methods;
 	methods = {
-		amount = function (name, initial)
-			if initial then
-				send_gauge(name, initial);
+		amount = function (name, conf)
+			if conf and conf.initial then
+				send_gauge(name, conf.initial);
 			end
 			return function (new_v) send_gauge(name, new_v); end
 		end;
-		counter = function (name, initial) --luacheck: ignore 212/initial
+		counter = function (name, conf) --luacheck: ignore 212/conf
 			return function (delta)
 				send_gauge(name, delta, true);
 			end;
@@ -54,7 +54,7 @@ local function new(config)
 				send_counter(name, 1);
 			end;
 		end;
-		distribution = function (name, unit, type) --luacheck: ignore 212/unit 212/type
+		distribution = function (name, conf) --luacheck: ignore 212/conf
 			return function (value)
 				send_histogram_sample(name, value);
 			end;
diff --git a/util/termcolours.lua b/util/termcolours.lua
index 829d84af..2c13d0ff 100644
--- a/util/termcolours.lua
+++ b/util/termcolours.lua
@@ -83,7 +83,7 @@ end
 setmetatable(stylemap, { __index = function(_, style)
 	if type(style) == "string" and style:find("%x%x%x%x%x%x") == 1 then
 		local g = style:sub(7) == " background" and "48;5;" or "38;5;";
-		return g .. color(hex2rgb(style));
+		return format("%s%d", g, color(hex2rgb(style)));
 	end
 end } );
 
diff --git a/util/timer.lua b/util/timer.lua
index 4670e196..cbfc1992 100644
--- a/util/timer.lua
+++ b/util/timer.lua
@@ -16,6 +16,11 @@ local tostring = tostring;
 local xpcall = require "util.xpcall".xpcall;
 local math_max = math.max;
 
+if server.timer then
+	-- The selected net.server implements this API, so defer to that
+	return server.timer;
+end
+
 local _ENV = nil;
 -- luacheck: std none
 
diff --git a/util/x509.lua b/util/x509.lua
index 15cc4d3c..342dafde 100644
--- a/util/x509.lua
+++ b/util/x509.lua
@@ -20,9 +20,12 @@
 
 local nameprep = require "util.encodings".stringprep.nameprep;
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
+local idna_to_unicode = require "util.encodings".idna.to_unicode;
 local base64 = require "util.encodings".base64;
 local log = require "util.logger".init("x509");
+local mt = require "util.multitable";
 local s_format = string.format;
+local ipairs = ipairs;
 
 local _ENV = nil;
 -- luacheck: std none
@@ -216,6 +219,60 @@ local function verify_identity(host, service, cert)
 	return false
 end
 
+-- TODO Support other SANs
+local function get_identities(cert) --> map of names to sets of services
+	if cert.setencode then
+		cert:setencode("utf8");
+	end
+
+	local names = mt.new();
+
+	local ext = cert:extensions();
+	local sans = ext[oid_subjectaltname];
+	if sans then
+		if sans["dNSName"] then -- Valid for any service
+			for _, name in ipairs(sans["dNSName"]) do
+				name = idna_to_unicode(nameprep(name));
+				if name then
+					names:set(name, "*", true);
+				end
+			end
+		end
+		if sans[oid_xmppaddr] then
+			for _, name in ipairs(sans[oid_xmppaddr]) do
+				name = nameprep(name);
+				if name then
+					names:set(name, "xmpp-client", true);
+					names:set(name, "xmpp-server", true);
+				end
+			end
+		end
+		if sans[oid_dnssrv] then
+			for _, srvname in ipairs(sans[oid_dnssrv]) do
+				local srv, name = srvname:match("^_([^.]+)%.(.*)");
+				if srv then
+					name = nameprep(name);
+					if name then
+						names:set(name, srv, true);
+					end
+				end
+			end
+		end
+	end
+
+	local subject = cert:subject();
+	for i = 1, #subject do
+		local dn = subject[i];
+		if dn.oid == oid_commonname then
+			local name = nameprep(dn.value);
+			if name and idna_to_ascii(name) then
+				names:set(name, "*", true);
+			end
+		end
+	end
+	return names.data;
+end
+
 local pat = "%-%-%-%-%-BEGIN ([A-Z ]+)%-%-%-%-%-\r?\n"..
 "([0-9A-Za-z+/=\r\n]*)\r?\n%-%-%-%-%-END %1%-%-%-%-%-";
 
@@ -237,6 +294,7 @@ end
 
 return {
 	verify_identity = verify_identity;
+	get_identities = get_identities;
 	pem2der = pem2der;
 	der2pem = der2pem;
 };
diff --git a/util/xml.lua b/util/xml.lua
index dac3f6fe..f826b6bf 100644
--- a/util/xml.lua
+++ b/util/xml.lua
@@ -71,7 +71,7 @@ local parse_xml = (function()
 		if ok then
 			return stanza.tags[1];
 		else
-			return ok, err.." (line "..line..", col "..col..")";
+			return ok, ("%s (line %d, col %d))"):format(err, line, col);
 		end
 	end;
 end)();
diff --git a/util/xmppstream.lua b/util/xmppstream.lua
index 58cbd18e..6aa1def3 100644
--- a/util/xmppstream.lua
+++ b/util/xmppstream.lua
@@ -64,6 +64,8 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 
 	local stream_default_ns = stream_callbacks.default_ns;
 
+	local stream_lang = "en";
+
 	local stack = {};
 	local chardata, stanza = {};
 	local stanza_size = 0;
@@ -101,6 +103,7 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 			if session.notopen then
 				if tagname == stream_tag then
 					non_streamns_depth = 0;
+					stream_lang = attr["xml:lang"] or stream_lang;
 					if cb_streamopened then
 						if lxp_supports_bytecount then
 							cb_handleprogress(stanza_size);
@@ -178,6 +181,9 @@ local function new_sax_handlers(session, stream_callbacks, cb_handleprogress)
 					cb_handleprogress(stanza_size);
 				end
 				stanza_size = 0;
+				if stanza.attr["xml:lang"] == nil then
+					stanza.attr["xml:lang"] = stream_lang;
+				end
 				if tagname ~= stream_error_tag then
 					cb_handlestanza(session, stanza);
 				else