diff options
Diffstat (limited to 'tools')
-rw-r--r-- | tools/build-env/Containerfile | 31 | ||||
-rwxr-xr-x | tools/build-env/build.sh | 16 | ||||
-rwxr-xr-x | tools/build-env/here.sh | 19 | ||||
-rwxr-xr-x | tools/cfgdump.lua | 11 | ||||
-rw-r--r-- | tools/dnsregistry.lua | 7 | ||||
-rw-r--r-- | tools/ejabberdsql2prosody.lua | 13 | ||||
-rw-r--r-- | tools/form2table.lua | 6 | ||||
-rw-r--r-- | tools/http-status-codes.lua | 5 | ||||
-rw-r--r-- | tools/jabberd14sql2prosody.lua | 11 | ||||
-rw-r--r-- | tools/migration/migrator.cfg.lua | 10 | ||||
-rw-r--r-- | tools/migration/migrator/jabberd14.lua | 7 | ||||
-rw-r--r-- | tools/migration/prosody-migrator.lua | 27 | ||||
-rwxr-xr-x | tools/mod2spec.sh | 4 | ||||
-rw-r--r-- | tools/modtrace.lua | 4 | ||||
-rw-r--r-- | tools/openfire2prosody.lua | 8 | ||||
-rwxr-xr-x | tools/tb2err | 9 | ||||
-rwxr-xr-x | tools/test_mutants.sh.lua | 217 | ||||
-rwxr-xr-x | tools/xepchanges.sh | 14 |
18 files changed, 384 insertions, 35 deletions
diff --git a/tools/build-env/Containerfile b/tools/build-env/Containerfile new file mode 100644 index 00000000..6ba02ba0 --- /dev/null +++ b/tools/build-env/Containerfile @@ -0,0 +1,31 @@ +ARG os +ARG dist +FROM ${os:-debian}:${dist:-sid} +ENV DEBIAN_FRONTEND noninteractive +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ccache dh-lua libicu-dev libidn11-dev libssl-dev \ + lua-bitop lua-dbi-mysql lua-dbi-postgresql lua-dbi-sqlite3 \ + lua-event lua-expat lua-filesystem lua-ldap lua-sec lua-socket \ + luarocks shellcheck mercurial; \ + apt-get install -y ca-certificates dns-root-data; \ + apt-get install -y lua-bit32 || true; \ + apt-get install -y lua-busted || true; \ + apt-get install -y lua-check || true; \ + apt-get install -y lua-readline || true; \ + apt-get install -y lua-unbound || true; \ + update-alternatives --set lua-interpreter /usr/bin/lua5.4 || true \ + apt-get clean + +# Place this file in an empty directory and build the image with +# podman build . -t prosody.im/build-env +# +# Substituting podman for docker should work, where that is what's available. +# +# Then in a source directory, run: +# podman run -it --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint /bin/bash \ +# --userns=keep-id --network host prosody.im/build-env +# +# In the resulting environment everything required to compile and run prosody +# is available, so e.g. `./configure; make; ./prosody` should Just Work! diff --git a/tools/build-env/build.sh b/tools/build-env/build.sh new file mode 100755 index 00000000..4351f8b8 --- /dev/null +++ b/tools/build-env/build.sh @@ -0,0 +1,16 @@ +#!/bin/sh -eux + +cd "$(dirname "$0")" + +containerify="$(command -v podman || command -v docker)" + +if [ -z "$containerify" ]; then + echo "podman or docker required" >&2 + exit 1 +fi + +$containerify build -f ./Containerfile --squash \ + --build-arg os="${2:-debian}" \ + --build-arg dist="${1:-testing}" \ + -t "prosody.im/build-env:${1:-testing}" + diff --git a/tools/build-env/here.sh b/tools/build-env/here.sh new file mode 100755 index 00000000..1d5cb515 --- /dev/null +++ b/tools/build-env/here.sh @@ -0,0 +1,19 @@ +#!/bin/sh -eux + +tag="testing" + +if [ "$#" -gt 0 ]; then + tag="$1" + shift +fi + +containerify="$(command -v podman docker)" + +$containerify run -it --rm \ + -v "$PWD:$PWD" \ + -w "$PWD" \ + -v "$HOME/.cache:$PWD/.cache" \ + --entrypoint /bin/bash \ + --userns=keep-id \ + --network \ + host "prosody.im/build-env:$tag" "$@" diff --git a/tools/cfgdump.lua b/tools/cfgdump.lua index 8c331a58..eef13e6d 100755 --- a/tools/cfgdump.lua +++ b/tools/cfgdump.lua @@ -1,21 +1,24 @@ #!/usr/bin/env lua -- cfgdump.lua prosody.cfg.lua [[host] option] +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end local s_format, print = string.format, print; local printf = function(fmt, ...) return print(s_format(fmt, ...)); end -local it = require "util.iterators"; +local it = require "prosody.util.iterators"; local function sort_anything(a, b) local typeof_a, typeof_b = type(a), type(b); if typeof_a ~= typeof_b then return typeof_a < typeof_b end return a < b -- should work for everything in a config file end -local serialization = require "util.serialization"; +local serialization = require "prosody.util.serialization"; local serialize = serialization.new and serialization.new({ unquoted = true, table_iterator = function(t) return it.sorted_pairs(t, sort_anything); end, }) or serialization.serialize; -local configmanager = require"core.configmanager"; -local startup = require "util.startup"; +local configmanager = require"prosody.core.configmanager"; +local startup = require "prosody.util.startup"; startup.set_function_metatable(); local config_filename, onlyhost, onlyoption = ...; diff --git a/tools/dnsregistry.lua b/tools/dnsregistry.lua index 3fd26628..5a258c1a 100644 --- a/tools/dnsregistry.lua +++ b/tools/dnsregistry.lua @@ -1,5 +1,8 @@ -- Generate util/dnsregistry.lua from IANA HTTP status code registry -local xml = require "util.xml"; +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end +local xml = require "prosody.util.xml"; local registries = xml.parse(io.read("*a"), { allow_processing_instructions = true }); print("-- Source: https://www.iana.org/assignments/dns-parameters/dns-parameters.xml"); @@ -22,7 +25,7 @@ for registry in registries:childtags("registry") do local record_desc = record:get_child_text("description"); local record_code = tonumber(record:get_child_text("value")); - if tostring(record):lower():match("reserved") or tostring(record):lower():match("reserved") then + if tostring(record):lower():match("reserved") or tostring(record):lower():match("unassigned") then record_code = nil; end diff --git a/tools/ejabberdsql2prosody.lua b/tools/ejabberdsql2prosody.lua index d0ab71cf..272da0aa 100644 --- a/tools/ejabberdsql2prosody.lua +++ b/tools/ejabberdsql2prosody.lua @@ -16,13 +16,16 @@ if my_name:match("[/\\]") then package.path = package.path..";"..my_name:gsub("[^/\\]+$", "../?.lua"); package.cpath = package.cpath..";"..my_name:gsub("[^/\\]+$", "../?.so"); end +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end -local serialize = require "util.serialization".serialize; -local st = require "util.stanza"; -local parse_xml = require "util.xml".parse; -package.loaded["util.logger"] = {init = function() return function() end; end} -local dm = require "util.datamanager" +local serialize = require "prosody.util.serialization".serialize; +local st = require "prosody.util.stanza"; +local parse_xml = require "prosody.util.xml".parse; +package.loaded["prosody.util.logger"] = {init = function() return function() end; end} +local dm = require "prosody.util.datamanager" dm.set_data_path("data"); function parseFile(filename) diff --git a/tools/form2table.lua b/tools/form2table.lua index 49a6972b..2d80ad23 100644 --- a/tools/form2table.lua +++ b/tools/form2table.lua @@ -1,4 +1,7 @@ -- Read an XML dataform and spit out a serialized Lua table of it +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end local function from_stanza(stanza) local layout = { @@ -45,4 +48,5 @@ local function from_stanza(stanza) return layout; end -print("dataforms.new " .. require "util.serialization".serialize(from_stanza(require "util.xml".parse(io.read("*a"))), { unquoted = true })) +print("dataforms.new " .. require"prosody.util.serialization".serialize(from_stanza(require"prosody.util.xml".parse(io.read("*a"))), + { unquoted = true })) diff --git a/tools/http-status-codes.lua b/tools/http-status-codes.lua index bd5bb52a..38f44992 100644 --- a/tools/http-status-codes.lua +++ b/tools/http-status-codes.lua @@ -1,6 +1,9 @@ -- Generate net/http/codes.lua from IANA HTTP status code registry +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end -local xml = require "util.xml"; +local xml = require "prosody.util.xml"; local registry = xml.parse(io.read("*a"), { allow_processing_instructions = true }); io.write([[ diff --git a/tools/jabberd14sql2prosody.lua b/tools/jabberd14sql2prosody.lua index d1d66dc1..12658e50 100644 --- a/tools/jabberd14sql2prosody.lua +++ b/tools/jabberd14sql2prosody.lua @@ -1,4 +1,7 @@ #!/usr/bin/env lua +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end do @@ -447,13 +450,13 @@ elseif package.config:sub(1,1) == "/" then end package.loaded["util.logger"] = {init = function() return function() end; end} -local dm = require "util.datamanager"; +local dm = require "prosody.util.datamanager"; dm.set_data_path("data"); -local datetime = require "util.datetime"; +local datetime = require "prosody.util.datetime"; -local st = require "util.stanza"; -local parse_xml = require "util.xml".parse; +local st = require "prosody.util.stanza"; +local parse_xml = require "prosody.util.xml".parse; function store_password(username, host, password) -- create or update account for username@host diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua index b81389b3..6e29b9b8 100644 --- a/tools/migration/migrator.cfg.lua +++ b/tools/migration/migrator.cfg.lua @@ -3,6 +3,7 @@ local data_path = "../../data"; local vhost = { "accounts", "account_details", + "account_roles", "roster", "vcard", "private", @@ -12,18 +13,27 @@ local vhost = { "offline-archive", "pubsub_nodes-pubsub", "pep-pubsub", + "cron", + "smacks_h", } local muc = { "persistent", "config", "state", "muc_log-archive", + "cron", }; +local upload = { + "uploads-archive", + "upload_stats", + "cron", +} input { hosts = { ["example.com"] = vhost; ["conference.example.com"] = muc; + ["share.example.com"] = upload; }; type = "internal"; path = data_path; diff --git a/tools/migration/migrator/jabberd14.lua b/tools/migration/migrator/jabberd14.lua index a4eef3f7..7edb887c 100644 --- a/tools/migration/migrator/jabberd14.lua +++ b/tools/migration/migrator/jabberd14.lua @@ -1,7 +1,10 @@ +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end local lfs = require "lfs"; -local st = require "util.stanza"; -local parse_xml = require "util.xml".parse; +local st = require "prosody.util.stanza"; +local parse_xml = require "prosody.util.xml".parse; local os_getenv = os.getenv; local io_open = io.open; local assert = assert; diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 21eb32e7..467ecf32 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -43,7 +43,11 @@ local function usage() print("If no stores are specified, 'input' and 'output' are used."); end -local startup = require "util.startup"; +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end + +local startup = require "prosody.util.startup"; do startup.parse_args({ short_params = { v = "verbose", h = "help", ["?"] = "help" }; @@ -79,7 +83,7 @@ end -- Command-line parsing local options = prosody.opts; -local envloadfile = require "util.envload".envloadfile; +local envloadfile = require "prosody.util.envload".envloadfile; local config_file = options.config or default_config; local from_store = arg[1] or "input"; @@ -132,8 +136,8 @@ if have_err then os.exit(1); end -local async = require "util.async"; -local server = require "net.server"; +local async = require "prosody.util.async"; +local server = require "prosody.net.server"; local watchers = { error = function (_, err) error(err); @@ -143,10 +147,10 @@ local watchers = { end; }; -local cm = require "core.configmanager"; -local hm = require "core.hostmanager"; -local sm = require "core.storagemanager"; -local um = require "core.usermanager"; +local cm = require "prosody.core.configmanager"; +local hm = require "prosody.core.hostmanager"; +local sm = require "prosody.core.storagemanager"; +local um = require "prosody.core.usermanager"; local function users(store, host) if store.users then @@ -164,6 +168,11 @@ local function prepare_config(host, conf) elseif conf.type == "sql" then cm.set(host, "sql", conf); end + if type(conf.config) == "table" then + for option, value in pairs(conf.config) do + cm.set(host, option, value); + end + end end local function get_driver(host, conf) @@ -200,7 +209,7 @@ migrate_once.pubsub = function(origin, destination, user, prefix, input_driver, end if options["keep-going"] then - local xpcall = require "util.xpcall".xpcall; + local xpcall = require "prosody.util.xpcall".xpcall; for t, f in pairs(migrate_once) do migrate_once[t] = function (origin, destination, user, ...) local function log_err(err) diff --git a/tools/mod2spec.sh b/tools/mod2spec.sh new file mode 100755 index 00000000..98c0805a --- /dev/null +++ b/tools/mod2spec.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -eu + +echo "spec/${1//./_}_spec.lua" diff --git a/tools/modtrace.lua b/tools/modtrace.lua index 45fa9f6a..f1927077 100644 --- a/tools/modtrace.lua +++ b/tools/modtrace.lua @@ -8,9 +8,9 @@ -- local dbuffer = require "tools.modtrace".trace("util.dbuffer"); -- -local t_pack = require "util.table".pack; +local t_pack = table.pack; local serialize = require "util.serialization".serialize; -local unpack = table.unpack or unpack; --luacheck: ignore 113 +local unpack = table.unpack; local set = require "util.set"; local serialize_cfg = { diff --git a/tools/openfire2prosody.lua b/tools/openfire2prosody.lua index cd3e62e5..2d8c6867 100644 --- a/tools/openfire2prosody.lua +++ b/tools/openfire2prosody.lua @@ -15,6 +15,10 @@ if my_name:match("[/\\]") then package.cpath = package.cpath..";"..my_name:gsub("[^/\\]+$", "../?.so"); end +if not pcall(require, "prosody.loader") then + pcall(require, "loader"); +end + -- ugly workaround for getting datamanager to work outside of prosody :( prosody = { }; prosody.platform = "unknown"; @@ -24,12 +28,12 @@ elseif package.config:sub(1,1) == "/" then prosody.platform = "posix"; end -local parse_xml = require "util.xml".parse; +local parse_xml = require "prosody.util.xml".parse; ----------------------------------------------------------------------- package.loaded["util.logger"] = {init = function() return function() end; end} -local dm = require "util.datamanager" +local dm = require "prosody.util.datamanager" dm.set_data_path("data"); local arg = ...; diff --git a/tools/tb2err b/tools/tb2err index 7b676813..76c847dc 100755 --- a/tools/tb2err +++ b/tools/tb2err @@ -1,6 +1,7 @@ -#!/usr/bin/env lua-any --- Lua-Versions: 5.3 5.2 5.1 +#!/usr/bin/env lua -- traceback to errors.err for vim -q +-- e.g. curl https://prosody.im/paste/xxx | tb2err > errors.err && vim -q + local path_sep = package.config:sub(1,1); for line in io.lines() do local src, err = line:match("%s*(%S+)(:%d+: .*)") @@ -10,11 +11,13 @@ for line in io.lines() do or src:match("/()net/") or src:match("/()util/") or src:match("/()modules/") + or src:match("/()prosody%-modules/") or src:match("/()plugins/") - or src:match("/()prosody[ctl]*$") + or src:match("/()prosody[ctl]*$") if cut then src = src:sub(cut); end + src = src:gsub("prosody%-modules/", "../modules/") src = src:gsub("^modules/", "plugins/") io.write(src, err, "\n"); end diff --git a/tools/test_mutants.sh.lua b/tools/test_mutants.sh.lua new file mode 100755 index 00000000..6e2423db --- /dev/null +++ b/tools/test_mutants.sh.lua @@ -0,0 +1,217 @@ +#!/bin/bash + +POLYGLOT=1--[===[ + +set -o pipefail + +if [[ "$#" == "0" ]]; then + echo "Lua mutation testing tool" + echo + echo "Usage:" + echo " $BASH_SOURCE MODULE_NAME SPEC_FILE" + echo + echo "Requires 'lua', 'ltokenp' and 'busted' in PATH" + exit 1; +fi + +MOD_NAME="$1" +MOD_FILE="$(lua "$BASH_SOURCE" resolve "$MOD_NAME")" + +if [[ "$MOD_FILE" == "" || ! -f "$MOD_FILE" ]]; then + echo "EE: Failed to locate module '$MOD_NAME' ($MOD_FILE)"; + exit 1; +fi + +SPEC_FILE="$2" + +if [[ "$SPEC_FILE" == "" ]]; then + SPEC_FILE="spec/${MOD_NAME/./_}_spec.lua" +fi + +if [[ "$SPEC_FILE" == "" || ! -f "$SPEC_FILE" ]]; then + echo "EE: Failed to find test spec file ($SPEC_FILE)" + exit 1; +fi + +if ! busted --helper=loader "$SPEC_FILE"; then + echo "EE: Tests fail on original source. Fix it"\!; + exit 1; +fi + +export MUTANT_N=0 +LIVING_MUTANTS=0 + +FILE_PREFIX="${MOD_FILE%.*}.mutant-" +FILE_SUFFIX=".${MOD_FILE##*.}" + +gen_mutant () { + echo "Generating mutant $2 to $3..." + ltokenp -s "$BASH_SOURCE" "$1" > "$3" + return "$?" +} + +# $1 = MOD_NAME, $2 = MUTANT_N, $3 = SPEC_FILE +test_mutant () { + ( + ulimit -m 131072 # 128MB + ulimit -t 16 # 16s + ulimit -f 32768 # 128MB (?) + exec busted --helper="$BASH_SOURCE" -Xhelper mutate="$1":"$2" "$3" + ) >/dev/null + return "$?"; +} + +MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}" + +gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE" +while [[ "$?" == "0" ]]; do + if ! test_mutant "$MOD_NAME" "$MUTANT_N" "$SPEC_FILE"; then + echo "Tests successfully killed mutant $MUTANT_N"; + rm "$MUTANT_FILE"; + else + echo "Mutant $MUTANT_N lives on"\! + LIVING_MUTANTS=$((LIVING_MUTANTS+1)) + fi + MUTANT_N=$((MUTANT_N+1)) + MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}" + gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE" +done + +if [[ "$?" != "2" ]]; then + echo "Failed: $?" + exit "$?"; +fi + +MUTANT_SCORE="$(lua -e "print(('%0.2f'):format((1-($LIVING_MUTANTS/$MUTANT_N))*100))")" +if test -f mutant-scores.txt; then + echo "$MOD_NAME $MUTANT_SCORE" >> mutant-scores.txt +fi +echo "$MOD_NAME: All $MUTANT_N mutants generated, $LIVING_MUTANTS survived (score: $MUTANT_SCORE%)" +rm "$MUTANT_FILE"; # Last file is always unmodified +exit 0; +]===] + +-- busted helper that runs mutations +if arg then + if arg[1] == "resolve" then + local filename = package.searchpath(assert(arg[2], "no module name given"), package.path); + if filename then + print(filename); + end + os.exit(filename and 0 or 1); + end + local mutants = {}; + + for i = 1, #arg do + local opt = arg[i]; + print("LOAD", i, opt) + local module_name, mutant_n = opt:match("^mutate=([^:]+):(%d+)"); + if module_name then + mutants[module_name] = tonumber(mutant_n); + end + end + + local orig_lua_searcher = package.searchers[2]; + + local function mutant_searcher(module_name) + local mutant_n = mutants[module_name]; + if not mutant_n then + return orig_lua_searcher(module_name); + end + local base_file, err = package.searchpath(module_name, package.path); + if not base_file then + return base_file, err; + end + local mutant_file = base_file:gsub("%.lua$", (".mutant-%d.lua"):format(mutant_n)); + return loadfile(mutant_file), mutant_file; + end + + if next(mutants) then + table.insert(package.searchers, 1, mutant_searcher); + end +end + +-- filter for ltokenp to mutate scripts +do + local last_output = {}; + local function emit(...) + last_output = {...}; + io.write(...) + io.write(" ") + return true; + end + + local did_mutate = false; + local count = -1; + local threshold = tonumber(os.getenv("MUTANT_N")) or 0; + local function should_mutate() + count = count + 1; + return count == threshold; + end + + local function mutate(name, value) + if name == "if" then + -- Bypass conditionals + if should_mutate() then + return emit("if true or"); + elseif should_mutate() then + return emit("if false and"); + end + elseif name == "<integer>" then + -- Introduce off-by-one errors + if should_mutate() then + return emit(("%d"):format(tonumber(value)+1)); + elseif should_mutate() then + return emit(("%d"):format(tonumber(value)-1)); + end + elseif name == "and" then + if should_mutate() then + return emit("or"); + end + elseif name == "or" then + if should_mutate() then + return emit("and"); + end + end + end + + local current_line_n, current_line_input, current_line_output = 0, {}, {}; + function FILTER(line_n,token,name,value) + if current_line_n ~= line_n then -- Finished a line, moving to the next? + if did_mutate and did_mutate.line == current_line_n then + -- The line we finished was mutated. Store the original and modified outputs. + did_mutate.line_original_src = table.concat(current_line_input, " "); + did_mutate.line_modified_src = table.concat(current_line_output, " "); + end + current_line_input = {}; + current_line_output = {}; + end + current_line_n = line_n; + if name == "<file>" then return; end + if name == "<eof>" then + if not did_mutate then + return os.exit(2); + else + emit(("\n-- Mutated line %d (changed '%s' to '%s'):\n"):format(did_mutate.line, did_mutate.original, did_mutate.modified)) + emit( ("-- Original: %s\n"):format(did_mutate.line_original_src)) + emit( ("-- Modified: %s\n"):format(did_mutate.line_modified_src)); + return; + end + end + if name == "<string>" then + value = string.format("%q",value); + end + if mutate(name, value) then + did_mutate = { + original = value; + modified = table.concat(last_output); + line = line_n; + }; + else + emit(value); + end + table.insert(current_line_input, value); + table.insert(current_line_output, table.concat(last_output)); + end +end + diff --git a/tools/xepchanges.sh b/tools/xepchanges.sh new file mode 100755 index 00000000..0a0140a1 --- /dev/null +++ b/tools/xepchanges.sh @@ -0,0 +1,14 @@ +#!/bin/sh -eu + +wget -N https://xmpp.org/extensions/xeplist.xml +xml2 <xeplist.xml | + 2csv xep-infos/xep number version | + grep -v ^xxxx,| + sort -g > xepinfos.csv + +xml2 < doc/doap.xml | + 2csv -d ' ' xmpp:SupportedXep @rdf:resource xmpp:version | + sed -r 's/https?:\/\/xmpp\.org\/extensions\/xep-0*([1-9][0-9]*)\.html/\1/' | + while read -r xep ver ; do + grep "^$xep," xepinfos.csv | awk -F, "\$2 != \"$ver\" { print (\"XEP-\"\$1\" updated to \"\$2\" from $ver\") }" + done |