diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/cfgdump.lua | 128 | ||||
-rwxr-xr-x | tools/ejabberd2prosody.lua | 8 | ||||
-rw-r--r-- | tools/form2table.lua | 48 | ||||
-rw-r--r-- | tools/jabberd14sql2prosody.lua | 4 | ||||
-rw-r--r-- | tools/linedebug.lua | 18 | ||||
-rw-r--r-- | tools/make_repo.lua | 44 | ||||
-rw-r--r-- | tools/migration/Makefile | 12 | ||||
-rw-r--r-- | tools/migration/migrator.cfg.lua | 34 | ||||
-rw-r--r-- | tools/migration/migrator/mtools.lua | 58 | ||||
-rw-r--r-- | tools/migration/migrator/prosody_files.lua | 144 | ||||
-rw-r--r-- | tools/migration/migrator/prosody_sql.lua | 190 | ||||
-rw-r--r-- | tools/migration/prosody-migrator.lua | 187 | ||||
-rw-r--r-- | tools/modtrace.lua | 159 | ||||
-rwxr-xr-x | tools/tb2err | 21 | ||||
-rwxr-xr-x | tools/xep227toprosody.lua | 4 |
15 files changed, 601 insertions, 458 deletions
diff --git a/tools/cfgdump.lua b/tools/cfgdump.lua new file mode 100755 index 00000000..8c331a58 --- /dev/null +++ b/tools/cfgdump.lua @@ -0,0 +1,128 @@ +#!/usr/bin/env lua + +-- cfgdump.lua prosody.cfg.lua [[host] option] + +local s_format, print = string.format, print; +local printf = function(fmt, ...) return print(s_format(fmt, ...)); end +local it = require "util.iterators"; +local function sort_anything(a, b) + local typeof_a, typeof_b = type(a), type(b); + if typeof_a ~= typeof_b then return typeof_a < typeof_b end + return a < b -- should work for everything in a config file +end +local serialization = require "util.serialization"; +local serialize = serialization.new and serialization.new({ + unquoted = true, table_iterator = function(t) return it.sorted_pairs(t, sort_anything); end, +}) or serialization.serialize; +local configmanager = require"core.configmanager"; +local startup = require "util.startup"; + +startup.set_function_metatable(); +local config_filename, onlyhost, onlyoption = ...; + +local ok, _, err = configmanager.load(config_filename or "./prosody.cfg.lua", "lua"); +assert(ok, err); + +if onlyhost then + if not onlyoption then + onlyhost, onlyoption = "*", onlyhost; + end + if onlyhost ~= "*" then + local component_module = configmanager.get(onlyhost, "component_module"); + + if component_module == "component" then + printf("Component %q", onlyhost); + elseif component_module then + printf("Component %q %q", onlyhost, component_module); + else + printf("VirtualHost %q", onlyhost); + end + end + printf("%s = %s", onlyoption or "?", serialize(configmanager.get(onlyhost, onlyoption))); + return; +end + +local config = configmanager.getconfig(); + + +for host, hostcfg in it.sorted_pairs(config) do + local fixed = {}; + for option, value in it.sorted_pairs(hostcfg) do + fixed[option] = value; + if option:match("ports?$") or option:match("interfaces?$") then + if option:match("s$") then + if type(value) ~= "table" then + fixed[option] = { value }; + end + else + if type(value) == "table" and #value > 1 then + fixed[option] = nil; + fixed[option.."s"] = value; + end + end + end + end + config[host] = fixed; +end + +local globals = config["*"]; config["*"] = nil; + +local function printsection(section) + local out, n = {}, 1; + for k,v in it.sorted_pairs(section) do + out[n], n = s_format("%s = %s", k, serialize(v)), n + 1; + end + table.sort(out); + print(table.concat(out, "\n")); +end + +print("-------------- Prosody Exported Configuration File -------------"); +print(); +print("------------------------ Global section ------------------------"); +print(); +printsection(globals); +print(); + +local has_components = nil; + +print("------------------------ Virtual hosts -------------------------"); + +for host, hostcfg in it.sorted_pairs(config) do + setmetatable(hostcfg, nil); + hostcfg.defined = nil; + + if hostcfg.component_module == nil then + print(); + printf("VirtualHost %q", host); + printsection(hostcfg); + else + has_components = true + end +end + +print(); + +if has_components then +print("------------------------- Components ---------------------------"); + + for host, hostcfg in it.sorted_pairs(config) do + local component_module = hostcfg.component_module; + hostcfg.component_module = nil; + + if component_module then + print(); + if component_module == "component" then + printf("Component %q", host); + else + printf("Component %q %q", host, component_module); + hostcfg.component_module = nil; + hostcfg.load_global_modules = nil; + end + printsection(hostcfg); + end + end +end + +print() +print("------------------------- End of File --------------------------"); + diff --git a/tools/ejabberd2prosody.lua b/tools/ejabberd2prosody.lua index 46a48f57..8b95aaef 100755 --- a/tools/ejabberd2prosody.lua +++ b/tools/ejabberd2prosody.lua @@ -85,7 +85,13 @@ function password(node, host, password) data.stored_key = hex(unb64(password[2])); data.server_key = hex(unb64(password[3])); data.salt = unb64(password[4]); - data.iteration_count = password[5]; + if type(password[6]) == "number" then + assert(password[5] == "sha", "unexpected passwd entry hash: "..tostring(password[5])); + data.iteration_count = password[6]; + else + assert(type(password[5]) == "number", "unexpected passwd entry in source data"); + data.iteration_count = password[5]; + end end local ret, err = dm.store(node, host, "accounts", data); print("["..(err or "success").."] accounts: "..node.."@"..host); diff --git a/tools/form2table.lua b/tools/form2table.lua new file mode 100644 index 00000000..49a6972b --- /dev/null +++ b/tools/form2table.lua @@ -0,0 +1,48 @@ +-- Read an XML dataform and spit out a serialized Lua table of it + +local function from_stanza(stanza) + local layout = { + title = stanza:get_child_text("title"); + instructions = stanza:get_child_text("instructions"); + }; + for tag in stanza:childtags("field") do + local field = { + name = tag.attr.var; + type = tag.attr.type; + label = tag.attr.label; + desc = tag:get_child_text("desc"); + required = tag:get_child("required") and true or nil; + value = tag:get_child_text("value"); + options = nil; + }; + + if field.type == "list-single" or field.type == "list-multi" then + local options = {}; + for option in tag:childtags("option") do + options[#options+1] = { label = option.attr.label, value = option:get_child_text("value") }; + end + field.options = options; + end + + if field.type == "jid-multi" or field.type == "list-multi" or field.type == "text-multi" then + local values = {}; + for value in tag:childtags("value") do + values[#values+1] = value:get_text(); + end + if field.type == "text-multi" then + values = table.concat(values, "\n"); + end + field.value = values; + end + + if field.type == "boolean" then + field.value = field.value == "true" or field.value == "1"; + end + + layout[#layout+1] = field; + + end + return layout; +end + +print("dataforms.new " .. require "util.serialization".serialize(from_stanza(require "util.xml".parse(io.read("*a"))), { unquoted = true })) diff --git a/tools/jabberd14sql2prosody.lua b/tools/jabberd14sql2prosody.lua index e43dc296..d1d66dc1 100644 --- a/tools/jabberd14sql2prosody.lua +++ b/tools/jabberd14sql2prosody.lua @@ -468,7 +468,7 @@ function store_vcard(username, host, stanza) end function store_roster(username, host, roster_items) - -- fetch current roster-table for username@host if he already has one + -- fetch current roster-table for username@host if they already have one local roster = dm.load(username, host, "roster") or {}; -- merge imported roster-items with loaded roster for item_tag in roster_items:childtags() do @@ -508,7 +508,7 @@ end function store_subscription_request(username, host, presence_stanza) local from_bare = presence_stanza.attr.from; - -- fetch current roster-table for username@host if he already has one + -- fetch current roster-table for username@host if they already have one local roster = dm.load(username, host, "roster") or {}; local item = roster[from_bare]; diff --git a/tools/linedebug.lua b/tools/linedebug.lua new file mode 100644 index 00000000..96214cc8 --- /dev/null +++ b/tools/linedebug.lua @@ -0,0 +1,18 @@ +local data = {} +local getinfo = debug.getinfo; +local function linehook(ev, li) + local S = getinfo(2, "S"); + if S and S.source and S.source:match"^@" then + local file = S.source:sub(2); + local lines = data[file]; + if not lines then + lines = {}; + data[file] = lines; + for line in io.lines(file) do + lines[#lines+1] = line; + end + end + io.stderr:write(ev, " ", file, " ", li, " ", lines[li], "\n"); + end +end +debug.sethook(linehook, "l"); diff --git a/tools/make_repo.lua b/tools/make_repo.lua new file mode 100644 index 00000000..5b5e6234 --- /dev/null +++ b/tools/make_repo.lua @@ -0,0 +1,44 @@ +print("Getting all the available modules") +if os.execute '[ -e "./downloaded_modules" ]' then + os.execute("rm -rf downloaded_modules") +end +os.execute("hg clone https://hg.prosody.im/prosody-modules/ downloaded_modules") +local i, popen = 0, io.popen +local flag = "mod_" +if os.execute '[ -e "./repository" ]' then + os.execute("mkdir repository") +end +local pfile = popen('ls -a "downloaded_modules"') +for filename in pfile:lines() do + i = i + 1 + if filename:sub(1, #flag) == flag then + local file = io.open("repository/"..filename.."-scm-1.rockspec", "w") + file:write('package = "'..filename..'"', '\n') + file:write('version = "scm-1"', '\n') + file:write('source = {', '\n') + file:write('\turl = "hg+https://hg.prosody.im/prosody-modules",', '\n') + file:write('\tdir = "prosody-modules"', '\n') + file:write('}', '\n') + file:write('description = {', '\n') + file:write('\thomepage = "https://prosody.im/",', '\n') + file:write('\tlicense = "MIT"', '\n') + file:write('}', '\n') + file:write('dependencies = {', '\n') + file:write('\t"lua >= 5.1"', '\n') + file:write('}', '\n') + file:write('build = {', '\n') + file:write('\ttype = "builtin",', '\n') + file:write('\tmodules = {', '\n') + file:write('\t\t["'..filename..'.'..filename..'"] = "'..filename..'/'..filename..'.lua"', '\n') + file:write('\t}', '\n') + file:write('}', '\n') + file:close() + end +end +pfile:close() +os.execute("cd repository/ && luarocks-admin make_manifest ./ && chmod -R 644 ./*") +print("") +print("Done!. Modules' sources are locally available at ./downloaded_modules") +print("Repository is available at ./repository") +print("The repository contains all of prosody modules' respective rockspecs, as well as manifest files and an html Index") +print("You can now either point your server to this folder, or copy its contents to another configured folder.") diff --git a/tools/migration/Makefile b/tools/migration/Makefile index 713831d2..fce0f7d9 100644 --- a/tools/migration/Makefile +++ b/tools/migration/Makefile @@ -12,16 +12,12 @@ INSTALLEDCONFIG = $(SYSCONFDIR) INSTALLEDMODULES = $(LIBDIR)/prosody/modules INSTALLEDDATA = $(DATADIR) -SOURCE_FILES = migrator/*.lua - -all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES) +all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua install: prosody-migrator.install migrator.cfg.lua.install - install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator + install -d $(BIN) $(CONFIG) $(SOURCE) install -d $(MAN)/man1 - install -d $(SOURCE)/migrator install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator - install -m644 $(SOURCE_FILES) $(SOURCE)/migrator test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua clean: @@ -31,7 +27,9 @@ clean: prosody-migrator.install: prosody-migrator.lua sed "1s/\blua\b/$(RUNWITH)/; \ s|^CFG_SOURCEDIR=.*;$$|CFG_SOURCEDIR='$(INSTALLEDSOURCE)';|; \ - s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|;" \ + s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|; \ + s|^CFG_DATADIR=.*;$$|CFG_DATADIR='$(INSTALLEDDATA)';|; \ + s|^CFG_PLUGINDIR=.*;$$|CFG_PLUGINDIR='$(INSTALLEDMODULES)/';|;" \ < prosody-migrator.lua > prosody-migrator.install migrator.cfg.lua.install: migrator.cfg.lua diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua index fa37f2a3..c26fa869 100644 --- a/tools/migration/migrator.cfg.lua +++ b/tools/migration/migrator.cfg.lua @@ -1,12 +1,38 @@ local data_path = "../../data"; +local vhost = { + "accounts", + "account_details", + "roster", + "vcard", + "private", + "blocklist", + "privacy", + "archive-archive", + "offline-archive", + "pubsub_nodes", + -- "pubsub_*-archive", + "pep", + -- "pep_*-archive", +} +local muc = { + "persistent", + "config", + "state", + "muc_log-archive", +}; + input { - type = "prosody_files"; + hosts = { + ["example.com"] = vhost; + ["conference.example.com"] = muc; + }; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } @@ -14,11 +40,11 @@ output { --[[ input { - type = "prosody_files"; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua deleted file mode 100644 index cfbfcce8..00000000 --- a/tools/migration/migrator/mtools.lua +++ /dev/null @@ -1,58 +0,0 @@ - - -local print = print; -local t_insert = table.insert; -local t_sort = table.sort; - - -local function sorted(params) - - local reader = params.reader; -- iterator to get items from - local sorter = params.sorter; -- sorting function - local filter = params.filter; -- filter function - - local cache = {}; - for item in reader do - if filter then item = filter(item); end - if item then t_insert(cache, item); end - end - if sorter then - t_sort(cache, sorter); - end - local i = 0; - return function() - i = i + 1; - return cache[i]; - end; - -end - -local function merged(reader, merger) - - local item1 = reader(); - local merged = { item1 }; - return function() - while true do - if not item1 then return nil; end - local item2 = reader(); - if not item2 then item1 = nil; return merged; end - if merger(item1, item2) then - --print("merged") - item1 = item2; - t_insert(merged, item1); - else - --print("unmerged", merged) - item1 = item2; - local tmp = merged; - merged = { item1 }; - return tmp; - end - end - end; - -end - -return { - sorted = sorted; - merged = merged; -} diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua deleted file mode 100644 index 4de09273..00000000 --- a/tools/migration/migrator/prosody_files.lua +++ /dev/null @@ -1,144 +0,0 @@ - -local print = print; -local assert = assert; -local setmetatable = setmetatable; -local tonumber = tonumber; -local char = string.char; -local coroutine = coroutine; -local lfs = require "lfs"; -local loadfile = loadfile; -local pcall = pcall; -local mtools = require "migrator.mtools"; -local next = next; -local pairs = pairs; -local json = require "util.json"; -local os_getenv = os.getenv; -local error = error; - -prosody = {}; -local dm = require "util.datamanager" - - -local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end -local function is_file(path) return lfs.attributes(path, "mode") == "file"; end -local function clean_path(path) - return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~"); -end -local encode, decode; do - local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end }); - decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end - encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end -end -local function decode_dir(x) - if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x); - end -end -local function decode_file(x) - if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x:gsub("%.dat$", "")); - end -end -local function prosody_dir(path, ondir, onfile, ...) - for x in lfs.dir(path) do - local xpath = path.."/"..x; - if decode_dir(x) and is_dir(xpath) then - ondir(xpath, x, ...); - elseif decode_file(x) and is_file(xpath) then - onfile(xpath, x, ...); - end - end -end - -local function handle_root_file(path, name) - --print("root file: ", decode_file(name)) - coroutine.yield { user = nil, host = nil, store = decode_file(name) }; -end -local function handle_host_file(path, name, host) - --print("host file: ", decode_dir(host).."/"..decode_file(name)) - coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) }; -end -local function handle_store_file(path, name, store, host) - --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store)) - coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) }; -end -local function handle_host_store(path, name, host) - prosody_dir(path, function() end, handle_store_file, name, host); -end -local function handle_host_dir(path, name) - prosody_dir(path, handle_host_store, handle_host_file, name); -end -local function handle_root_dir(path) - prosody_dir(path, handle_host_dir, handle_root_file); -end - -local function decode_user(item) - local userdata = { - user = item[1].user; - host = item[1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - userdata.stores[store.store] = store.data; - store.user = nil; store.host = nil; store.store = nil; - end - return userdata; -end - -local function reader(input) - local path = clean_path(assert(input.path, "no input.path specified")); - assert(is_dir(path), "input.path is not a directory"); - local iter = coroutine.wrap(function()handle_root_dir(path);end); - -- get per-user stores, sorted - local iter = mtools.sorted { - reader = function() - local x = iter(); - while x do - dm.set_data_path(path); - local err; - x.data, err = dm.load(x.user, x.host, x.store); - if x.data == nil and err then - local p = dm.getpath(x.user, x.host, x.store); - print(("Error loading data at path %s for %s@%s (%s store): %s") - :format(p, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>", err or "<nil>")); - else - return x; - end - x = iter(); - end - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user); - end); - - return function() - local x = iter(); - return x and decode_user(x); - end -end - -local function writer(output) - local path = clean_path(assert(output.path, "no output.path specified")); - assert(is_dir(path), "output.path is not a directory"); - return function(item) - if not item then return; end -- end of input - dm.set_data_path(path); - for store, data in pairs(item.stores) do - assert(dm.store(item.user, item.host, store, data)); - end - end -end - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua deleted file mode 100644 index 6df2b025..00000000 --- a/tools/migration/migrator/prosody_sql.lua +++ /dev/null @@ -1,190 +0,0 @@ - -local assert = assert; -local have_DBI = pcall(require,"DBI"); -local print = print; -local type = type; -local next = next; -local pairs = pairs; -local t_sort = table.sort; -local json = require "util.json"; -local mtools = require "migrator.mtools"; -local tostring = tostring; -local tonumber = tonumber; - -if not have_DBI then - error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0); -end - -local sql = require "util.sql"; - -local function create_table(engine, name) -- luacheck: ignore 431/engine - local Table, Column, Index = sql.Table, sql.Column, sql.Index; - - local ProsodyTable = Table { - name= name or "prosody"; - Column { name="host", type="TEXT", nullable=false }; - Column { name="user", type="TEXT", nullable=false }; - Column { name="store", type="TEXT", nullable=false }; - Column { name="key", type="TEXT", nullable=false }; - Column { name="type", type="TEXT", nullable=false }; - Column { name="value", type="MEDIUMTEXT", nullable=false }; - Index { name="prosody_index", "host", "user", "store", "key" }; - }; - engine:transaction(function() - ProsodyTable:create(engine); - end); - -end - -local function serialize(value) - local t = type(value); - if t == "string" or t == "boolean" or t == "number" then - return t, tostring(value); - elseif t == "table" then - local value,err = json.encode(value); - if value then return "json", value; end - return nil, err; - end - return nil, "Unhandled value type: "..t; -end -local function deserialize(t, value) - if t == "string" then return value; - elseif t == "boolean" then - if value == "true" then return true; - elseif value == "false" then return false; end - elseif t == "number" then return tonumber(value); - elseif t == "json" then - return json.decode(value); - end -end - -local function decode_user(item) - local userdata = { - user = item[1][1].user; - host = item[1][1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - for i=1,#store do -- loop over store data - local row = store[i]; - local k = row.key; - local v = deserialize(row.type, row.value); - if k and v then - if k ~= "" then result[k] = v; elseif type(v) == "table" then - for a,b in pairs(v) do - result[a] = b; - end - end - end - userdata.stores[store[1].store] = result; - end - end - return userdata; -end - -local function needs_upgrade(engine, params) - if params.driver == "MySQL" then - local success = engine:transaction(function() - local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'"); - assert(result:rowcount() == 0); - - -- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already - local check_encoding_query = [[ - SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME" - FROM "information_schema"."columns" - WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' ); - ]]; - check_encoding_query = check_encoding_query:format(engine.charset, engine.charset); - local result = engine:execute(check_encoding_query); - assert(result:rowcount() == 0) - end); - if not success then - -- Upgrade required - return true; - end - end - return false; -end - -local function reader(input) - local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, input) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - end)); - local keys = {"host", "user", "store", "key", "type", "value"}; - assert(engine:connect()); - local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";")); - -- get SQL rows, sorted - local iter = mtools.sorted { - reader = function() val = f(s, val); return val; end; - filter = function(x) - for i=1,#keys do - x[ keys[i] ] = x[i]; - end - if x.host == "" then x.host = nil; end - if x.user == "" then x.user = nil; end - if x.store == "" then x.store = nil; end - return x; - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge rows to get stores - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user and a.store == b.store); - end); - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a[1].host == b[1].host and a[1].user == b[1].user); - end); - return function() - local x = iter(); - return x and decode_user(x); - end; -end - -local function writer(output, iter) - local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, output) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - create_table(engine); - end)); - assert(engine:connect()); - assert(engine:delete("DELETE FROM \"prosody\"")); - local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)"; - - return function(item) - if not item then assert(engine.conn:commit()) return end -- end of input - local host = item.host or ""; - local user = item.user or ""; - for store, data in pairs(item.stores) do - -- TODO transactions - local extradata = {}; - for key, value in pairs(data) do - if type(key) == "string" and key ~= "" then - local t, value = assert(serialize(value)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value)); - else - extradata[key] = value; - end - end - if next(extradata) ~= nil then - local t, extradata = assert(serialize(extradata)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata)); - end - end - end; -end - - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 1219d891..26b9ef58 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -1,41 +1,67 @@ #!/usr/bin/env lua -CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR"); -CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR"); +CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR"); +CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR"); +CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR"); +CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR"); --- Substitute ~ with path to home directory in paths -if CFG_CONFIGDIR then - CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME")); +-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- + +local function is_relative(path) + local path_sep = package.config:sub(1,1); + return ((path_sep == "/" and path:sub(1,1) ~= "/") + or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\"))) end +-- Tell Lua where to find our libraries if CFG_SOURCEDIR then - CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME")); + local function filter_relative_paths(path) + if is_relative(path) then return ""; end + end + local function sanitise_paths(paths) + return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";")); + end + package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path); + package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath); +end + +-- Substitute ~ with path to home directory in data path +if CFG_DATADIR then + if os.getenv("HOME") then + CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME")); + end end local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua"; --- Command-line parsing -local options = {}; -local i = 1; -while arg[i] do - if arg[i]:sub(1,2) == "--" then - local opt, val = arg[i]:match("([%w-]+)=?(.*)"); - if opt then - options[(opt:sub(3):gsub("%-", "_"))] = #val > 0 and val or true; - end - table.remove(arg, i); - else - i = i + 1; +local startup = require "util.startup"; +do + startup.parse_args(); + startup.init_global_state(); + prosody.process_type = "migrator"; + startup.force_console_logging(); + startup.init_logging(); + startup.init_gc(); + startup.init_errors(); + startup.setup_plugindir(); + startup.setup_plugin_install_path(); + startup.setup_datadir(); + startup.chdir(); + startup.read_version(); + startup.switch_user(); + startup.check_dependencies(); + startup.log_startup_warnings(); + prosody.config_loaded = true; + startup.load_libraries(); + startup.init_http_client(); + prosody.core_post_stanza = function () + -- silence assert in core.moduleapi + error("Attempt to send stanzas from inside migrator.", 0); end end -if CFG_SOURCEDIR then - package.path = CFG_SOURCEDIR.."/?.lua;"..package.path; - package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath; -else - package.path = "../../?.lua;"..package.path - package.cpath = "../../?.so;"..package.cpath -end +-- Command-line parsing +local options = prosody.opts; local envloadfile = require "util.envload".envloadfile; @@ -69,24 +95,14 @@ if not config[to_store] then print("Error: Output store '"..to_store.."' not found in the config file."); end -function load_store_handler(name) - local store_type = config[name].type; - if not store_type then - print("Error: "..name.." store type not specified in the config file"); - return false; - else - local ok, err = pcall(require, "migrator."..store_type); - if not ok then - print(("Error: Failed to initialize '%s' store:\n\t%s") - :format(name, err)); - return false; - end +for store, conf in pairs(config) do -- COMPAT + if conf.type == "prosody_files" then + conf.type = "internal"; + elseif conf.type == "prosody_sql" then + conf.type = "sql"; end - return true; end -have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output")); - if have_err then print(""); print("Usage: "..arg[0].." FROM_STORE TO_STORE"); @@ -101,17 +117,88 @@ if have_err then os.exit(1); end -local itype = config[from_store].type; -local otype = config[to_store].type; -local reader = require("migrator."..itype).reader(config[from_store]); -local writer = require("migrator."..otype).writer(config[to_store]); +local async = require "util.async"; +local server = require "net.server"; +local watchers = { + error = function (_, err) + error(err); + end; + waiting = function () + server.loop(); + end; +}; + +local cm = require "core.configmanager"; +local hm = require "core.hostmanager"; +local sm = require "core.storagemanager"; +local um = require "core.usermanager"; + +local function users(store, host) + if store.users then + return store:users(); + else + return um.users(host); + end +end -local json = require "util.json"; +local function prepare_config(host, conf) + if conf.type == "internal" then + sm.olddm.set_data_path(conf.path or prosody.paths.data); + elseif conf.type == "sql" then + cm.set(host, "sql", conf); + end +end -io.stderr:write("Migrating...\n"); -for x in reader do - --print(json.encode(x)) - writer(x); +local function get_driver(host, conf) + prepare_config(host, conf); + return assert(sm.load_driver(host, conf.type)); end -writer(nil); -- close + +local migration_runner = async.runner(function (job) + for host, stores in pairs(job.input.hosts) do + prosody.hosts[host] = startup.make_host(host); + sm.initialize_host(host); + um.initialize_host(host); + + local input_driver = get_driver(host, job.input); + + local output_driver = get_driver(host, job.output); + + for _, store in ipairs(stores) do + local p, typ = store:match("()%-(%w+)$"); + if typ then store = store:sub(1, p-1); else typ = "keyval"; end + log("info", "Migrating host %s store %s (%s)", host, store, typ); + + local origin = assert(input_driver:open(store, typ)); + local destination = assert(output_driver:open(store, typ)); + + if typ == "keyval" then -- host data + local data, err = origin:get(nil); + assert(not err, err); + assert(destination:set(nil, data)); + end + + for user in users(origin, host) do + if typ == "keyval" then + local data, err = origin:get(user); + assert(not err, err); + assert(destination:set(user, data)); + elseif typ == "archive" then + local iter, err = origin:find(user); + assert(iter, err); + for id, item, when, with in iter do + assert(destination:append(user, id, item, when, with)); + end + else + error("Don't know how to migrate data of type '"..typ.."'."); + end + end + end + end +end, watchers); + +io.stderr:write("Migrating...\n"); + +migration_runner:run({ input = config[from_store], output = config[to_store] }); + io.stderr:write("Done!\n"); diff --git a/tools/modtrace.lua b/tools/modtrace.lua new file mode 100644 index 00000000..45fa9f6a --- /dev/null +++ b/tools/modtrace.lua @@ -0,0 +1,159 @@ +-- Trace module calls and method calls on created objects +-- +-- Very rough and for debugging purposes only. It makes many +-- assumptions and there are many ways it could fail. +-- +-- Example use: +-- +-- local dbuffer = require "tools.modtrace".trace("util.dbuffer"); +-- + +local t_pack = require "util.table".pack; +local serialize = require "util.serialization".serialize; +local unpack = table.unpack or unpack; --luacheck: ignore 113 +local set = require "util.set"; + +local serialize_cfg = { + preset = "oneline"; + freeze = true; + fatal = false; + fallback = function (v) return "<"..tostring(v)..">" end; +}; + +local function stringify_value(v) + if type(v) == "string" and #v > 20 then + return ("<string(%d)>"):format(#v); + elseif type(v) == "function" then + return tostring(v); + end + return serialize(v, serialize_cfg); +end + +local function stringify_params(...) + local n = select("#", ...); + local r = {}; + for i = 1, n do + table.insert(r, stringify_value((select(i, ...)))); + end + return table.concat(r, ", "); +end + +local function stringify_result(ret) + local r = {}; + for i = 1, ret.n do + table.insert(r, stringify_value(ret[i])); + end + return table.concat(r, ", "); +end + +local function stringify_call(method_name, ...) + return ("%s(%s)"):format(method_name, stringify_params(...)); +end + +local function wrap_method(original_obj, original_method, method_name) + method_name = ("<%s>:%s"):format(getmetatable(original_obj).__name or "object", method_name); + return function (new_obj_self, ...) + local opts = new_obj_self._modtrace_opts; + local f = opts.output or io.stderr; + f:write(stringify_call(method_name, ...)); + local ret = t_pack(original_method(original_obj, ...)); + if ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + return unpack(ret, 1, ret.n); + end; +end + +local function wrap_function(original_function, function_name, opts) + local f = opts.output or io.stderr; + return function (...) + f:write(stringify_call(function_name, ...)); + local ret = t_pack(original_function(...)); + if ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + return unpack(ret, 1, ret.n); + end; +end + +local function wrap_metamethod(name, method) + if name == "__index" then + return function (new_obj, k) + local original_method; + if type(method) == "table" then + original_method = new_obj._modtrace_original_obj[k]; + else + original_method = method(new_obj._modtrace_original_obj, k); + end + if original_method == nil then + return nil; + end + return wrap_method(new_obj._modtrace_original_obj, original_method, k); + end; + end + return function (new_obj, ...) + return method(new_obj._modtrace_original_obj, ...); + end; +end + +local function wrap_mt(original_mt) + local new_mt = {}; + for k, v in pairs(original_mt) do + new_mt[k] = wrap_metamethod(k, v); + end + return new_mt; +end + +local function wrap_obj(original_obj, opts) + local new_mt = wrap_mt(getmetatable(original_obj)); + return setmetatable({_modtrace_original_obj = original_obj, _modtrace_opts = opts}, new_mt); +end + +local function wrap_new(original_new, function_name, opts) + local f = opts.output or io.stderr; + return function (...) + f:write(stringify_call(function_name, ...)); + local ret = t_pack(original_new(...)); + local obj = ret[1]; + + if ret.n == 1 and type(ret[1]) == "table" then + f:write(" = <", getmetatable(ret[1]).__name or "object", ">", "\n"); + elseif ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + + if obj then + ret[1] = wrap_obj(obj, opts); + end + return unpack(ret, 1, ret.n); + end; +end + +local function trace(module, opts) + if type(module) == "string" then + module = require(module); + end + opts = opts or {}; + local new_methods = set.new(opts.new_methods or {"new"}); + local fake_module = setmetatable({}, { + __index = function (_, k) + if new_methods:contains(k) then + return wrap_new(module[k], k, opts); + else + return wrap_function(module[k], k, opts); + end + end; + }); + return fake_module; +end + +return { + wrap = trace; + trace = trace; +} diff --git a/tools/tb2err b/tools/tb2err new file mode 100755 index 00000000..7b676813 --- /dev/null +++ b/tools/tb2err @@ -0,0 +1,21 @@ +#!/usr/bin/env lua-any +-- Lua-Versions: 5.3 5.2 5.1 +-- traceback to errors.err for vim -q +local path_sep = package.config:sub(1,1); +for line in io.lines() do + local src, err = line:match("%s*(%S+)(:%d+: .*)") + if src then + src = src:gsub("\\", path_sep); + local cut = src:match("/()core/") + or src:match("/()net/") + or src:match("/()util/") + or src:match("/()modules/") + or src:match("/()plugins/") + or src:match("/()prosody[ctl]*$") + if cut then + src = src:sub(cut); + end + src = src:gsub("^modules/", "plugins/") + io.write(src, err, "\n"); + end +end diff --git a/tools/xep227toprosody.lua b/tools/xep227toprosody.lua index 521851e9..af8e3f1c 100755 --- a/tools/xep227toprosody.lua +++ b/tools/xep227toprosody.lua @@ -67,7 +67,7 @@ function store_password(username, host, password) end function store_roster(username, host, roster_items) - -- fetch current roster-table for username@host if he already has one + -- fetch current roster-table for username@host if they already have one local roster = dm.load(username, host, "roster") or {}; -- merge imported roster-items with loaded roster for item_tag in roster_items:childtags("item") do @@ -129,7 +129,7 @@ end function store_subscription_request(username, host, presence_stanza) local from_bare = presence_stanza.attr.from; - -- fetch current roster-table for username@host if he already has one + -- fetch current roster-table for username@host if they already have one local roster = dm.load(username, host, "roster") or {}; local item = roster[from_bare]; |