From 6041d3e4e83ca60fb16fb4f61d83084a76632da9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 21:32:34 +0200 Subject: migrator: Rewrite to use storage modules This allows migrating to and from any storage module that supports the right methods. Based on experimental mod_migrate work. --- tools/migration/Makefile | 7 +- tools/migration/migrator.cfg.lua | 34 +++++- tools/migration/migrator/mtools.lua | 58 --------- tools/migration/migrator/prosody_files.lua | 144 ---------------------- tools/migration/migrator/prosody_sql.lua | 190 ----------------------------- tools/migration/prosody-migrator.lua | 148 ++++++++++++++++------ 6 files changed, 142 insertions(+), 439 deletions(-) delete mode 100644 tools/migration/migrator/mtools.lua delete mode 100644 tools/migration/migrator/prosody_files.lua delete mode 100644 tools/migration/migrator/prosody_sql.lua (limited to 'tools') diff --git a/tools/migration/Makefile b/tools/migration/Makefile index 713831d2..dc14e0da 100644 --- a/tools/migration/Makefile +++ b/tools/migration/Makefile @@ -12,16 +12,13 @@ INSTALLEDCONFIG = $(SYSCONFDIR) INSTALLEDMODULES = $(LIBDIR)/prosody/modules INSTALLEDDATA = $(DATADIR) -SOURCE_FILES = migrator/*.lua - -all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES) +all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua install: prosody-migrator.install migrator.cfg.lua.install - install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator + install -d $(BIN) $(CONFIG) $(SOURCE) install -d $(MAN)/man1 install -d $(SOURCE)/migrator install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator - install -m644 $(SOURCE_FILES) $(SOURCE)/migrator test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua clean: diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua index fa37f2a3..c26fa869 100644 --- a/tools/migration/migrator.cfg.lua +++ b/tools/migration/migrator.cfg.lua @@ -1,12 +1,38 @@ local data_path = "../../data"; +local vhost = { + "accounts", + "account_details", + "roster", + "vcard", + "private", + "blocklist", + "privacy", + "archive-archive", + "offline-archive", + "pubsub_nodes", + -- "pubsub_*-archive", + "pep", + -- "pep_*-archive", +} +local muc = { + "persistent", + "config", + "state", + "muc_log-archive", +}; + input { - type = "prosody_files"; + hosts = { + ["example.com"] = vhost; + ["conference.example.com"] = muc; + }; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } @@ -14,11 +40,11 @@ output { --[[ input { - type = "prosody_files"; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua deleted file mode 100644 index cfbfcce8..00000000 --- a/tools/migration/migrator/mtools.lua +++ /dev/null @@ -1,58 +0,0 @@ - - -local print = print; -local t_insert = table.insert; -local t_sort = table.sort; - - -local function sorted(params) - - local reader = params.reader; -- iterator to get items from - local sorter = params.sorter; -- sorting function - local filter = params.filter; -- filter function - - local cache = {}; - for item in reader do - if filter then item = filter(item); end - if item then t_insert(cache, item); end - end - if sorter then - t_sort(cache, sorter); - end - local i = 0; - return function() - i = i + 1; - return cache[i]; - end; - -end - -local function merged(reader, merger) - - local item1 = reader(); - local merged = { item1 }; - return function() - while true do - if not item1 then return nil; end - local item2 = reader(); - if not item2 then item1 = nil; return merged; end - if merger(item1, item2) then - --print("merged") - item1 = item2; - t_insert(merged, item1); - else - --print("unmerged", merged) - item1 = item2; - local tmp = merged; - merged = { item1 }; - return tmp; - end - end - end; - -end - -return { - sorted = sorted; - merged = merged; -} diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua deleted file mode 100644 index 4de09273..00000000 --- a/tools/migration/migrator/prosody_files.lua +++ /dev/null @@ -1,144 +0,0 @@ - -local print = print; -local assert = assert; -local setmetatable = setmetatable; -local tonumber = tonumber; -local char = string.char; -local coroutine = coroutine; -local lfs = require "lfs"; -local loadfile = loadfile; -local pcall = pcall; -local mtools = require "migrator.mtools"; -local next = next; -local pairs = pairs; -local json = require "util.json"; -local os_getenv = os.getenv; -local error = error; - -prosody = {}; -local dm = require "util.datamanager" - - -local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end -local function is_file(path) return lfs.attributes(path, "mode") == "file"; end -local function clean_path(path) - return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~"); -end -local encode, decode; do - local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end }); - decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end - encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end -end -local function decode_dir(x) - if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x); - end -end -local function decode_file(x) - if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x:gsub("%.dat$", "")); - end -end -local function prosody_dir(path, ondir, onfile, ...) - for x in lfs.dir(path) do - local xpath = path.."/"..x; - if decode_dir(x) and is_dir(xpath) then - ondir(xpath, x, ...); - elseif decode_file(x) and is_file(xpath) then - onfile(xpath, x, ...); - end - end -end - -local function handle_root_file(path, name) - --print("root file: ", decode_file(name)) - coroutine.yield { user = nil, host = nil, store = decode_file(name) }; -end -local function handle_host_file(path, name, host) - --print("host file: ", decode_dir(host).."/"..decode_file(name)) - coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) }; -end -local function handle_store_file(path, name, store, host) - --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store)) - coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) }; -end -local function handle_host_store(path, name, host) - prosody_dir(path, function() end, handle_store_file, name, host); -end -local function handle_host_dir(path, name) - prosody_dir(path, handle_host_store, handle_host_file, name); -end -local function handle_root_dir(path) - prosody_dir(path, handle_host_dir, handle_root_file); -end - -local function decode_user(item) - local userdata = { - user = item[1].user; - host = item[1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - userdata.stores[store.store] = store.data; - store.user = nil; store.host = nil; store.store = nil; - end - return userdata; -end - -local function reader(input) - local path = clean_path(assert(input.path, "no input.path specified")); - assert(is_dir(path), "input.path is not a directory"); - local iter = coroutine.wrap(function()handle_root_dir(path);end); - -- get per-user stores, sorted - local iter = mtools.sorted { - reader = function() - local x = iter(); - while x do - dm.set_data_path(path); - local err; - x.data, err = dm.load(x.user, x.host, x.store); - if x.data == nil and err then - local p = dm.getpath(x.user, x.host, x.store); - print(("Error loading data at path %s for %s@%s (%s store): %s") - :format(p, x.user or "", x.host or "", x.store or "", err or "")); - else - return x; - end - x = iter(); - end - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user); - end); - - return function() - local x = iter(); - return x and decode_user(x); - end -end - -local function writer(output) - local path = clean_path(assert(output.path, "no output.path specified")); - assert(is_dir(path), "output.path is not a directory"); - return function(item) - if not item then return; end -- end of input - dm.set_data_path(path); - for store, data in pairs(item.stores) do - assert(dm.store(item.user, item.host, store, data)); - end - end -end - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua deleted file mode 100644 index 6df2b025..00000000 --- a/tools/migration/migrator/prosody_sql.lua +++ /dev/null @@ -1,190 +0,0 @@ - -local assert = assert; -local have_DBI = pcall(require,"DBI"); -local print = print; -local type = type; -local next = next; -local pairs = pairs; -local t_sort = table.sort; -local json = require "util.json"; -local mtools = require "migrator.mtools"; -local tostring = tostring; -local tonumber = tonumber; - -if not have_DBI then - error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0); -end - -local sql = require "util.sql"; - -local function create_table(engine, name) -- luacheck: ignore 431/engine - local Table, Column, Index = sql.Table, sql.Column, sql.Index; - - local ProsodyTable = Table { - name= name or "prosody"; - Column { name="host", type="TEXT", nullable=false }; - Column { name="user", type="TEXT", nullable=false }; - Column { name="store", type="TEXT", nullable=false }; - Column { name="key", type="TEXT", nullable=false }; - Column { name="type", type="TEXT", nullable=false }; - Column { name="value", type="MEDIUMTEXT", nullable=false }; - Index { name="prosody_index", "host", "user", "store", "key" }; - }; - engine:transaction(function() - ProsodyTable:create(engine); - end); - -end - -local function serialize(value) - local t = type(value); - if t == "string" or t == "boolean" or t == "number" then - return t, tostring(value); - elseif t == "table" then - local value,err = json.encode(value); - if value then return "json", value; end - return nil, err; - end - return nil, "Unhandled value type: "..t; -end -local function deserialize(t, value) - if t == "string" then return value; - elseif t == "boolean" then - if value == "true" then return true; - elseif value == "false" then return false; end - elseif t == "number" then return tonumber(value); - elseif t == "json" then - return json.decode(value); - end -end - -local function decode_user(item) - local userdata = { - user = item[1][1].user; - host = item[1][1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - for i=1,#store do -- loop over store data - local row = store[i]; - local k = row.key; - local v = deserialize(row.type, row.value); - if k and v then - if k ~= "" then result[k] = v; elseif type(v) == "table" then - for a,b in pairs(v) do - result[a] = b; - end - end - end - userdata.stores[store[1].store] = result; - end - end - return userdata; -end - -local function needs_upgrade(engine, params) - if params.driver == "MySQL" then - local success = engine:transaction(function() - local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'"); - assert(result:rowcount() == 0); - - -- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already - local check_encoding_query = [[ - SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME" - FROM "information_schema"."columns" - WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' ); - ]]; - check_encoding_query = check_encoding_query:format(engine.charset, engine.charset); - local result = engine:execute(check_encoding_query); - assert(result:rowcount() == 0) - end); - if not success then - -- Upgrade required - return true; - end - end - return false; -end - -local function reader(input) - local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, input) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - end)); - local keys = {"host", "user", "store", "key", "type", "value"}; - assert(engine:connect()); - local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";")); - -- get SQL rows, sorted - local iter = mtools.sorted { - reader = function() val = f(s, val); return val; end; - filter = function(x) - for i=1,#keys do - x[ keys[i] ] = x[i]; - end - if x.host == "" then x.host = nil; end - if x.user == "" then x.user = nil; end - if x.store == "" then x.store = nil; end - return x; - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge rows to get stores - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user and a.store == b.store); - end); - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a[1].host == b[1].host and a[1].user == b[1].user); - end); - return function() - local x = iter(); - return x and decode_user(x); - end; -end - -local function writer(output, iter) - local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, output) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - create_table(engine); - end)); - assert(engine:connect()); - assert(engine:delete("DELETE FROM \"prosody\"")); - local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)"; - - return function(item) - if not item then assert(engine.conn:commit()) return end -- end of input - local host = item.host or ""; - local user = item.user or ""; - for store, data in pairs(item.stores) do - -- TODO transactions - local extradata = {}; - for key, value in pairs(data) do - if type(key) == "string" and key ~= "" then - local t, value = assert(serialize(value)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value)); - else - extradata[key] = value; - end - end - if next(extradata) ~= nil then - local t, extradata = assert(serialize(extradata)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata)); - end - end - end; -end - - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 1219d891..3cc10ecf 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -1,19 +1,43 @@ #!/usr/bin/env lua -CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR"); -CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR"); +CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR"); +CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR"); +CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR"); +CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR"); --- Substitute ~ with path to home directory in paths -if CFG_CONFIGDIR then - CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME")); +-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- + +local function is_relative(path) + local path_sep = package.config:sub(1,1); + return ((path_sep == "/" and path:sub(1,1) ~= "/") + or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\"))) end +-- Tell Lua where to find our libraries if CFG_SOURCEDIR then - CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME")); + local function filter_relative_paths(path) + if is_relative(path) then return ""; end + end + local function sanitise_paths(paths) + return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";")); + end + package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path); + package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath); +end + +-- Substitute ~ with path to home directory in data path +if CFG_DATADIR then + if os.getenv("HOME") then + CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME")); + end end local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua"; +local startup = require "util.startup"; +startup.prosodyctl(); +-- TODO startup.migrator ? + -- Command-line parsing local options = {}; local i = 1; @@ -29,13 +53,6 @@ while arg[i] do end end -if CFG_SOURCEDIR then - package.path = CFG_SOURCEDIR.."/?.lua;"..package.path; - package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath; -else - package.path = "../../?.lua;"..package.path - package.cpath = "../../?.so;"..package.cpath -end local envloadfile = require "util.envload".envloadfile; @@ -69,24 +86,14 @@ if not config[to_store] then print("Error: Output store '"..to_store.."' not found in the config file."); end -function load_store_handler(name) - local store_type = config[name].type; - if not store_type then - print("Error: "..name.." store type not specified in the config file"); - return false; - else - local ok, err = pcall(require, "migrator."..store_type); - if not ok then - print(("Error: Failed to initialize '%s' store:\n\t%s") - :format(name, err)); - return false; - end +for store, conf in pairs(config) do -- COMPAT + if conf.type == "prosody_files" then + conf.type = "internal"; + elseif conf.type == "prosody_sql" then + conf.type = "sql"; end - return true; end -have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output")); - if have_err then print(""); print("Usage: "..arg[0].." FROM_STORE TO_STORE"); @@ -101,17 +108,82 @@ if have_err then os.exit(1); end -local itype = config[from_store].type; -local otype = config[to_store].type; -local reader = require("migrator."..itype).reader(config[from_store]); -local writer = require("migrator."..otype).writer(config[to_store]); +local async = require "util.async"; +local server = require "net.server"; +local watchers = { + error = function (_, err) + error(err); + end; + waiting = function () + server.loop(); + end; +}; + +local cm = require "core.configmanager"; +local hm = require "core.hostmanager"; +local sm = require "core.storagemanager"; +local um = require "core.usermanager"; + +local function users(store, host) + if store.users then + return store:users(); + else + return um.users(host); + end +end -local json = require "util.json"; +local function prepare_config(host, conf) + if conf.type == "internal" then + sm.olddm.set_data_path(conf.path or prosody.paths.data); + elseif conf.type == "sql" then + cm.set(host, "sql", conf); + end +end -io.stderr:write("Migrating...\n"); -for x in reader do - --print(json.encode(x)) - writer(x); +local function get_driver(host, conf) + prepare_config(host, conf); + return assert(sm.load_driver(host, conf.type)); end -writer(nil); -- close + +local migration_runner = async.runner(function (job) + for host, stores in pairs(job.input.hosts) do + prosody.hosts[host] = startup.make_host(host); + sm.initialize_host(host); + um.initialize_host(host); + + local input_driver = get_driver(host, job.input); + + local output_driver = get_driver(host, job.output); + + for _, store in ipairs(stores) do + local p, typ = store:match("()%-(%w+)$"); + if typ then store = store:sub(1, p-1); else typ = "keyval"; end + log("info", "Migrating host %s store %s (%s)", host, store, typ); + + local origin = assert(input_driver:open(store, typ)); + local destination = assert(output_driver:open(store, typ)); + + if typ == "keyval" then -- host data + local data, err = origin:get(nil); + assert(not err, err); + assert(destination:set(nil, data)); + end + + for user in users(origin, host) do + if typ == "keyval" then + local data, err = origin:get(user); + assert(not err, err); + assert(destination:set(user, data)); + else + error("Don't know how to migrate data of type '"..typ.."'."); + end + end + end + end +end, watchers); + +io.stderr:write("Migrating...\n"); + +migration_runner:run({ input = config[from_store], output = config[to_store] }); + io.stderr:write("Done!\n"); -- cgit v1.2.3 From 037fc25b4c87c9ee1168635335c3ac5ee3af7353 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 21:31:15 +0200 Subject: migrator: Add support for archives (fixes #651) --- tools/migration/prosody-migrator.lua | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'tools') diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 3cc10ecf..5c4800ad 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -174,6 +174,12 @@ local migration_runner = async.runner(function (job) local data, err = origin:get(user); assert(not err, err); assert(destination:set(user, data)); + elseif typ == "archive" then + local iter, err = origin:find(user); + assert(iter, err); + for id, item, when, with in iter do + assert(destination:append(user, id, item, when, with)); + end else error("Don't know how to migrate data of type '"..typ.."'."); end -- cgit v1.2.3 From 8398643365bfd23a8b78c45946915c403882cbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Mon, 29 Jul 2019 10:32:03 -0700 Subject: make_repo.lua: Moved to /tools/make_repo.lua --- tools/make_repo.lua | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tools/make_repo.lua (limited to 'tools') diff --git a/tools/make_repo.lua b/tools/make_repo.lua new file mode 100644 index 00000000..5b5e6234 --- /dev/null +++ b/tools/make_repo.lua @@ -0,0 +1,44 @@ +print("Getting all the available modules") +if os.execute '[ -e "./downloaded_modules" ]' then + os.execute("rm -rf downloaded_modules") +end +os.execute("hg clone https://hg.prosody.im/prosody-modules/ downloaded_modules") +local i, popen = 0, io.popen +local flag = "mod_" +if os.execute '[ -e "./repository" ]' then + os.execute("mkdir repository") +end +local pfile = popen('ls -a "downloaded_modules"') +for filename in pfile:lines() do + i = i + 1 + if filename:sub(1, #flag) == flag then + local file = io.open("repository/"..filename.."-scm-1.rockspec", "w") + file:write('package = "'..filename..'"', '\n') + file:write('version = "scm-1"', '\n') + file:write('source = {', '\n') + file:write('\turl = "hg+https://hg.prosody.im/prosody-modules",', '\n') + file:write('\tdir = "prosody-modules"', '\n') + file:write('}', '\n') + file:write('description = {', '\n') + file:write('\thomepage = "https://prosody.im/",', '\n') + file:write('\tlicense = "MIT"', '\n') + file:write('}', '\n') + file:write('dependencies = {', '\n') + file:write('\t"lua >= 5.1"', '\n') + file:write('}', '\n') + file:write('build = {', '\n') + file:write('\ttype = "builtin",', '\n') + file:write('\tmodules = {', '\n') + file:write('\t\t["'..filename..'.'..filename..'"] = "'..filename..'/'..filename..'.lua"', '\n') + file:write('\t}', '\n') + file:write('}', '\n') + file:close() + end +end +pfile:close() +os.execute("cd repository/ && luarocks-admin make_manifest ./ && chmod -R 644 ./*") +print("") +print("Done!. Modules' sources are locally available at ./downloaded_modules") +print("Repository is available at ./repository") +print("The repository contains all of prosody modules' respective rockspecs, as well as manifest files and an html Index") +print("You can now either point your server to this folder, or copy its contents to another configured folder.") -- cgit v1.2.3 From 8dceed3e9e3dda2a11e16ab8cf200d97561857d2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 19 Apr 2020 01:09:21 +0200 Subject: migrator: Inject data- and plugin paths during build Same way as with the prosody and prosodyctl executables --- tools/migration/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/migration/Makefile b/tools/migration/Makefile index dc14e0da..4d916a9f 100644 --- a/tools/migration/Makefile +++ b/tools/migration/Makefile @@ -28,7 +28,9 @@ clean: prosody-migrator.install: prosody-migrator.lua sed "1s/\blua\b/$(RUNWITH)/; \ s|^CFG_SOURCEDIR=.*;$$|CFG_SOURCEDIR='$(INSTALLEDSOURCE)';|; \ - s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|;" \ + s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|; \ + s|^CFG_DATADIR=.*;$$|CFG_DATADIR='$(INSTALLEDDATA)';|; \ + s|^CFG_PLUGINDIR=.*;$$|CFG_PLUGINDIR='$(INSTALLEDMODULES)/';|;" \ < prosody-migrator.lua > prosody-migrator.install migrator.cfg.lua.install: migrator.cfg.lua -- cgit v1.2.3 From 25685a3d1bf426a1f93fffaaf3d847b7e68bc9e9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 19 Apr 2020 01:17:48 +0200 Subject: migrator: Don't create unused directory I guess this contained the old per-store migrators --- tools/migration/Makefile | 1 - 1 file changed, 1 deletion(-) (limited to 'tools') diff --git a/tools/migration/Makefile b/tools/migration/Makefile index 4d916a9f..fce0f7d9 100644 --- a/tools/migration/Makefile +++ b/tools/migration/Makefile @@ -17,7 +17,6 @@ all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua install: prosody-migrator.install migrator.cfg.lua.install install -d $(BIN) $(CONFIG) $(SOURCE) install -d $(MAN)/man1 - install -d $(SOURCE)/migrator install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua -- cgit v1.2.3 From 4cb9c49d9522b827cca3ff9906b4ca2348c988b1 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 28 Oct 2020 22:42:43 +0100 Subject: tools/tb2err: Formats Lua traceback in errors.err format Manually opening to the files and line numbers from a Lua traceback is tedious. This tool converts tracebacks into a format that many compilers and such tools use, which is also compatible with Vim (and possibly other editors). Thus if someone sends you a pastebin link with a traceback, a command like the following gets you right to the relevant lines: curl paste.example/abc123.txt | tb2err > errors.err; vim -q --- tools/tb2err | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 tools/tb2err (limited to 'tools') diff --git a/tools/tb2err b/tools/tb2err new file mode 100755 index 00000000..7b676813 --- /dev/null +++ b/tools/tb2err @@ -0,0 +1,21 @@ +#!/usr/bin/env lua-any +-- Lua-Versions: 5.3 5.2 5.1 +-- traceback to errors.err for vim -q +local path_sep = package.config:sub(1,1); +for line in io.lines() do + local src, err = line:match("%s*(%S+)(:%d+: .*)") + if src then + src = src:gsub("\\", path_sep); + local cut = src:match("/()core/") + or src:match("/()net/") + or src:match("/()util/") + or src:match("/()modules/") + or src:match("/()plugins/") + or src:match("/()prosody[ctl]*$") + if cut then + src = src:sub(cut); + end + src = src:gsub("^modules/", "plugins/") + io.write(src, err, "\n"); + end +end -- cgit v1.2.3 From 43ec58831c05914393b9931503d1f32f0ba72549 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 28 Oct 2020 22:48:31 +0100 Subject: tools/cfgdump: Reads Prosody config file and pretty-prints it back out Useful for comparing what you think you have in your config with what Prosody sees, e.g. wrt (lack of) significance of indentation, order of options vs scope etc. (global options do not go at the end!) Could probably be turned into a prosodyctl command, especially if it learns to redact secrets and passwords. --- tools/cfgdump.lua | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 tools/cfgdump.lua (limited to 'tools') diff --git a/tools/cfgdump.lua b/tools/cfgdump.lua new file mode 100755 index 00000000..b7ca0716 --- /dev/null +++ b/tools/cfgdump.lua @@ -0,0 +1,120 @@ +#!/usr/bin/env lua + +-- cfgdump.lua prosody.cfg.lua [[host] option] + +local s_format, print = string.format, print; +local printf = function(fmt, ...) return print(s_format(fmt, ...)); end +local serialization = require"util.serialization"; +local serialize = serialization.new and serialization.new({ unquoted = true }) or serialization.serialize; +local configmanager = require"core.configmanager"; +local startup = require "util.startup"; + +startup.set_function_metatable(); +local config_filename, onlyhost, onlyoption = ...; + +local ok, _, err = configmanager.load(config_filename or "./prosody.cfg.lua", "lua"); +assert(ok, err); + +if onlyhost then + if not onlyoption then + onlyhost, onlyoption = "*", onlyhost; + end + if onlyhost ~= "*" then + local component_module = configmanager.get(onlyhost, "component_module"); + + if component_module == "component" then + printf("Component %q", onlyhost); + elseif component_module then + printf("Component %q %q", onlyhost, component_module); + else + printf("VirtualHost %q", onlyhost); + end + end + printf("%s = %s", onlyoption or "?", serialize(configmanager.get(onlyhost, onlyoption))); + return; +end + +local config = configmanager.getconfig(); + + +for host, hostcfg in pairs(config) do + local fixed = {}; + for option, value in pairs(hostcfg) do + fixed[option] = value; + if option:match("ports?$") or option:match("interfaces?$") then + if option:match("s$") then + if type(value) ~= "table" then + fixed[option] = { value }; + end + else + if type(value) == "table" and #value > 1 then + fixed[option] = nil; + fixed[option.."s"] = value; + end + end + end + end + config[host] = fixed; +end + +local globals = config["*"]; config["*"] = nil; + +local function printsection(section) + local out, n = {}, 1; + for k,v in pairs(section) do + out[n], n = s_format("%s = %s", k, serialize(v)), n + 1; + end + table.sort(out); + print(table.concat(out, "\n")); +end + +print("-------------- Prosody Exported Configuration File -------------"); +print(); +print("------------------------ Global section ------------------------"); +print(); +printsection(globals); +print(); + +local has_components = nil; + +print("------------------------ Virtual hosts -------------------------"); + +for host, hostcfg in pairs(config) do + setmetatable(hostcfg, nil); + hostcfg.defined = nil; + + if hostcfg.component_module == nil then + print(); + printf("VirtualHost %q", host); + printsection(hostcfg); + else + has_components = true + end +end + +print(); + +if has_components then +print("------------------------- Components ---------------------------"); + + for host, hostcfg in pairs(config) do + local component_module = hostcfg.component_module; + hostcfg.component_module = nil; + + if component_module then + print(); + if component_module == "component" then + printf("Component %q", host); + else + printf("Component %q %q", host, component_module); + hostcfg.component_module = nil; + hostcfg.load_global_modules = nil; + end + printsection(hostcfg); + end + end +end + +print() +print("------------------------- End of File --------------------------"); + -- cgit v1.2.3 From 4d9e30aff34d5c37c4a43e7ed6af8ed4c8b621ca Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 28 Oct 2020 23:05:15 +0100 Subject: tools/linedebug: Print each line of source executed The extremely verbose debug output. Nice for getting some idea which code runs most often, or where it pauses for no reason etc. --- tools/linedebug.lua | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tools/linedebug.lua (limited to 'tools') diff --git a/tools/linedebug.lua b/tools/linedebug.lua new file mode 100644 index 00000000..96214cc8 --- /dev/null +++ b/tools/linedebug.lua @@ -0,0 +1,18 @@ +local data = {} +local getinfo = debug.getinfo; +local function linehook(ev, li) + local S = getinfo(2, "S"); + if S and S.source and S.source:match"^@" then + local file = S.source:sub(2); + local lines = data[file]; + if not lines then + lines = {}; + data[file] = lines; + for line in io.lines(file) do + lines[#lines+1] = line; + end + end + io.stderr:write(ev, " ", file, " ", li, " ", lines[li], "\n"); + end +end +debug.sethook(linehook, "l"); -- cgit v1.2.3 From 53e6579aaba5b9d1ec4e75c3d323f6e52d23bf80 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 28 Oct 2020 23:15:52 +0100 Subject: tools/form2table: Convert XEP-0004 dataform from XML to util.dataforms Lua format Used this to generate code for a number of PubSub forms IIRC --- tools/form2table.lua | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tools/form2table.lua (limited to 'tools') diff --git a/tools/form2table.lua b/tools/form2table.lua new file mode 100644 index 00000000..49a6972b --- /dev/null +++ b/tools/form2table.lua @@ -0,0 +1,48 @@ +-- Read an XML dataform and spit out a serialized Lua table of it + +local function from_stanza(stanza) + local layout = { + title = stanza:get_child_text("title"); + instructions = stanza:get_child_text("instructions"); + }; + for tag in stanza:childtags("field") do + local field = { + name = tag.attr.var; + type = tag.attr.type; + label = tag.attr.label; + desc = tag:get_child_text("desc"); + required = tag:get_child("required") and true or nil; + value = tag:get_child_text("value"); + options = nil; + }; + + if field.type == "list-single" or field.type == "list-multi" then + local options = {}; + for option in tag:childtags("option") do + options[#options+1] = { label = option.attr.label, value = option:get_child_text("value") }; + end + field.options = options; + end + + if field.type == "jid-multi" or field.type == "list-multi" or field.type == "text-multi" then + local values = {}; + for value in tag:childtags("value") do + values[#values+1] = value:get_text(); + end + if field.type == "text-multi" then + values = table.concat(values, "\n"); + end + field.value = values; + end + + if field.type == "boolean" then + field.value = field.value == "true" or field.value == "1"; + end + + layout[#layout+1] = field; + + end + return layout; +end + +print("dataforms.new " .. require "util.serialization".serialize(from_stanza(require "util.xml".parse(io.read("*a"))), { unquoted = true })) -- cgit v1.2.3 From 7312a610d4a9c1ca879feb7d271b98d33541ca3a Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Fri, 30 Oct 2020 13:53:24 +0000 Subject: tools.modtrace: Library for tracing/debugging Lua module and method calls --- tools/modtrace.lua | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tools/modtrace.lua (limited to 'tools') diff --git a/tools/modtrace.lua b/tools/modtrace.lua new file mode 100644 index 00000000..6360fb61 --- /dev/null +++ b/tools/modtrace.lua @@ -0,0 +1,152 @@ +-- Trace module calls and method calls on created objects +-- +-- Very rough and for debugging purposes only. It makes many +-- assumptions and there are many ways it could fail. +-- +-- Example use: +-- +-- local dbuffer = require "tools.modtrace".trace("util.dbuffer"); +-- + +local t_pack = require "util.table".pack; +local serialize = require "util.serialization".serialize; +local unpack = table.unpack or unpack; --luacheck: ignore 113 +local set = require "util.set"; + +local function stringify_value(v) + if type(v) == "string" and #v > 20 then + return (""):format(#v); + elseif type(v) == "function" then + return tostring(v); + end + return serialize(v, "debug"); +end + +local function stringify_params(...) + local n = select("#", ...); + local r = {}; + for i = 1, n do + table.insert(r, stringify_value((select(i, ...)))); + end + return table.concat(r, ", "); +end + +local function stringify_result(ret) + local r = {}; + for i = 1, ret.n do + table.insert(r, stringify_value(ret[i])); + end + return table.concat(r, ", "); +end + +local function stringify_call(method_name, ...) + return ("%s(%s)"):format(method_name, stringify_params(...)); +end + +local function wrap_method(original_obj, original_method, method_name) + method_name = ("<%s>:%s"):format(getmetatable(original_obj).__name or "object", method_name); + return function (new_obj_self, ...) + local opts = new_obj_self._modtrace_opts; + local f = opts.output or io.stderr; + f:write(stringify_call(method_name, ...)); + local ret = t_pack(original_method(original_obj, ...)); + if ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + return unpack(ret, 1, ret.n); + end; +end + +local function wrap_function(original_function, function_name, opts) + local f = opts.output or io.stderr; + return function (...) + f:write(stringify_call(function_name, ...)); + local ret = t_pack(original_function(...)); + if ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + return unpack(ret, 1, ret.n); + end; +end + +local function wrap_metamethod(name, method) + if name == "__index" then + return function (new_obj, k) + local original_method; + if type(method) == "table" then + original_method = new_obj._modtrace_original_obj[k]; + else + original_method = method(new_obj._modtrace_original_obj, k); + end + if original_method == nil then + return nil; + end + return wrap_method(new_obj._modtrace_original_obj, original_method, k); + end; + end + return function (new_obj, ...) + return method(new_obj._modtrace_original_obj, ...); + end; +end + +local function wrap_mt(original_mt) + local new_mt = {}; + for k, v in pairs(original_mt) do + new_mt[k] = wrap_metamethod(k, v); + end + return new_mt; +end + +local function wrap_obj(original_obj, opts) + local new_mt = wrap_mt(getmetatable(original_obj)); + return setmetatable({_modtrace_original_obj = original_obj, _modtrace_opts = opts}, new_mt); +end + +local function wrap_new(original_new, function_name, opts) + local f = opts.output or io.stderr; + return function (...) + f:write(stringify_call(function_name, ...)); + local ret = t_pack(original_new(...)); + local obj = ret[1]; + + if ret.n == 1 and type(ret[1]) == "table" then + f:write(" = <", getmetatable(ret[1]).__name or "object", ">", "\n"); + elseif ret.n > 0 then + f:write(" = ", stringify_result(ret), "\n"); + else + f:write("\n"); + end + + if obj then + ret[1] = wrap_obj(obj, opts); + end + return unpack(ret, 1, ret.n); + end; +end + +local function trace(module, opts) + if type(module) == "string" then + module = require(module); + end + opts = opts or {}; + local new_methods = set.new(opts.new_methods or {"new"}); + local fake_module = setmetatable({}, { + __index = function (_, k) + if new_methods:contains(k) then + return wrap_new(module[k], k, opts); + else + return wrap_function(module[k], k, opts); + end + end; + }); + return fake_module; +end + +return { + wrap = trace; + trace = trace; +} -- cgit v1.2.3 From 1ab96f2289a63f6d1fa55b5d131ebf3296206bc2 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Fri, 30 Oct 2020 14:04:40 +0000 Subject: tools.modtrace: Pass config to serialize() --- tools/modtrace.lua | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/modtrace.lua b/tools/modtrace.lua index 6360fb61..45fa9f6a 100644 --- a/tools/modtrace.lua +++ b/tools/modtrace.lua @@ -13,13 +13,20 @@ local serialize = require "util.serialization".serialize; local unpack = table.unpack or unpack; --luacheck: ignore 113 local set = require "util.set"; +local serialize_cfg = { + preset = "oneline"; + freeze = true; + fatal = false; + fallback = function (v) return "<"..tostring(v)..">" end; +}; + local function stringify_value(v) if type(v) == "string" and #v > 20 then return (""):format(#v); elseif type(v) == "function" then return tostring(v); end - return serialize(v, "debug"); + return serialize(v, serialize_cfg); end local function stringify_params(...) -- cgit v1.2.3