diff options
Diffstat (limited to 'tools/migration')
| -rw-r--r-- | tools/migration/Makefile | 39 | ||||
| -rw-r--r-- | tools/migration/migrator.cfg.lua | 26 | ||||
| -rw-r--r-- | tools/migration/migrator/jabberd14.lua | 142 | ||||
| -rw-r--r-- | tools/migration/migrator/mtools.lua | 56 | ||||
| -rw-r--r-- | tools/migration/migrator/prosody_files.lua | 138 | ||||
| -rw-r--r-- | tools/migration/migrator/prosody_sql.lua | 200 | ||||
| -rw-r--r-- | tools/migration/prosody-migrator.lua | 132 |
7 files changed, 733 insertions, 0 deletions
diff --git a/tools/migration/Makefile b/tools/migration/Makefile new file mode 100644 index 00000000..ae402bd2 --- /dev/null +++ b/tools/migration/Makefile @@ -0,0 +1,39 @@ + +include ../../config.unix + +BIN = $(DESTDIR)$(PREFIX)/bin +CONFIG = $(DESTDIR)$(SYSCONFDIR) +SOURCE = $(DESTDIR)$(PREFIX)/lib/prosody +DATA = $(DESTDIR)$(DATADIR) +MAN = $(DESTDIR)$(PREFIX)/share/man + +INSTALLEDSOURCE = $(PREFIX)/lib/prosody +INSTALLEDCONFIG = $(SYSCONFDIR) +INSTALLEDMODULES = $(PREFIX)/lib/prosody/modules +INSTALLEDDATA = $(DATADIR) + +SOURCE_FILES = migrator/*.lua + +all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES) + +install: prosody-migrator.install migrator.cfg.lua.install + install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator + install -d $(MAN)/man1 + install -d $(SOURCE)/migrator + install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator + install -m644 $(SOURCE_FILES) $(SOURCE)/migrator + test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua + +clean: + rm -f prosody-migrator.install + rm -f migrator.cfg.lua.install + +prosody-migrator.install: prosody-migrator.lua + sed "1s/\blua\b/$(RUNWITH)/; \ + s|^CFG_SOURCEDIR=.*;$$|CFG_SOURCEDIR='$(INSTALLEDSOURCE)';|; \ + s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|;" \ + < prosody-migrator.lua > prosody-migrator.install + +migrator.cfg.lua.install: migrator.cfg.lua + sed "s|^local data_path = .*;$$|local data_path = '$(INSTALLEDDATA)';|;" \ + < migrator.cfg.lua > migrator.cfg.lua.install diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua new file mode 100644 index 00000000..fa37f2a3 --- /dev/null +++ b/tools/migration/migrator.cfg.lua @@ -0,0 +1,26 @@ +local data_path = "../../data"; + +input { + type = "prosody_files"; + path = data_path; +} + +output { + type = "prosody_sql"; + driver = "SQLite3"; + database = data_path.."/prosody.sqlite"; +} + +--[[ + +input { + type = "prosody_files"; + path = data_path; +} +output { + type = "prosody_sql"; + driver = "SQLite3"; + database = data_path.."/prosody.sqlite"; +} + +]] diff --git a/tools/migration/migrator/jabberd14.lua b/tools/migration/migrator/jabberd14.lua new file mode 100644 index 00000000..2f0b0b78 --- /dev/null +++ b/tools/migration/migrator/jabberd14.lua @@ -0,0 +1,142 @@ + +local lfs = require "lfs"; +local st = require "util.stanza"; +local parse_xml = require "util.xml".parse; +local os_getenv = os.getenv; +local io_open = io.open; +local assert = assert; +local ipairs = ipairs; +local coroutine = coroutine; +local print = print; + +module "jabberd14" + +local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end +local function is_file(path) return lfs.attributes(path, "mode") == "file"; end +local function clean_path(path) + return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~"); +end + +local function load_xml(path) + local f, err = io_open(path); + if not f then return f, err; end + local data = f:read("*a"); + f:close(); + if not data then return; end + return parse_xml(data); +end + +local function load_spool_file(host, filename, path) + local xml = load_xml(path); + if not xml then return; end + + local register_element = xml:get_child("query", "jabber:iq:register"); + local username_element = register_element and register_element:get_child("username", "jabber:iq:register"); + local password_element = register_element and register_element:get_child("password", "jabber:iq:auth"); + local username = username_element and username_element:get_text(); + local password = password_element and password_element:get_text(); + if not username then + print("[warn] Missing /xdb/{jabber:iq:register}register/username> in file "..filename) + return; + elseif username..".xml" ~= filename then + print("[warn] Missing /xdb/{jabber:iq:register}register/username does not match filename "..filename); + return; + end + + local userdata = { + user = username; + host = host; + stores = {}; + }; + local stores = userdata.stores; + stores.accounts = { password = password }; + + for i=1,#xml.tags do + local tag = xml.tags[i]; + local xname = (tag.attr.xmlns or "")..":"..tag.name; + if tag.attr.j_private_flag == "1" and tag.attr.xmlns then + -- Private XML + stores.private = stores.private or {}; + tag.attr.j_private_flag = nil; + stores.private[tag.attr.xmlns] = st.preserialize(tag); + elseif xname == "jabber:iq:auth:password" then + if stores.accounts.password ~= tag:get_text() then + if password then + print("[warn] conflicting passwords") + else + stores.accounts.password = tag:get_text(); + end + end + elseif xname == "jabber:iq:register:query" then + -- already processed + elseif xname == "jabber:xdb:nslist:foo" then + -- ignore + elseif xname == "jabber:iq:auth:0k:zerok" then + -- ignore + elseif xname == "jabber:iq:roster:query" then + -- Roster + local roster = {}; + local subscription_types = { from = true, to = true, both = true, none = true }; + for _,item_element in ipairs(tag.tags) do + assert(item_element.name == "item"); + assert(item_element.attr.jid); + assert(subscription_types[item_element.attr.subscription]); + assert((item_element.attr.ask or "subscribe") == "subscribe") + if item_element.name == "item" then + local groups = {}; + for _,group_element in ipairs(item_element.tags) do + assert(group_element.name == "group"); + groups[group_element:get_text()] = true; + end + local item = { + name = item_element.attr.name; + subscription = item_element.attr.subscription; + ask = item_element.attr.ask; + groups = groups; + }; + roster[item_element.attr.jid] = item; + end + end + stores.roster = roster; + elseif xname == "jabber:iq:last:query" then + -- Last activity + elseif xname == "jabber:x:offline:foo" then + -- Offline messages + elseif xname == "vcard-temp:vCard" then + -- vCards + stores.vcard = st.preserialize(tag); + else + print("[warn] Unknown tag: "..xname); + end + end + return userdata; +end + +local function loop_over_users(path, host, cb) + for file in lfs.dir(path) do + if file:match("%.xml$") then + local user = load_spool_file(host, file, path.."/"..file); + if user then cb(user); end + end + end +end +local function loop_over_hosts(path, cb) + for host in lfs.dir(path) do + if host ~= "." and host ~= ".." and is_dir(path.."/"..host) then + loop_over_users(path.."/"..host, host, cb); + end + end +end + +function reader(input) + local path = clean_path(assert(input.path, "no input.path specified")); + assert(is_dir(path), "input.path is not a directory"); + + if input.host then + return coroutine.wrap(function() loop_over_users(input.path, input.host, coroutine.yield) end); + else + return coroutine.wrap(function() loop_over_hosts(input.path, coroutine.yield) end); + end +end + +return _M; diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua new file mode 100644 index 00000000..e7b774bb --- /dev/null +++ b/tools/migration/migrator/mtools.lua @@ -0,0 +1,56 @@ + + +local print = print; +local t_insert = table.insert; +local t_sort = table.sort; + +module "mtools" + +function sorted(params) + + local reader = params.reader; -- iterator to get items from + local sorter = params.sorter; -- sorting function + local filter = params.filter; -- filter function + + local cache = {}; + for item in reader do + if filter then item = filter(item); end + if item then t_insert(cache, item); end + end + if sorter then + t_sort(cache, sorter); + end + local i = 0; + return function() + i = i + 1; + return cache[i]; + end; + +end + +function merged(reader, merger) + + local item1 = reader(); + local merged = { item1 }; + return function() + while true do + if not item1 then return nil; end + local item2 = reader(); + if not item2 then item1 = nil; return merged; end + if merger(item1, item2) then + --print("merged") + item1 = item2; + t_insert(merged, item1); + else + --print("unmerged", merged) + item1 = item2; + local tmp = merged; + merged = { item1 }; + return tmp; + end + end + end; + +end + +return _M; diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua new file mode 100644 index 00000000..4462fb3e --- /dev/null +++ b/tools/migration/migrator/prosody_files.lua @@ -0,0 +1,138 @@ + +local print = print; +local assert = assert; +local setmetatable = setmetatable; +local tonumber = tonumber; +local char = string.char; +local coroutine = coroutine; +local lfs = require "lfs"; +local loadfile = loadfile; +local pcall = pcall; +local mtools = require "migrator.mtools"; +local next = next; +local pairs = pairs; +local json = require "util.json"; +local os_getenv = os.getenv; + +prosody = {}; +local dm = require "util.datamanager" + +module "prosody_files" + +local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end +local function is_file(path) return lfs.attributes(path, "mode") == "file"; end +local function clean_path(path) + return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~"); +end +local encode, decode; do + local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end }); + decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end + encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end +end +local function decode_dir(x) + if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then + return decode(x); + end +end +local function decode_file(x) + if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then + return decode(x:gsub("%.dat$", "")); + end +end +local function prosody_dir(path, ondir, onfile, ...) + for x in lfs.dir(path) do + local xpath = path.."/"..x; + if decode_dir(x) and is_dir(xpath) then + ondir(xpath, x, ...); + elseif decode_file(x) and is_file(xpath) then + onfile(xpath, x, ...); + end + end +end + +local function handle_root_file(path, name) + --print("root file: ", decode_file(name)) + coroutine.yield { user = nil, host = nil, store = decode_file(name) }; +end +local function handle_host_file(path, name, host) + --print("host file: ", decode_dir(host).."/"..decode_file(name)) + coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) }; +end +local function handle_store_file(path, name, store, host) + --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store)) + coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) }; +end +local function handle_host_store(path, name, host) + prosody_dir(path, function() end, handle_store_file, name, host); +end +local function handle_host_dir(path, name) + prosody_dir(path, handle_host_store, handle_host_file, name); +end +local function handle_root_dir(path) + prosody_dir(path, handle_host_dir, handle_root_file); +end + +local function decode_user(item) + local userdata = { + user = item[1].user; + host = item[1].host; + stores = {}; + }; + for i=1,#item do -- loop over stores + local result = {}; + local store = item[i]; + userdata.stores[store.store] = store.data; + store.user = nil; store.host = nil; store.store = nil; + end + return userdata; +end + +function reader(input) + local path = clean_path(assert(input.path, "no input.path specified")); + assert(is_dir(path), "input.path is not a directory"); + local iter = coroutine.wrap(function()handle_root_dir(path);end); + -- get per-user stores, sorted + local iter = mtools.sorted { + reader = function() + local x = iter(); + if x then + dm.set_data_path(path); + local err; + x.data, err = dm.load(x.user, x.host, x.store); + if x.data == nil and err then + error(("Error loading data at path %s for %s@%s (%s store)") + :format(path, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>"), 0); + end + return x; + end + end; + sorter = function(a, b) + local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; + local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; + return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); + end; + }; + -- merge stores to get users + iter = mtools.merged(iter, function(a, b) + return (a.host == b.host and a.user == b.user); + end); + + return function() + local x = iter(); + return x and decode_user(x); + end +end + +function writer(output) + local path = clean_path(assert(output.path, "no output.path specified")); + assert(is_dir(path), "output.path is not a directory"); + return function(item) + if not item then return; end -- end of input + dm.set_data_path(path); + for store, data in pairs(item.stores) do + assert(dm.store(item.user, item.host, store, data)); + end + end +end + +return _M; diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua new file mode 100644 index 00000000..27b5835e --- /dev/null +++ b/tools/migration/migrator/prosody_sql.lua @@ -0,0 +1,200 @@ + +local assert = assert; +local have_DBI, DBI = pcall(require,"DBI"); +local print = print; +local type = type; +local next = next; +local pairs = pairs; +local t_sort = table.sort; +local json = require "util.json"; +local mtools = require "migrator.mtools"; +local tostring = tostring; +local tonumber = tonumber; + +if not have_DBI then + error("LuaDBI (required for SQL support) was not found, please see http://prosody.im/doc/depends#luadbi", 0); +end + +module "prosody_sql" + +local function create_table(connection, params) + local create_sql = "CREATE TABLE `prosody` (`host` TEXT, `user` TEXT, `store` TEXT, `key` TEXT, `type` TEXT, `value` TEXT);"; + if params.driver == "PostgreSQL" then + create_sql = create_sql:gsub("`", "\""); + elseif params.driver == "MySQL" then + create_sql = create_sql:gsub("`value` TEXT", "`value` MEDIUMTEXT"); + end + + local stmt = connection:prepare(create_sql); + if stmt then + local ok = stmt:execute(); + local commit_ok = connection:commit(); + if ok and commit_ok then + local index_sql = "CREATE INDEX `prosody_index` ON `prosody` (`host`, `user`, `store`, `key`)"; + if params.driver == "PostgreSQL" then + index_sql = index_sql:gsub("`", "\""); + elseif params.driver == "MySQL" then + index_sql = index_sql:gsub("`([,)])", "`(20)%1"); + end + local stmt, err = connection:prepare(index_sql); + local ok, commit_ok, commit_err; + if stmt then + ok, err = assert(stmt:execute()); + commit_ok, commit_err = assert(connection:commit()); + end + elseif params.driver == "MySQL" then -- COMPAT: Upgrade tables from 0.8.0 + -- Failed to create, but check existing MySQL table here + local stmt = connection:prepare("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'"); + local ok = stmt:execute(); + local commit_ok = connection:commit(); + if ok and commit_ok then + if stmt:rowcount() > 0 then + local stmt = connection:prepare("ALTER TABLE prosody MODIFY COLUMN `value` MEDIUMTEXT"); + local ok = stmt:execute(); + local commit_ok = connection:commit(); + if ok and commit_ok then + print("Database table automatically upgraded"); + end + end + repeat until not stmt:fetch(); + end + end + end +end + +local function serialize(value) + local t = type(value); + if t == "string" or t == "boolean" or t == "number" then + return t, tostring(value); + elseif t == "table" then + local value,err = json.encode(value); + if value then return "json", value; end + return nil, err; + end + return nil, "Unhandled value type: "..t; +end +local function deserialize(t, value) + if t == "string" then return value; + elseif t == "boolean" then + if value == "true" then return true; + elseif value == "false" then return false; end + elseif t == "number" then return tonumber(value); + elseif t == "json" then + return json.decode(value); + end +end + +local function decode_user(item) + local userdata = { + user = item[1][1].user; + host = item[1][1].host; + stores = {}; + }; + for i=1,#item do -- loop over stores + local result = {}; + local store = item[i]; + for i=1,#store do -- loop over store data + local row = store[i]; + local k = row.key; + local v = deserialize(row.type, row.value); + if k and v then + if k ~= "" then result[k] = v; elseif type(v) == "table" then + for a,b in pairs(v) do + result[a] = b; + end + end + end + userdata.stores[store[1].store] = result; + end + end + return userdata; +end + +function reader(input) + local dbh = assert(DBI.Connect( + assert(input.driver, "no input.driver specified"), + assert(input.database, "no input.database specified"), + input.username, input.password, + input.host, input.port + )); + assert(dbh:ping()); + local stmt = assert(dbh:prepare("SELECT * FROM prosody")); + assert(stmt:execute()); + local keys = {"host", "user", "store", "key", "type", "value"}; + local f,s,val = stmt:rows(true); + -- get SQL rows, sorted + local iter = mtools.sorted { + reader = function() val = f(s, val); return val; end; + filter = function(x) + for i=1,#keys do + if not x[keys[i]] then return false; end -- TODO log error, missing field + end + if x.host == "" then x.host = nil; end + if x.user == "" then x.user = nil; end + if x.store == "" then x.store = nil; end + return x; + end; + sorter = function(a, b) + local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; + local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; + return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); + end; + }; + -- merge rows to get stores + iter = mtools.merged(iter, function(a, b) + return (a.host == b.host and a.user == b.user and a.store == b.store); + end); + -- merge stores to get users + iter = mtools.merged(iter, function(a, b) + return (a[1].host == b[1].host and a[1].user == b[1].user); + end); + return function() + local x = iter(); + return x and decode_user(x); + end; +end + +function writer(output, iter) + local dbh = assert(DBI.Connect( + assert(output.driver, "no output.driver specified"), + assert(output.database, "no output.database specified"), + output.username, output.password, + output.host, output.port + )); + assert(dbh:ping()); + create_table(dbh, output); + local stmt = assert(dbh:prepare("SELECT * FROM prosody")); + assert(stmt:execute()); + local stmt = assert(dbh:prepare("DELETE FROM prosody")); + assert(stmt:execute()); + local insert_sql = "INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)"; + if output.driver == "PostgreSQL" then + insert_sql = insert_sql:gsub("`", "\""); + end + local insert = assert(dbh:prepare(insert_sql)); + + return function(item) + if not item then assert(dbh:commit()) return dbh:close(); end -- end of input + local host = item.host or ""; + local user = item.user or ""; + for store, data in pairs(item.stores) do + -- TODO transactions + local extradata = {}; + for key, value in pairs(data) do + if type(key) == "string" and key ~= "" then + local t, value = assert(serialize(value)); + local ok, err = assert(insert:execute(host, user, store, key, t, value)); + else + extradata[key] = value; + end + end + if next(extradata) ~= nil then + local t, extradata = assert(serialize(extradata)); + local ok, err = assert(insert:execute(host, user, store, "", t, extradata)); + end + end + end; +end + + +return _M; diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua new file mode 100644 index 00000000..7c933b88 --- /dev/null +++ b/tools/migration/prosody-migrator.lua @@ -0,0 +1,132 @@ +#!/usr/bin/env lua + +CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR"); +CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR"); + +-- Substitute ~ with path to home directory in paths +if CFG_CONFIGDIR then + CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME")); +end + +if CFG_SOURCEDIR then + CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME")); +end + +local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua"; + +-- Command-line parsing +local options = {}; +local handled_opts = 0; +for i = 1, #arg do + if arg[i]:sub(1,2) == "--" then + local opt, val = arg[i]:match("([%w-]+)=?(.*)"); + if opt then + options[(opt:sub(3):gsub("%-", "_"))] = #val > 0 and val or true; + end + handled_opts = i; + else + break; + end +end +table.remove(arg, handled_opts); + +if CFG_SOURCEDIR then + package.path = CFG_SOURCEDIR.."/?.lua;"..package.path; + package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath; +else + package.path = "../../?.lua;"..package.path + package.cpath = "../../?.so;"..package.cpath +end + +local envloadfile = require "util.envload".envloadfile; + +-- Load config file +local function loadfilein(file, env) + if loadin then + return loadin(env, io.open(file):read("*a")); + else + return envloadfile(file, env); + end +end + +local config_file = options.config or default_config; +local from_store = arg[1] or "input"; +local to_store = arg[2] or "output"; + +config = {}; +local config_env = setmetatable({}, { __index = function(t, k) return function(tbl) config[k] = tbl; end; end }); +local config_chunk, err = loadfilein(config_file, config_env); +if not config_chunk then + print("There was an error loading the config file, check the file exists"); + print("and that the syntax is correct:"); + print("", err); + os.exit(1); +end + +config_chunk(); + +local have_err; +if #arg > 0 and #arg ~= 2 then + have_err = true; + print("Error: Incorrect number of parameters supplied."); +end +if not config[from_store] then + have_err = true; + print("Error: Input store '"..from_store.."' not found in the config file."); +end +if not config[to_store] then + have_err = true; + print("Error: Output store '"..to_store.."' not found in the config file."); +end + +function load_store_handler(name) + local store_type = config[name].type; + if not store_type then + print("Error: "..name.." store type not specified in the config file"); + return false; + else + local ok, err = pcall(require, "migrator."..store_type); + if not ok then + if package.loaded["migrator."..store_type] then + print(("Error: Failed to initialize '%s' store:\n\t%s") + :format(name, err)); + else + print(("Error: Unrecognised store type for '%s': %s") + :format(from_store, store_type)); + end + return false; + end + end + return true; +end + +have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output")); + +if have_err then + print(""); + print("Usage: "..arg[0].." FROM_STORE TO_STORE"); + print("If no stores are specified, 'input' and 'output' are used."); + print(""); + print("The available stores in your migrator config are:"); + print(""); + for store in pairs(config) do + print("", store); + end + print(""); + os.exit(1); +end + +local itype = config[from_store].type; +local otype = config[to_store].type; +local reader = require("migrator."..itype).reader(config[from_store]); +local writer = require("migrator."..otype).writer(config[to_store]); + +local json = require "util.json"; + +io.stderr:write("Migrating...\n"); +for x in reader do + --print(json.encode(x)) + writer(x); +end +writer(nil); -- close +io.stderr:write("Done!\n"); |