From b6a24a9bb00a9461eace9ad2172d3774d7a2a71b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 8 Feb 2016 20:23:12 +0100 Subject: mod_storage_sql: Support non-string keys in map store --- plugins/mod_storage_sql.lua | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 57d964e2..31d83f68 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -134,7 +134,10 @@ function map_store:get(username, key) return deserialize(row[1], row[2]); end else - error("TODO: non-string keys"); + for row in engine:select("SELECT `type`, `value` FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", host, username or "", self.store, "") do + local data = deserialize(row[1], row[2]); + return data and data[key] or nil; + end end end); if not ok then return nil, result; end @@ -150,7 +153,16 @@ function map_store:set(username, key, data) engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, key, t, value); end else - error("TODO: non-string keys"); + local extradata = {}; + for row in engine:select("SELECT `type`, `value` FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", host, username or "", self.store, "") do + extradata = deserialize(row[1], row[2]); + break; + end + engine:delete("DELETE FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", + host, username or "", self.store, ""); + extradata[key] = data; + local t, value = assert(serialize(extradata)); + engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, "", t, value); end return true; end); -- cgit v1.2.3 From d96a0ef55c6de851369bd0a562caf349d4435c75 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 7 Aug 2014 12:15:15 -0400 Subject: storagemanager: When map store isn't available, fallback to keyval store [backported from trunk] --- core/storagemanager.lua | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/core/storagemanager.lua b/core/storagemanager.lua index c312cf05..39eae120 100644 --- a/core/storagemanager.lua +++ b/core/storagemanager.lua @@ -99,11 +99,47 @@ local function get_driver(host, store) return driver, driver_name; end -local function open(host, store, typ) +local map_shim_mt = { + __index = { + get = function(self, username, key) + local ret, err = self.keyval_store:get(username); + if ret == nil and err then return nil, err end + return ret[key]; + end; + set = function(self, username, key, data) + local current, err = self.keyval_store:get(username); + if current == nil then + if err then + return nil, err; + else + current = {}; + end + end + current[key] = data; + return self.keyval_store:set(username, current); + end; + }; +} + +local open; + +local function create_map_shim(host, store) + local keyval_store, err = open(host, store, "keyval"); + if keyval_store == nil then return nil, err end + return setmetatable({ + keyval_store = keyval_store; + }, map_shim_mt); +end + +function open(host, store, typ) local driver, driver_name = get_driver(host, store); local ret, err = driver:open(store, typ); if not ret then if err == "unsupported-store" then + if typ == "map" then -- Use shim on top of keyval store + log("debug", "map storage driver unavailable, using shim on top of keyval store."); + return create_map_shim(host, store); + end log("debug", "Storage driver %s does not support store %s (%s), falling back to null driver", driver_name, store, typ or ""); ret = null_storage_driver; -- cgit v1.2.3 From 07aeb61310bc5187677798f0fcf38e4241ddaae6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 8 Aug 2014 12:38:35 +0200 Subject: storagemanager: Fix map store shim if store is empty --- core/storagemanager.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storagemanager.lua b/core/storagemanager.lua index 39eae120..cf2c65d5 100644 --- a/core/storagemanager.lua +++ b/core/storagemanager.lua @@ -103,7 +103,7 @@ local map_shim_mt = { __index = { get = function(self, username, key) local ret, err = self.keyval_store:get(username); - if ret == nil and err then return nil, err end + if ret == nil then return nil, err end return ret[key]; end; set = function(self, username, key, data) -- cgit v1.2.3 From 38cee98c50dbbddd39cfd2cc91370807d22323a3 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 9 Feb 2016 16:54:56 +0100 Subject: storagemanager: Add set_keys method to map store shim --- core/storagemanager.lua | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/storagemanager.lua b/core/storagemanager.lua index cf2c65d5..4f04e594 100644 --- a/core/storagemanager.lua +++ b/core/storagemanager.lua @@ -118,6 +118,23 @@ local map_shim_mt = { current[key] = data; return self.keyval_store:set(username, current); end; + set_keys = function (self, username, keydatas) + local current, err = self.keyval_store:get(username); + if current == nil then + if err then + return nil, err; + else + current = keydatas; + end + else + for k,v in pairs(keydatas) do + if v == self.remove then v = nil; end + current[k] = v; + end + end + return self.keyval_store:set(username, current); + end; + remove = {}; }; } -- cgit v1.2.3 From 315a1fc1ca3d6e316088cf0aec66a738a8955cbf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 9 Feb 2016 16:56:27 +0100 Subject: mod_storage_sql: Implement map:set_keys, allowing multiple keys to be set in the same transaction --- plugins/mod_storage_sql.lua | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 31d83f68..5f25c563 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -127,6 +127,7 @@ end local map_store = {}; map_store.__index = map_store; +map_store.remove = {}; function map_store:get(username, key) local ok, result = engine:transaction(function() if type(key) == "string" and key ~= "" then @@ -144,25 +145,30 @@ function map_store:get(username, key) return result; end function map_store:set(username, key, data) + return self:set_keys(username, { [key] = data or self.remove }); +end +function map_store:set_keys(username, keydatas) local ok, result = engine:transaction(function() - if type(key) == "string" and key ~= "" then - engine:delete("DELETE FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", - host, username or "", self.store, key); - if data ~= nil then - local t, value = assert(serialize(data)); - engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, key, t, value); - end - else - local extradata = {}; - for row in engine:select("SELECT `type`, `value` FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", host, username or "", self.store, "") do - extradata = deserialize(row[1], row[2]); - break; + for key, data in pairs(keydatas) do + if type(key) == "string" and key ~= "" then + engine:delete("DELETE FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", + host, username or "", self.store, key); + if data ~= self.remove then + local t, value = assert(serialize(data)); + engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, key, t, value); + end + else + local extradata = {}; + for row in engine:select("SELECT `type`, `value` FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", host, username or "", self.store, "") do + extradata = deserialize(row[1], row[2]); + break; + end + engine:delete("DELETE FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", + host, username or "", self.store, ""); + extradata[key] = data; + local t, value = assert(serialize(extradata)); + engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, "", t, value); end - engine:delete("DELETE FROM `prosody` WHERE `host`=? AND `user`=? AND `store`=? AND `key`=?", - host, username or "", self.store, ""); - extradata[key] = data; - local t, value = assert(serialize(extradata)); - engine:insert("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)", host, username or "", self.store, "", t, value); end return true; end); -- cgit v1.2.3 From 3977731fad75dd16a6a08a5e7ebc73bc340938ab Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 9 Feb 2016 17:01:04 +0100 Subject: rostermanager: Use storagemanager --- core/rostermanager.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/rostermanager.lua b/core/rostermanager.lua index 0b72d0a6..e6c9ff03 100644 --- a/core/rostermanager.lua +++ b/core/rostermanager.lua @@ -18,7 +18,6 @@ local type = type; local hosts = hosts; local bare_sessions = prosody.bare_sessions; -local datamanager = require "util.datamanager" local um_user_exists = require "core.usermanager".user_exists; local st = require "util.stanza"; @@ -109,7 +108,8 @@ local function load_roster(username, host) else -- Attempt to load roster for non-loaded user log("debug", "load_roster: loading for offline user: %s@%s", username, host); end - local data, err = datamanager.load(username, host, "roster"); + local roster_store = require "core.storagemanager".open(host, "roster", "keyval"); + local data, err = roster_store:get(username); roster = data or {}; if user then user.roster = roster; end roster_metadata(roster, err); @@ -143,7 +143,8 @@ function save_roster(username, host, roster) metadata.version = (metadata.version or 0) + 1; end if metadata.broken then return nil, "Not saving broken roster" end - return datamanager.store(username, host, "roster", roster); + local roster_store = require "core.storagemanager".open(host, "roster", "keyval"); + return roster_store:set(username, roster); end log("warn", "save_roster: user had no roster to save"); return nil; -- cgit v1.2.3 From a687ceb76b1e3cc6c7f3a01969d311d4692dfd8c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 9 Feb 2016 17:09:01 +0100 Subject: rostermanager: Use map stores when only one contact is changed --- core/rostermanager.lua | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/core/rostermanager.lua b/core/rostermanager.lua index e6c9ff03..176eba5b 100644 --- a/core/rostermanager.lua +++ b/core/rostermanager.lua @@ -29,7 +29,7 @@ local function add_to_roster(session, jid, item) if session.roster then local old_item = session.roster[jid]; session.roster[jid] = item; - if save_roster(session.username, session.host) then + if save_roster(session.username, session.host, nil, jid) then return true; else session.roster[jid] = old_item; @@ -44,7 +44,7 @@ local function remove_from_roster(session, jid) if session.roster then local old_item = session.roster[jid]; session.roster[jid] = nil; - if save_roster(session.username, session.host) then + if save_roster(session.username, session.host, nil, jid) then return true; else session.roster[jid] = old_item; @@ -123,13 +123,13 @@ local function load_roster(username, host) return roster, err; end -function save_roster(username, host, roster) +function save_roster(username, host, roster, jid) if not um_user_exists(username, host) then log("debug", "not saving roster for %s@%s: the user doesn't exist", username, host); return nil; end - log("debug", "save_roster: saving roster for %s@%s", username, host); + log("debug", "save_roster: saving roster for %s@%s, (%s)", username, host, jid or "all contacts"); if not roster then roster = hosts[host] and hosts[host].sessions[username] and hosts[host].sessions[username].roster; --if not roster then @@ -143,8 +143,13 @@ function save_roster(username, host, roster) metadata.version = (metadata.version or 0) + 1; end if metadata.broken then return nil, "Not saving broken roster" end - local roster_store = require "core.storagemanager".open(host, "roster", "keyval"); - return roster_store:set(username, roster); + if jid == nil then + local roster_store = require "core.storagemanager".open(host, "roster", "keyval"); + return roster_store:set(username, roster); + else + local roster_store = require "core.storagemanager".open(host, "roster", "map"); + return roster_store:set_keys(username, { [false] = metadata, [jid] = roster[jid] or roster_store.remove }); + end end log("warn", "save_roster: user had no roster to save"); return nil; @@ -160,7 +165,7 @@ local function process_inbound_subscription_approval(username, host, jid) item.subscription = "both"; end item.ask = nil; - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end end @@ -184,7 +189,7 @@ local function process_inbound_subscription_cancellation(username, host, jid) end end if changed then - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end end @@ -208,7 +213,7 @@ local function process_inbound_unsubscribe(username, host, jid) end end if changed then - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end end @@ -241,7 +246,7 @@ local function set_contact_pending_in(username, host, jid) return; -- false end roster[false].pending[jid] = true; - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end function is_contact_pending_out(username, host, jid) local roster = load_roster(username, host); @@ -260,7 +265,7 @@ local function set_contact_pending_out(username, host, jid) -- subscribe end item.ask = "subscribe"; log("debug", "set_contact_pending_out: saving roster; set %s@%s.roster[%q].ask=subscribe", username, host, jid); - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end local function unsubscribe(username, host, jid) local roster = load_roster(username, host); @@ -275,7 +280,7 @@ local function unsubscribe(username, host, jid) elseif item.subscription == "to" then item.subscription = "none"; end - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end local function subscribed(username, host, jid) if is_contact_pending_in(username, host, jid) then @@ -291,7 +296,7 @@ local function subscribed(username, host, jid) item.subscription = "both"; end roster[false].pending[jid] = nil; - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end -- TODO else implement optional feature pre-approval (ask = subscribed) end local function unsubscribed(username, host, jid) @@ -311,7 +316,7 @@ local function unsubscribed(username, host, jid) is_subscribed = true; end end - local success = (pending or is_subscribed) and save_roster(username, host, roster); + local success = (pending or is_subscribed) and save_roster(username, host, roster, jid); return success, pending, subscribed; end @@ -320,7 +325,7 @@ local function process_outbound_subscription_request(username, host, jid) local item = roster[jid]; if item and (item.subscription == "none" or item.subscription == "from") then item.ask = "subscribe"; - return save_roster(username, host, roster); + return save_roster(username, host, roster, jid); end end -- cgit v1.2.3 From f856a201b70f1cbacb1d9715d44a6ab442a7fa06 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 9 Feb 2016 17:24:04 +0100 Subject: mod_storage_sql: Silence some harmless luacheck warnings --- plugins/mod_storage_sql.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 5f25c563..34fcacee 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -1,4 +1,6 @@ +-- luacheck: ignore 212/self + local json = require "util.json"; local sql = require "util.sql"; local xml_parse = require "util.xml".parse; @@ -125,6 +127,7 @@ end --- Archive store API +-- luacheck: ignore 512 431/user 431/store local map_store = {}; map_store.__index = map_store; map_store.remove = {}; -- cgit v1.2.3