From 630035a369f5e5fe9783435011cda4bb78e7f889 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 21 Nov 2021 15:50:36 +0100 Subject: mod_cron: Initial commit of periodic task runner A number of modules now have periodic tasks that need to run, e.g. for cleaning out old messages or files. This has highlighted a need for coordinating and optimizing scheduling of such tasks. --- plugins/mod_cron.lua | 59 ++++++++++++++++++++++++++ teal-src/plugins/mod_cron.tl | 98 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 plugins/mod_cron.lua create mode 100644 teal-src/plugins/mod_cron.tl diff --git a/plugins/mod_cron.lua b/plugins/mod_cron.lua new file mode 100644 index 00000000..257bf399 --- /dev/null +++ b/plugins/mod_cron.lua @@ -0,0 +1,59 @@ +module:set_global(); + +local async = require("util.async"); + +local periods = { hourly = 3600; daily = 86400 } + +local active_hosts = {} + +function module.add_host(host_module) + + local last_run_times = host_module:open_store("cron", "map"); + active_hosts[host_module.host] = true; + + local function save_task(task, started_at) last_run_times:set(nil, task.id, started_at); end + + local function task_added(event) + local task = event.item; + if task.name == nil then task.name = task.when; end + if task.id == nil then task.id = event.source.name .. "/" .. task.name:gsub("%W", "_"):lower(); end + if task.last == nil then task.last = last_run_times:get(nil, task.id); end + task.save = save_task; + module:log("debug", "%s task %s added, last run %s", task.when, task.id, + task.last and require("util.datetime").datetime(task.last) or "never"); + return true + end + + local function task_removed(event) + local task = event.item; + host_module:log("debug", "Task %s removed", task.id); + return true + end + + host_module:handle_items("task", task_added, task_removed, true); + + function host_module.unload() active_hosts[host_module.host] = nil; end +end + +local function should_run(when, last) return not last or last + periods[when] <= os.time() end + +local function run_task(task) + local started_at = os.time(); + task:run(started_at); + task:save(started_at); +end + +local task_runner = async.runner(run_task); +module:add_timer(1, function() + module:log("info", "Running periodic tasks"); + local delay = 3600; + for host in pairs(active_hosts) do + module:log("debug", "Running periodic tasks for host %s", host); + for _, task in ipairs(module:context(host):get_host_items("task")) do + module:log("debug", "Considering %s task %s (%s)", task.when, task.id, task.run); + if should_run(task.when, task.last) then task_runner:run(task); end + end + end + module:log("debug", "Wait %ds", delay); + return delay +end); diff --git a/teal-src/plugins/mod_cron.tl b/teal-src/plugins/mod_cron.tl new file mode 100644 index 00000000..fc9e9934 --- /dev/null +++ b/teal-src/plugins/mod_cron.tl @@ -0,0 +1,98 @@ +module:set_global(); + +local async = require "util.async"; + +local record map_store + -- TODO move to somewhere sensible + get : function (map_store, string, K) : V + set : function (map_store, string, K, V) +end + +local enum frequency + "hourly" + "daily" +end + +local record task_spec + id : string -- unique id + name : string -- name or short description + when : frequency + last : integer + run : function (task_spec, integer) + save : function (task_spec, integer) +end + +local record task_event + source : module + item : task_spec +end + +local periods : { frequency : integer } = { hourly = 3600, daily = 86400 } + +local active_hosts : { string : boolean } = { } + +function module.add_host(host_module : moduleapi) + + local last_run_times = host_module:open_store("cron", "map") as map_store; + active_hosts[host_module.host] = true; + + local function save_task(task : task_spec, started_at : integer) + last_run_times:set(nil, task.id, started_at); + end + + local function task_added(event : task_event) : boolean + local task = event.item; + if task.name == nil then + task.name = task.when; + end + if task.id == nil then + task.id = event.source.name .. "/" .. task.name:gsub("%W", "_"):lower(); + end + if task.last == nil then + task.last = last_run_times:get(nil, task.id); + end + task.save = save_task; + module:log("debug", "%s task %s added, last run %s", task.when, task.id, + task.last and require"util.datetime".datetime(task.last) or "never"); + return true; + end + + local function task_removed(event : task_event) : boolean + local task = event.item; + host_module:log("debug", "Task %s removed", task.id); + return true; + end + + host_module:handle_items("task", task_added, task_removed, true); + + function host_module.unload() + active_hosts[host_module.host]=nil; + end +end + +local function should_run(when : frequency, last : integer) : boolean + return not last or last + periods[when] <= os.time(); +end + +local function run_task(task : task_spec) + local started_at = os.time(); + task:run(started_at); + task:save(started_at); +end + +local task_runner = async.runner(run_task); +module:add_timer(1, function() : integer + module:log("info", "Running periodic tasks"); + local delay = 3600; + for host in pairs(active_hosts) do + module:log("debug", "Running periodic tasks for host %s", host); + for _, task in ipairs(module:context(host):get_host_items("task") as { task_spec } ) do + module:log("debug", "Considering %s task %s (%s)", task.when, task.id, task.run); + if should_run(task.when, task.last) then task_runner:run(task); end + end + end + module:log("debug", "Wait %ds", delay); + return delay; +end); + +-- TODO measure load, pick a good time to do stuff -- cgit v1.2.3