tildeverse
/
cryptpad
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

archive inactive data in one of the server's workers

This commit is contained in:
ansuz 2020-09-30 14:42:14 +05:30
parent 6ada5fd751
commit 04d1bc9cd7
5 changed files with 79 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lib/eviction.js
View File

@ -15,14 +15,11 @@ Env = {
limits: {
<unsafeKey>: <limit>,
},
config: {
inactiveTime: <number of days>,
archiveRetentionTime: <number of days>,
accountRetentionTime: <number of days>,
pinPath: <filesystem path>,
customLimits: <custom limits map>,
archiveRetentionTime: <number of days>,
accountRetentionTime: <number of days>,
inactiveTime: <number of days>,
paths: {
pin: <path to pin storage>
},
store,
pinStore,
@ -35,11 +32,9 @@ Env = {
module.exports = function (Env, cb) {
var complete = Util.once(Util.mkAsync(cb));
var config = Env.config;
// the administrator should have set an 'inactiveTime' in their config
// if they didn't, just exit.
if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") {
if (!Env.inactiveTime || typeof(Env.inactiveTime) !== "number") {
return void complete("NO_INACTIVE_TIME");
}
@ -55,10 +50,10 @@ module.exports = function (Env, cb) {
});
// files which have not been changed since before this date can be considered inactive
var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
var inactiveTime = +new Date() - (Env.inactiveTime * 24 * 3600 * 1000);
// files which were archived before this date can be considered safe to remove
var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
var retentionTime = +new Date() - (Env.archiveRetentionTime * 24 * 3600 * 1000);
var store;
var pinStore;
@ -111,7 +106,7 @@ module.exports = function (Env, cb) {
// if the admin has not set an 'archiveRetentionTime', this block makes no sense
// so just skip it
if (typeof(config.archiveRetentionTime) !== "number") { return; }
if (typeof(Env.archiveRetentionTime) !== "number") { return; }
// count the number of files which have been removed in this run
var removed = 0;
@ -165,7 +160,7 @@ module.exports = function (Env, cb) {
};
var removeArchivedBlobProofs = function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
if (typeof(Env.archiveRetentionTime) !== "number") { return; }
// Iterate over archive blob ownership proofs and remove them
// if they are older than the specified retention time
var removed = 0;
@ -190,7 +185,7 @@ module.exports = function (Env, cb) {
};
var removeArchivedBlobs = function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
if (typeof(Env.archiveRetentionTime) !== "number") { return; }
// Iterate over archived blobs and remove them
// if they are older than the specified retention time
var removed = 0;
@ -280,8 +275,8 @@ module.exports = function (Env, cb) {
var inactive = 0;
var accountRetentionTime;
if (typeof(config.accountRetentionTime) === 'number' && config.accountRetentionTime > 0) {
accountRetentionTime = +new Date() - (24 * 3600 * 1000 * config.accountRetentionTime);
if (typeof(Env.accountRetentionTime) === 'number' && Env.accountRetentionTime > 0) {
accountRetentionTime = +new Date() - (24 * 3600 * 1000 * Env.accountRetentionTime);
} else {
accountRetentionTime = -1;
}
@ -297,7 +292,7 @@ module.exports = function (Env, cb) {
};
var accountIsActive = function (mtime, pinList, id) {
// console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1); // XXX
// console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1);
// if their pin log has changed recently then consider them active
if (mtime && mtime > accountRetentionTime) {
return true;
@ -360,7 +355,7 @@ module.exports = function (Env, cb) {
};
Pins.load(w(done), {
pinPath: config.pinPath,
pinPath: Env.paths.pin,
handler: handler,
});
};
@ -515,9 +510,6 @@ module.exports = function (Env, cb) {
};
nThen(loadStorage)
.nThen(function () {
Log.info("EVICT_TIME_TO_LOAD_PINS", msSinceStart());
})
.nThen(removeArchivedChannels)
.nThen(removeArchivedBlobProofs)
.nThen(removeArchivedBlobs)

38
lib/historyKeeper.js
View File

@ -11,6 +11,7 @@ const Core = require("./commands/core");
const Store = require("./storage/file");
const BlobStore = require("./storage/blob");
const Workers = require("./workers/index");
//const Eviction = require("./eviction");
module.exports.create = function (config, cb) {
const Log = config.log;
@ -26,14 +27,15 @@ module.exports.create = function (config, cb) {
Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE');
// TODO populate Env with everything that you use from config
// so that you can stop passing around your raw config
// and more easily share state between historyKeeper and rpc
const Env = {
Log: Log,
// store
id: Crypto.randomBytes(8).toString('hex'),
inactiveTime: config.inactiveTime,
archiveRetentionTime: config.archiveRetentionTime,
accountRetentionTime: config.accountRetentionTime,
metadata_cache: {},
channel_cache: {},
queueStorage: WriteQueue(),
@ -254,6 +256,10 @@ module.exports.create = function (config, cb) {
verbose: config.verbose,
openFileLimit: config.openFileLimit,
inactiveTime: config.inactiveTime,
archiveRetentionTime: config.archiveRetentionTime,
accountRetentionTime: config.accountRetentionTime,
maxWorkers: config.maxWorkers,
}, w(function (err) {
if (err) {
@ -261,8 +267,8 @@ module.exports.create = function (config, cb) {
}
}));
}).nThen(function () {
if (config.disableIntegratedTasks) { return; }
config.intervals = config.intervals || {};
if (config.disableIntegratedTasks) { return; }
var tasks_running;
config.intervals.taskExpiration = setInterval(function () {
@ -275,6 +281,30 @@ module.exports.create = function (config, cb) {
tasks_running = false;
});
}, 1000 * 60 * 5); // run every five minutes
}).nThen(function () {
if (config.disableIntegratedEviction) { return; }
const ONE_DAY = 24 * 1000 * 60 * 60;
// setting the time of the last eviction to "now"
// effectively makes it so that we'll start evicting after the server
// has been up for at least one day
var last_eviction = +new Date();
var active = false;
config.intervals.eviction = setInterval(function () {
if (active) { return; }
var now = +new Date();
// evict inactive data once per day
if (last_eviction && (now - ONE_DAY) < last_eviction) { return; }
active = true;
Env.evictInactive(function (err) {
if (err) {
// NO_INACTIVE_TIME
Log.error('EVICT_INACTIVE_MAIN_ERROR', err);
}
active = false;
last_eviction = now;
});
}, 60 * 1000);
}).nThen(function () {
RPC.create(Env, function (err, _rpc) {
if (err) { throw err; }

20
lib/workers/db-worker.js
View File

@ -13,6 +13,7 @@ const Saferphore = require("saferphore");
const Logger = require("../log");
const Tasks = require("../storage/tasks");
const Nacl = require('tweetnacl/nacl-fast');
const Eviction = require("../eviction");
const Env = {
Log: {},
@ -39,13 +40,21 @@ const init = function (config, _cb) {
return void cb('E_INVALID_CONFIG');
}
Env.paths = {
pin: config.pinPath,
};
Env.inactiveTime = config.inactiveTime;
Env.archiveRetentionTime = config.archiveRetentionTime;
Env.accountRetentionTime = config.accountRetentionTime;
nThen(function (w) {
Store.create(config, w(function (err, _store) {
if (err) {
w.abort();
return void cb(err);
}
store = _store;
Env.store = store = _store;
}));
Store.create({
filePath: config.pinPath,
@ -54,7 +63,7 @@ const init = function (config, _cb) {
w.abort();
return void cb(err);
}
pinStore = _pinStore;
Env.pinStore = pinStore = _pinStore;
}));
BlobStore.create({
blobPath: config.blobPath,
@ -66,7 +75,7 @@ const init = function (config, _cb) {
w.abort();
return void cb(err);
}
blobStore = blob;
Env.blobStore = blobStore = blob;
}));
}).nThen(function (w) {
Tasks.create({
@ -436,6 +445,10 @@ const writeTask = function (data, cb) {
Env.tasks.write(data.time, data.task_command, data.args, cb);
};
const evictInactive = function (data, cb) {
Eviction(Env, cb);
};
const COMMANDS = {
COMPUTE_INDEX: computeIndex,
COMPUTE_METADATA: computeMetadata,
@ -449,6 +462,7 @@ const COMMANDS = {
REMOVE_OWNED_BLOB: removeOwnedBlob,
RUN_TASKS: runTasks,
WRITE_TASK: writeTask,
EVICT_INACTIVE: evictInactive,
};
COMMANDS.INLINE = function (data, cb) {

6
lib/workers/index.js
View File

@ -350,6 +350,12 @@ Workers.initialize = function (Env, config, _cb) {
}, cb);
};
Env.evictInactive = function (cb) {
sendCommand({
command: 'EVICT_INACTIVE',
}, cb);
};
Env.runTasks = function (cb) {
sendCommand({
command: 'RUN_TASKS',

10
scripts/evict-inactive.js
View File

@ -5,13 +5,17 @@ var BlobStore = require("../lib/storage/blob");
var Quota = require("../lib/commands/quota");
var config = require("../lib/load-config");
var Env = {
config: require("../lib/load-config"),
inactiveTime: config.inactiveTime,
archiveRetentionTime: config.archiveRetentionTime,
accountRetentionTime: config.accountRetentionTime,
paths: {
pin: config.pinPath,
},
};
var prepareEnv = function (Env, cb) {
var config = Env.config;
Env.customLimits = config.customLimits;
Quota.applyCustomLimits(Env);