From b65730b853036b181041f010cd52980624c6ba88 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 7 Mar 2022 18:42:00 +0530 Subject: [PATCH] allow admins to enable configurable disk I/O profiling --- lib/commands/admin-rpc.js | 3 ++ lib/decrees.js | 12 +++++ lib/env.js | 13 +++++ lib/hk-util.js | 6 ++- lib/storage/blob.js | 2 + lib/storage/block.js | 2 + server.js | 7 +++ www/admin/inner.js | 104 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 148 insertions(+), 1 deletion(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 59750571e..3fc87bce5 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -315,6 +315,9 @@ var instanceStatus = function (Env, Server, cb) { disableIntegratedEviction: Env.disableIntegratedEviction, disableIntegratedTasks: Env.disableIntegratedTasks, + enableProfiling: Env.enableProfiling, + profilingWindow: Env.profilingWindow, + maxUploadSize: Env.maxUploadSize, premiumUploadSize: Env.premiumUploadSize, diff --git a/lib/decrees.js b/lib/decrees.js index 5f599705e..371a4f8d7 100644 --- a/lib/decrees.js +++ b/lib/decrees.js @@ -24,6 +24,8 @@ SET_PREMIUM_UPLOAD_SIZE // BACKGROUND PROCESSES DISABLE_INTEGRATED_TASKS DISABLE_INTEGRATED_EVICTION +ENABLE_PROFILING +SET_PROFILING_WINDOW // BROADCAST SET_LAST_BROADCAST_HASH @@ -143,6 +145,16 @@ var makeIntegerSetter = function (attr) { return makeGenericSetter(attr, args_isInteger); }; +var arg_isPositiveInteger = function (args) { + return Array.isArray(args) && isInteger(args[0]) && args[0] > 0; +}; + +// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['ENABLE_PROFILING', [true]]], console.log) +commands.ENABLE_PROFILING = makeBooleanSetter('enableProfiling'); + +// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PROFILING_WINDOW', [10000]]], console.log) +commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', arg_isPositiveInteger); + // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAX_UPLOAD_SIZE', [50 * 1024 * 1024]]], console.log) commands.SET_MAX_UPLOAD_SIZE = makeIntegerSetter('maxUploadSize'); diff --git a/lib/env.js b/lib/env.js index 9970bc4f9..656855396 100644 --- a/lib/env.js +++ b/lib/env.js @@ -54,6 +54,10 @@ module.exports.create = function (config) { launchTime: +new Date(), + enableProfiling: false, + profilingWindow: 10000, + bytesWritten: 0, + inactiveTime: config.inactiveTime, archiveRetentionTime: config.archiveRetentionTime, accountRetentionTime: config.accountRetentionTime, @@ -222,6 +226,15 @@ module.exports.create = function (config) { return typeof(config[key]) === 'string'? config[key]: def; }; + Env.incrementBytesWritten = function (n) { + if (!Env.enableProfiling) { return; } + if (!n || typeof(n) !== 'number' || n < 0) { return; } + Env.bytesWritten += n; + setTimeout(function () { + Env.bytesWritten -= n; + }, Env.profilingWindow); + }; + paths.pin = keyOrDefaultString('pinPath', './pins'); paths.block = keyOrDefaultString('blockPath', './block'); paths.data = keyOrDefaultString('filePath', './datastore'); diff --git a/lib/hk-util.js b/lib/hk-util.js index 276ac3e6f..a95eb19a6 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -380,10 +380,14 @@ const storeMessage = function (Env, channel, msg, isCp, optionalMessageHash, cb) // Message stored, call back cb(); - index.size += msgBin.length; + var msgLength = msgBin.length; + index.size += msgLength; // handle the next element in the queue next(); + + // keep track of how many bytes are written + Env.incrementBytesWritten(msgLength); })); }); }); diff --git a/lib/storage/blob.js b/lib/storage/blob.js index e5c7a2fce..a48f669af 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -131,11 +131,13 @@ var upload = function (Env, safeKey, content, cb) { blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); + Env.incrementBytesWritten(len); }); } else { session.blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); + Env.incrementBytesWritten(len); } }; diff --git a/lib/storage/block.js b/lib/storage/block.js index 1078f6d2e..90228c6bc 100644 --- a/lib/storage/block.js +++ b/lib/storage/block.js @@ -49,6 +49,7 @@ Block.archive = function (Env, publicKey, _cb) { return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH'); } + // TODO Env.incrementBytesWritten Fse.move(currentPath, archivePath, { overwrite: true, }, cb); @@ -83,6 +84,7 @@ Block.write = function (Env, publicKey, buffer, _cb) { })); }).nThen(function () { Fs.writeFile(path, buffer, { encoding: 'binary' }, cb); + Env.incrementBytesWritten(buffer && buffer.length); }); }; diff --git a/server.js b/server.js index 73f63a3e1..204cd5f6e 100644 --- a/server.js +++ b/server.js @@ -284,6 +284,13 @@ var send404 = function (res, path) { send404(res); }); }; +app.get('/api/profiling', function (req, res, next) { + if (!Env.enableProfiling) { return void send404(res); } + res.setHeader('Content-Type', 'text/javascript'); + res.send(JSON.stringify({ + bytesWritten: Env.bytesWritten, + })); +}); app.use(function (req, res, next) { res.status(404); diff --git a/www/admin/inner.js b/www/admin/inner.js index 98213b384..59e410b55 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -66,6 +66,7 @@ define([ ], 'stats': [ // Msg.admin_cat_stats 'cp-admin-refresh-stats', + 'cp-admin-uptime', 'cp-admin-active-sessions', 'cp-admin-active-pads', 'cp-admin-open-files', @@ -85,6 +86,8 @@ define([ 'performance': [ // Msg.admin_cat_performance 'cp-admin-refresh-performance', 'cp-admin-performance-profiling', + 'cp-admin-enable-disk-measurements', + 'cp-admin-bytes-written', ], 'network': [ // Msg.admin_cat_network 'cp-admin-update-available', @@ -644,6 +647,29 @@ define([ return $div; }; + Messages.admin_uptimeTitle = 'Launch time'; + Messages.admin_uptimeHint = 'Date and time at which the server was launched'; + + create['uptime'] = function () { + var key = 'uptime'; + var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle + var pre = h('pre'); + + var set = function () { + var uptime = APP.instanceStatus.launchTime; + if (typeof(uptime) !== 'number') { return; } + pre.innerText = new Date(uptime); + }; + + set(); + + $div.append(pre); + onRefreshStats.reg(function () { + set(); + }); + return $div; + }; + create['active-sessions'] = function () { var key = 'active-sessions'; var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle @@ -1739,6 +1765,84 @@ define([ return $div; }; + Messages.admin_enableDiskMeasurementsTitle = "Measure disk performance"; // XXX + Messages.admin_enableDiskMeasurementsHint = "If enabled, a JSON endpoint will be exposed under /api/profiling which keeps a running measurement of disk I/O within a configurable window. This setting can impact server performance and may reveal data you'd rather keep hidden. It is recommended that you leave it disabled unless you know what you are doing."; // XXX + + create['enable-disk-measurements'] = makeAdminCheckbox({ + key: 'enable-disk-measurements', + getState: function () { + return APP.instanceStatus.enableProfiling; + }, + query: function (val, setState) { + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'ADMIN_DECREE', + data: ['ENABLE_PROFILING', [val]] + }, function (e, response) { + if (e || response.error) { + UI.warn(Messages.error); + console.error(e, response); + } + APP.updateStatus(function () { + setState(APP.instanceStatus.enableProfiling); + }); + }); + }, + }); + + Messages.admin_bytesWrittenTitle = "Disk performance measurement window"; + Messages.admin_bytesWrittenHint = "If you have enabled disk performance measurements then the duration of the window can be configured below."; // XXX + Messages.admin_bytesWrittenDuration = "Duration of the window in milliseconds: {0}"; // XXX + Messages.admin_defaultDuration = "admin_defaultDuration"; // XXX + Messages.admin_setDuration = "Set duration"; // XXX + + var isPositiveInteger = function (n) { + return n && typeof(n) === 'number' && n % 1 === 0 && n > 0; + }; + + create['bytes-written'] = function () { + var key = 'bytes-written'; + var $div = makeBlock(key); + + var duration = APP.instanceStatus.profilingWindow; + if (!isPositiveInteger(duration)) { duration = 10000; } + var newDuration = h('input', {type: 'number', min: 0, value: duration}); + var set = h('button.btn.btn-primary', Messages.admin_setDuration); + $div.append(h('div', [ + h('span.cp-admin-bytes-written-duration', Messages._getKey('admin_bytesWrittenDuration', [duration])), + h('div.cp-admin-setlimit-form', [ + newDuration, + h('nav', [set]) + ]) + ])); + + UI.confirmButton(set, { + classes: 'btn-primary', + multiple: true, + validate: function () { + var l = parseInt($(newDuration).val()); + if (isNaN(l)) { return false; } + return true; + } + }, function () { + var d = parseInt($(newDuration).val()); + if (!isPositiveInteger(d)) { return void UI.warn(Messages.error); } + + var data = [d]; + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'ADMIN_DECREE', + data: ['SET_PROFILING_WINDOW', data] + }, function (e, response) { + if (e || response.error) { + UI.warn(Messages.error); + return void console.error(e, response); + } + $div.find('.cp-admin-bytes-written-duration').text(Messages._getKey('admin_limit', [d])); + }); + }); + + return $div; + }; + create['update-available'] = function () { // Messages.admin_updateAvailableTitle.admin_updateAvailableHint.admin_updateAvailableLabel.admin_updateAvailableButton if (!APP.instanceStatus.updateAvailable) { return; } var $div = makeBlock('update-available', true);