diff --git a/CHANGES b/CHANGES index 02fe2ac77..4f83b4792 100644 --- a/CHANGES +++ b/CHANGES @@ -2870,4 +2870,5 @@ * mail: update Haraka to 3.0.5 * Add sqlite3 addon * docker: update docker to 27.3.1 +* du: add exclude file to skip filesystem usage checks diff --git a/setup/start.sh b/setup/start.sh index 9495f2d92..47076ff9a 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -86,6 +86,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/firewall" mkdir -p "${PLATFORM_DATA_DIR}/sshfs" mkdir -p "${PLATFORM_DATA_DIR}/cifs" mkdir -p "${PLATFORM_DATA_DIR}/oidc" +mkdir -p "${PLATFORM_DATA_DIR}/diskusage" # ensure backups folder exists and is writeable mkdir -p /var/backups @@ -231,11 +232,14 @@ if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_pa exit 1 fi +# migrate disk usage cache file +[[ -f "${PLATFORM_DATA_DIR}/diskusage.json" ]] && mv "${PLATFORM_DATA_DIR}/diskusage.json" "${PLATFORM_DATA_DIR}/diskusage/cache.json" + log "Changing ownership" # note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change chown -R "${USER}" /etc/cloudron -chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs" "${PLATFORM_DATA_DIR}/tls" "${PLATFORM_DATA_DIR}/oidc" +chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs" "${PLATFORM_DATA_DIR}/tls" "${PLATFORM_DATA_DIR}/oidc" "${PLATFORM_DATA_DIR}/diskusage" chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}" chown "${USER}:${USER}" "${APPS_DATA_DIR}" diff --git a/src/paths.js b/src/paths.js index 4aa684396..4c55974a8 100644 --- a/src/paths.js +++ b/src/paths.js @@ -40,7 +40,8 @@ exports = module.exports = { BACKUP_INFO_DIR: path.join(baseDir(), 'platformdata/backup'), UPDATE_DIR: path.join(baseDir(), 'platformdata/update'), UPDATE_CHECKER_FILE: path.join(baseDir(), 'platformdata/update/updatechecker.json'), - DISK_USAGE_FILE: path.join(baseDir(), 'platformdata/diskusage.json'), + DISK_USAGE_CACHE_FILE: path.join(baseDir(), 'platformdata/diskusage/cache.json'), + DISK_USAGE_EXCLUDE_FILE: path.join(baseDir(), 'platformdata/diskusage/exclude'), SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'), DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'), DHPARAMS_FILE: path.join(baseDir(), 'platformdata/dhparams.pem'), diff --git a/src/routes/test/system-test.js b/src/routes/test/system-test.js index 70ca35075..da76e3a92 100644 --- a/src/routes/test/system-test.js +++ b/src/routes/test/system-test.js @@ -130,7 +130,7 @@ describe('System', function () { describe('disk usage', function () { it('get succeeds with no cache', async function () { - safe.fs.unlinkSync(paths.DISK_USAGE_FILE); + safe.fs.unlinkSync(paths.DISK_USAGE_CACHE_FILE); const response = await superagent.get(`${serverUrl}/api/v1/system/disk_usage`) .query({ access_token: owner.token }) diff --git a/src/system.js b/src/system.js index 9cbc112f2..0d5702795 100644 --- a/src/system.js +++ b/src/system.js @@ -217,7 +217,7 @@ async function getMemory() { } async function getDiskUsage() { - const cache = safe.JSON.parse(safe.fs.readFileSync(paths.DISK_USAGE_FILE, 'utf8')); + const cache = safe.JSON.parse(safe.fs.readFileSync(paths.DISK_USAGE_CACHE_FILE, 'utf8')); if (cache?.disks) { cache.filesystems = cache.disks; // legacy cache file had "disks" delete cache.disks; @@ -234,6 +234,7 @@ async function updateDiskUsage(progressCallback) { let percent = 1; const dockerDf = await docker.df(); + const excludePaths = (safe.fs.readFileSync(paths.DISK_USAGE_EXCLUDE_FILE, 'utf8') || '').split('\n'); const fsCount = Object.keys(filesystems).length; for (const fsPath in filesystems) { @@ -256,6 +257,9 @@ async function updateDiskUsage(progressCallback) { content.usage = dockerDf.LayersSize; } else if (content.id === 'docker-volumes') { content.usage = dockerDf.Volumes.map((v) => v.UsageData.Size).reduce((a,b) => a + b, 0); + } else if (excludePaths.includes(fsPath)) { + debug(`updateDiskUsage: skipping since path ${fsPath} is excluded`); + content.usage = 0; } else { const [error, usage] = await safe(du(content.path)); if (error) progressCallback({ message: `du error: ${error.message}`}); // can happen if app is installing etc @@ -265,7 +269,7 @@ async function updateDiskUsage(progressCallback) { } } - if (!safe.fs.writeFileSync(paths.DISK_USAGE_FILE, JSON.stringify({ ts: now, filesystems }, null, 4), 'utf8')) throw new BoxError(BoxError.FS_ERROR, `Could not write du cache file: ${safe.error.message}`); + if (!safe.fs.writeFileSync(paths.DISK_USAGE_CACHE_FILE, JSON.stringify({ ts: now, filesystems }, null, 4), 'utf8')) throw new BoxError(BoxError.FS_ERROR, `Could not write du cache file: ${safe.error.message}`); return filesystems; } diff --git a/src/test/common.js b/src/test/common.js index 4e44b9868..53bf0fe39 100644 --- a/src/test/common.js +++ b/src/test/common.js @@ -228,7 +228,7 @@ async function domainSetup() { } async function setup() { - await fs.promises.rm(paths.DISK_USAGE_FILE, { force: true }); + await fs.promises.rm(paths.DISK_USAGE_CACHE_FILE, { force: true }); await domainSetup(); const ownerId = await users.createOwner(admin.email, admin.username, admin.password, admin.displayName, auditSource); admin.id = ownerId;