From d81ee7d99a4bb9362bd2150512bcf2041e021b52 Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Tue, 2 Mar 2021 23:05:41 -0800 Subject: [PATCH] timestamp the setup and installer logs at some point, mysql disconnects the box code and it becomes hard to debug without the timestamps --- scripts/installer.sh | 42 +++++++++++++++++++++++------------------- setup/start.sh | 42 +++++++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/scripts/installer.sh b/scripts/installer.sh index 94d506681..ee0804f29 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -11,6 +11,10 @@ if [[ ${EUID} -ne 0 ]]; then exit 1 fi +function log() { + echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1" +} + readonly user=yellowtent readonly box_src_dir=/home/${user}/box @@ -23,9 +27,9 @@ readonly ubuntu_codename=$(lsb_release -cs) readonly is_update=$(systemctl is-active box && echo "yes" || echo "no") -echo "==> installer: Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION) <==" +log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)" -echo "==> installer: updating docker" +log "updating docker" readonly docker_version=20.10.3 if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then @@ -34,24 +38,24 @@ if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb - echo "==> installer: Waiting for all dpkg tasks to finish..." + log "Waiting for all dpkg tasks to finish..." while fuser /var/lib/dpkg/lock; do sleep 1 done while ! dpkg --force-confold --configure -a; do - echo "==> installer: Failed to fix packages. Retry" + log "Failed to fix packages. Retry" sleep 1 done # the latest docker might need newer packages while ! apt update -y; do - echo "==> installer: Failed to update packages. Retry" + log "Failed to update packages. Retry" sleep 1 done while ! apt install -y /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb; do - echo "==> installer: Failed to install docker. Retry" + log "Failed to install docker. Retry" sleep 1 done @@ -60,14 +64,14 @@ fi readonly nginx_version=$(nginx -v 2>&1) if [[ "${nginx_version}" != *"1.18."* ]]; then - echo "==> installer: installing nginx 1.18" + log "installing nginx 1.18" curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb # apt install with install deps (as opposed to dpkg -i) apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb rm /tmp/nginx.deb fi -echo "==> installer: updating node" +log "updating node" readonly node_version=14.15.4 if [[ "$(node --version)" != "v${node_version}" ]]; then mkdir -p /usr/local/node-${node_version} @@ -85,31 +89,31 @@ for try in `seq 1 10`; do # however by default npm drops privileges for npm rebuild # https://docs.npmjs.com/misc/config#unsafe-perm if cd "${box_src_tmp_dir}" && npm rebuild --unsafe-perm; then break; fi - echo "==> installer: Failed to rebuild, trying again" + log "Failed to rebuild, trying again" sleep 5 done if [[ ${try} -eq 10 ]]; then - echo "==> installer: npm rebuild failed, giving up" + log "npm rebuild failed, giving up" exit 4 fi -echo "==> installer: downloading new addon images" +log "downloading new addon images" images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));") -echo -e "\tPulling docker images: ${images}" +log "\tPulling docker images: ${images}" for image in ${images}; do if ! docker pull "${image}"; then # this pulls the image using the sha256 - echo "==> installer: Could not pull ${image}" + log "Could not pull ${image}" exit 5 fi if ! docker pull "${image%@sha256:*}"; then # this will tag the image for readability - echo "==> installer: Could not pull ${image%@sha256:*}" + log "Could not pull ${image%@sha256:*}" exit 6 fi done -echo "==> installer: update cloudron-syslog" +log "update cloudron-syslog" CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog" CLOUDRON_SYSLOG_VERSION="1.0.3" @@ -117,7 +121,7 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO rm -rf "${CLOUDRON_SYSLOG_DIR}" mkdir -p "${CLOUDRON_SYSLOG_DIR}" if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi - echo "===> installer: Failed to install cloudron-syslog, trying again" + log "Failed to install cloudron-syslog, trying again" sleep 5 done @@ -126,17 +130,17 @@ if ! id "${user}" 2>/dev/null; then fi if [[ "${is_update}" == "yes" ]]; then - echo "==> installer: stop box service for update" + log "stop box service for update" ${box_src_dir}/setup/stop.sh fi # ensure we are not inside the source directory, which we will remove now cd /root -echo "==> installer: switching the box code" +log "switching the box code" rm -rf "${box_src_dir}" mv "${box_src_tmp_dir}" "${box_src_dir}" chown -R "${user}:${user}" "${box_src_dir}" -echo "==> installer: calling box setup script" +log "calling box setup script" "${box_src_dir}/setup/start.sh" diff --git a/setup/start.sh b/setup/start.sh index 779600d6a..7f8996aa5 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -5,7 +5,11 @@ set -eu -o pipefail # This script is run after the box code is switched. This means that this script # should pretty much always succeed. No network logic/download code here. -echo "==> Cloudron Start" +function log() { + echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> start: $1" +} + +log "Cloudron Start" readonly USER="yellowtent" readonly HOME_DIR="/home/${USER}" @@ -26,7 +30,7 @@ if ! getent group media; then addgroup --gid 500 --system media fi -echo "==> Configuring docker" +log "Configuring docker" cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app systemctl enable apparmor systemctl restart apparmor @@ -39,7 +43,7 @@ mkdir -p "${BOX_DATA_DIR}" mkdir -p "${APPS_DATA_DIR}" # keep these in sync with paths.js -echo "==> Ensuring directories" +log "Ensuring directories" mkdir -p "${PLATFORM_DATA_DIR}/graphite" mkdir -p "${PLATFORM_DATA_DIR}/mysql" @@ -71,7 +75,7 @@ mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys mkdir -p /var/backups chmod 777 /var/backups -echo "==> Configuring journald" +log "Configuring journald" sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \ -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \ -i /etc/systemd/journald.conf @@ -92,7 +96,7 @@ setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal # Give user access to nginx logs (uses adm group) usermod -a -G adm ${USER} -echo "==> Setting up unbound" +log "Setting up unbound" # DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org) # We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!) # We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound @@ -102,7 +106,7 @@ cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-ne # update the root anchor after a out-of-disk-space situation (see #269) unbound-anchor -a /var/lib/unbound/root.key -echo "==> Adding systemd services" +log "Adding systemd services" cp -r "${script_dir}/start/systemd/." /etc/systemd/system/ [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service systemctl daemon-reload @@ -124,11 +128,11 @@ systemctl restart unbound # ensure cloudron-syslog runs systemctl restart cloudron-syslog -echo "==> Configuring sudoers" +log "Configuring sudoers" rm -f /etc/sudoers.d/${USER} cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER} -echo "==> Configuring collectd" +log "Configuring collectd" rm -rf /etc/collectd /var/log/collectd.log ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf" @@ -140,7 +144,7 @@ if [[ "${ubuntu_version}" == "20.04" ]]; then fi systemctl restart collectd -echo "==> Configuring logrotate" +log "Configuring logrotate" if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf fi @@ -150,10 +154,10 @@ cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/" # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/" -echo "==> Adding motd message for admins" +log "Adding motd message for admins" cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron -echo "==> Configuring nginx" +log "Configuring nginx" # link nginx config to system config unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx ln -s "${PLATFORM_DATA_DIR}/nginx" /etc/nginx @@ -181,12 +185,12 @@ if [[ ! -f /etc/mysql/mysql.cnf ]] || ! diff -q "${script_dir}/start/mysql.cnf" cp "${script_dir}/start/mysql.cnf" /etc/mysql/mysql.cnf while true; do if ! systemctl list-jobs | grep mysql; then break; fi - echo "Waiting for mysql jobs..." + log "Waiting for mysql jobs..." sleep 1 done while true; do if systemctl restart mysql; then break; fi - echo "Restarting MySql again after sometime since this fails randomly" + log "Restarting MySql again after sometime since this fails randomly" sleep 1 done else @@ -203,17 +207,17 @@ mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box' # set HOME explicity, because it's not set when the installer calls it. this is done because # paths.js uses this env var and some of the migrate code requires box code -echo "==> Migrating data" +log "Migrating data" cd "${BOX_SRC_DIR}" if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then - echo "DB migration failed" + log "DB migration failed" exit 1 fi rm -f /etc/cloudron/cloudron.conf if [[ ! -f "${BOX_DATA_DIR}/dhparams.pem" ]]; then - echo "==> Generating dhparams (takes forever)" + log "Generating dhparams (takes forever)" openssl dhparam -out "${BOX_DATA_DIR}/dhparams.pem" 2048 cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem" else @@ -230,7 +234,7 @@ if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then fi fi -echo "==> Changing ownership" +log "Changing ownership" # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change chown -R "${USER}" /etc/cloudron chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" @@ -244,9 +248,9 @@ find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail" chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys -echo "==> Starting Cloudron" +log "Starting Cloudron" systemctl start box sleep 2 # give systemd sometime to start the processes -echo "==> Almost done" +log "Almost done"