cloudron-support: check service uptime

scripts/cloudron-support

@@ -61,6 +61,24 @@ function enable_remote_access() {
     echo "Done"
 }
 
+function wait_systemd_service() {
+    local -r service="$1"
+
+    for i in {1..3}; do
+        ts=$(systemctl show "${service}" -p ActiveEnterTimestamp | sed 's/ActiveEnterTimestamp=//g')
+        start=$(date '+%s' --date="${ts}")
+        now=$(date '+%s')
+
+        up_time=$(( $now - $start ))
+        (( up_time > 10 )) && return 0
+
+        info "Service '${service}' just started $up_time secs ago, checking health again in 10s"
+        sleep 10
+    done
+
+    return 1
+}
+
 function check_host_mysql() {
     if ! systemctl is-active -q mysql; then
         info "MySQL is down. Trying to restart MySQL ..."
@@ -73,6 +91,11 @@ function check_host_mysql() {
         fi
     fi
 
+    if ! wait_systemd_service mysql; then
+        fail "MySQL keeps restarting, please investigate the error by inspecting /var/log/mysql/error.log"
+        exit 1
+    fi
+
     success "MySQL is running"
 }
 
@@ -87,11 +110,16 @@ function check_box() {
         systemctl start box
 
         if ! systemctl is-active -q box; then
-            fail "box is still down, please investigate the error by inspecting /home/yellowtent/platformdata/logs/box.log"
+            fail "box service is still down, please investigate the error by inspecting /home/yellowtent/platformdata/logs/box.log"
             exit 1
         fi
     fi
 
+    if ! wait_systemd_service box; then
+        fail "box service keeps restarting, please investigate the error by inspecting /home/yellowtent/platformdata/logs/box.log"
+        exit 1
+    fi
+
     success "box v${version} is running"
 }
 
@@ -183,6 +211,11 @@ function check_unbound() {
         fi
     fi
 
+    if ! wait_systemd_service unbound; then
+        fail "unbound service keeps restarting, please investigate the error using 'journalctl -u unbound'"
+        exit 1
+    fi
+
     test_resolve=$(dig cloudron.io @127.0.0.1 +short)
     if [[ -z "test_resolve" ]]; then
         fail "DNS is not resolving, maybe try forwarding all DNS requests using the --use-external-dns option"
@@ -219,11 +252,16 @@ function check_nginx() {
         systemctl restart nginx
 
         if ! systemctl is-active -q nginx; then
-            fail "nginx is still down, please investigate the error by inspecting /var/log/nginx/error.log"
+            fail "nginx is still down, please investigate the error by inspecting 'journalctl -u nginx' and /var/log/nginx/error.log"
             exit 1
         fi
     fi
 
+    if ! wait_systemd_service nginx; then
+        fail "nginx service keeps restarting, please investigate the error using 'journalctl -u nginx' and /var/log/nginx/error.log"
+        exit 1
+    fi
+
     success "nginx is running"
 }
 
@@ -264,6 +302,11 @@ function check_docker() {
         fi
     fi
 
+    if ! wait_systemd_service docker; then
+        fail "Docker keeps restarting, please investigate the error using 'journalctl -u docker'"
+        exit 1
+    fi
+
     success "docker is running"
 }
 
@@ -287,20 +330,6 @@ function check_node() {
     success "node version is correct"
 }
 
-function check_docker() {
-    if ! systemctl is-active -q docker; then
-        info "Docker is down. Trying to restart docker ..."
-        systemctl restart docker
-
-        if ! systemctl is-active -q docker; then
-            fail "Docker is still down, please investigate the error using 'journalctl -u docker'"
-            exit 1
-        fi
-    fi
-
-    success "docker is running"
-}
-
 function check_dashboard_site_domain() {
     local -r dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='dashboard_domain'" 2>/dev/null)
     local -r domain_provider=$(mysql -NB -uroot -ppassword -e "SELECT provider FROM box.domains WHERE domain='${dashboard_domain}'" 2>/dev/null)