cloudron-support: troubleshoot

scripts/cloudron-support

@@ -16,6 +16,7 @@ readonly HELP_MESSAGE="
  Options:
    --enable-remote-access    Enable SSH Remote Access for the Cloudron support team
    --send-diagnostics        Collects server diagnostics and uploads it to ${PASTEBIN}
+   --troubleshoot            Dashboard down? Run tests to identify the potential problem
    --owner-login             Login as owner
    --help                    Show this message
 "
@@ -37,7 +38,24 @@ function enable_remote_access() {
     echo "Done"
 }
 
+function check_host_mysql() {
+    if ! systemctl is-active -q mysql; then
+        echo "MySQL is down. Maybe restarting fixes it"
+
+        systemctl restart mysql
+
+        if ! systemctl is-active -q mysql; then
+            echo "MySQL is still down, please investigate the error by inspecting /var/log/mysql/error.log"
+            exit 1
+        fi
+    fi
+
+    echo "MySQL is OK"
+}
+
 function owner_login() {
+    check_host_mysql
+
     local -r owner_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL AND active=1 ORDER BY creationTime LIMIT 1" 2>/dev/null)
     local -r owner_password=$(pwgen -1s 12)
     local -r dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='dashboard_domain'" 2>/dev/null)
@@ -102,6 +120,99 @@ function send_diagnostics() {
     echo -e "\nPlease email the following link to support@cloudron.io : ${PASTEBIN}/${paste_key}"
 }
 
+function check_unbound() {
+    if ! systemctl is-active -q unbound; then
+        echo "unbound is down. updating root anchor to see if it fixes it"
+        unbound-anchor -a /var/lib/unbound/root.key
+        systemctl restart unbound
+
+        if ! systemctl is-active -q unbound; then
+            echo "unbound is still down, please investigate the error using 'journalctl -u unbound'"
+            exit 1
+        fi
+    fi
+
+    test_resolve=$(dig cloudron.io @127.0.0.1 +short)
+    if [[ -z "test_resolve" ]]; then
+        echo "DNS is not resolving, maybe try forwarding all DNS requests using the --use-external-dns option"
+        exit 1
+    fi
+
+    echo "unbound is OK"
+}
+
+function check_nginx() {
+    if ! systemctl is-active -q nginx; then
+        echo "nginx is down. checking if this is because of invalid certs"
+        # TODO: delete config files that use invalid certs and restart nginx
+    fi
+
+    echo "nginx is OK"
+}
+
+function check_docker() {
+    if ! systemctl is-active -q docker; then
+        echo "Docker is down. Maybe restarting fixes it"
+        systemctl restart docker
+
+        if ! systemctl is-active -q docker; then
+            echo "Docker is still down, please investigate the error using 'journalctl -u docker'"
+            exit 1
+        fi
+    fi
+
+    echo "docker is OK"
+}
+
+function check_hairpin_nat() {
+    local -r dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='dashboard_domain'" 2>/dev/null)
+    if ! curl --fail -s https://my.${dashboard_domain} >/dev/null; then
+        echo "Could not query dashboard domain. Is Hairpin NAT functional?"
+        exit 1
+    fi
+
+    echo "Hairpin NAT is OK"
+}
+
+function check_expired_domain() {
+    local -r dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='dashboard_domain'" 2>/dev/null)
+    local -r expdate=$(whois ${dashboard_domain} | egrep -i 'Expiration Date:|Expires on|Expiry Date:' | head -1 | awk '{print $NF}')
+    local -r expdate_secs=$(date -d"$expdate" +%s)
+    local -r curdate_secs="$(date +%s)"
+
+    if (( curdate_secs > expdate_secs )); then
+        echo "Domain appears to be expired"
+        exit 1
+    fi
+
+    echo "Domain is valid and has not expired"
+}
+
+function use_external_dns() {
+    local -r $conf_file="/etc/unbound/unbound.conf.d/forward-everything.conf"
+
+    echo "Forwarding all DNS requests to Google/Cloudflare DNS. To remove the forwarding, please delete $conf and 'systemctl restart unbound'"
+
+    cat > $conf_file <<EOF
+forward-zone:
+    name: "."
+    forward-addr: 1.1.1.1
+    forward-addr: 8.8.8.8
+EOF
+
+    systemctl restart unbound
+}
+
+function troubleshoot() {
+    # note: disk space test has already been run globally
+    check_nginx
+    check_docker
+    check_host_mysql
+    check_unbound
+    check_hairpin_nat # requires mysql to be checked
+    check_expired_domain
+}
+
 function check_disk_space() {
     # check if at least 10mb root partition space is available
     if [[ "`df --output="avail" / | sed -n 2p`" -lt "10240" ]]; then
@@ -124,7 +235,7 @@ function check_disk_space() {
 
 check_disk_space
 
-args=$(getopt -o "" -l "admin-login,enable-ssh,enable-remote-access,help,owner-login,send-diagnostics" -n "$0" -- "$@")
+args=$(getopt -o "" -l "admin-login,enable-ssh,enable-remote-access,help,owner-login,send-diagnostics,use-external-dns,troubleshoot" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -138,6 +249,8 @@ while true; do
         ;&
     --owner-login) owner_login; exit 0;;
     --send-diagnostics) send_diagnostics; exit 0;;
+    --troubleshoot) troubleshoot; exit 0;;
+    --use-external-dns) use_external_dns; exit 0;;
     --help) break;;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;