app.portBindings and newManifest.tcpPorts may be null

This commit is contained in:
Girish Ramakrishnan
2015-07-20 00:09:47 -07:00
commit df9d321ac3
243 changed files with 42623 additions and 0 deletions

279
setup/start/collectd.conf Normal file
View File

@@ -0,0 +1,279 @@
# Config file for collectd(1).
#
# Some plugins need additional configuration and are disabled by default.
# Please read collectd.conf(5) for details.
#
# You should also read /usr/share/doc/collectd-core/README.Debian.plugins
# before enabling any more plugins.
##############################################################################
# Global #
#----------------------------------------------------------------------------#
# Global settings for the daemon. #
##############################################################################
Hostname "localhost"
#FQDNLookup true
#BaseDir "/var/lib/collectd"
#PluginDir "/usr/lib/collectd"
#TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
#----------------------------------------------------------------------------#
# When enabled, plugins are loaded automatically with the default options #
# when an appropriate <Plugin ...> block is encountered. #
# Disabled by default. #
#----------------------------------------------------------------------------#
#AutoLoadPlugin false
#----------------------------------------------------------------------------#
# Interval at which to query values. This may be overwritten on a per-plugin #
# base by using the 'Interval' option of the LoadPlugin block: #
# <LoadPlugin foo> #
# Interval 60 #
# </LoadPlugin> #
#----------------------------------------------------------------------------#
# IMPORTANT: changing this value requires a change in whisper schema as well
Interval 20
#Timeout 2
#ReadThreads 5
#WriteThreads 5
# Limit the size of the write queue. Default is no limit. Setting up a limit
# is recommended for servers handling a high volume of traffic.
#WriteQueueLimitHigh 1000000
#WriteQueueLimitLow 800000
##############################################################################
# Logging #
#----------------------------------------------------------------------------#
# Plugins which provide logging functions should be loaded first, so log #
# messages generated when loading or configuring other plugins can be #
# accessed. #
##############################################################################
LoadPlugin logfile
#LoadPlugin syslog
<Plugin logfile>
LogLevel "info"
File "/var/log/collectd.log"
Timestamp true
PrintSeverity false
</Plugin>
#<Plugin syslog>
# LogLevel info
#</Plugin>
##############################################################################
# LoadPlugin section #
#----------------------------------------------------------------------------#
# Specify what features to activate. #
##############################################################################
LoadPlugin aggregation
#LoadPlugin amqp
#LoadPlugin apache
#LoadPlugin apcups
#LoadPlugin ascent
#LoadPlugin battery
#LoadPlugin bind
#LoadPlugin cgroups
#LoadPlugin conntrack
#LoadPlugin contextswitch
LoadPlugin cpu
#LoadPlugin cpufreq
#LoadPlugin csv
#LoadPlugin curl
#LoadPlugin curl_json
#LoadPlugin curl_xml
#LoadPlugin dbi
LoadPlugin df
#LoadPlugin disk
#LoadPlugin dns
#LoadPlugin email
#LoadPlugin entropy
#LoadPlugin ethstat
#LoadPlugin exec
#LoadPlugin filecount
#LoadPlugin fscache
#LoadPlugin gmond
#LoadPlugin hddtemp
LoadPlugin interface
#LoadPlugin ipmi
#LoadPlugin iptables
#LoadPlugin ipvs
#LoadPlugin irq
#LoadPlugin java
#LoadPlugin libvirt
LoadPlugin load
#LoadPlugin lvm
#LoadPlugin madwifi
#LoadPlugin mbmon
#LoadPlugin md
#LoadPlugin memcachec
#LoadPlugin memcached
LoadPlugin memory
#LoadPlugin modbus
#LoadPlugin multimeter
#LoadPlugin mysql
#LoadPlugin netlink
#LoadPlugin network
#LoadPlugin nfs
LoadPlugin nginx
#LoadPlugin notify_desktop
#LoadPlugin notify_email
#LoadPlugin ntpd
#LoadPlugin numa
#LoadPlugin nut
#LoadPlugin olsrd
#LoadPlugin openvpn
#<LoadPlugin perl>
# Globals true
#</LoadPlugin>
#LoadPlugin pinba
LoadPlugin ping
#LoadPlugin postgresql
#LoadPlugin powerdns
LoadPlugin processes
#LoadPlugin protocols
#<LoadPlugin python>
# Globals true
#</LoadPlugin>
#LoadPlugin rrdcached
#LoadPlugin rrdtool
#LoadPlugin sensors
#LoadPlugin serial
#LoadPlugin snmp
#LoadPlugin statsd
LoadPlugin swap
#LoadPlugin table
LoadPlugin tail
#LoadPlugin tail_csv
#LoadPlugin tcpconns
#LoadPlugin teamspeak2
#LoadPlugin ted
#LoadPlugin thermal
#LoadPlugin tokyotyrant
#LoadPlugin unixsock
#LoadPlugin uptime
#LoadPlugin users
#LoadPlugin uuid
#LoadPlugin varnish
LoadPlugin vmem
#LoadPlugin vserver
#LoadPlugin wireless
LoadPlugin write_graphite
#LoadPlugin write_http
#LoadPlugin write_riemann
##############################################################################
# Plugin configuration #
#----------------------------------------------------------------------------#
# In this section configuration stubs for each plugin are provided. A desc- #
# ription of those options is available in the collectd.conf(5) manual page. #
##############################################################################
<Plugin "aggregation">
<Aggregation>
Plugin "cpu"
Type "cpu"
GroupBy "Host"
GroupBy "TypeInstance"
CalculateNum false
CalculateSum true
CalculateAverage true
CalculateMinimum false
CalculateMaximum true
CalculateStddev false
</Aggregation>
</Plugin>
<Plugin df>
Device "/dev/vda1"
Device "/dev/loop0"
Device "/dev/loop1"
ReportByDevice true
IgnoreSelected false
ValuesAbsolute true
ValuesPercentage true
</Plugin>
<Plugin interface>
Interface "eth0"
IgnoreSelected false
</Plugin>
<Plugin nginx>
URL "http://127.0.0.1/nginx_status"
</Plugin>
<Plugin ping>
Host "google.com"
Interval 1.0
Timeout 0.9
TTL 255
</Plugin>
<Plugin processes>
ProcessMatch "app" "node app.js"
</Plugin>
<Plugin swap>
ReportByDevice false
ReportBytes true
</Plugin>
<Plugin "tail">
<File "/var/log/nginx/error.log">
Instance "nginx"
<Match>
Regex ".*"
DSType "CounterInc"
Type counter
Instance "errors"
</Match>
</File>
<File "/var/log/nginx/access.log">
Instance "nginx"
<Match>
Regex ".*"
DSType "CounterInc"
Type counter
Instance "requests"
</Match>
<Match>
Regex " \".*\" [0-9]+ [0-9]+ ([0-9]+)"
DSType GaugeAverage
Type delay
Instance "response"
</Match>
</File>
</Plugin>
<Plugin vmem>
Verbose false
</Plugin>
<Plugin write_graphite>
<Node "graphing">
Host "localhost"
Port "2003"
Protocol "tcp"
LogSendErrors true
Prefix "collectd."
StoreRates true
AlwaysAppendDS false
EscapeCharacter "_"
</Node>
</Plugin>
<Include "/etc/collectd/collectd.conf.d">
Filter "*.conf"
</Include>

View File

@@ -0,0 +1,115 @@
# http://nginx.org/en/docs/http/websocket.html
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen 443;
server_name <%= vhost %>;
ssl on;
# paths are relative to prefix and not to this file
ssl_certificate cert/host.cert;
ssl_certificate_key cert/host.key;
ssl_session_timeout 5m;
ssl_session_cache shared:SSL:50m;
# https://bettercrypto.org/static/applied-crypto-hardening.pdf
# https://mozilla.github.io/server-side-tls/ssl-config-generator/
# https://cipherli.st/
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
ssl_ciphers 'AES128+EECDH:AES128+EDH';
add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
proxy_http_version 1.1;
proxy_intercept_errors on;
proxy_read_timeout 3500;
proxy_connect_timeout 3250;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto https;
# upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
error_page 500 502 503 504 =200 @appstatus;
location @appstatus {
internal;
root <%= sourceDir %>/webadmin/dist;
rewrite ^/$ /appstatus.html break;
}
location / {
# increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
# Disable check to allow unlimited body sizes
client_max_body_size 0;
<% if ( endpoint === 'admin' ) { %>
location /api/ {
proxy_pass http://127.0.0.1:3000;
client_max_body_size 1m;
}
# graphite paths
location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
proxy_pass http://127.0.0.1:8000;
client_max_body_size 1m;
}
location / {
root <%= sourceDir %>/webadmin/dist;
index index.html index.htm;
}
<% } else if ( endpoint === 'oauthproxy' ) { %>
proxy_pass http://127.0.0.1:4000;
proxy_set_header X-Cloudron-Proxy-Port <%= port %>;
<% } else if ( endpoint === 'app' ) { %>
proxy_pass http://127.0.0.1:<%= port %>;
<% } else if ( endpoint === 'splash' ) { %>
root <%= sourceDir %>;
error_page 503 /update.html;
location /update.html {
add_header Cache-Control no-cache;
}
location /theme.css {
add_header Cache-Control no-cache;
}
location /3rdparty/ {
add_header Cache-Control no-cache;
}
location /js/ {
add_header Cache-Control no-cache;
}
location /progress.json {
add_header Cache-Control no-cache;
}
location /api/v1/cloudron/progress {
add_header Cache-Control no-cache;
default_type application/json;
alias <%= sourceDir %>/progress.json;
}
location / {
return 503;
}
<% } %>
}
}

View File

@@ -0,0 +1,80 @@
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
image/webp webp;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/ogg ogg;
audio/x-m4a m4a;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-m4v m4v;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}

View File

@@ -0,0 +1,64 @@
user www-data;
worker_processes 1;
pid /run/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
# the collectd config depends on this log format
log_format combined2 '$remote_addr - [$time_local] '
'"$request" $status $body_bytes_sent $request_time '
'"$http_referer" "$http_user_agent"';
access_log access.log combined2;
sendfile on;
keepalive_timeout 65;
# HTTP server
server {
listen 80;
# collectd
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
location / {
# redirect everything to HTTPS
return 301 https://$host$request_uri;
}
}
# We have to enable https for nginx to read in the vhost in http request
# and send a 404. This is a side-effect of using wildcard DNS
server {
listen 443 default_server;
ssl on;
ssl_certificate cert/host.cert;
ssl_certificate_key cert/host.key;
error_page 404 = @fallback;
location @fallback {
internal;
root <%= sourceDir %>/webadmin/dist;
rewrite ^/$ /nakeddomain.html break;
}
return 404;
}
include applications/*.conf;
}

93
setup/start/setup_infra.sh Executable file
View File

@@ -0,0 +1,93 @@
#!/bin/bash
set -eu -o pipefail
readonly DATA_DIR="/home/yellowtent/data"
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
arg_fqdn="$1"
# removing containers ensures containers are launched with latest config updates
# restore code in appatask does not delete old containers
infra_version="none"
[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
echo "Infrastructure is upto date"
exit 0
fi
echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
existing_containers=$(docker ps -qa)
echo "Remove containers: ${existing_containers}"
if [[ -n "${existing_containers}" ]]; then
echo "${existing_containers}" | xargs docker rm -f
fi
# graphite
docker run --restart=always -d --name="graphite" \
-p 127.0.0.1:2003:2003 \
-p 127.0.0.1:2004:2004 \
-p 127.0.0.1:8000:8000 \
-v "${DATA_DIR}/graphite:/app/data" cloudron/graphite:0.3.1
# mail
mail_container_id=$(docker run --restart=always -d --name="mail" \
-p 127.0.0.1:25:25 \
-h "${arg_fqdn}" \
-e "DOMAIN_NAME=${arg_fqdn}" \
-v "${DATA_DIR}/box/mail:/app/data" \
cloudron/mail:0.3.0)
echo "Mail container id: ${mail_container_id}"
# mysql
mysql_addon_root_password=$(pwgen -1 -s)
docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
readonly MYSQL_ROOT_HOST='${docker0_ip}'
EOF
mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-h "${arg_fqdn}" \
-v "${DATA_DIR}/mysql:/var/lib/mysql" \
-v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:r" \
cloudron/mysql:0.3.0)
echo "MySQL container id: ${mysql_container_id}"
# postgresql
postgresql_addon_root_password=$(pwgen -1 -s)
cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
EOF
postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-h "${arg_fqdn}" \
-v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:r" \
cloudron/postgresql:0.3.0)
echo "PostgreSQL container id: ${postgresql_container_id}"
# mongodb
mongodb_addon_root_password=$(pwgen -1 -s)
cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
EOF
mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-h "${arg_fqdn}" \
-v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:r" \
cloudron/mongodb:0.3.0)
echo "Mongodb container id: ${mongodb_container_id}"
if [[ "${infra_version}" == "none" ]]; then
# if no existing infra was found (for new and restoring cloudons), download app backups
echo "Marking installed apps for restore"
mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore" WHERE installationState = "installed"' box
else
# if existing infra was found, just mark apps for reconfiguration
mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure" WHERE installationState = "installed"' box
fi
echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"