Remove dummy record

Part of #113

.gitattributes

@@ -1,7 +1,6 @@
 # following files are skipped when exporting using git archive
-/release export-ignore
-/admin export-ignore
 test export-ignore
+docs export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
 

.gitignore

@@ -1,6 +1,5 @@
 node_modules/
 coverage/
-docs/
 webadmin/dist/
 setup/splash/website/
 installer/src/certs/server.key

CHANGES

@@ -583,3 +583,92 @@
 - Keep eventlogs only for a week
 - Throttle OOM mails
 
+[0.17.4]
+- Add warning for users moving to custom domains
+- Out of disk space and certificate renewal mails are now sent to cloudron owner for selfhosters
+- Fix a bug where selfhosted Cloudrons do not start because of a MySQL error
+- Implement new app versioning & update scheme
+
+[0.17.5]
+- Fix migration interface issue
+- Allow self hosted Cloudron to login to the Cloudron Store
+- Send mail to self hosted Cloudron admins about OOM and App died errors
+- Fix bug where box update emails are sent repeatedly
+
+[0.18.0]
+- Fix app bundle installation
+- Fix RBL lookup in mail server
+- Add spam filter for email
+
+[0.19.0]
+* New base image 0.19.0
+* Upgrade PostgreSQL and MySQL
+
+[0.19.1]
+* Make email optional (settings -> enable/disable mail)
+* Make PostgresSQL behave better in low memory cloudrons
+* Add demo mode check
+* Fix plan listing
+
+[0.20.0]
+* Fix bug where crash reports where not being sent to support@cloudron.io (#29)
+* Do not overwrite existing DNS records during app installation (#27)
+* Add UI to configure app's memory limit (#18)
+* Fix OAuth proxy support (#6)
+
+[0.20.1]
+* Fix bug where oauth proxy was installed for apps with customAuth
+
+[0.20.2]
+* Fix memory limit slider to start from the minimum memory (#43)
+* Save user certs separately from automatic certs (#44)
+* Fix access control display for email apps (#45)
+
+[0.20.3]
+* Make DigitalOcean selfhosting independent
+
+[0.21.0]
+* Delivery of email to aliases is now case insensitive (#35)
+* Mailing list support via Groups (#15)
+* Fix issue where non-admin users could not update their profile
+
+[0.21.1]
+* Fix app clone error (mailbox was not allocated)
+* Do not allow "-" in group names
+
+[0.22.0]
+* Rebuild server instances instead of recreating
+
+[0.50.0]
+* Add UI to configure backup location
+* Add DNS backend to make it easy to run on any server with SSH access
+* Update wildcard certificate
+* Fix crash in mail container with SPF plugin
+* Fix postgresql addon to restore correctly
+* Periodically cleanup file system backups
+* Improve invitation emails
+* Fix bug where mailbox name was generated incorrectly for nake domain (#81)
+
+[0.60.0]
+* Implement new approach to selfhosting. `cloudron machine create` is now deprecated.
+  Please see the [selfhosting guide](https://cloudron.io/references/selfhosting.html)
+  for more details
+* Send email to admins if backup fails
+* Add UI to set digitalocean as DNS provider
+
+[0.60.1]
+* Apply less strict hostname checking for email
+* Fix bug in Cloudron plan listing
+* Improved storage provider interface
+
+[0.70.0]
+* Remove standalone installer daemon
+
+[0.70.1]
+* Add additional platform healthcheck
+
+[0.80.0]
+* Add optional SSO for apps
+* Improve app status page
+* Several webinterface improvements
+

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    box
+    Copyright (C) 2016  yellowtent
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.

README.md

@@ -1,17 +1,79 @@
-Cloudron a Smart Server
-=======================
+# Cloudron
 
+[Cloudron](https://cloudron.io) is the best way to run apps on your server.
 
+Web applications like email, contacts, blog, chat are the backbone of the modern
+internet. Yet, we live in a world where hosting these essential applications is
+a complex task.
 
-Selfhost Instructions
----------------------
+We are building the ultimate platform for self-hosting web apps. The Cloudron allows
+anyone to effortlessly host web applications on their server on their own terms.
 
-The smart server currently relies on an AWS account with access to Route53 and S3 and is tested on DigitalOcean and EC2.
+Support us on
+[![Flattr Cloudron](https://button.flattr.com/flattr-badge-large.png)](https://flattr.com/submit/auto?user_id=cloudron&url=https://cloudron.io&title=Cloudron&tags=opensource&category=software)
+or [pay us a coffee](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8982CKNM46D8U)
 
-First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
+## Features
+
+* Single click install for apps. Check out the [App Store](https://cloudron.io/appstore.html).
+
+* Per-app encrypted backups and restores.
+
+* App updates delivered via the App Store.
+
+* Secure - Cloudron manages the firewall. All apps are secured with HTTPS. Certificates are
+  installed and renewed automatically.
+
+* Centralized User & Group management. Control who can access which app.
+
+* Single Sign On. Use same credentials across all apps.
+
+* Automatic updates for the Cloudron platform.
+
+* Trivially migrate to another server keeping your apps and data (for example, switch your
+  infrastructure provider or move to a bigger server).
+
+* Comprehensive [REST API](https://cloudron.io/references/api.html).
+
+* [CLI](https://git.cloudron.io/cloudron/cloudron-cli) to configure apps.
+
+* Alerts, audit logs, graphs, dns management ... and much more
+
+## Demo
+
+Try our demo at https://my-demo.cloudron.me (username: cloudron password: cloudron).
+
+## Installing
+
+You can install the Cloudron platform on your own server or get a managed server
+from cloudron.io.
+
+* [Selfhosting](https://cloudron.io/references/selfhosting.html)
+* [Managed Hosting](https://cloudron.io/pricing.html)
+
+## Documentation
+
+* [User manual](https://cloudron.io/references/usermanual.html)
+* [Developer docs](https://cloudron.io/documentation.html)
+* [Architecture](https://cloudron.io/references/architecture.html)
+
+## Related repos
+
+The [base image repo](https://git.cloudron.io/cloudron/docker-base-image) is the parent image of all
+the containers in the Cloudron.
+
+The [graphite repo](https://git.cloudron.io/cloudron/docker-graphite) contains the graphite code
+that collects metrics for graphs.
+
+The addons are located in separate repositories
+* [Redis](https://git.cloudron.io/cloudron/redis-addon)
+* [Postgresql](https://git.cloudron.io/cloudron/postgresql-addon)
+* [MySQL](https://git.cloudron.io/cloudron/mysql-addon)
+* [Mongodb](https://git.cloudron.io/cloudron/mongodb-addon)
+* [Mail](https://git.cloudron.io/cloudron/mail-addon)
+
+## Community
+
+* [Chat](https://chat.cloudron.io/)
+* [Support](mailto:support@cloudron.io)
 
-```
-curl https://s3.amazonaws.com/prod-cloudron-releases/installer.sh -o installer.sh
-chmod +x installer.sh
-./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
-```

baseimage/createDOImage

@@ -138,7 +138,7 @@ cd "${SOURCE_DIR}"
 git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /tmp/box.tar.gz"
 
 echo "Executing init script"
-if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision} caas"; then
     echo "Init script failed"
     exit 1
 fi

baseimage/createEC2Image

@@ -1,185 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-export JSON="${SOURCE_DIR}/node_modules/.bin/json"
-
-installer_revision=$(git rev-parse HEAD)
-instance_id=""
-server_ip=""
-destroy_server="yes"
-
-ami_id="ami-f9e30f96"
-region="eu-central-1"
-aws_credentials="baseimage"
-security_group="sg-b9a473d1"
-instance_type="t2.small"
-subnet_id="subnet-801402e9"
-key_pair_name="id_rsa_yellowtent"
-
-# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
-# brew install gnu-getopt to get the GNU getopt on OS X
-[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-args=$(${GNU_GETOPT} -o "" -l "revisio0n:,no-destroy" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --revision) installer_revision="$2"; shift 2;;
-    --no-destroy) destroy_server="no"; shift 2;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
-readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-if [[ ! -f "${ssh_keys}" ]]; then
-    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
-    exit 1
-fi
-
-function debug() {
-    echo "$@" >&2
-}
-
-function get_pretty_revision() {
-    local git_rev="$1"
-    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
-
-    echo "${sha1}"
-}
-
-now=$(date "+%Y-%m-%d-%H%M%S")
-pretty_revision=$(get_pretty_revision "${installer_revision}")
-
-echo "Creating EC2 instance"
-instance_id=$(aws ec2 run-instances --image-id ${ami_id} --region ${region} --profile ${aws_credentials} --security-group-ids ${security_group} --instance-type ${instance_type} --key-name ${key_pair_name} --subnet-id ${subnet_id} --associate-public-ip-address | $JSON Instances[0].InstanceId)
-echo "Got InstanceId: ${instance_id}"
-
-# name the instance
-aws ec2 create-tags --profile ${aws_credentials} --resources ${instance_id} --tags "Key=Name,Value=baseimage-${pretty_revision}"
-
-echo "Waiting for instance to be running..."
-while true; do
-    event_status=`aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].State.Name`
-    if [[ "${event_status}" == "running" ]]; then
-        break
-    fi
-    debug -n "."
-    sleep 10
-done
-
-server_ip=$(aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].PublicIpAddress)
-echo "Server IP is: ${server_ip}"
-
-while true; do
-    echo "Trying to copy init script to server"
-    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" ubuntu@${server_ip}:.; then
-        break
-    fi
-    echo "Timedout, trying again in 30 seconds"
-    sleep 30
-done
-
-echo "Copying infra_version.js"
-$scp22 "${SCRIPT_DIR}/../src/infra_version.js" ubuntu@${server_ip}:.
-
-echo "Copying box source"
-cd "${SOURCE_DIR}"
-git archive --format=tar HEAD | $ssh22 "ubuntu@${server_ip}" "cat - > /tmp/box.tar.gz"
-
-echo "Enabling root ssh access"
-if ! $ssh22 "ubuntu@${server_ip}" "sudo sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys"; then
-    echo "Unable to enable root access"
-    echo "Make sure to cleanup the ec2 instance ${instance_id}"
-    exit 1
-fi
-
-echo "Executing init script"
-if ! $ssh22 "root@${server_ip}" "/bin/bash /home/ubuntu/initializeBaseUbuntuImage.sh ${installer_revision}"; then
-    echo "Init script failed"
-    echo "Make sure to cleanup the ec2 instance ${instance_id}"
-    exit 1
-fi
-
-snapshot_name="cloudron-${pretty_revision}-${now}"
-
-echo "Creating ami image ${snapshot_name}"
-image_id=$(aws ec2 create-image --region ${region} --profile ${aws_credentials} --instance-id ${instance_id} --name ${snapshot_name} | $JSON ImageId)
-
-echo "Image creation started for image id: ${image_id}"
-
-echo "Waiting for image creation to finish..."
-while true; do
-    event_status=`aws ec2 describe-images --region ${region} --profile ${aws_credentials} --image-id ${image_id} | $JSON Images[0].State`
-    if [[ "${event_status}" == "available" ]]; then
-        break
-    fi
-    debug -n "."
-    sleep 10
-done
-
-echo "Terminating instance"
-aws ec2 terminate-instances --region ${region} --profile ${aws_credentials} --instance-ids ${instance_id}
-
-echo "Make image public"
-aws ec2 modify-image-attribute --region ${region} --profile ${aws_credentials} --image-id ${image_id} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
-
-
-# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
-# Images are currently created in eu-central-1
-echo "Coping image to other regions"
-ec2_regions=( "us-east-1" "us-west-1" "us-west-2" "ap-south-1" "ap-northeast-2" "ap-southeast-1" "ap-southeast-2" "ap-northeast-1" "eu-west-1" "sa-east-1" )
-ec2_amis=( )
-
-for r in ${ec2_regions[@]}; do
-    echo "=> ${r}"
-    ami_id=$(aws ec2 copy-image --region ${r} --profile ${aws_credentials} --source-image-id ${image_id} --source-region ${region} --name ${snapshot_name} | $JSON ImageId)
-
-    # append in the same order as the regions
-    ec2_amis+=( ${ami_id} )
-done
-
-# wait for all images to be available
-echo "Waiting for images to be ready (first will take the longest)..."
-region_string="${region}=${image_id}"
-i=0
-while [ $i -lt ${#ec2_regions[*]} ]; do
-    echo "=> ${ec2_regions[$i]} ${ec2_amis[$i]}"
-    while true; do
-        event_status=`aws ec2 describe-images --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} | $JSON Images[0].State`
-        if [[ "${event_status}" == "available" ]]; then
-            echo "done"
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-
-    # now make it public
-    aws ec2 modify-image-attribute --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
-
-    # append to output string for release tool
-    region_string+=",${ec2_regions[$i]}=${ec2_amis[$i]}"
-
-    # inc the iteration counter
-    i=$(( $i + 1));
-done
-
-echo ""
-echo "--------------------------------------------------"
-echo "New image id is: ${image_id}"
-echo "Image region string for release:"
-echo "${region_string}"
-echo "--------------------------------------------------"
-echo ""

baseimage/initializeBaseUbuntuImage.sh

@@ -5,7 +5,8 @@ set -euv -o pipefail
 readonly USER=yellowtent
 readonly USER_HOME="/home/${USER}"
 readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
-readonly INSTALLER_REVISION="$1"
+readonly INSTALLER_REVISION="${1:-master}"
+readonly PROVIDER="${2:-generic}"
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -23,20 +24,11 @@ if ! id "${USER}"; then
     useradd "${USER}" -m
 fi
 
-echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ==="
-
-echo "=== Prepare installer source ==="
-rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}"
-rm -rf /tmp/box && mkdir -p /tmp/box
-tar xvf /tmp/box.tar.gz -C /tmp/box && rm /tmp/box.tar.gz
-cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}"
-echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
-
 export DEBIAN_FRONTEND=noninteractive
 
 echo "=== Upgrade ==="
-apt-get update
-apt-get dist-upgrade -y
+apt-get -o Dpkg::Options::="--force-confdef" update -y
+apt-get -o Dpkg::Options::="--force-confdef" dist-upgrade -y
 apt-get install -y curl
 
 # Setup firewall before everything. docker creates it's own chain and the -X below will remove it
@@ -46,14 +38,19 @@ echo "=== Setting up firewall ==="
 iptables -F # flush all chains
 iptables -X # delete all chains
 # default policy for filter table
-iptables -P INPUT DROP
+iptables -P INPUT ACCEPT # accept by default to allow network drives to persist
 iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject
 iptables -P OUTPUT ACCEPT
 
 # NOTE: keep these in sync with src/apps.js validatePortBindings
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
+# caas has ssh on port 202
+if [[ "${PROVIDER}" == "caas" ]]; then
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
+else
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,22,443,587,993,4190 -j ACCEPT
+fi
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -A INPUT -p udp --sport 53 -j ACCEPT
@@ -68,7 +65,7 @@ iptables -A OUTPUT -o lo -j ACCEPT
 
 # log dropped incoming. keep this at the end of all the rules
 iptables -N LOGGING # new chain
-iptables -A INPUT -j LOGGING # last rule in INPUT chain
+iptables -A INPUT -j LOGGING # last rule in INPUT chain (log and drop)
 iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
 iptables -A LOGGING -j DROP
 
@@ -77,6 +74,7 @@ apt-get -y install btrfs-tools
 
 echo "==== Install docker ===="
 # install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
+# IMPORTANT: docker 1.11.x breaks the --dns option hack that we use below
 curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
 apt-get -y install aufs-tools
 chmod +x /usr/bin/docker
@@ -102,7 +100,7 @@ After=network.target docker.socket
 Requires=docker.socket
 
 [Service]
-ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs
+ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --dns 127.0.0.1
 MountFlags=slave
 LimitNOFILE=1048576
 LimitNPROC=1048576
@@ -113,7 +111,12 @@ WantedBy=multi-user.target
 EOF
 
 echo "=== Setup btrfs data ==="
-truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
+if ! grep -q loop.ko /lib/modules/`uname -r`/modules.builtin; then
+    # on scaleway loop is not built-in
+    echo "loop" >> /etc/modules
+    modprobe loop
+fi
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by cloudron-system-setup.service)
 mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
 mkdir -p "${USER_DATA_DIR}"
 mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
@@ -133,8 +136,13 @@ iptables -I FORWARD -d 169.254.169.254 -j DROP
 mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4
 
 echo "=== Enable memory accounting =="
-sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
-update-grub
+if [[ "${PROVIDER}" == "digitalocean" ]] || [[ "${PROVIDER}" == "caas" ]]; then
+    sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="console=tty1 root=LABEL=DOROOT notsc clocksource=kvm-clock net.ifnames=0 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+    update-grub
+elif [[ "${PROVIDER}" == "ec2" ]] || [[ "${PROVIDER}" == "generic" ]]; then
+    sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+    update-grub
+fi
 
 # now add the user to the docker group
 usermod "${USER}" -a -G docker
@@ -149,12 +157,16 @@ apt-get install -y python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
 echo "==== Downloading docker images ===="
-images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+if [ -f ${SOURCE_DIR}/infra_version.js ]; then
+    images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImages.join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-echo "Pulling images: ${images}"
-for image in ${images}; do
-    docker pull "${image}"
-done
+    echo "Pulling images: ${images}"
+    for image in ${images}; do
+        docker pull "${image}"
+    done
+else
+    echo "No infra_versions.js found, skipping image download"
+fi
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
@@ -185,28 +197,16 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron
 
-echo "=== Rebuilding npm packages ==="
-cd "${INSTALLER_SOURCE_DIR}" && npm install --production
+echo "=== Prepare installer revision - ${INSTALLER_REVISION}) ==="
+rm -rf /tmp/box && mkdir -p /tmp/box
+curl "https://git.cloudron.io/cloudron/box/repository/archive.tar.gz?ref=${INSTALLER_REVISION}" | tar zxvf - --strip-components=1 -C /tmp/box
+mkdir -p "${INSTALLER_SOURCE_DIR}"
+cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}" && rm -rf /tmp/box
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
+echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
 
-echo "==== Install installer systemd script ===="
-cat > /etc/systemd/system/cloudron-installer.service <<EOF
-[Unit]
-Description=Cloudron Installer
-; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
-BindsTo=systemd-journald.service
-
-[Service]
-Type=idle
-ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
-Environment="DEBUG=installer*,connect-lastmile"
-; kill any child (installer.sh) as well
-KillMode=control-group
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target
-EOF
+echo "==== Install cloudron-version tool ===="
+npm install -g cloudron-version@0.1.1
 
 # Restore iptables before docker
 echo "==== Install iptables-restore systemd script ===="
@@ -227,16 +227,16 @@ EOF
 # Allocate swap files
 # https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
 # On ubuntu ec2 we use cloud-init https://wiki.archlinux.org/index.php/Cloud-init
-echo "==== Install box-setup systemd script ===="
-cat > /etc/systemd/system/box-setup.service <<EOF
+echo "==== Install cloudron-system-setup systemd script ===="
+cat > /etc/systemd/system/cloudron-system-setup.service <<EOF
 [Unit]
 Description=Box Setup
-Before=docker.service collectd.service mysql.service
-After=do-resize.service cloud-init.service
+Before=docker.service collectd.service mysql.service sshd.service nginx.service
+After=cloud-init.service
 
 [Service]
 Type=oneshot
-ExecStart="${INSTALLER_SOURCE_DIR}/systemd/box-setup.sh"
+ExecStart="${INSTALLER_SOURCE_DIR}/systemd/cloudron-system-setup.sh"
 RemainAfterExit=yes
 
 [Install]
@@ -244,9 +244,8 @@ WantedBy=multi-user.target
 EOF
 
 systemctl daemon-reload
-systemctl enable cloudron-installer
 systemctl enable iptables-restore
-systemctl enable box-setup
+systemctl enable cloudron-system-setup
 
 # Configure systemd
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
@@ -275,12 +274,22 @@ setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
 echo "==== Install ssh ==="
 apt-get -y install openssh-server
-# https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
-sed -e 's/^#\?Port .*/Port 202/g' \
-    -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
-    -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
-    -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
-    -i /etc/ssh/sshd_config
+
+# caas has ssh on port 202 and we disable password login
+if [[ "${PROVIDER}" == "caas" ]]; then
+    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+    sed -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+        -e 's/^#\?Port .*/Port 202/g' \
+        -i /etc/ssh/sshd_config
+fi
+
+# DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
+# We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
+echo "==== Install unbound DNS ==="
+apt-get -y install unbound
 
 # required so we can connect to this machine since port 22 is blocked by iptables by now
 systemctl reload sshd
+

box.js

@@ -14,7 +14,6 @@ var appHealthMonitor = require('./src/apphealthmonitor.js'),
     async = require('async'),
     config = require('./src/config.js'),
     ldap = require('./src/ldap.js'),
-    oauthproxy = require('./src/oauthproxy.js'),
     server = require('./src/server.js'),
     simpleauth = require('./src/simpleauth.js');
 
@@ -37,12 +36,12 @@ async.series([
     ldap.start,
     simpleauth.start,
     appHealthMonitor.start,
-    oauthproxy.start
 ], function (error) {
     if (error) {
         console.error('Error starting server', error);
         process.exit(1);
     }
+    console.log('Cloudron is up and running');
 });
 
 var NOOP_CALLBACK = function () { };
@@ -51,7 +50,6 @@ process.on('SIGINT', function () {
     server.stop(NOOP_CALLBACK);
     ldap.stop(NOOP_CALLBACK);
     simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
     setTimeout(process.exit.bind(process), 3000);
 });
 
@@ -59,6 +57,5 @@ process.on('SIGTERM', function () {
     server.stop(NOOP_CALLBACK);
     ldap.stop(NOOP_CALLBACK);
     simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
     setTimeout(process.exit.bind(process), 3000);
 });

docs/makeDocs

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-./node_modules/.bin/apidoc -i src/routes -o docs

docs/references/addons.md

@@ -0,0 +1,384 @@
+# Overview
+
+Addons are services like database, authentication, email, caching that are part of the
+Cloudron runtime. Setup, provisioning, scaling and maintanence of addons is taken care of
+by the runtime.
+
+The fundamental idea behind addons is to allow sharing of Cloudron resources across applications.
+For example, a single MySQL server instance can be used across multiple apps. The Cloudron
+runtime sets up addons in such a way that apps are isolated from each other.
+
+# Using Addons
+
+Addons are opt-in and must be specified in the [Cloudron Manifest](/references/manifest.html).
+When the app runs, environment variables contain the necessary information to access the addon.
+For example, the mysql addon sets the `MYSQL_URL` environment variable which is the
+connection string that can be used to connect to the database.
+
+When working with addons, developers need to remember the following:
+* Environment variables are subject to change every time the app restarts. This can happen if the
+Cloudron is rebooted or restored or the app crashes or an addon is re-provisioned. For this reason,
+applications must never cache the value of environment variables across restarts.
+
+* Addons must be setup or updated on each application start up. Most applications use DB migration frameworks
+for this purpose to setup and update the DB schema.
+
+* Addons are configured in the [addons section](/references/manifest.html#addons) of the manifest as below:
+```
+    {
+      ...
+      "addons": {
+        "oauth": { },
+        "redis" : { }
+      }
+    }
+```
+
+# All addons
+
+## email
+
+This addon allows an app to send and recieve emails on behalf of the user. The intended use case is webmail applications.
+
+If an app wants to send mail (e.g notifications), it must use the [sendmail](/references/addons#sendmail)
+addon. If the app wants to receive email (e.g user replying to notification), it must use the
+[recvmail](/references/addons#recvmail) addon instead.
+
+Apps using the IMAP and ManageSieve services below must be prepared to accept self-signed certificates (this is not a problem
+because these are addresses internal to the Cloudron).
+
+Exported environment variables:
+```
+MAIL_SMTP_SERVER=       # SMTP server IP or hostname. Supports STARTTLS (TLS upgrade is enforced).
+MAIL_SMTP_PORT=         # SMTP server port
+MAIL_IMAP_SERVER=       # IMAP server IP or hostname. TLS required.
+MAIL_IMAP_PORT=         # IMAP server port
+MAIL_SIEVE_SERVER=      # ManageSieve server IP or hostname. TLS required.
+MAIL_SIEVE_PORT=        # ManageSieve server port
+MAIL_DOMAIN=            # Domain of the mail server
+```
+
+## ldap
+
+This addon provides LDAP based authentication via LDAP version 3.
+
+Exported environment variables:
+```
+LDAP_SERVER=                                # ldap server IP
+LDAP_PORT=                                  # ldap server port
+LDAP_URL=                                   # ldap url of the form ldap://ip:port
+LDAP_USERS_BASE_DN=                         # ldap users base dn of the form ou=users,dc=cloudron
+LDAP_GROUPS_BASE_DN=                        # ldap groups base dn of the form ou=groups,dc=cloudron
+LDAP_BIND_DN=                               # DN to perform LDAP requests
+LDAP_BIND_PASSWORD=                         # Password to perform LDAP requests
+```
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `ldapsearch` client within the context of the app:
+```
+cloudron exec
+
+# list users
+> ldapsearch -x -h "${LDAP_SERVER}" -p "${LDAP_PORT}" -b  "${LDAP_USERS_BASE_DN}"
+
+# list users with authentication (Substitute username and password below)
+> ldapsearch -x -D cn=<username>,${LDAP_USERS_BASE_DN} -w <password> -h "${LDAP_SERVER}" -p "${LDAP_PORT}" -b  "${LDAP_USERS_BASE_DN}"
+
+# list admins
+> ldapsearch -x -h "${LDAP_SERVER}" -p "${LDAP_PORT}" -b  "${LDAP_USERS_BASE_DN}" "memberof=cn=admins,${LDAP_GROUPS_BASE_DN}"
+
+# list groups
+> ldapsearch -x -h "${LDAP_SERVER}" -p "${LDAP_PORT}" -b  "${LDAP_GROUPS_BASE_DN}"
+```
+
+## localstorage
+
+Since all Cloudron apps run within a read-only filesystem, this addon provides a writeable folder under `/app/data/`.
+All contents in that folder are included in the backup. On first run, this folder will be empty. File added in this path
+as part of the app's image (Dockerfile) won't be present. A common pattern is to create the directory structure required
+the app as part of the app's startup script.
+
+The permissions and ownership of data within that directory are not guranteed to be preserved. For this reason, each app
+has to restore permissions as required by the app as part of the app's startup script.
+
+If the app is running under the recommeneded `cloudron` user, this can be achieved with:
+```
+chown -R cloudron:cloudron /app/data
+```
+
+## mongodb
+
+By default, this addon provide mongodb 2.6.3.
+
+Exported environment variables:
+```
+MONGODB_URL=          # mongodb url
+MONGODB_USERNAME=     # username
+MONGODB_PASSWORD=     # password
+MONGODB_HOST=         # server IP/hostname
+MONGODB_PORT=         # server port
+MONGODB_DATABASE=     # database name
+```
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `mongo` shell within the context of the app:
+```
+cloudron exec
+
+# mongo -u "${MONGODB_USERNAME}" -p "${MONGODB_PASSWORD}" ${MONGODB_HOST}:${MONGODB_PORT}/${MONGODB_DATABASE}
+
+```
+## mysql
+
+By default, this addon provides a single database on MySQL 5.6.19. The database is already created and the application
+only needs to create the tables.
+
+Exported environment variables:
+```
+MYSQL_URL=            # the mysql url (only set when using a single database, see below)
+MYSQL_USERNAME=       # username
+MYSQL_PASSWORD=       # password
+MYSQL_HOST=           # server IP/hostname
+MYSQL_PORT=           # server port
+MYSQL_DATABASE=       # database name (only set when using a single database, see below)
+```
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `mysql` client within the context of the app:
+```
+cloudron exec
+
+> mysql --user=${MYSQL_USERNAME} --password=${MYSQL_PASSWORD} --host=${MYSQL_HOST} ${MYSQL_DATABASE}
+
+```
+
+The `multipleDatabases` option can be set to `true` if the app requires more than one database. When enabled,
+the following environment variables are injected:
+
+```
+MYSQL_DATABASE_PREFIX=      # prefix to use to create databases
+```
+
+## oauth
+
+The Cloudron OAuth 2.0 provider can be used in an app to implement Single Sign-On.
+
+Exported environment variables:
+```
+OAUTH_CLIENT_ID=      # client id
+OAUTH_CLIENT_SECRET=  # client secret
+```
+
+The callback url required for the OAuth transaction can be contructed from the environment variables below:
+
+```
+APP_DOMAIN=           # hostname of the app
+APP_ORIGIN=           # origin of the app of the form https://domain
+API_ORIGIN=      # origin of the OAuth provider of the form https://my-cloudrondomain
+```
+
+OAuth2 URLs can be constructed as follows:
+
+```
+AuthorizationURL = ${API_ORIGIN}/api/v1/oauth/dialog/authorize # see above for API_ORIGIN
+TokenURL = ${API_ORIGIN}/api/v1/oauth/token
+```
+
+The token obtained via OAuth has a restricted scope wherein they can only access the [profile API](/references/api.html#profile). This restriction
+is so that apps cannot make undesired changes to the user's Cloudron.
+
+We currently provide OAuth2 integration for Ruby [omniauth](https://github.com/cloudron-io/omniauth-cloudron) and Node.js [passport](https://github.com/cloudron-io/passport-cloudron).
+
+## postgresql
+
+By default, this addon provides PostgreSQL 9.4.4.
+
+Exported environment variables:
+```
+POSTGRESQL_URL=       # the postgresql url
+POSTGRESQL_USERNAME=  # username
+POSTGRESQL_PASSWORD=  # password
+POSTGRESQL_HOST=      # server name
+POSTGRESQL_PORT=      # server port
+POSTGRESQL_DATABASE=  # database name
+```
+
+The postgresql addon whitelists the hstore and pg_trgm extensions to be installable by the database owner.
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `psql` client within the context of the app:
+```
+cloudron exec
+
+> PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d ${POSTGRESQL_DATABASE}
+```
+
+## recvmail
+
+The recvmail addon can be used to receive email for the application.
+
+Exported environment variables:
+```
+MAIL_IMAP_SERVER=     # the IMAP server. this can be an IP or DNS name
+MAIL_IMAP_PORT=       # the IMAP server port
+MAIL_IMAP_USERNAME=   # the username to use for authentication
+MAIL_IMAP_PASSWORD=   # the password to use for authentication
+MAIL_TO=              # the "To" address to use
+MAIL_DOMAIN=          # the mail for which email will be received
+```
+
+The IMAP server only accepts TLS connections. The app must be prepared to accept self-signed certs (this is not a problem because the
+imap address is internal to the Cloudron).
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `openssl` tool within the context of the app:
+```
+cloudron exec
+
+> openssl s_client -connect "${MAIL_IMAP_SERVER}:${MAIL_IMAP_PORT}" -crlf
+```
+
+The IMAP command `? LOGIN username password` can then be used to test the authentication.
+
+## redis
+
+By default, this addon provides redis 2.8.13. The redis is configured to be persistent and data is preserved across updates
+and restarts.
+
+Exported environment variables:
+```
+REDIS_URL=            # the redis url
+REDIS_HOST=           # server name
+REDIS_PORT=           # server port
+REDIS_PASSWORD=       # password
+```
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `redis-cli` client within the context of the app:
+```
+cloudron exec
+
+> redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}"
+```
+
+## scheduler
+
+The scheduler addon can be used to run tasks at periodic intervals (cron).
+
+Scheduler can be configured as below:
+```
+    "scheduler": {
+        "update_feeds": {
+            "schedule": "*/5 * * * *",
+            "command": "/app/code/update_feed.sh"
+        }
+    }
+```
+
+In the above example, `update_feeds` is the name of the task and is an arbitrary string.
+
+`schedule` values must fall within the following ranges:
+
+ * Minutes: 0-59
+ * Hours: 0-23
+ * Day of Month: 1-31
+ * Months: 0-11
+ * Day of Week: 0-6
+
+_NOTE_: scheduler does not support seconds
+
+`schedule` supports ranges (like standard cron):
+
+ * Asterisk. E.g. *
+ * Ranges. E.g. 1-3,5
+ * Steps. E.g. */2
+
+`command` is executed through a shell (sh -c). The command runs in the same launch environment
+as the application. Environment variables, volumes (`/tmp` and `/run`) are all
+shared with the main application.
+
+If a task is still running when a new instance of the task is scheduled to be started, the previous
+task instance is killed.
+
+
+## sendmail
+
+The sendmail addon can be used to send email from the application.
+
+Exported environment variables:
+```
+MAIL_SMTP_SERVER=     # the mail server (relay) that apps can use. this can be an IP or DNS name
+MAIL_SMTP_PORT=       # the mail server port
+MAIL_SMTP_USERNAME=   # the username to use for authentication as well as the `from` username when sending emails
+MAIL_SMTP_PASSWORD=   # the password to use for authentication
+MAIL_FROM=            # the "From" address to use
+MAIL_DOMAIN=          # the domain name to use for email sending (i.e username@domain)
+```
+
+The SMTP server does not require STARTTLS. If STARTTLS is used, the app must be prepared to accept self-signed certs.
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `swaks` tool within the context of the app:
+```
+cloudron exec
+
+> swaks --server "${MAIL_SMTP_SERVER}" -p "${MAIL_SMTP_PORT}" --from "${MAIL_SMTP_USERNAME}@${MAIL_DOMAIN}" --body "Test mail from cloudron app at $(hostname -f)" --auth-user "${MAIL_SMTP_USERNAME}" --auth-password "${MAIL_SMTP_PASSWORD}"
+```
+
+## simpleauth
+
+Simple Auth can be used for authenticating users with a HTTP request. This method of authentication is targeted
+at applications, which for whatever reason can't use the ldap addon.
+The response contains an `accessToken` which can then be used to access the [Cloudron API](/references/api.html).
+
+Exported environment variables:
+```
+SIMPLE_AUTH_SERVER=    # the simple auth HTTP server
+SIMPLE_AUTH_PORT=      # the simple auth server port
+SIMPLE_AUTH_URL=       # the simple auth server URL. same as "http://SIMPLE_AUTH_SERVER:SIMPLE_AUTH_PORT
+SIMPLE_AUTH_CLIENT_ID  # a client id for identifying the request originator with the auth server
+```
+
+This addons provides two REST APIs:
+
+**POST /api/v1/login**
+
+Request JSON body:
+```
+{
+  "username": "<username> or <email>",
+  "password": "<password>"
+}
+```
+
+Response 200 with JSON body:
+```
+{
+  "accessToken": "<accessToken>",
+  "user": {
+    "id": "<userId>",
+    "username": "<username>",
+    "email": "<email>",
+    "admin": <admin boolean>,
+    "displayName": "<display name>"
+  }
+}
+```
+
+**GET /api/v1/logout**
+
+Request params:
+```
+?access_token=<accessToken>
+```
+
+Response 200 with JSON body:
+```
+{}
+```
+
+For debugging, [cloudron exec](https://www.npmjs.com/package/cloudron) can be used to run the `curl` tool within the context of the app:
+```
+cloudron exec
+
+> USERNAME=<enter username>
+
+> PASSWORD=<enter password>
+
+> PAYLOAD="{\"clientId\":\"${SIMPLE_AUTH_CLIENT_ID}\", \"username\":\"${USERNAME}\", \"password\":\"${PASSWORD}\"}"
+
+> curl -H "Content-Type: application/json" -X POST -d "${PAYLOAD}" "${SIMPLE_AUTH_ORIGIN}/api/v1/login"
+```

docs/references/architecture.md

@@ -0,0 +1,88 @@
+# Introduction
+
+The Cloudron platform is designed to easily install and run web applications.
+The application architecture is designed to let the Cloudron take care of system 
+operations like updates, backups, firewalls, domain management, certificate management
+etc. This allows app developers to focus on their application logic instead of deployment.
+
+At a high level, an application provides an `image` and a `manifest`. The image is simply
+a docker image that is a bundle of the application code and it's dependencies.  The manifest 
+file specifies application runtime requirements like database type and authentication scheme.
+It also provides meta information for display purposes in the [Cloudron Store](/appstore.html) 
+like the title, icon and pricing.
+
+Web applications like blogs, wikis, password managers, code hosting, document editing, 
+file syncers, notes, email, forums are a natural fit for the Cloudron. Decentralized "social" 
+networks are also good app candidates for the Cloudron.
+
+# Image
+
+Application images are created using [Docker](https://www.docker.io). Docker provides a way
+to package (and containerize) the application as a filesystem which contains it's code, system libraries 
+and just about anything the app requires. This flexible approach allows the application to use just 
+about any language or framework.
+
+Application images are instantiated as `containers`. Cloudron can run one or more isolated instances
+of the same application as one or more containers.
+
+Containerizing your application provides the following benefits:
+* Apps run in the familiar environment that they were packaged for and can have libraries
+and packages that are independent of the host OS.
+* Containers isolate applications from one another.
+
+The [base image](/references/baseimage.html) is the parent of all app images.
+
+# Cloudron Manifest
+
+Each app provides a `CloudronManifest.json` that specifies information required for the
+`Cloudron Store` and for the installation of the image in the Cloudron.
+
+Information required for container installation includes:
+* List of `addons` like databases, caches, authentication mechanisms and file systems
+* The http port on which the container is listening for incoming requests
+* Additional TCP ports on which the application is listening to (for e.g., git, ssh,
+irc protocols)
+
+Information required for the Cloudron Store includes:
+* Unique App Id
+* Title
+* Version
+* Logo
+
+See the [manifest reference](/references/manifest.html) for more information.
+
+# Addons
+
+Addons are services like database, authentication, email, caching that are part of the
+Cloudron. Setup, provisioning, scaling and maintenance of addons is taken care of by the
+Cloudron.
+
+The fundamental idea behind addons is to allow resource sharing across applications.
+For example, a single MySQL server instance can be used across multiple apps. The Cloudron
+sets up addons in such a way that apps are isolated from each other.
+
+Addons are opt-in and must be specified in the Cloudron Manifest. When the app runs, environment
+variables contain the necessary information to access the addon. See the
+[addon reference](/references/addons.html) for more information.
+
+# Authentication
+
+The Cloudron provides a centralized dashboard to manage users, roles and permissions. Applications
+do not create or manage user credentials on their own and instead use one of the various
+authentication strategies provided by the Cloudron.
+
+Authentication strategies include OAuth 2.0, LDAP or Simple Auth. See the
+[Authentication Reference](/references/authentication.html) for more information.
+
+Authorizing users is application specific and it is only authentication that is delegated to the
+Cloudron.
+
+# Cloudron Store
+
+Cloudron Store provides a market place to publish and optionally monetize your app. Submitting to the
+Cloudron Store enables any Cloudron user to discover, purchase and install your application with
+a few clicks.
+
+# What next?
+
+* [Package an existing app for the Cloudron](/tutorials/packaging.html)

docs/references/authentication.md

@@ -0,0 +1,113 @@
+# Overview
+
+Cloudron provides a centralized dashboard to manage users, roles and permissions. Applications
+do not create or manage user credentials on their own and instead use one of the various
+authentication strategies provided by the Cloudron.
+
+Note that authentication only identifies a user and does not indicate if the user is authorized
+to perform an action in the application. Authorizing users is application specific and must be
+implemented by the application.
+
+# Users & Admins
+
+Cloudron user management is intentionally very simple. The owner (first user) of the
+Cloudron is `admin` by default. The `admin` role allows one to install, uninstall and reconfigure
+applications on the Cloudron.
+
+A Cloudron `admin` can create one or more users. Cloudron users can login and use any of the installed
+apps in the Cloudron. In general, adding a cloudron user is akin to adding a person from one's family
+or organization or team because such users gain access to all apps in the Cloudron. Removing a user
+immediately revokes access from all apps.
+
+A Cloudron `admin` can give admin privileges to one or more Cloudron users.
+
+Each Cloudron user has an unique `username` and an `email`.
+
+# Strategies
+
+Cloudron provides multiple authentication strategies.
+
+* OAuth 2.0 provided by the [OAuth addon](/references/addons.html#oauth)
+* LDAP provided by the [LDAP addon](/references/addons.html#ldap)
+* Simple Auth provided by [Simple Auth addon](/references/addons.html#simpleauth)
+
+# Choosing a strategy
+
+Applications can be broadly categorized based on their user management as follows:
+
+* Multi-user aware
+  * Such apps have a full fledged user system and support multiple users and groups.
+  * These apps should use OAuth or LDAP.
+  * LDAP and OAuth APIs allow apps to detect if the user is a cloudron `admin`. Apps should use this flag
+    to show the application's admin panel for such users.
+
+
+* No user
+  * Such apps have no concept of logged-in user.
+  * The Cloudron provides a `website visibility` setting that allows a Cloudron admin to optionally
+    install an OAuth proxy in front of such applications. In such a case, a user visiting the website first
+    authenticates with the OAuth proxy and once authenticated is allowed into the application.
+  * When an OAuth proxy is installed, such applications can use the `X-Authenticated-User` header from the
+    [ICAP Extensions](https://tools.ietf.org/html/draft-stecher-icap-subid-00#section-3.4) de facto standard.
+    This value can be used for display purposes or creating meta data for a document.
+
+
+* Single user
+  * Such apps only have a single user who is usually also the `admin`.
+  * These apps can use Simple Auth or LDAP since they can authenticate users with a simple HTTP or LDAP request.
+  * Such apps _must_ set the `singleUser` property in the manifest which will restrict login to a single user
+    (configurable through the Cloudron's admin panel).
+
+# Public and Private apps
+
+`Private` apps display content only when they have a signed-in user. These apps can choose one of the
+authentication strategies listed above.
+
+`Public` apps display content to any visiting user (e.g a blog). These apps have a `login` url to allow
+the editors & admins to login. This path can be optionally set as the `configurePath` in the manifest for
+discoverability (for example, some blogs hide the login link).
+
+Some apps allow the user to choose `private` or `public` mode or some other combination. Such configuration
+is done at app install time and cannot be changed using a settings interface. It is tempting to show the user
+a configuration dialog on first installation to switch the modes. This, however, leads the user to believe that
+this configuration can be changed at any time later. In the case where this setting can be changed dynamically
+from a settings ui in the app, it's better to simply put some sensible defaults and let the user discover
+the settings. In the case where such settings cannot be changed dynamically, it is best to simply publish two
+separate apps in the Cloudron store each with a different configuration.
+
+# External User Registration
+
+Some apps allow external users to register and create accounts. For example, a public company chat that
+can invite anyone to join or a blog allowing registered commenters.
+
+Such applications must track Cloudron users and external registered users independently (for example, using a flag).
+As a thumb rule, apps must provide separate login buttons for each of the possible user sources. Such a design prevents
+external users from (inadvertently) spoofing Cloudron users.
+
+Naively handling user registration enables attacks of the following kind:
+* An external user named `foo` registers in the app.
+* A LDAP user named `foo` is later created on the Cloudron.
+* When a user named `foo` logs in, the app cannot determine the correct `foo` anymore. Making separate login buttons for each
+login source clears the confusion for both the user and the app.
+
+# Userid
+
+The preferred approach to track users in an application is a uuid or the Cloudron `username`.
+The `username` in Cloudron is unique and cannot be changed.
+
+Tracking users using `email` field is error prone since that may be changed by the user anytime.
+
+# Single Sign-on
+
+Single sign-on (SSO) is a property where a user logged in one application automatically logs into
+another application without having to re-enter his credentials. When applications implement the
+OAuth strategy, they automatically take part in Cloudron SSO. When a user signs in one application with
+OAuth, they will automatically log into any other app implementing OAuth.
+
+Conversely, signing off from one app, logs them off from all the apps.
+
+# Security
+
+The LDAP and Simple Auth strategies require the user to provide their plain text passwords to the
+application. This might be a cause of concern and app developers are thus highly encouraged to integrate
+with OAuth. OAuth also has the advantage of supporting Single Sign On.

docs/references/baseimage.md

@@ -0,0 +1,94 @@
+# Overview
+
+The application's Dockerfile must specify the FROM base image to be `cloudron/base:0.9.0`.
+
+The base image already contains most popular software packages including node, nginx, apache,
+ruby, PHP. Using the base image greatly reduces the size of app images.
+
+The goal of the base image is simply to provide pre-downloaded software packages. The packages
+are not configured in any way and it's up to the application to configure them as they choose.
+For example, while `apache` is installed, there are no meaningful site configurations that the
+application can use.
+
+# Packages
+
+The following packages are part of the base image. If you need another version, you will have to
+install it yourself.
+
+* Apache 2.4.18
+* Composer 1.2.0
+* Go 1.5.4, 1.6.3
+* Gunicorn 19.4.5
+* Java 1.8
+* Maven 3.3.9
+* Mongo 2.6.10
+* MySQL Client 5.7.13
+* nginx 1.10.0
+* Node 0.10.40, 0.12.7, 4.2.6, 4.4.7 (installed under `/usr/local/node-<version>`) [more information](#node-js)
+* Perl 5.22.1
+* PHP 7.0.8
+* Postgresql client 9.5.4
+* Python 2.7.12
+* Redis 3.0.6
+* Ruby 2.3.1
+* sqlite3 3.11.0
+* Supervisor 3.2.0
+* uwsgi 2.0.12
+
+# Inspecting the base image
+
+The base image can be inspected by installing [Docker](https://docs.docker.com/installation/).
+
+Once installed, pull down the base image locally using the following command:
+```
+    docker pull cloudron/base:0.9.0
+```
+
+To inspect the base image:
+```
+    docker run -ti cloudron/base:0.9.0 /bin/bash
+```
+
+*Note:* Please use `docker 1.9.0` or above to pull the base image. Doing otherwise results in a base
+image with an incorrect image id. The image id of `cloudron/base:0.9.0` is `d038af182821`.
+
+# The `cloudron` user
+
+The base image contains a user named `cloudron` that apps can use to run their app.
+
+It is good security practice to run apps as a non-previleged user.
+
+# Env vars
+
+The following environment variables are set as part of the application runtime.
+
+## API_ORIGIN
+
+API_ORIGIN is set to the HTTP(S) origin of this Cloudron's API. For example,
+`https://my-girish.cloudron.us`.
+
+## APP_DOMAIN
+
+APP_DOMAIN is set to the domain name of the application. For example, `app-girish.cloudron.us`.
+
+## APP_ORIGIN
+
+APP_ORIGIN is set to the HTTP(S) origin on the application. This is origin which the
+user can use to reach the application. For example, `https://app-girish.cloudron.us`.
+
+## CLOUDRON
+
+CLOUDRON is always set to '1'. This is useful to write Cloudron specific code.
+
+## WEBADMIN_ORIGIN
+
+WEBADMIN_ORIGIN is set to the HTTP(S) origin of the Cloudron's web admin. For example,
+`https://my-girish.cloudron.us`.
+
+# Node.js
+
+The base image comes pre-installed with various node.js versions.
+
+They can be used by adding `ENV PATH /usr/local/node-<version>/bin:$PATH`.
+
+See [Packages](/references/baseimage.html#packages) for available versions.

docs/references/bestpractices.md

@@ -0,0 +1,93 @@
+# Best practices
+
+## Overview
+
+This document explains the spirit of what makes a Cloudron app.
+
+## No Setup
+
+Cloudron apps do not show a setup screen after installation and should choose reasonable
+defaults.
+
+Databases, email configuration should be automatically picked up using [addons](/references/addons.html).
+
+Admin role for the application can be detected dynamically using one of the [authentication](/references/authentication.html)
+strategies.
+
+## Image
+
+The Dockerfile contains a specification for building an application image.
+
+  * Install any required software packages in the Dockerfile.
+
+  * Create static configuration files in the Dockerfile.
+
+  * Create symlinks to dynamic configuration files under `/run` in the Dockerfile.
+
+  * Docker supports restarting processes natively. Should your application crash, it will
+    be restarted automatically. If your application is a single process, you do not require
+    any process manager.
+
+  * The main process must handle `SIGTERM` and forward it as required to child processes. `bash`
+    does not automatically forward signals to child processes. For this reason, when using a startup
+    shell script, remember to use `exec <app>` as the last line. Doing so will replace bash with your
+    program and allows your program to handle signals as required.
+
+  * Use `supervisor`, `pm2` or any of the other process managers if you application has more
+    then one component. This excludes web servers like apache, nginx which can already manage their
+    children by themselves. Be sure to pick a process manager that forwards signals to child processes.
+
+  * Disable auto updates for apps. Updates must be triggered through the Cloudron Store. This allows the admin
+    to manage updates and downtime in a central location (the Cloudron Webadmin).
+
+## File system
+
+The Cloudron runs the application image as read-only. The app can only write to the following directories:
+
+  * `/tmp` - use this for temporary files.
+
+  * `/run` - use this for runtime configration and any dynamic data.
+
+  * `/app/data` - When the `localstorage` addon is enabled, any data under this directory is automatically backed up.
+
+## Logging
+
+Cloudron applications stream their logs to stdout and stderr. In contrast to logging
+to files, this approach has many advantages:
+
+  * App does not need to rotate logs and the Cloudron takes care of managing logs
+  * App does not need special mechanism to release log file handles (on a log rotate)
+  * Integrates better with tooling like `cloudron cli`
+
+This document gives you some recipes for configuring popular libraries to log to stdout. See 
+[base image](/references/baseimage.html#configuring) on how to configure various libraries to log to stdout/stderr.
+
+
+## Memory
+
+By default, applications get 256MB RAM (including swap). This can be changed using the `memoryLimit` field in the manifest.
+
+Design your application runtime for concurrent use by 10s of users. The Cloudron is not designed for concurrent access by
+100s or 1000s of users.
+
+## Startup
+
+  * Apps must not present a post-installation screen on first run. It should be already pre-configured for
+    a specific purpose.
+
+  * Do not run as `root`. Apps can use the `cloudron` user which is part of the [base image](/references/baseimage.html)
+    for this purpose or create their own.
+
+  * When using the `localstorage` addon, the application must change the ownership of files in `/app/data` as desired using `chown`. This
+    is necessary because file permissions may not be correctly preserved across backup, restore, application and base image
+    updates.
+
+  * Addon information (mail, database) is exposed as environment variables. An application must use these values directly
+    and not cache them across restarts. If the variables are stored in a configuration file, then the configuration file
+    must be regenerated on every application start. This is usually done using a configuration template that is patched
+    on every startup.
+
+## Authentication
+
+Apps should integrate with one of the [authentication strategies](/references/authentication.html).
+This saves the user from having to manage separate set of users for different apps.

docs/references/button.md

@@ -0,0 +1,47 @@
+# Cloudron Button
+
+The `Cloudron Button` allows anyone to install an application with
+the click of a button on their Cloudron.
+
+The button can be added to just about any website including the application's website
+and README.md files in GitHub repositories.
+
+## Prerequisites
+
+The `Cloudron Button` is intended to work only for applications that have been
+published on the Cloudron Store. The [basic tutorial](/tutorials/basic.html#publishing)
+gives an overview of how to package and publish your application for the
+Cloudron Store.
+
+## HTML Snippet
+
+```
+<img src="https://cloudron.io/img/button32.png" href="https://cloudron.io/button.html?app=<appid>">
+```
+
+_Note_: Replace `<appid>` with your application's id.
+
+## Markdown Snippet
+
+```
+[![Install](https://cloudron.io/img/button32.png)](https://cloudron.io/button.html?app=<appid>)
+```
+
+_Note_: Replace `<appid>` with your application's id.
+
+
+## Button Height
+
+The button may be used in different heights - 32, 48 and 64 pixels.
+
+[![Install](/img/button32.png)](https://cloudron.io/button.html?app=io.gogs.cloudronapp)
+
+[![Install](/img/button48.png)](https://cloudron.io/button.html?app=io.gogs.cloudronapp)
+
+[![Install](/img/button64.png)](https://cloudron.io/button.html?app=io.gogs.cloudronapp)
+
+or as SVG
+
+[![Install](/img/button.svg)](https://cloudron.io/button.html?app=io.gogs.cloudronapp)
+
+_Note_: Clicking the buttons above will install [Gogs](http://gogs.io/) on your Cloudron.

docs/references/manifest.md

@@ -0,0 +1,469 @@
+# Overview
+
+Every Cloudron Application contains a `CloudronManifest.json`.
+
+The manifest contains two categories of information:
+
+* Information about displaying the app on the Cloudron Store. For example,
+  the title, author information, description etc
+
+* Information for installing the app on the Cloudron. This includes fields
+  like httpPort, tcpPorts.
+
+A CloudronManifest.json can **only** contain fields that are listed as part of this
+specification. The Cloudron Store and the Cloudron *may* reject applications that have
+extra fields.
+
+Here is an example manifest:
+
+```
+{
+  "id": "com.example.test",
+  "title": "Example Application",
+  "author": "Girish Ramakrishnan <girish@cloudron.io>",
+  "description": "This is an example app",
+  "tagline": "A great beginning",
+  "version": "0.0.1",
+  "healthCheckPath": "/",
+  "httpPort": 8000,
+  "addons": {
+    "localstorage": {}
+  },
+  "manifestVersion": 1,
+  "website": "https://www.example.com",
+  "contactEmail": "support@clourdon.io",
+  "icon": "file://icon.png",
+  "tags": [ "test", "collaboration" ],
+  "mediaLinks": [ "www.youtube.com/watch?v=dQw4w9WgXcQ" ]
+}
+```
+
+# Fields
+
+## addons
+
+Type: object
+
+Required: no
+
+Allowed keys
+* [ldap](addons.html#ldap)
+* [localstorage](addons.html#localstorage)
+* [mongodb](addons.html#mongodb)
+* [mysql](addons.html#mysql)
+* [oauth](addons.html#oauth)
+* [postgresql](addons.html#postgresql)
+* [redis](addons.html#redis)
+* [sendmail](addons.html#sendmail)
+
+The `addons` object lists all the [addons](addons.html) and the addon configuration used by the application.
+
+Example:
+```
+  "addons": {
+    "localstorage": {},
+    "mongodb": {}
+  }
+```
+
+## author
+
+Type: string
+
+Required: yes
+
+The `author` field contains the name and email of the app developer (or company).
+
+Example:
+```
+  "author": "Cloudron Inc <girish@cloudron.io>"
+```
+
+## changelog
+
+Type: markdown string
+
+Required: no
+
+The `changelog` field contains the changes in this version of the application. This string
+can be a markdown style bulleted list.
+
+Example:
+```
+  "changelog": "* Add support for IE8 \n* New logo"
+```
+
+## configurePath
+
+Type: path string
+
+Required: no
+
+The `configurePath` can be used to specify the absolute path to the configuration / settings
+page of the app. When this path is present, an absoluted URL is constructed from the app's
+install location this path and presented to the user in the configuration dialog of the app.
+
+This is useful for apps that have a main page which does not display a configuration / settings
+url (i.e) it's hidden for aesthetic reasons. For example, a blogging app like wordpress might
+keep the admin page url hidden in the main page. Setting the configurationPath makes the
+configuration url discoverable by the user.
+
+Example:
+```
+  "configurePath": "/wp-admin"
+```
+
+## contactEmail
+
+Type: email
+
+Required: yes
+
+The `contactEmail` field contains the email address that Cloudron users can contact for any
+bug reports and suggestions.
+
+Example:
+```
+  "contactEmail": "support@testapp.com"
+```
+
+## description
+
+Type: markdown string
+
+Required: yes
+
+The `description` field contains a detailed description of the app. This information is shown
+to the user when they install the app from the Cloudron Store.
+
+Example:
+```
+  "description": "This is a detailed description of this app."
+```
+
+A large `description` can be unweildy to manage and edit inside the CloudronManifest.json. For
+this reason, the `description` can also contain a file reference. The Cloudron CLI tool fills up
+the description from this file when publishing your application.
+
+Example:
+```
+  "description:": "file://DESCRIPTION.md"
+```
+
+## developmentMode
+
+Type: boolean
+
+Required: no
+
+Setting `developmentMode` to true disables readonly rootfs and the default memory limit. In addition,
+the application *pauses* on start and can be started manually using `cloudron exec`.  Note that you
+cannot submit an app to the store with this field turned on.
+
+This mode can be used to identify the files being modified by your application - often required to
+debug situations where your app does not run on a readonly rootfs. Run your app using `cloudron exec`
+and use `find / -mmin -30` to find file that have been changed or created in the last 30 minutes.
+
+## healthCheckPath
+
+Type: url path
+
+Required: yes
+
+The `healthCheckPath` field is used by the Cloudron Runtime to determine if your app is running and
+responsive. The app must return a 2xx HTTP status code as a response when this path is queried. In
+most cases, the default "/" will suffice but there might be cases where periodically querying "/"
+is an expensive operation. In addition, the app might want to use a specialized route should it
+want to perform some specialized internal checks.
+
+Example:
+```
+  "healthCheckPath": "/"
+```
+## httpPort
+
+Type: positive integer
+
+Required: yes
+
+The `httpPort` field contains the TCP port on which your app is listening for HTTP requests. This port
+is exposed to the world via subdomain/location that the user chooses at installation time. While not
+required, it is good practice to mark this port as `EXPOSE` in the Dockerfile.
+
+Cloudron Apps are containerized and thus two applications can listen on the same port. In reality,
+they are in different network namespaces and do not conflict with each other.
+
+Note that this port has to be HTTP and not HTTPS or any other non-HTTP protocol. HTTPS proxying is
+handled by the Cloudron platform (since it owns the certificates).
+
+Example:
+```
+  "httpPort": 8080
+```
+
+## icon
+
+Type: local image filename
+
+Required: no
+
+The `icon` field is used to display the application icon/logo in the Cloudron Store. Icons are expected
+to be square of size 256x256.
+
+```
+  "icon": "file://icon.png"
+```
+
+## id
+
+Type: reverse domain string
+
+Required: yes
+
+The `id` is a unique human friendly Cloudron Store id. This is similar to reverse domain string names used
+as java package names. The convention is to base the `id` based on a domain that you own.
+
+The Cloudron tooling allows you to build applications with any `id`. However, you will be unable to publish
+the application if the id is already in use by another application.
+
+```
+  "id": "io.cloudron.testapp"
+```
+
+## manifestVersion
+
+Type: integer
+
+Required: yes
+
+`manifestVersion` specifies the version of the manifest and is always set to 1.
+
+```
+  "manifestVersion": 1
+```
+
+## mediaLinks
+
+Type: array of urls
+
+Required: no
+
+The `mediaLinks` field contains an array of links that the Cloudron Store uses to display a slide show of pictures
+and videos of the application.
+
+They have to be publicly reachable via `https` and should have an aspect ratio of 3 to 1.
+For example `600px by 200px` (with/height).
+
+```
+  "mediaLinks": [
+    "www.youtube.com/watch?v=dQw4w9WgXcQ",
+    "https://images.rapgenius.com/fd0175ef780e2feefb30055be9f2e022.520x343x1.jpg"
+  ]
+```
+
+## memoryLimit
+
+Type: bytes (integer)
+
+Required: no
+
+The `memoryLimit` field is the maximum amount of memory (including swap) in bytes an app is allowed to consume before it
+gets killed and restarted.
+
+By default, all apps have a memoryLimit of 256MB. For example, to have a limit of 500MB,
+
+```
+  "memoryLimit": 524288000
+```
+
+## maxBoxVersion
+
+Type: semver string
+
+Required: no
+
+The `maxBoxVersion` field is the maximum box version that the app can possibly run on. Attempting to install the app on
+a box greater than `maxBoxVersion` will fail.
+
+This is useful when a new box release introduces features which are incompatible with the app. This situation is quite
+unlikely and it is recommended to leave this unset.
+
+## minBoxVersion
+
+Type: semver string
+
+Required: no
+
+The `minBoxVersion` field is the minimum box version that the app can possibly run on. Attempting to install the app on
+a box lesser than `minBoxVersion` will fail.
+
+This is useful when the app relies on features that are only available from a certain version of the box. If unset, the
+default value is `0.0.1`.
+
+## postInstallMessage
+
+Type: markdown string
+
+Required: no
+
+The `postInstallMessageField` is a message that is displayed to the user after an app is installed.
+
+The intended use of this field is to display some post installation steps that the user has to carry out to
+complete the installation. For example, displaying the default admin credentials and informing the user to
+to change it.
+
+## singleUser
+
+Type: boolean
+
+Required: no
+
+The `singleUser` field can be set to true for apps that are meant to be used only a single user.
+
+When set, the Cloudron will display a user selection dialog at installation time. The selected user is the sole user
+who can access the app.
+
+## tagline
+
+Type: one-line string
+
+Required: no
+
+The `tagline` is used by the Cloudron Store to display a single line short description of the application.
+
+```
+  "tagline": "The very best note keeper"
+```
+
+## tags
+
+Type: Array of strings
+
+Required: no
+
+The `tags` are used by the Cloudron Store for filtering searches by keyword.
+
+```
+  "tags": [ "git", "version control", "scm" ]
+```
+
+## targetBoxVersion
+
+Type: semver string
+
+Required: no
+
+The `targetBoxVersion` field is the box version that the app was tested on. By definition, this version has to be greater
+than the `minBoxVersion`.
+
+The box uses this value to enable compatibility behavior of APIs. For example, an app sets the targetBoxVersion to 0.0.5
+and is published on the store. Later, box version 0.0.10 introduces a new feature that conflicts with how apps used
+to run in 0.0.5 (say SELinux was enabled for apps). When the box runs such an app, it ensures compatible behavior
+and will disable the SELinux feature for the app.
+
+If unspecified, this value defaults to `minBoxVersion`.
+
+## tcpPorts
+
+Type: object
+
+Required: no
+
+Syntax: Each key is the environment variable. Each value is an object containing `title`, `description` and `defaultValue`.
+An optional `containerPort` may be specified.
+
+The `tcpPorts` field provides information on the non-http TCP ports/services that your application is listening on. During
+installation, the user can decide how these ports are exposed from their Cloudron.
+
+For example, if the application runs an SSH server at port 29418, this information is listed here. At installation time,
+the user can decide any of the following:
+* Expose the port with the suggested `defaultValue` to the outside world. This will only work if no other app is being exposed at same port.
+* Provide an alternate value on which the port is to be exposed to outside world.
+* Disable the port/service.
+
+To illustrate, the application lists the ports as below:
+```
+  "tcpPorts": {
+    "SSH_PORT": {
+      "title": "SSH Port",
+      "description": "SSH Port over which repos can be pushed & pulled",
+      "defaultValue": 29418,
+      "containerPort": 22
+    }
+  },
+```
+
+In the above example:
+* `SSH_PORT` is an app specific environment variable. Only strings, numbers and _ (underscore) are allowed. The author has to ensure that they don't clash with platform profided variable names.
+
+* `title` is a short one line information about this port/service.
+
+* `description` is a multi line description about this port/service.
+
+* `defaultValue` is the recommended port value to be shown in the app installation UI.
+
+* `containerPort` is the port that the app is listening on (recall that each app has it's own networking namespace).
+
+In more detail:
+
+* If the user decides to disable the SSH service, this environment variable `SSH_PORT` is absent. Applications _must_ detect this on
+  start up and disable these services.
+
+* `SSH_PORT` is set to the value of the exposed port. Should the user choose to expose the SSH server on port 6000, then the
+  value of SSH_PORT is 6000.
+
+* `defaultValue` is **only** used for display purposes in the app installation UI.  This value is independent of the value
+   that the app is listening on. For example, the app can run an SSH server at port 22 but still recommend a value of 29418 to the user.
+
+* `containerPort` is the port that the app is listening on. The Cloudron runtime will _bridge_ the user chosen external port
+  with the app specific `containerPort`. Cloudron Apps are containerized and each app has it's own networking namespace.
+  As a result, different apps can have the same `containerPort` value because these values are namespaced.
+
+* The environment variable `SSH_PORT` may be used by the app to display external URLs. For example, the app might want to display
+  the SSH URL. In such a case, it would be incorrect to use the `containerPort` 22 or the `defaultValue` 29418 since this is not
+  the value chosen by the user.
+
+* `containerPort` is optional and can be omitted, in which case the bridged port numbers are the same internally and externally.
+  Some apps use the same variable (in their code) for listen port and user visible display strings. When packaging these apps,
+  it might be simpler to listen on `SSH_PORT` internally. In such cases, the app can omit the `containerPort` value and should
+  instead reconfigure itself to listen internally on `SSH_PORT` on each start up.
+
+## title
+
+Type: string
+
+Required: yes
+
+The `title` is the primary application title displayed on the Cloudron Store.
+
+Example:
+```
+  "title": "Gitlab"
+```
+
+## version
+
+Type: semver string
+
+Required: yes
+
+The `version` field specifies a [semver](http://semver.org/) string. The version is used by the Cloudron to compare versions and to
+determine if an update is available.
+
+Example:
+```
+  "version": "1.1.0"
+```
+
+## website
+
+Type: url
+
+Required: yes
+
+The `website` field is a URL where the user can read more about the application.
+
+Example:
+```
+  "website": "https://example.com/myapp"
+```

docs/references/recipes.md

@@ -0,0 +1,61 @@
+# Configuration Recipes
+
+## nginx
+
+`nginx` is often used as a reverse proxy in front of the application, to dispatch to different backend programs based on the request route or other characteristics. In such a case it is recommended to run nginx and the application through a process manager like `supervisor`.
+
+Example nginx supervisor configuration file:
+```
+[program:nginx]
+directory=/tmp
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/supervisor/%(program_name)s.log
+stderr_logfile=/var/log/supervisor/%(program_name)s.log
+```
+
+The nginx configuration, provided with the base image, can be used by adding an application specific config file under `/etc/nginx/sites-enabled/` when building the docker image.
+
+```
+ADD <app config file> /etc/nginx/sites-enabled/<app config file>
+```
+
+Since the base image nginx configuration is unpatched from the ubuntu package, the application configuration has to ensure nginx is using `/run/` instead of `/var/lib/nginx/` to support the read-only filesystem nature of a Cloudron application.
+
+Example nginx app config file:
+```
+client_body_temp_path /run/client_body;
+proxy_temp_path /run/proxy_temp;
+fastcgi_temp_path /run/fastcgi_temp;
+scgi_temp_path /run/scgi_temp;
+uwsgi_temp_path /run/uwsgi_temp;
+
+server {
+  listen 8000;
+
+  root /app/code/dist;
+
+  location /api/v1/ {
+    proxy_pass http://127.0.0.1:8001;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_read_timeout 86400;
+  }
+}
+
+```
+
+## supervisor
+
+Use this in the program's config:
+
+```
+[program:app]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+```

docs/references/selfhosting.md

@@ -0,0 +1,346 @@
+# Overview
+
+The Cloudron platform can be installed on public cloud servers from EC2, Digital Ocean, Hetzner,
+Linode, OVH, Scaleway, Vultr etc. Running Cloudron on a home server or company intranet is work
+in progress.
+
+If you run into any trouble following this guide, ask us at our [chat](https://chat.cloudron.io).
+
+# Understand
+
+Before installing the Cloudron, it is helpful to understand Cloudron's design. The Cloudron
+intends to make self-hosting effortless. It takes care of updates, backups, firewall, dns setup,
+certificate management etc. All app and user configuration is carried out using the web interface.
+
+This approach to self-hosting means that the Cloudron takes complete ownership of the server and
+only tracks changes that were made via the web interface. Any external changes made to the server
+(i.e other than via the Cloudron web interface or API) may be lost across updates.
+
+The Cloudron requires a domain name when it is installed. Apps are installed into subdomains.
+The `my` subdomain is special and is the location of the Cloudron web interface. For this to
+work, the Cloudron requires a way to programmatically configure the DNS entries of the domain.
+Note that the Cloudron will never overwrite _existing_ DNS entries and refuse to install
+apps on existing subdomains.
+
+# CLI Tool
+
+The [Cloudron tool](https://git.cloudron.io/cloudron/cloudron-cli) is useful for managing
+a Cloudron. <b class="text-danger">The Cloudron CLI tool has to be run on a Laptop or PC</b>
+
+## Linux & OS X
+
+Installing the CLI tool requires node.js and npm. The CLI tool can be installed using the following command:
+
+```
+npm install -g cloudron
+```
+
+Depending on your setup, you may need to run this as root.
+
+On OS X, it is known to work with the `openssl` package from homebrew.
+
+See [#14](https://git.cloudron.io/cloudron/cloudron-cli/issues/14) for more information.
+
+## Windows
+
+The CLI tool does not work on Windows. Please contact us on our [chat](https://chat.cloudron.io) if you want to help with Windows support.
+
+# Provider
+
+Both DigitalOcean and EC2 from Amazon Web Services are frequently tested by us.
+
+In addition to those, the Cloudron community has successfully installed the platform on those providers:
+* [hosttech](https://www.hosttech.ch/)
+* [Linode](https://www.linode.com/)
+* [OVH](https://www.ovh.com/)
+* [Scaleway](https://www.scaleway.com/)
+* [So you Start](https://www.soyoustart.com/)
+* [Vultr](https://www.vultr.com/)
+
+Please let us know if any of them requires tweaks or adjustments.
+
+# Installing
+
+## Choose Domain
+
+A domain name is required when installing the Cloudron. Currently, only Second Level Domains
+are supported. For example, `example.com`, `example.co.uk` will work fine. Choosing a domain
+name at any other level like `cloudron.example.com` will not work.
+
+The domain name must use one of the following name servers:
+* AWS Route 53
+* Digital Ocean
+* Wildcard - If your domain does not use any of the name servers above, you can manually add
+a wildcard (`*`) DNS entry.
+
+You will have to provide the DNS API credentials after you complete the installation.
+
+## Create server
+
+Create an `Ubuntu 16.04 (Xenial)` server with at-least `1gb` RAM. Do not make any changes
+to vanilla ubuntu. Be sure to allocate a static IPv4 address for your server.
+
+### Linode
+
+Since Linode does not manage SSH keys, be sure to add the public key to
+`/root/.ssh/authorized_keys`.
+
+### Scaleway
+
+Use the [boot script](https://github.com/scaleway-community/scaleway-docker/issues/2) to
+enable memory accouting.
+
+## Setup `my` subdomain
+
+The Cloudron web interface is installed at the `my` subdomain of your domain.
+Add a `A` DNS record for the `my` subdomain with the IP of the server created
+above. Doing this will allow the Cloudron to start up with a valid TLS certificate.
+
+## Run setup
+
+SSH into your server and run the following commands:
+
+```
+wget https://git.cloudron.io/cloudron/box/raw/master/scripts/cloudron-setup
+chmod +x cloudron-setup
+./cloudron-setup --domain <domain> --provider <digitalocean|ec2|generic|scaleway> --encryption-key <key>
+```
+
+The setup will take around 10-15 minutes.
+
+`cloudron-setup` takes the following arguments:
+
+* `--domain` is the domain name in which apps are installed. Currently, only Second Level
+Domains are supported. For example, `example.com`, `example.co.uk`, `example.rocks` will
+work fine. Choosing a domain name at any other level like `cloudron.example.com` will not
+work.
+
+* `--provider` is the name of your VPS provider. If the name is not on the list, simply
+choose `generic`. If the Cloudron does not complete initialization, it may mean that
+we have to add some vendor specific quirks. Please open a
+[bug report](https://git.cloudron.io/cloudron/box/issues) in that case.
+
+* `--encryption-key` is the key to be used for encrypting backup data.
+
+Optional arguments used for update and restore:
+
+* `--version` is the version of Cloudron to install. By default, the setup script installs
+the latest version. This is useful when restoring a Cloudron from a backup.
+
+* `--restore-url` is an URL to the backup to restore to.
+
+
+## Finish setup
+
+Once the setup script completes, visit `https://my.<domain>` to complete the installation.
+
+Please note the following:
+
+1. The website should already have a valid TLS certificate. If you see any certificate warnings, it means your Cloudron was not created correctly.
+
+2. If you see a login screen, instead of a setup screen, it means that someone else got to your Cloudron first and set it up
+already! In this unlikely case, simply delete the server and start over.
+
+Once the setup is done, you can access the admin page in the future at `https://my.<domain>`.
+
+**If apps do not start after installation, a server restart may be required to let bootloader changes come into action.**
+
+## DNS
+
+Cloudron has to be given the API credentials for configuring your domain under `Certs & Domains`
+in the web UI.
+
+### Route 53
+
+Create root or IAM credentials and choose `Route 53` as the DNS provider.
+
+* For root credentials:
+  * In AWS Console, under your name in the menu bar, click `Security Credentials`
+  * Click on `Access Keys` and create a key pair.
+* For IAM credentials:
+    * You can use the following policy to create IAM credentials:
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "route53:*",
+            "Resource": [
+                "arn:aws:route53:::hostedzone/<hosted zone id>"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "route53:ListHostedZones",
+                "route53:GetChange"
+            ],
+            "Resource": [
+                "*"
+            ]
+        }
+    ]
+}
+```
+
+### Digital Ocean
+
+Create an API token with read+write access and choose `Digital Ocean` as the DNS provider.
+
+### Other
+
+If your domain *does not* use Route 53 or Digital Ocean, setup a wildcard (`*`) DNS `A` record that points to the
+IP of the server created above. If your DNS provider has an API, please open an
+[issue](https://git.cloudron.io/cloudron/box/issues) and we may be able to support it.
+
+## Backups
+
+The Cloudron creates encrypted backups once a day. Each app is backed up independently and these
+backups have the prefix `appbackup_`. The platform state is backed up independently with the
+prefix `backup_`.
+
+By default, backups reside in `/var/backups`. Having backups reside in the same location as the
+server instance is dangerous and it must be changed to an external storage location like `S3`
+as soon as possible.
+
+### S3
+
+Provide S3 backup credentials in the `Settings` page.
+
+Create a bucket in S3 (You have to have an account at [AWS](https://aws.amazon.com/)). The bucket can be setup to periodically delete old backups by
+adding a lifecycle rule using the AWS console. S3 supports both permanent deletion
+or moving objects to the cheaper Glacier storage class based on an age attribute.
+With the current daily backup schedule a setting of two days should be sufficient
+for most use-cases.
+
+* For root credentials:
+    * In AWS Console, under your name in the menu bar, click `Security Credentials`
+    * Click on `Access Keys` and create a key pair.
+* For IAM credentials:
+* You can use the following policy to create IAM credentials:
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": [
+                "arn:aws:s3:::<your bucket name>",
+                "arn:aws:s3:::<your bucket name>/*"
+            ]
+        }
+    ]
+}
+```
+
+# Email
+
+Cloudron has a built-in email server. By default, it only sends out email on behalf of apps
+(for example, password reset or notification). You can enable the email server for sending
+and receiving mail on the `settings` page. This feature is only available if you have setup
+a DNS provider like Digital Ocean or Route53.
+
+Your server's IP plays a big role in how emails from our Cloudron get handled. Spammers
+frequently abuse public IP addresses and as a result your Cloudron might possibly start
+out with a bad reputation. The good news is that most IP based blacklisting services cool
+down over time. The Cloudron sets up DNS entries for SPF, DKIM, DMARC automatically and
+reputation should be easy to get back.
+
+## Checklist
+
+* Once your Cloudron is ready, setup a Reverse DNS PTR record to be setup for the `my` subdomain.
+
+    * AWS/EC2 - Fill the PTR [request form](https://aws-portal.amazon.com/gp/aws/html-forms-controller/contactus/ec2-email-limit-rdns-request.
+
+    * Digital Ocean - Digital Ocean sets up a PTR record based on the droplet's name. So, simply rename
+    your droplet to `my.<domain>`.
+
+    * Scaleway - Edit your security group to allow email. You can also set a PTR record on the interface with your
+    `my.<domain>`.
+
+* Check if your IP is listed in any DNSBL list [here](http://multirbl.valli.org/). In most cases,
+you can apply for removal of your IP by filling out a form at the DNSBL manager site.
+
+* Finally, check your spam score at [mail-tester.com](https://www.mail-tester.com/). The Cloudron
+should get 100%, if not please let us know.
+
+# Updates
+
+Apps installed from the Cloudron Store are automatically updated every night.
+
+The Cloudron platform itself updates in two ways: update or upgrade.
+
+### Update
+
+An **update** is applied onto the running server instance. Such updates are performed
+every night. You can also use the Cloudron UI to initiate an update immediately.
+
+The Cloudron will always make a complete backup before attempting an update. In the unlikely
+case an update fails, it can be [restored](/references/selfhosting.html#restore).
+
+### Upgrade
+
+An **upgrade** requires a new OS image and thus involves creating the Cloudron from scratch.
+This process involves creating a new server with the latest code and restoring it from the
+last backup. Currently only Cloudrons using the **S3 backup storage** support upgrades.
+Read more about [backup storage](#s3), otherwise contact us in our [chat](https://chat.cloudron.io).
+
+To upgrade follow these steps closely:
+
+* Create a new backup - `cloudron machine backup create <domain>`
+
+* List the latest backup - `cloudron machine backup list <domain>`
+
+* Make the latest box backup (files starting with `backup_`) public. This can be done from the AWS S3 console as seen here:
+
+  <img src="/docs/img/aws_backup_public.png" class="shadow haze"><br/>
+
+* Copy the new public URL of the latest backup for use as the `--restore-url` below.
+
+  <img src="/docs/img/aws_backup_link.png" class="shadow haze"><br/>
+
+* Create a new Cloudron by following the [installing](/references/selfhosting.html#installing) section.
+  When running the setup script, pass in the `--encryption-key` and `--restore-url` flags.
+  The `--encryption-key` is the backup encryption key. It can be displayed with `cloudron machine info`
+
+Similar to the initial installation, a Cloudron upgrade looks like:
+```
+$ ssh root@newserverip
+> wget https://git.cloudron.io/cloudron/box/raw/master/scripts/cloudron-setup
+> chmod +x cloudron-setup
+> ./cloudron-setup --domain <domain> --provider <digitalocean|ec2|generic|scaleway> --encryption-key <key> --restore-url <publicS3Url>
+```
+
+ * Finally, once you see the newest version being displayed in your Cloudron webinterface, you can safely delete the old server instance.
+
+# Restore
+
+To restore a Cloudron from a specific backup:
+
+* Select the backup - `cloudron machine backup list <domain>`
+
+* Make the box backup public (this can be done from the S3 console). Also, copy the URL of
+  the backup for use as the `restore-url` below.
+
+* Create a new Cloudron by following the [installing](/references/selfhosting.html#installing) section.
+  When running the setup script, pass in the `version`, `restore-key` and `restore-url` flags.
+  The `version` field is the version of the Cloudron that the backup corresponds to (it is embedded
+  in the backup file name).
+
+* Make the box backup private, once the upgrade is complete.
+
+# Debug
+
+You can SSH into your Cloudron and collect logs:
+
+* `journalctl -a -u box` to get debug output of box related code.
+* `docker ps` will give you the list of containers. The addon containers are named as `mail`, `postgresql`,
+   `mysql` etc. If you want to get a specific container's log output, `journalctl -a CONTAINER_ID=<container_id>`.
+
+# Help
+
+If you run into any problems, join us at our [chat](https://chat.cloudron.io) or [email us](mailto:support@cloudron.io).

docs/references/usermanual.md

@@ -0,0 +1,354 @@
+# Introduction
+
+The Cloudron is the best platform self-hosting web applications on your server. You
+can easily install apps on it, add users, manage access restriction and keep your
+server and apps updated with no effort.
+
+You might wonder that there are so many 1-click app solutions out there and what is so special
+about Cloudron? As the name implies, 1-click installers simply install code into a server
+and leave it at that. There's so much more to do:
+
+1. Configure a domain to point to your server
+2. Setup SSL certificates and renew them periodically
+3. Ensure apps are backed up correctly
+4. Ensure apps are uptodate and secure
+5. Have a mechanism to quickly restore apps from a backup
+6. Manage users across all your apps
+7. Get alerts and notifications about the status of apps
+
+... and so on ...
+
+We made the Cloudron to dramatically lower the bar for people to run apps on servers. Just provide
+a domain name, install apps and add users. All the server management tasks listed above is
+completely automated.
+
+If you want to learn more about the secret sauce that makes the Cloudron, please read our
+[architecture overview](/references/architecture.html).
+
+# Use cases
+
+Here are some of the apps you can run on a Cloudron:
+
+* RSS Reader
+* Chat, IRC, Jabber servers
+* Public forum
+* Blog
+* File syncing and sharing
+* Code hosting
+* Email
+
+Our list of apps is growing everyday, so be sure to [follow us on twitter](https://twitter.com/cloudron_io).
+
+# Activation
+
+When you first create the Cloudron, the setup wizard will ask you to setup an administrator
+account. Don't worry, a Cloudron adminstrator doesn't need to know anything about maintaining
+a server! It's the whole reason why we made the Cloudron. Being a Cloudron administrator is
+more analagous to being the owner of a smartphone. You can always add more administrators to
+the Cloudron from the `Users` menu item.
+
+<img src="/docs/img/webadmin_domain.png" class="shadow">
+
+The Cloudron administration page is located at the `my` subdomain. You might want to bookmark
+this link!
+
+# Apps
+
+## Installation
+
+You can install apps on the Cloudron by choosing the `App Store` menu item. Use the 'Search' bar
+to search for apps.
+
+Clicking on app gives you information about the app.
+
+<img src="/docs/img/app_info.png" class="shadow">
+
+Clicking the `Install` button will show an install dialog like below:
+
+<img src="/docs/img/app_install.png" class="shadow">
+
+The `Location` field is the subdomain in which your app will be installed. For example, if you use the
+`mail` location for your web mail client, then it will be accessible at `mail.<domain>`.
+
+Tip: You can access the apps directly on your browser using `mail.<domain>`. You don't have to
+visit the Cloudron administration panel.
+
+`Access control` specifies who can access this app.
+
+* `Every Cloudron user` - Any user in your Cloudron can access the app. Initially, you are the only
+   user in your Cloudron. Unless you explicitly invite others, nobody else can access these apps.
+   Note that the term 'access' depends on the app. For a blog, this means that nobody can post new
+   blog posts (but anybody can view them). For a chat server, this might mean that nobody can access
+   your chat server.
+
+* `Restrict to groups` - Only users in the groups can access the app.
+
+## Updates
+
+All your apps automatically update as and when the application author releases an update. The Cloudron
+will attempt to update around midnight of your timezone.
+
+Some app updates are not automatic. This can happen if a new version of the app has removed some features
+that you were relying on. In such a case, the update has to be manually approved. This is simply a matter
+of clicking the `Update` button (the green star) after you read about the changes.
+
+<img src="/docs/img/app_update.png" class="shadow">
+
+## Backups
+
+<i>If you self-host, please refer to the [self-hosting documentation](/references/selfhosting.html#backups) for backups.</i>
+
+All apps are automatically backed up every day. Backups are stored encrypted in Amazon S3. You don't have
+to do anything about it. The [Cloudron CLI](https://git.cloudron.io/cloudron/cloudron-cli) tool can be used
+to download application backups.
+
+## Configuration
+
+Apps can be reconfigured using the `Configure` button.
+
+<img src="/docs/img/app_configure_button.png" class="shadow">
+
+Click on the wrench button will bring up the configure dialog.
+
+<img src="/docs/img/app_configure.png" class="shadow">
+
+You can do the following:
+* Change the location to move the app to another subdomain. Say, you want to move your blog from `blog` to `about`.
+* Change who can access the app.
+
+Changing an app's configuration has a small downtime (usually around a minute).
+
+## Restore
+
+Apps can be restored to a previous backup by clicking on the `Restore` button.
+
+<img src="/docs/img/app_restore_button.png" class="shadow">
+
+Note that restoring previous data might also restore the previous version of the software. For example, you might
+be currently using Version 5 of the app. If you restore to a backup that was made with Version 3 of the app, then the restore
+operation will install Version 3 of the app. This is because the latest version may not be able to handle old data.
+
+## Uninstall
+
+You can uninstall an app by clicking the `Uninstall` button.
+
+<img src="/docs/img/app_uninstall_button.png" class="shadow">
+
+Note that all data associated with the app will be immediately removed from the Cloudron. App data might still
+persist in your old backups and the [CLI tool](https://git.cloudron.io/cloudron/cloudron-cli) provides a way to
+restore from those old backups should it be required.
+
+## Embedding Apps
+
+It is possible to embed Cloudron apps into other websites. By default, this is disabled to prevent
+[Clickjacking](https://cloudron.io/blog/2016-07-15-site-embedding.html).
+
+You can set a website that is allowed to embed your Cloudron app using the app's [Configure dialog](#configuration).
+Click on 'Show Advanced Settings...' and enter the embedder website name.
+
+# Custom domain
+
+When you create a Cloudron from cloudron.io, we provide a subdomain under `cloudron.me` like `girish.cloudron.me`.
+Apps are available under that subdomain using a hyphenated name like `blog-girish.cloudron.me`.
+
+Domain names are a thing of pride and the Cloudron makes it easy to make your apps accessible from memorable locations like `blog.girish.in`.
+
+## Single app on a custom domain
+
+This approach is applicable if you desire that only a single app be accessing from a custom
+domain. For this, open the app's configure dialog and choose `External Domain` in the location dropdown.
+
+<img src="/docs/img/app_external_domain.png" class="shadow">
+
+This dialog will suggest you to add a `CNAME` record. Once you setup a CNAME record with your DNS provider,
+the app will be accessible from that external domain.
+
+## Entire Cloudron on a custom domain
+
+This approach is applicable if you want all your apps to be accessible from subdomains of your custom domain.
+For example, `blog.girish.in`, `notes.girish.in`, `owncloud.girish.in`, `mail.girish.in` and so on. This
+approach is also the only way that the Cloudron supports for sending and receiving emails from your domain.
+
+For this, go to the 'Domains & Certs' menu item.
+
+<img src="/docs/img/custom_domain_menu.png" class="shadow">
+
+Change the domain name to your custom domain. Currently, we require that your domain be hosted on AWS Route53.
+
+<img src="/docs/img/custom_domain_change.png" class="shadow">
+
+Moving to a custom domain will retain all your apps and data and will take around 15 minutes. If you require assistance with another provider,
+<a href="mailto:support@cloudron.io">just let us know</a>.
+
+# User management
+
+## Users
+
+You can invite new users (friends, family, colleagues) with their email address from the `Users` menu. They will
+receive an invite to sign up with your Cloudron. They can now access the apps that you have given them access
+to.
+
+<img src="/docs/img/users.png" class="shadow">
+
+To remove a user, simply remove them from the list. Note that the removed user cannot access any app anymore.
+
+## Administrators
+
+A Cloudron administrator is a special right given to an existing Cloudron user allowing them to manage
+apps and users. To make an existing user an administator, click the edit (pencil) button corresponding to
+the user and check the `Allow this user to manage apps, groups and other users` checkbox.
+
+<img src="/docs/img/administrator.png" class="shadow">
+
+## Groups
+
+Groups provide a convenient way to group users. It's purpose is two-fold:
+
+* You can assign one or more groups to apps to restrict who can access for an app.
+* Each group is a mailing list (forwarding address) constituting of it's members.
+
+You can create a group by using the `Groups` menu item.
+
+<img src="/docs/img/groups.png" class="shadow">
+
+To set the access restriction use the app's configure dialog.
+
+<img src="/docs/img/app_access_control.png" class="shadow">
+
+You can now send mails to `groupname@<domain>` to address all the group members.
+
+# Login
+
+## Cloudron admin
+
+The Cloudron admin page is always located at the `my` subdomain of your Cloudron domain. For custom domains,
+this will be like `my.girish.in`. For domains from cloudron.io, this will be like `my-girish.cloudron.me`.
+
+## Apps (single sign-on)
+
+An important feature of the Cloudron is Single Sign-On. You use the same username & password for logging in
+to all your apps. No more having to manage separate set of credentials for each service!
+
+## Single user apps
+
+Some apps only work with a single user. For example, a notes app might allow only a single user to login and add
+notes. For such apps, you will be prompted during installation to select the single user who can access the app.
+
+<img src="/docs/img/app_single_user.png" class="shadow">
+
+If you want multiple users to use the app independently, simply install the app multiple times to different locations.
+
+# Email
+
+The Cloudron has a built-in email server. The primary email address is the same as the username. Emails can be sent
+and received from `<username>@<domain>`. The Cloudron does not allow masquerading - one user cannot send email
+pretending to be another user.
+
+## Enabling Email
+
+By default, Cloudron's email server only allows apps to send email. To enable users to send and receive email,
+turn on the option under `Settings`. Turning on this option also allows apps to _receive_ email.
+
+Once email is enabled, the Cloudron will keep the the `MX` DNS record updated.
+
+<img src="/docs/img/enable_email.png" class="shadow">
+
+## Receiving email using IMAP
+
+Use the following settings to receive email.
+
+  * Server Name - Use the `my` subdomain of your Cloudron
+  * Port - 993
+  * Connection Security - TLS
+  * Username/password - Same as your Cloudron credentials
+
+## Sending email using SMTP
+
+Use the following settings to send email.
+
+  * Server Name - Use the `my` subdomain of your Cloudron
+  * Port - 587
+  * Connection Security - STARTTLS
+  * Username/password - Same as your Cloudron credentials
+
+## Email filters using Sieve
+
+Use the following settings to setup email filtering users via Manage Sieve.
+
+  * Server Name - Use the `my` subdomain of your Cloudron
+  * Port - 4190
+  * Connection Security - TLS
+  * Username/password - Same as your Cloudron credentials
+
+The [Rainloop](https://cloudron.io/appstore.html?app=net.rainloop.cloudronapp) and [Roundcube](https://cloudron.io/appstore.html?app=net.roundcube.cloudronapp)
+apps are already pre-configured to use the above settings.
+
+## Aliases
+
+You can configure one or more aliases alongside the primary email address of each user. You can set aliases by editing the
+user's settings, available behind the edit button in the user listing. Note that aliases cannot conflict with existing user names.
+
+<img src="/docs/img/email_alias.png" class="shadow">
+
+Currently, it is not possible to login using the alias for SMTP/IMAP/Sieve services. Instead, add the alias as an identity in
+your mail client but login using the Cloudron credentials.
+
+## Subaddresses
+
+Emails addressed to `<username>+tag@<domain>` will be delivered to the `username` mailbox. You can use this feature to give out emails of the form
+`username+kayak@<domain>`, `username+aws@<domain>` and so on and have them all delivered to your mailbox.
+
+## Forwarding addresses
+
+Each group on the Cloudron is also a forwarding address. Mails can be addressed to `group@<domain>` and the mail will
+be sent to each user who is part of the group.
+
+## Marking Spam
+
+The spam detection agent on the Cloudron requires training to identify spam. To do this, simply move your junk mails
+to a pre-created folder named `Spam`. Most mail clients have a Junk or Spam button which does this automatically.
+
+# Graphs
+
+The Graphs view shows an overview of the disk and memory usage on your Cloudron.
+
+<img src="/docs/img/graphs.png" class="shadow">
+
+The `Disk Usage` graph shows you how much disk space you have left. Note that the Cloudron will
+send the Cloudron admins an email notification when the disk is ~90% full.
+
+The `Apps` Memory graph shows the memory consumed by each installed app. You can click on each segment
+on the graph to see the memory consumption over time in the chart below it.
+
+The `System` Memory graph shows the overall memory consumption on the entire Cloudron. If you see
+the Free memory < 50MB frequently, you should consider upgrading to a Cloudron with more memory.
+
+# Activity log
+
+The `Activity` view shows the activity on your Cloudron. It includes information about who is using
+the apps on your Cloudron and also tracks configuration changes.
+
+<img src="/docs/img/activity.png" class="shadow">
+
+# Domains and SSL Certificates
+
+All apps on the Cloudron can only be reached by `https`. The Cloudron automatically installs and
+renews certificates for your apps as needed. Should installation of certificate fail for reasons
+beyond it's control, Cloudron admins will get a notification about it.
+
+# API Access
+
+All the operations listed in this manual like installing app, configuring users and groups, are
+completely programmable with a [REST API](/references/api.html).
+
+# Moving to a larger Cloudron
+
+When using a Cloudron from cloudron.io, it is easy to migrate your apps and data to a bigger server.
+In the `Settings` page, you can change the plan.
+
+<insert picture>
+
+# Command line tool
+
+If you are a software developer or a sysadmin, the Cloudron comes with a CLI tool that can be
+used to develop custom apps for the Cloudron. Read more about it [here](https://git.cloudron.io/cloudron/cloudron-cli).

docs/tutorials/node.md

@@ -0,0 +1,621 @@
+# Overview
+
+This tutorial provides an introduction to developing applications
+for the Cloudron using node.js.
+
+# Installation
+
+## Install CLI tool
+
+The Cloudron CLI tool allows you to install, configure and test apps on your Cloudron.
+
+Installing the CLI tool requires [node.js](https://nodejs.org/) and
+[npm](https://www.npmjs.com/). You can then install the CLI tool using the following
+command:
+
+```
+    sudo npm install -g cloudron
+```
+
+Note: Depending on your setup, you can run the above command without `sudo`.
+
+## Testing your installation
+
+The `cloudron` command should now be available in your path.
+
+Let's login to the Cloudron as follows:
+
+```
+$ cloudron login
+Cloudron Hostname: craft.selfhost.io
+
+Enter credentials for craft.selfhost.io:
+Username: girish
+Password:
+Login successful.
+```
+
+## Your First Application
+
+Creating an application for Cloudron can be summarized as follows:
+
+1. Create a web application using any language/framework. This web application must run a HTTP server
+   and can optionally provide other services using custom protocols (like git, ssh, TCP etc).
+
+2. Create a [Dockerfile](http://docs.docker.com/engine/reference/builder/) that specifies how to create 
+   an application ```image```. An ```image``` is essentially a bundle of the application source code
+   and it's dependencies.
+
+3. Create a [CloudronManifest.json](/references/manifest.html) file that provides essential information
+   about the app. This includes information required for the Cloudron Store like title, version, icon and 
+   runtime requirements like `addons`.
+
+## Simple Web application
+
+To keep things simple, we will start by deploying a trivial node.js server running on port 8000.
+
+Create a new project folder `tutorial/` and add a file named `tutorial/server.js` with the following content:
+```javascript
+var http = require("http");
+
+var server = http.createServer(function (request, response) {
+  response.writeHead(200, {"Content-Type": "text/plain"});
+  response.end("Hello World\n");
+});
+
+server.listen(8000);
+
+console.log("Server running at port 8000");
+```
+
+## Dockerfile
+
+A Dockerfile contains commands to assemble an image.
+
+Create a file named `tutorial/Dockerfile` with the following content:
+
+```dockerfile
+FROM cloudron/base:0.9.0
+
+ADD server.js /app/code/server.js
+
+CMD [ "/usr/local/node-0.12.7/bin/node", "/app/code/server.js" ]
+```
+
+The `FROM` command specifies that we want to start off with Cloudron's [base image](/references/baseimage.html).
+All Cloudron apps **must** start from this base image.
+
+The `ADD` command copies the source code of the app into the directory `/app/code`.
+While this example only copies a single file, the ADD command can be used to copy directory trees as well.
+See the [Dockerfile](https://docs.docker.com/reference/builder/#add) documentation for more details.
+
+The `CMD` command specifies how to run the server. There are multiple versions of node available under `/usr/local`. We
+choose node v0.12.7 for our app.
+
+## CloudronManifest.json
+
+The `CloudronManifest.json` specifies
+
+* Information about displaying the app on the Cloudron Store. For example,
+  the title, author information, description etc
+
+* Information for installing the app on the Cloudron. This includes fields
+  like httpPort, tcpPorts.
+
+Create the CloudronManifest.json using the following command:
+
+```
+$ cloudron init
+id: io.cloudron.tutorial                       # unique id for this app. use reverse domain name convention
+author: John Doe                               # developer or company name of the for user <email>
+title: Tutorial App                            # Cloudron Store title of this app
+description: App that uses node.js             # A string or local file reference like file://DESCRIPTION.md
+tagline: Changing the world one app at a time  # A tag line for this app for the Cloudron Store
+website: https://cloudron.io                   # A link to this app's website
+contactEmail: support@cloudron.io              # Contact email of developer or company
+httPort: 8000                                  # The http port on which this application listens to
+```
+
+The above command creates a CloudronManifest.json:
+
+File ```tutorial/CloudronManifest.json```
+
+```json
+{
+  "id": "io.cloudron.tutorial",
+  "author": "John Doe",
+  "title": "Tutorial App",
+  "description": "App that uses node.js",
+  "tagline": "Changing the world one app at a time",
+  "version": "0.0.1",
+  "healthCheckPath": "/",
+  "httpPort": 8000,
+  "addons": {
+    "localstorage": {}
+  },
+  "minBoxVersion": "0.0.1",
+  "manifestVersion": 1,
+  "website": "https://cloudron.io",
+  "contactEmail": "support@cloudron.io",
+  "icon": "",
+  "mediaLinks": []
+}
+```
+
+You can read in more detail about each field in the [Manifest reference](/references/manifest.html).
+
+# Installing
+
+## Building
+
+We now have all the necessary files in place to build and deploy the app to the Cloudron.
+Building creates an image of the app using the Dockerfile which can then be used to deploy
+to the Cloudron.
+
+Building, pushing and pulling docker images is very bandwidth and CPU intensive. To alleviate this
+problem, apps are built using the `build service` which uses `cloudron.io` account credentials.
+
+**Warning**: As of this writing, the build service uses the public Docker registry and the images that are built
+can be downloaded by anyone. This means that your source code will be viewable by others.
+
+Initiate a build using ```cloudron build```:
+```
+$ cloudron build
+Building io.cloudron.tutorial@0.0.1
+
+Appstore login:
+Email: ramakrishnan.girish@gmail.com         # cloudron.io account
+Password:                                    # Enter password
+Login successful.
+
+Build scheduled with id 76cebfdd-7822-4f3d-af17-b3eb393ae604
+Downloading source
+Building
+Step 0 : FROM cloudron/base:0.9.0
+ ---> 97583855cc0c
+Step 1 : ADD server.js /app/code
+ ---> b09b97ecdfbc
+Removing intermediate container 03c1e1f77acb
+Step 2 : CMD /usr/local/node-0.12.7/bin/node /app/code/main.js
+ ---> Running in 370f59d87ab2
+ ---> 53b51eabcb89
+Removing intermediate container 370f59d87ab2
+Successfully built 53b51eabcb89
+The push refers to a repository [cloudron/img-2074d69134a7e0da3d6cdf3c53e241c4] (len: 1)
+Sending image list
+Pushing repository cloudron/img-2074d69134a7e0da3d6cdf3c53e241c4 (1 tags)
+Image already pushed, skipping 57f52d167bbb
+Image successfully pushed b09b97ecdfbc
+Image successfully pushed 53b51eabcb89
+Pushing tag for rev [53b51eabcb89] on {https://cdn-registry-1.docker.io/v1/repositories/cloudron/img-2074d69134a7e0da3d6cdf3c53e241c4/tags/76cebfdd-7822-4f3d-af17-b3eb393ae604}
+Build succeeded
+```
+
+## Installing
+
+Now that we have built the image, we can install our latest build on the Cloudron
+using the following command:
+
+```
+$ cloudron install
+Using cloudron craft.selfhost.io
+Using build 76cebfdd-7822-4f3d-af17-b3eb393ae604 from 1 hour ago
+Location: tutorial                         # This is the location into which the application installs
+App is being installed with id: 4dedd3bb-4bae-41ef-9f32-7f938995f85e
+
+ => Waiting to start installation
+ => Registering subdomain .
+ => Verifying manifest .
+ => Downloading image ..............
+ => Creating volume .
+ => Creating container
+ => Setting up collectd profile ................
+ => Waiting for DNS propagation ...
+
+App is installed.
+```
+
+This makes the app available at https://tutorial-craft.selfhost.io.
+
+Open the app in your default browser:
+```
+cloudron open
+```
+
+You should see `Hello World`.
+
+# Testing
+
+The application testing cycle involves `cloudron build` and `cloudron install`.
+Note that `cloudron install` updates an existing app in place.
+
+You can view the logs using `cloudron logs`. When the app is running you can follow the logs
+using `cloudron logs -f`.
+
+For example, you can see the console.log output in our server.js with the command below:
+
+```
+$ cloudron logs
+Using cloudron craft.selfhost.io
+2015-05-08T03:28:40.233940616Z Server running at port 8000
+```
+
+It is also possible to run a *shell* and *execute* arbitrary commands in the context of the application
+process by using `cloudron exec`. By default, exec simply drops you into an interactive bash shell with
+which you can inspect the file system and the environment.
+
+```
+$ cloudron exec
+```
+
+You can also execute arbitrary commands:
+```
+$ cloudron exec env # display the env variables that your app is running with
+```
+
+# Storing data
+
+For file system storage, an app can use the `localstorage` addon to store data under `/app/data`.
+When the `localstorage` addon is active, any data under /app/data is automatically backed up. When an
+app is updated, /app/data already contains the data generated by the previous version.
+
+*Note*: For convenience, the initial CloudronManifest.json generated by `cloudron init` already contains this
+addon.
+
+Let us put this theory into action by saving a *visit counter* as a file.
+*server.js* has been modified to count the number of visitors on the site by storing a counter
+in a file named ```counter.dat```.
+
+File ```tutorial/server.js```
+
+```javascript
+var http = require('http'),
+    fs = require('fs'),
+    util = require('util');
+
+var COUNTER_FILE = '/app/data/counter.dat';
+
+var server = http.createServer(function (request, response) {
+    var counter = 0;
+    if (fs.existsSync(COUNTER_FILE)) {
+        // read existing counter if it exists
+        counter = parseInt(fs.readFileSync(COUNTER_FILE, 'utf8'), 10);
+    }
+
+    response.writeHead(200, {"Content-Type": "text/plain"});
+    response.end(util.format("Hello World. %s visitors have visited this page\n", counter));
+    ++counter; // bump the counter
+    fs.writeFileSync(COUNTER_FILE, counter + '', 'utf8'); // save back counter
+});
+
+server.listen(8000);
+
+console.log("Server running at port 8000");
+```
+
+Now every time you refresh the page you will notice that the counter bumps up. You will
+also notice that if you make changes to the app and do a `cloudron install`, the `counter.dat`
+is *retained* across updates.
+
+# Database
+
+Most web applications require a database of some form. In theory, it is possible to run any
+database you want as part of the application image. This is, however, a waste of server resources
+should every app runs it's own database server.
+
+To solve this, the Cloudron provides shareable resources like databases in form of ```addons```.
+The database server is managed by the Cloudron and the application simply needs to request access to
+the database in the CloudronManifest.json. While the database server itself is a shared resource, the
+databases are exclusive to the application. Each database is password protected and accessible only
+to the application. Databases and tables can be configured without restriction as the application
+requires.
+
+Cloudron currently provides `mysql`, `postgresql`, `mongodb`, `redis` database addons.
+
+For this tutorial, let us try to save the counter in `redis` addon. For this, we make use of the
+[redis](https://www.npmjs.com/package/redis) module.
+
+Since this is a node.js app, let's add a very basic `package.json` containing the `redis` module dependency.
+
+File `tutorial/package.json`
+```json
+{
+  "name": "tutorial",
+  "version": "1.0.0",
+  "dependencies": {
+    "redis": "^0.12.1"
+  }
+}
+```
+
+and modify our Dockerfile to look like this:
+
+File `tutorial/Dockerfile`
+
+```dockerfile
+FROM cloudron/base:0.9.0
+
+ADD server.js /app/code/server.js
+ADD package.json /app/code/package.json
+
+WORKDIR /app/code
+RUN npm install --production
+
+CMD [ "/usr/local/node-0.12.7/bin/node", "/app/code/server.js" ]
+```
+
+Notice the new `RUN` command which installs the node module dependencies in package.json using `npm install`.
+
+Since we want to use redis, we have to modify the CloudronManifest.json to make redis available for this app.
+
+File `tutorial/CloudronManifest.json`
+
+```json
+{
+  "id": "io.cloudron.tutorial",
+  "author": "John Doe",
+  "title": "Tutorial App",
+  "description": "App that uses node.js",
+  "tagline": "Changing the world one app at a time",
+  "version": "0.0.1",
+  "healthCheckPath": "/",
+  "httpPort": 8000,
+  "addons": {
+    "localstorage": {},
+    "redis": {}
+  },
+  "minBoxVersion": "0.0.1",
+  "manifestVersion": 1,
+  "website": "https://cloudron.io",
+  "contactEmail": "support@cloudron.io",
+  "icon": "",
+  "mediaLinks": []
+}
+```
+
+When the application runs, environment variables `REDIS_HOST`, `REDIS_PORT` and
+`REDIS_PASSWORD` are injected. You can read about the environment variables in the
+[Redis reference](/references/addons.html#redis).
+
+Let's change `server.js` to use redis instead of file backed counting:
+
+File ```tutorial/server.js```
+
+```javascript
+var http = require('http'),
+    fs = require('fs'),
+    util = require('util'),
+    redis = require('redis');
+
+var redisClient = redis.createClient(process.env.REDIS_PORT, process.env.REDIS_HOST);
+redisClient.auth(process.env.REDIS_PASSWORD);
+redisClient.on("error", function (err) {
+  console.log("Redis Client Error " + err);
+});
+
+var COUNTER_KEY = 'counter';
+
+var server = http.createServer(function (request, response) {
+  redisClient.get(COUNTER_KEY, function (err, reply) {
+    var counter = (!err && reply) ? parseInt(reply, 10) : 0;
+    response.writeHead(200, {"Content-Type": "text/plain"});
+    response.end(util.format("Hello World. %s visitors have visited this page\n", counter));
+    redisClient.incr(COUNTER_KEY);
+  });
+});
+
+server.listen(8000);
+
+console.log("Server running at port 8000");
+```
+
+Simply `cloudron build` and `cloudron install` to test your app!
+
+# Authentication
+
+The Cloudron has a centralized panel for managing users and groups. Apps can integrate Single Sign-On
+authentication using LDAP or OAuth.
+
+Note that apps that are single user can skip Single Sign-On support. The Cloudron implements an `OAuth
+proxy` (accessed through the app configuration dialog) that optionally lets the Cloudron admin make the
+app visible only for logged in users.
+
+## LDAP
+
+Let's start out by adding the [ldap](/references/addons.html#ldap) addon to the manifest.
+
+File `tutorial/CloudronManifest.json`
+```json
+{
+  "id": "io.cloudron.tutorial",
+  "author": "John Doe",
+  "title": "Tutorial App",
+  "description": "App that uses node.js",
+  "tagline": "Changing the world one app at a time",
+  "version": "0.0.1",
+  "healthCheckPath": "/",
+  "httpPort": 8000,
+  "addons": {
+    "localstorage": {},
+    "ldap": {}
+  },
+  "minBoxVersion": "0.0.1",
+  "manifestVersion": 1,
+  "website": "https://cloudron.io",
+  "contactEmail": "support@cloudron.io",
+  "icon": "",
+  "mediaLinks": []
+}
+```
+
+Building and installing the app shows that the app gets new LDAP specific environment variables.
+
+```
+$ cloudron build
+$ cloudron install
+$ cloudron exec env | grep LDAP
+LDAP_SERVER=172.17.42.1
+LDAP_PORT=3002
+LDAP_URL=ldap://172.17.42.1:3002
+LDAP_USERS_BASE_DN=ou=users,dc=cloudron
+LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron
+```
+
+Let's test the environment variables to use by using the [ldapjs](http://www.ldapjs.org) npm module.
+We start by adding ldapjs to package.json.
+
+File `tutorial/package.json`
+```json
+{
+  "name": "tutorial",
+  "version": "1.0.0",
+  "dependencies": {
+    "ldapjs": "^0.7.1"
+  }
+}
+```
+
+The server code has been modified to authenticate using the `X-Username` and `X-Password` headers for
+any path other than '/'.
+
+File `tutorial/server.js`
+```javascript
+var http = require("http"),
+    ldap = require('ldapjs');
+
+var ldapClient = ldap.createClient({ url: process.env.LDAP_URL });
+
+var server = http.createServer(function (request, response) {
+  if (request.url === '/') {
+    response.writeHead(200, {"Content-Type": "text/plain"});
+    return response.end();
+  }
+
+  var username = request.headers['x-username'] || '';
+  var password = request.headers['x-password'] || '';
+  var ldapDn = 'cn=' + username + ',' + process.env.LDAP_USERS_BASE_DN;
+
+  ldapClient.bind(ldapDn, password, function (error) {
+    if (error) {
+      response.writeHead(401, {"Content-Type": "text/plain"});
+      response.end('Failed to authenticate: ' + error);
+    } else {
+      response.writeHead(200, {"Content-Type": "text/plain"});
+      response.end('Successfully authenticated');
+    }
+  });
+});
+
+server.listen(8000);
+
+console.log("Server running at port 8000");
+```
+
+Once we have used `cloudron build` and `cloudron install`, you can use `curl` to test
+credentials as follows:
+
+```bash
+  # Test with various credentials here. Your cloudon admin username and password should succeed.
+  curl -X 'X-Username: admin' -X 'X-Password: pass' https://tutorial-craft.selfhost.io/login
+```
+
+## OAuth
+
+An app can integrate with OAuth 2.0 Authorization code grant flow by adding
+[oauth](/references/addons.html#oauth) to CloudronManifest.json `addons` section.
+
+Doing so will get the following environment variables:
+```
+$ cloudron exec env
+OAUTH_CLIENT_ID=cid-addon-4089f65a-2adb-49d2-a6d1-e519b7d85e8d
+OAUTH_CLIENT_SECRET=5af99a9633283aa15f5e6df4a108ff57f82064e4845de8bce8ad3af54dfa9dda
+OAUTH_ORIGIN=https://my-craft.selfhost.io
+API_ORIGIN=https://my-craft.selfhost.io
+HOSTNAME=tutorial-craft.selfhost.io
+```
+
+OAuth Authorization code grant flow works as follows:
+* App starts the flow by redirecting the user to Cloudron authorization endpoint of the following format:
+```
+https://API_ORIGIN/api/v1/oauth/dialog/authorize?response_type=code&client_id=OAUTH_CLIENT_ID&redirect_uri=CALLBACK_URL&scope=profile
+```
+
+  In the above URL, API_ORIGIN and OAUTH_CLIENT_ID are environment variables. CALLBACK_URL is a url of the app
+to which the user will be redirected back to after successful authentication. CALLBACK_URL has to have the
+same origin as the app.
+
+* The Cloudron OAuth server authenticates the user (using a password form) at the above URL. It also establishes
+that the user grants the client's access request.
+
+* If the user authenticated successfully, it will redirect the browser to CALLBACK_URL with a `code` query parameter.
+
+* The app can exchange the `code` above for a `access token` by using the `OAUTH_CLIENT_SECRET`. It does so by making
+  a _POST_ request to the following url:
+```
+https://API_ORIGIN/api/v1/oauth/token?response_type=token&client_id=OAUTH_CLIENT_ID
+```
+with the following request body (json):
+```json
+{
+    "grant_type": "authorization_code",
+    "code": "<the code received in CALLBACK_URL query parameter>",
+    "redirect_uri": "https://<HOSTNAME>",
+    "client_id": "<OAUTH_CLIENT_ID>",
+    "client_secret": "<OAUTH_CLIENT_SECRET>"
+}
+```
+
+  In the above URL, API_ORIGIN, OAUTH_CLIENT_ID and HOSTNAME are environment variables. The response contains
+the `access_token` in the body.
+
+* The `access_token` can be used to get the [user's profile](/references/api.html#profile) using the following url:
+```
+https://API_ORIGIN/api/v1/profile?access_token=ACCESS_TOKEN
+```
+
+  The `access_token` may also be provided in the `Authorization` header as `Bearer: <token>`.
+
+An implementation of the above OAuth logic is at [ircd-app](https://github.com/cloudron-io/ircd-app/blob/master/settings/app.js).
+
+The following libraries implement Cloudron OAuth for Ruby and Javascript.
+
+ * [omniauth-cloudron](https://github.com/cloudron-io/omniauth-cloudron)
+ * [passport-cloudron](https://github.com/cloudron-io/passport-cloudron)
+
+# Beta Testing
+
+Once your app is ready, you can upload it to the store for `beta testing` by
+other Cloudron users. This can be done using:
+
+```
+  cloudron upload
+```
+
+The app should now be visible in the Store view of your cloudron under
+the 'Testing' section. You can check if the icon, description and other details
+appear correctly.
+
+Other Cloudron users can install your app on their Cloudron's using
+`cloudron install --appstore-id <appid@version>`. Note that this currently
+requires your beta testers to install the CLI tool and put their Cloudron in
+developer mode.
+
+# Publishing
+
+Once you are satisfied with the beta testing, you can submit it for review.
+
+```
+  cloudron submit
+```
+
+The cloudron.io team will review the app and publish the app to the store.
+
+# Next steps
+
+Congratulations! You are now well equipped to build web applications for the Cloudron.
+
+# Samples
+
+  * [Lets Chat](https://github.com/cloudron-io/letschat-app)
+  * [Haste bin](https://github.com/cloudron-io/haste-app)
+  * [Pasteboard](https://github.com/cloudron-io/pasteboard-app)

docs/tutorials/packaging.md

@@ -0,0 +1,492 @@
+# Overview
+
+This tutorial outlines how to package an existing web application for the Cloudron.
+
+If you are aware of Docker and Heroku, you should feel at home packaging for the
+Cloudron. Roughly, the steps involved are:
+
+* Create a Dockerfile for your application. If your application already has
+  a Dockerfile, you should able to reuse most of it. By virtue of Docker, the Cloudron
+  is able to run apps written in any language/framework.
+
+* Create a CloudronManifest.json that provides information like title, author, description
+   etc. You can also specify the addons (like database) required
+  to run your app. When the app runs on the Cloudron, it will have environment
+  variables set for connecting to the addon.
+
+* Test the app on your Cloudron with the CLI tool.
+
+* Optionally, submit the app to [Cloudron Store](/appstore.html).
+
+# Prerequisites
+
+## Install CLI tool
+
+The Cloudron CLI tool allows you to install, configure and test apps on your Cloudron.
+
+Installing the CLI tool requires [node.js](https://nodejs.org/) and
+[npm](https://www.npmjs.com/). You can then install the CLI tool using the following
+command:
+
+```
+    sudo npm install -g cloudron
+```
+
+Note: Depending on your setup, you can run the above command without `sudo`.
+
+## Login to Cloudron
+
+The `cloudron` command should now be available in your path.
+
+You can login to your Cloudron now:
+
+```
+$ cloudron login
+Cloudron Hostname: craft.selfhost.io
+
+Enter credentials for craft.selfhost.io:
+Username: girish
+Password:
+Login successful.
+```
+
+# Basic app
+
+We will first package a very simple app to understand how the packaging works.
+You can clone this app from https://git.cloudron.io/cloudron/tutorial-basic.
+
+## The server
+
+The basic app server is a very simple HTTP server that runs on port 8000.
+While the server in this tutorial uses node.js, you can write your server
+in any language you want.
+
+```server.js
+var http = require("http");
+
+var server = http.createServer(function (request, response) {
+  response.writeHead(200, {"Content-Type": "text/plain"});
+  response.end("Hello World\n");
+});
+
+server.listen(8000);
+
+console.log("Server running at port 8000");
+```
+
+## Dockerfile
+
+The Dockerfile contains instructions on how to create an image for your application.
+
+```Dockerfile
+FROM cloudron/base:0.9.0
+
+ADD server.js /app/code/server.js
+
+CMD [ "/usr/local/node-4.2.1/bin/node", "/app/code/server.js" ]
+```
+
+The `FROM` command specifies that we want to start off with Cloudron's [base image](/references/baseimage.html).
+All Cloudron apps **must** start from this base image. This approach conserves space on the Cloudron since
+Docker images tend to be quiet large.
+
+The `ADD` command copies the source code of the app into the directory `/app/code`. There is nothing special
+about the `/app/code` directory and it is merely a convention we use to store the application code.
+
+The `CMD` command specifies how to run the server. The base image already contains many different versions of
+node.js. We use Node 4.2.1 here.
+
+This Dockerfile can be built and run locally as:
+```
+docker build -t tutorial .
+docker run -p 8000:8000 -ti tutorial
+```
+
+## Manifest
+
+The `CloudronManifest.json` specifies
+
+* Information for installing and running the app on the Cloudron. This includes fields like addons, httpPort, tcpPorts.
+
+* Information about displaying the app on the Cloudron Store. For example, fields like title, author, description.
+
+Create the CloudronManifest.json using `cloudron init` as follows:
+
+```
+$ cloudron init
+id: io.cloudron.tutorial                       # unique id for this app. use reverse domain name convention
+author: John Doe                               # developer or company name of the for user <email>
+title: Tutorial App                            # Cloudron Store title of this app
+description: App that uses node.js             # A string or local file reference like file://DESCRIPTION.md
+tagline: Changing the world one app at a time  # A tag line for this app for the Cloudron Store
+website: https://cloudron.io                   # A link to this app's website
+contactEmail: support@cloudron.io              # Contact email of developer or company
+httPort: 8000                                  # The http port on which this application listens to
+```
+
+The above command creates a CloudronManifest.json:
+
+File ```tutorial/CloudronManifest.json```
+
+```json
+{
+  "id": "io.cloudron.tutorial",
+  "title": "Tutorial App",
+  "author": "John Doe",
+  "description": "file://DESCRIPTION.md",
+  "changelog": "file://CHANGELOG",
+  "tagline": "Changing the world one app at a time",
+  "version": "0.0.1",
+  "healthCheckPath": "/",
+  "httpPort": 8000,
+  "addons": {
+    "localstorage": {}
+  },
+  "manifestVersion": 1,
+  "website": "https://cloudron.io",
+  "contactEmail": "support@cloudron.io",
+  "icon": "",
+  "tags": [
+    "changme"
+  ],
+  "mediaLinks": [ ]
+}
+```
+
+You can read in more detail about each field in the [Manifest reference](/references/manifest.html). The
+`localstorage` addon allows the app to store files in `/app/data`. We will explore addons further further
+down in this tutorial.
+
+Additional files created by `init` are:
+* `DESCRIPTION.md` - A markdown file providing description of the app for the Cloudron Store.
+* `CHANGELOG` - A file containing change information for each version released to the Cloudron Store. This
+  information is shown when the user updates the app.
+
+# Installing
+
+We now have all the necessary files in place to build and deploy the app to the Cloudron.
+
+## Building
+
+Building, pushing and pulling docker images can be very bandwidth and CPU intensive. To alleviate this
+problem, apps are built using the `build service` which uses `cloudron.io` account credentials.
+
+**Warning**: As of this writing, the build service uses the public Docker registry and the images that are built
+can be downloaded by anyone. This means that your source code will be viewable by others.
+
+Initiate a build using ```cloudron build```:
+```
+$ cloudron build
+Building io.cloudron.tutorial@0.0.1
+
+Appstore login:
+Email: ramakrishnan.girish@gmail.com         # cloudron.io account
+Password:                                    # Enter password
+Login successful.
+
+Build scheduled with id e7706847-f2e3-4ba2-9638-3f334a9453a5
+Waiting for build to begin, this may take a bit...
+Downloading source
+Building
+Step 1 : FROM cloudron/base:0.9.0
+ ---> be9fc6312b2d
+Step 2 : ADD server.js /app/code/server.js
+ ---> 10513e428d7a
+Removing intermediate container 574573f6ed1c
+Step 3 : CMD /usr/local/node-4.2.1/bin/node /app/code/server.js
+ ---> Running in b541d149b6b9
+ ---> 51aa796ea6e5
+Removing intermediate container b541d149b6b9
+Successfully built 51aa796ea6e5
+Pushing
+The push refers to a repository [docker.io/cloudron/img-062037096d69bbf3ffb5b9316ad89cb9] (len: 1)
+Pushed 51aa796ea6e5
+Pushed 10513e428d7a
+Image already exists be9fc6312b2d
+Image already exists a0261a2a7c75
+Image already exists f9d4f0f1eeed
+Image already exists 2b650158d5d8
+e7706847-f2e3-4ba2-9638-3f334a9453a5: digest: sha256:8241d68b65874496191106ecf2ee8f3df2e05a953cd90ff074a6f8815a49389c size: 26098
+Build succeeded
+Success
+```
+
+## Installing
+
+Now that we have built the image, we can install our latest build on the Cloudron
+using the following command:
+
+```
+$ cloudron install
+Using cloudron craft.selfhost.io
+Using build 76cebfdd-7822-4f3d-af17-b3eb393ae604 from 1 hour ago
+Location: tutorial                         # This is the location into which the application installs
+App is being installed with id: 4dedd3bb-4bae-41ef-9f32-7f938995f85e
+
+ => Waiting to start installation
+ => Registering subdomain .
+ => Verifying manifest .
+ => Downloading image ..............
+ => Creating volume .
+ => Creating container
+ => Setting up collectd profile ................
+ => Waiting for DNS propagation ...
+
+App is installed.
+```
+
+Open the app in your default browser:
+```
+cloudron open
+```
+
+You should see `Hello World`.
+
+# Testing
+
+The application testing cycle involves `cloudron build` and `cloudron install`.
+Note that `cloudron install` updates an existing app in place.
+
+You can view the logs using `cloudron logs`. When the app is running you can follow the logs
+using `cloudron logs -f`.
+
+For example, you can see the console.log output in our server.js with the command below:
+
+```
+$ cloudron logs
+Using cloudron craft.selfhost.io
+16:44:11 [main] Server running at port 8000
+```
+
+It is also possible to run a *shell* and *execute* arbitrary commands in the context of the application
+process by using `cloudron exec`. By default, exec simply drops you into an interactive bash shell with
+which you can inspect the file system and the environment.
+
+```
+$ cloudron exec
+```
+
+You can also execute arbitrary commands:
+```
+$ cloudron exec env # display the env variables that your app is running with
+```
+
+### DevelopmentMode
+
+When debugging complex startup scripts, one can specify `"developmentMode": true,` in the CloudronManifest.json.
+This will ignore the `RUN` command, specified in the Dockerfile and allows the developer to interactively test
+the startup scripts using `cloudron exec`.
+
+**Note:** that an app running in this mode has full read/write access to the filesystem and all memory limits are lifted.
+
+
+# Addons
+
+## Filesystem
+
+The application container created on the Cloudron has a `readonly` file system. Writing to any location
+other than the below will result in an error:
+
+* `/tmp` - Use this location for temporary files. The Cloudron will cleanup any files in this directory
+  periodically.
+
+* `/run` - Use this location for runtime configuration and dynamic data. These files should not be expected
+  to persist across application restarts (for example, after an update or a crash).
+
+* `/app/data` - Use this location to store application data that is to be backed up. To use this location,
+  you must use the [localstorage](/references/addons.html#localstorage) addon. For convenience, the initial CloudronManifest.json generated by
+  `cloudron init` already contains this addon.
+
+## Database
+
+Most web applications require a database of some form. In theory, it is possible to run any
+database you want as part of the application image. This is, however, a waste of server resources
+should every app runs it's own database server.
+
+Cloudron currently provides [mysql](/references/addons.html#mysql), [postgresql](/references/addons.html#postgresql),
+[mongodb](/references/addons.html#mongodb), [redis](/references/addons.html#redis) database addons. When choosing
+these addons, the Cloudron will inject environment variables that contain information on how to connect
+to the addon.
+
+See https://git.cloudron.io/cloudron/tutorial-redis for a simple example of how redis can be used by
+an application. The server simply uses the environment variables to connect to redis.
+
+## Email
+
+Cloudron applications can send email using the `sendmail` addon. Using the `sendmail` addon provides
+the SMTP server and authentication credentials in environment variables.
+
+Cloudron applications can also receive mail via IMAP using the `recvmail` addon.
+
+## Authentication
+
+The Cloudron has a centralized panel for managing users and groups. Apps can integrate Single Sign-On
+authentication using LDAP or OAuth.
+
+Apps can integrate with the Cloudron authentication system using LDAP, OAuth or Simple Auth. See the
+[authentication](/references/authentication.html) reference page for more details.
+
+See https://git.cloudron.io/cloudron/tutorial-ldap for a simple example of how to authenticate via LDAP.
+
+For apps that are single user can skip Single Sign-On support by setting the `"singleUser": true`
+in the manifest. By doing so, the Cloudron will installer will show a dialog to choose a user.
+
+For app that have no user management at all, the Cloudron implements an `OAuth proxy` that
+optionally lets the Cloudron admin make the app visible only for logged in users.
+
+# Best practices
+
+## No Setup
+
+A Cloudron app is meant to instantly usable after installation. For this reason, Cloudron apps must not
+show any setup screen after installation and should simply choose reasonable defaults.
+
+Databases, email configuration should be automatically picked up from the environment variables using
+addons.
+
+## Dockerfile
+
+The app is run as a read-only docker container. Because of this:
+* Install any required packages in the Dockerfile.
+* Create static configuration files in the Dockerfile.
+* Create symlinks to dynamic configuration files under /run in the Dockerfile.
+
+## Process manager
+
+Docker supports restarting processes natively. Should your application crash, it will be restarted
+automatically. If your application is a single process, you do not require any process manager.
+
+Use supervisor, pm2 or any of the other process managers if you application has more then one component.
+This **excludes** web servers like apache, nginx which can already manage their children by themselves.
+Be sure to pick a process manager that forwards signals to child processes.
+
+## Automatic updates
+
+Some apps support automatic updates by overwriting themselves. A Cloudron app cannot overwrite itself
+because of the read-only file system. For this reason, disable auto updates for app and let updates be
+triggered through the Cloudron Store. This ties in better to the Cloudron's update and restore approach
+should something go wrong with the update.
+
+## Logging
+
+Cloudron applications stream their logs to stdout and stderr. In practice, this ideal is hard to achieve.
+Some programs like apache simply don't log to stdout. In those cases, simply log to `/tmp` or `/run`.
+
+Logging to stdout has many advantages:
+* App does not need to rotate logs and the Cloudron takes care of managing logs.
+* App does not need special mechanism to release log file handles (on a log rotate).
+* Integrates better with tooling like cloudron cli.
+
+## Memory
+
+By default, applications get 256MB RAM (including swap). This can be changed using the `memoryLimit`
+field in the manifest.
+
+Design your application runtime for concurrent use by 50 users. The Cloudron is not designed for
+concurrent access by 100s or 1000s of users.
+
+## Authentication
+
+Apps should integrate with one of the [authentication strategies](/references/authentication.html).
+This saves the user from having to manage separate set of credentials for each app.
+
+## Startup Script
+
+Many apps do not launch the server directly, as we did in our basic example. Instead, they execute
+a `start.sh` script (named so by convention) which launches the server. Before starting the server,
+the `start.sh` script does the following:
+
+  * When using the `localstorage` addon, it changes the ownership of files in `/app/data` as desired using `chown`. This
+    is necessary because file permissions may not be correctly preserved across backup, restore, application and base image
+    updates.
+
+  * Addon information (mail, database) exposed as environment  are subject to change across restarts and an application
+    must use these values directly (i.e not cache them across restarts). For this reason, it usually regenerates
+    any config files with the current database settings on each invocation.
+
+  * Finally, it starts the server as a non-root user.
+
+The app's main process must handle SIGTERM and forward it as required to child processes. bash does not
+automatically forward signals to child processes. For this reason, when using a startup shell script,
+remember to use exec <app> as the last line. Doing so will replace bash with your program and allows
+your program to handle signals as required.
+
+# Beta Testing
+
+## Metadata
+
+Publishing to the Cloudron Store requires apps to have meta data specified in the `CloudronManifest.json`.
+
+The `cloudron` tool will notify if any such information is missing, prior to uploading.
+See more information for each field [here](/references/manifest.html).
+
+## Upload for Testing
+
+Once your app is ready, you can upload it to the store for `beta testing` by
+other Cloudron users. This can be done using:
+
+```
+  cloudron upload
+```
+
+The app should now be visible in the Store view of your cloudron under
+the 'Testing' section. You can check if the icon, description and other details
+appear correctly.
+
+Other Cloudron users can install your app on their Cloudron's using
+`cloudron install --appstore-id <appid@version>`.
+
+# Publishing
+
+Once you are satisfied with the beta testing, you can submit it for review.
+
+```
+  cloudron submit
+```
+
+The cloudron.io team will review the app and publish the app to the store.
+
+# Updating the app
+
+## Versioning
+
+To create an update for an app, simply bump up the [semver version](/references/manifest.html#version) field in
+the manifest and publish a new version to the store.
+
+The Cloudron chooses the next app version to update to based on the following algorithm:
+* Choose the maximum `patch` version matching the app's current `major` and `minor` version.
+* Failing the above, choose the maximum patch version of the next minor version matching the app's current `major` version.
+* Failing the above, choose the maximum patch and minor version of the next major version
+
+For example, let's assume the versions 1.1.3, 1.1.4, 1.1.5, 1.2.4, 1.2.6, 1.3.0, 2.0.0 are published.
+
+* If the app is running 1.1.3, then app will directly update to 1.1.5 (skipping 1.1.4)
+* Once in 1.1.5, the app will update to 1.2.6 (skipping 1.2.4)
+* Once in 1.2.6, the app will update to 1.3.0
+* Once in 1.3.0, the app will update to 2.0.0
+
+The Cloudron admins get notified by email for any major or minor app releases.
+
+## Failed updates
+
+The Cloudron always makes a backup of the app before making an update. Should the
+update fail, the user can restore to the backup (which will also restore the app's
+code to the previous version).
+
+# Cloudron Button
+
+The [Cloudron Button](/references/button.html) allows anyone to install your application with the click of a button
+on their Cloudron.
+
+The button can be added to just about any website including the application's website
+and README.md files in GitHub repositories.
+
+# Next steps
+
+Congratulations! You are now well equipped to build web applications for the Cloudron.
+
+You can see some examples of how real apps are packaged here:
+
+  * [Lets Chat](https://git.cloudron.io/cloudron/letschat-app)
+  * [Haste bin](https://git.cloudron.io/cloudron/haste-app)
+  * [Pasteboard](https://git.cloudron.io/cloudron/pasteboard-app)

installer/README.md

@@ -0,0 +1,27 @@
+# Installer
+
+This subfolder contains all resources, which persist across a Cloudron update.
+Only code and assets, which are part of the updater belong here.
+
+Installer is the name which got inherited from times, where this folder contained
+much more infrastructure components, like a local webserver to facilitate updates.
+
+
+## installer.sh
+
+The main entry point for initial provisioning and also updates (not upgrades).
+
+It is called from:
+ * cloudron-setup (during initial provisioning, restoring or upgrade)
+ * cloudron.js in the box code (during an update)
+
+Two arguments need to be supplied in this order:
+ 1. The public url to download the box release tarball `--sourcetarballurl`
+ 2. JSON object which contains the user-data `--data`
+
+
+## cloudron-system-setup.sh
+
+This is the systemd unit file script hook, which persists Cloudron updates.
+Mostly it revolves around setting up various parts of the filesystem, like btrfs
+volumes and swap files

installer/npm-shrinkwrap.json

@@ -1,892 +0,0 @@
-{
-  "name": "installer",
-  "version": "0.0.1",
-  "dependencies": {
-    "async": {
-      "version": "1.5.0",
-      "from": "https://registry.npmjs.org/async/-/async-1.5.0.tgz",
-      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
-    },
-    "body-parser": {
-      "version": "1.14.1",
-      "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
-      "dependencies": {
-        "bytes": {
-          "version": "2.1.0",
-          "from": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
-        },
-        "content-type": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
-        },
-        "depd": {
-          "version": "1.1.0",
-          "from": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
-        },
-        "http-errors": {
-          "version": "1.3.1",
-          "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-          "dependencies": {
-            "inherits": {
-              "version": "2.0.1",
-              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
-              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-            },
-            "statuses": {
-              "version": "1.2.1",
-              "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
-              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-            }
-          }
-        },
-        "iconv-lite": {
-          "version": "0.4.12",
-          "from": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "qs": {
-          "version": "5.1.0",
-          "from": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
-        },
-        "raw-body": {
-          "version": "2.1.4",
-          "from": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
-          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
-          "dependencies": {
-            "unpipe": {
-              "version": "1.0.0",
-              "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
-              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
-            }
-          }
-        },
-        "type-is": {
-          "version": "1.6.9",
-          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "dependencies": {
-            "media-typer": {
-              "version": "0.3.0",
-              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
-            },
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "connect-lastmile": {
-      "version": "0.0.13",
-      "from": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
-      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
-      "dependencies": {
-        "debug": {
-          "version": "2.1.3",
-          "from": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
-          "dependencies": {
-            "ms": {
-              "version": "0.7.0",
-              "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz",
-              "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
-            }
-          }
-        }
-      }
-    },
-    "debug": {
-      "version": "2.2.0",
-      "from": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-      "dependencies": {
-        "ms": {
-          "version": "0.7.1",
-          "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-        }
-      }
-    },
-    "express": {
-      "version": "4.13.3",
-      "from": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
-      "dependencies": {
-        "accepts": {
-          "version": "1.2.13",
-          "from": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
-          "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
-          "dependencies": {
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            },
-            "negotiator": {
-              "version": "0.5.3",
-              "from": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz",
-              "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
-            }
-          }
-        },
-        "array-flatten": {
-          "version": "1.1.1",
-          "from": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
-          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
-        },
-        "content-disposition": {
-          "version": "0.5.0",
-          "from": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz",
-          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
-        },
-        "content-type": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
-        },
-        "cookie": {
-          "version": "0.1.3",
-          "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
-        },
-        "cookie-signature": {
-          "version": "1.0.6",
-          "from": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
-          "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
-        },
-        "depd": {
-          "version": "1.0.1",
-          "from": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
-          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
-        },
-        "escape-html": {
-          "version": "1.0.2",
-          "from": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz",
-          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
-        },
-        "etag": {
-          "version": "1.7.0",
-          "from": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz",
-          "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
-        },
-        "finalhandler": {
-          "version": "0.4.0",
-          "from": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
-          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
-          "dependencies": {
-            "unpipe": {
-              "version": "1.0.0",
-              "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
-              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
-            }
-          }
-        },
-        "fresh": {
-          "version": "0.3.0",
-          "from": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
-        },
-        "merge-descriptors": {
-          "version": "1.0.0",
-          "from": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz",
-          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
-        },
-        "methods": {
-          "version": "1.1.1",
-          "from": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "parseurl": {
-          "version": "1.3.0",
-          "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
-        },
-        "path-to-regexp": {
-          "version": "0.1.7",
-          "from": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
-          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
-        },
-        "proxy-addr": {
-          "version": "1.0.8",
-          "from": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
-          "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
-          "dependencies": {
-            "forwarded": {
-              "version": "0.1.0",
-              "from": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz",
-              "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
-            },
-            "ipaddr.js": {
-              "version": "1.0.1",
-              "from": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz",
-              "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz"
-            }
-          }
-        },
-        "qs": {
-          "version": "4.0.0",
-          "from": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
-        },
-        "range-parser": {
-          "version": "1.0.3",
-          "from": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz",
-          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
-        },
-        "send": {
-          "version": "0.13.0",
-          "from": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
-          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
-          "dependencies": {
-            "destroy": {
-              "version": "1.0.3",
-              "from": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz",
-              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
-            },
-            "http-errors": {
-              "version": "1.3.1",
-              "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-              "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-              "dependencies": {
-                "inherits": {
-                  "version": "2.0.1",
-                  "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
-                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-                }
-              }
-            },
-            "mime": {
-              "version": "1.3.4",
-              "from": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz",
-              "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
-            },
-            "ms": {
-              "version": "0.7.1",
-              "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
-              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-            },
-            "statuses": {
-              "version": "1.2.1",
-              "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
-              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-            }
-          }
-        },
-        "serve-static": {
-          "version": "1.10.0",
-          "from": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz",
-          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
-        },
-        "type-is": {
-          "version": "1.6.9",
-          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "dependencies": {
-            "media-typer": {
-              "version": "0.3.0",
-              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
-            },
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            }
-          }
-        },
-        "utils-merge": {
-          "version": "1.0.0",
-          "from": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz",
-          "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
-        },
-        "vary": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
-        }
-      }
-    },
-    "json": {
-      "version": "9.0.3",
-      "from": "https://registry.npmjs.org/json/-/json-9.0.3.tgz",
-      "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz"
-    },
-    "morgan": {
-      "version": "1.6.1",
-      "from": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
-      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
-      "dependencies": {
-        "basic-auth": {
-          "version": "1.0.3",
-          "from": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz",
-          "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz"
-        },
-        "depd": {
-          "version": "1.0.1",
-          "from": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
-          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "on-headers": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
-        }
-      }
-    },
-    "proxy-middleware": {
-      "version": "0.15.0",
-      "from": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz",
-      "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz"
-    },
-    "request": {
-      "version": "2.72.0",
-      "from": "request@*",
-      "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz",
-      "dependencies": {
-        "aws-sign2": {
-          "version": "0.6.0",
-          "from": "aws-sign2@>=0.6.0 <0.7.0",
-          "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
-        },
-        "aws4": {
-          "version": "1.4.1",
-          "from": "aws4@>=1.2.1 <2.0.0",
-          "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.4.1.tgz"
-        },
-        "bl": {
-          "version": "1.1.2",
-          "from": "bl@>=1.1.2 <1.2.0",
-          "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz",
-          "dependencies": {
-            "readable-stream": {
-              "version": "2.0.6",
-              "from": "readable-stream@>=2.0.5 <2.1.0",
-              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
-              "dependencies": {
-                "core-util-is": {
-                  "version": "1.0.2",
-                  "from": "core-util-is@>=1.0.0 <1.1.0",
-                  "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
-                },
-                "inherits": {
-                  "version": "2.0.1",
-                  "from": "inherits@>=2.0.1 <2.1.0",
-                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-                },
-                "isarray": {
-                  "version": "1.0.0",
-                  "from": "isarray@>=1.0.0 <1.1.0",
-                  "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
-                },
-                "process-nextick-args": {
-                  "version": "1.0.7",
-                  "from": "process-nextick-args@>=1.0.6 <1.1.0",
-                  "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz"
-                },
-                "string_decoder": {
-                  "version": "0.10.31",
-                  "from": "string_decoder@>=0.10.0 <0.11.0",
-                  "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
-                },
-                "util-deprecate": {
-                  "version": "1.0.2",
-                  "from": "util-deprecate@>=1.0.1 <1.1.0",
-                  "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
-                }
-              }
-            }
-          }
-        },
-        "caseless": {
-          "version": "0.11.0",
-          "from": "caseless@>=0.11.0 <0.12.0",
-          "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz"
-        },
-        "combined-stream": {
-          "version": "1.0.5",
-          "from": "combined-stream@>=1.0.5 <1.1.0",
-          "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
-          "dependencies": {
-            "delayed-stream": {
-              "version": "1.0.0",
-              "from": "delayed-stream@>=1.0.0 <1.1.0",
-              "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
-            }
-          }
-        },
-        "extend": {
-          "version": "3.0.0",
-          "from": "extend@>=3.0.0 <3.1.0",
-          "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
-        },
-        "forever-agent": {
-          "version": "0.6.1",
-          "from": "forever-agent@>=0.6.1 <0.7.0",
-          "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
-        },
-        "form-data": {
-          "version": "1.0.0-rc4",
-          "from": "form-data@>=1.0.0-rc3 <1.1.0",
-          "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz",
-          "dependencies": {
-            "async": {
-              "version": "1.5.2",
-              "from": "async@>=1.5.2 <2.0.0",
-              "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
-            }
-          }
-        },
-        "har-validator": {
-          "version": "2.0.6",
-          "from": "har-validator@>=2.0.6 <2.1.0",
-          "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz",
-          "dependencies": {
-            "chalk": {
-              "version": "1.1.3",
-              "from": "chalk@>=1.1.1 <2.0.0",
-              "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-              "dependencies": {
-                "ansi-styles": {
-                  "version": "2.2.1",
-                  "from": "ansi-styles@>=2.2.1 <3.0.0",
-                  "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
-                },
-                "escape-string-regexp": {
-                  "version": "1.0.5",
-                  "from": "escape-string-regexp@>=1.0.2 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
-                },
-                "has-ansi": {
-                  "version": "2.0.0",
-                  "from": "has-ansi@>=2.0.0 <3.0.0",
-                  "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
-                  "dependencies": {
-                    "ansi-regex": {
-                      "version": "2.0.0",
-                      "from": "ansi-regex@>=2.0.0 <3.0.0",
-                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
-                    }
-                  }
-                },
-                "strip-ansi": {
-                  "version": "3.0.1",
-                  "from": "strip-ansi@>=3.0.0 <4.0.0",
-                  "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-                  "dependencies": {
-                    "ansi-regex": {
-                      "version": "2.0.0",
-                      "from": "ansi-regex@>=2.0.0 <3.0.0",
-                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
-                    }
-                  }
-                },
-                "supports-color": {
-                  "version": "2.0.0",
-                  "from": "supports-color@>=2.0.0 <3.0.0",
-                  "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
-                }
-              }
-            },
-            "commander": {
-              "version": "2.9.0",
-              "from": "commander@>=2.9.0 <3.0.0",
-              "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz",
-              "dependencies": {
-                "graceful-readlink": {
-                  "version": "1.0.1",
-                  "from": "graceful-readlink@>=1.0.0",
-                  "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz"
-                }
-              }
-            },
-            "is-my-json-valid": {
-              "version": "2.13.1",
-              "from": "is-my-json-valid@>=2.12.4 <3.0.0",
-              "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz",
-              "dependencies": {
-                "generate-function": {
-                  "version": "2.0.0",
-                  "from": "generate-function@>=2.0.0 <3.0.0",
-                  "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz"
-                },
-                "generate-object-property": {
-                  "version": "1.2.0",
-                  "from": "generate-object-property@>=1.1.0 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz",
-                  "dependencies": {
-                    "is-property": {
-                      "version": "1.0.2",
-                      "from": "is-property@>=1.0.0 <2.0.0",
-                      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz"
-                    }
-                  }
-                },
-                "jsonpointer": {
-                  "version": "2.0.0",
-                  "from": "jsonpointer@2.0.0",
-                  "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz"
-                },
-                "xtend": {
-                  "version": "4.0.1",
-                  "from": "xtend@>=4.0.0 <5.0.0",
-                  "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
-                }
-              }
-            },
-            "pinkie-promise": {
-              "version": "2.0.1",
-              "from": "pinkie-promise@>=2.0.0 <3.0.0",
-              "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
-              "dependencies": {
-                "pinkie": {
-                  "version": "2.0.4",
-                  "from": "pinkie@>=2.0.0 <3.0.0",
-                  "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
-                }
-              }
-            }
-          }
-        },
-        "hawk": {
-          "version": "3.1.3",
-          "from": "hawk@>=3.1.3 <3.2.0",
-          "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
-          "dependencies": {
-            "hoek": {
-              "version": "2.16.3",
-              "from": "hoek@>=2.0.0 <3.0.0",
-              "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
-            },
-            "boom": {
-              "version": "2.10.1",
-              "from": "boom@>=2.0.0 <3.0.0",
-              "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
-            },
-            "cryptiles": {
-              "version": "2.0.5",
-              "from": "cryptiles@>=2.0.0 <3.0.0",
-              "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
-            },
-            "sntp": {
-              "version": "1.0.9",
-              "from": "sntp@>=1.0.0 <2.0.0",
-              "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
-            }
-          }
-        },
-        "http-signature": {
-          "version": "1.1.1",
-          "from": "http-signature@>=1.1.0 <1.2.0",
-          "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
-          "dependencies": {
-            "assert-plus": {
-              "version": "0.2.0",
-              "from": "assert-plus@>=0.2.0 <0.3.0",
-              "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
-            },
-            "jsprim": {
-              "version": "1.2.2",
-              "from": "jsprim@>=1.2.2 <2.0.0",
-              "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz",
-              "dependencies": {
-                "extsprintf": {
-                  "version": "1.0.2",
-                  "from": "extsprintf@1.0.2",
-                  "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz"
-                },
-                "json-schema": {
-                  "version": "0.2.2",
-                  "from": "json-schema@0.2.2",
-                  "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz"
-                },
-                "verror": {
-                  "version": "1.3.6",
-                  "from": "verror@1.3.6",
-                  "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz"
-                }
-              }
-            },
-            "sshpk": {
-              "version": "1.8.3",
-              "from": "sshpk@>=1.7.0 <2.0.0",
-              "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.8.3.tgz",
-              "dependencies": {
-                "asn1": {
-                  "version": "0.2.3",
-                  "from": "asn1@>=0.2.3 <0.3.0",
-                  "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
-                },
-                "assert-plus": {
-                  "version": "1.0.0",
-                  "from": "assert-plus@>=1.0.0 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
-                },
-                "dashdash": {
-                  "version": "1.14.0",
-                  "from": "dashdash@>=1.12.0 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.0.tgz"
-                },
-                "getpass": {
-                  "version": "0.1.6",
-                  "from": "getpass@>=0.1.1 <0.2.0",
-                  "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz"
-                },
-                "jsbn": {
-                  "version": "0.1.0",
-                  "from": "jsbn@>=0.1.0 <0.2.0",
-                  "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz"
-                },
-                "tweetnacl": {
-                  "version": "0.13.3",
-                  "from": "tweetnacl@>=0.13.0 <0.14.0",
-                  "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.13.3.tgz"
-                },
-                "jodid25519": {
-                  "version": "1.0.2",
-                  "from": "jodid25519@>=1.0.0 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz"
-                },
-                "ecc-jsbn": {
-                  "version": "0.1.1",
-                  "from": "ecc-jsbn@>=0.1.1 <0.2.0",
-                  "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz"
-                }
-              }
-            }
-          }
-        },
-        "is-typedarray": {
-          "version": "1.0.0",
-          "from": "is-typedarray@>=1.0.0 <1.1.0",
-          "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz"
-        },
-        "isstream": {
-          "version": "0.1.2",
-          "from": "isstream@>=0.1.2 <0.2.0",
-          "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
-        },
-        "json-stringify-safe": {
-          "version": "5.0.1",
-          "from": "json-stringify-safe@>=5.0.1 <5.1.0",
-          "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
-        },
-        "mime-types": {
-          "version": "2.1.11",
-          "from": "mime-types@>=2.1.7 <2.2.0",
-          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.11.tgz",
-          "dependencies": {
-            "mime-db": {
-              "version": "1.23.0",
-              "from": "mime-db@>=1.23.0 <1.24.0",
-              "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.23.0.tgz"
-            }
-          }
-        },
-        "node-uuid": {
-          "version": "1.4.7",
-          "from": "node-uuid@>=1.4.7 <1.5.0",
-          "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz"
-        },
-        "oauth-sign": {
-          "version": "0.8.2",
-          "from": "oauth-sign@>=0.8.1 <0.9.0",
-          "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz"
-        },
-        "qs": {
-          "version": "6.1.0",
-          "from": "qs@>=6.1.0 <6.2.0",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz"
-        },
-        "stringstream": {
-          "version": "0.0.5",
-          "from": "stringstream@>=0.0.4 <0.1.0",
-          "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
-        },
-        "tough-cookie": {
-          "version": "2.2.2",
-          "from": "tough-cookie@>=2.2.0 <2.3.0",
-          "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz"
-        },
-        "tunnel-agent": {
-          "version": "0.4.3",
-          "from": "tunnel-agent@>=0.4.1 <0.5.0",
-          "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
-        }
-      }
-    },
-    "safetydance": {
-      "version": "0.0.19",
-      "from": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz",
-      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
-    },
-    "semver": {
-      "version": "5.1.0",
-      "from": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
-    },
-    "superagent": {
-      "version": "0.21.0",
-      "from": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
-      "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
-      "dependencies": {
-        "qs": {
-          "version": "1.2.0",
-          "from": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz"
-        },
-        "formidable": {
-          "version": "1.0.14",
-          "from": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz",
-          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
-        },
-        "mime": {
-          "version": "1.2.11",
-          "from": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
-        },
-        "component-emitter": {
-          "version": "1.1.2",
-          "from": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz",
-          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
-        },
-        "methods": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
-        },
-        "cookiejar": {
-          "version": "2.0.1",
-          "from": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
-        },
-        "reduce-component": {
-          "version": "1.0.1",
-          "from": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz",
-          "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
-        },
-        "extend": {
-          "version": "1.2.1",
-          "from": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz",
-          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
-        },
-        "form-data": {
-          "version": "0.1.3",
-          "from": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
-          "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
-          "dependencies": {
-            "combined-stream": {
-              "version": "0.0.7",
-              "from": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
-              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
-              "dependencies": {
-                "delayed-stream": {
-                  "version": "0.0.5",
-                  "from": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz",
-                  "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
-                }
-              }
-            },
-            "async": {
-              "version": "0.9.2",
-              "from": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
-              "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
-            }
-          }
-        },
-        "readable-stream": {
-          "version": "1.0.27-1",
-          "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
-          "dependencies": {
-            "core-util-is": {
-              "version": "1.0.1",
-              "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz",
-              "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
-            },
-            "isarray": {
-              "version": "0.0.1",
-              "from": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-              "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
-            },
-            "string_decoder": {
-              "version": "0.10.31",
-              "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
-            },
-            "inherits": {
-              "version": "2.0.1",
-              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
-              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-            }
-          }
-        }
-      }
-    }
-  }
-}

installer/package.json

@@ -1,48 +0,0 @@
-{
-  "name": "installer",
-  "description": "Cloudron Installer",
-  "version": "0.0.1",
-  "private": "true",
-  "author": {
-    "name": "Cloudron authors"
-  },
-  "repository": {
-    "type": "git"
-  },
-  "engines": [
-    "node >=4.0.0 <=4.1.1"
-  ],
-  "dependencies": {
-    "async": "^1.5.0",
-    "body-parser": "^1.12.0",
-    "connect-lastmile": "0.0.13",
-    "debug": "^2.1.1",
-    "express": "^4.11.2",
-    "json": "^9.0.3",
-    "morgan": "^1.5.1",
-    "proxy-middleware": "^0.15.0",
-    "request": "^2.72.0",
-    "safetydance": "0.0.19",
-    "semver": "^5.1.0",
-    "superagent": "^0.21.0"
-  },
-  "devDependencies": {
-    "colors": "^1.1.2",
-    "commander": "^2.8.1",
-    "expect.js": "^0.3.1",
-    "istanbul": "^0.3.5",
-    "lodash": "^3.2.0",
-    "mocha": "^2.1.0",
-    "nock": "^0.59.1",
-    "sleep": "^3.0.0",
-    "superagent-sync": "^0.2.0",
-    "supererror": "^0.7.0",
-    "yesno": "0.0.1"
-  },
-  "scripts": {
-    "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test",
-    "precommit": "/bin/true",
-    "prepush": "npm test",
-    "postmerge": "/bin/true"
-  }
-}

installer/scripts/installer.sh

@@ -2,14 +2,20 @@
 
 set -eu -o pipefail
 
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
 readonly BOX_SRC_DIR=/home/yellowtent/box
 readonly DATA_DIR=/home/yellowtent/data
+readonly CLOUDRON_CONF=/home/yellowtent/configs/cloudron.conf
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="${script_dir}/../../node_modules/.bin/json"
 readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
 
-readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
+readonly is_update=$([[ -f "${CLOUDRON_CONF}" ]] && echo "yes" || echo "no")
 
 # create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@
 (echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh
@@ -17,14 +23,16 @@ chmod +x /home/yellowtent/provision.sh
 
 arg_source_tarball_url=""
 arg_data=""
+arg_data_file=""
 
-args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@")
+args=$(getopt -o "" -l "sourcetarballurl:,data:,data-file:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --sourcetarballurl) arg_source_tarball_url="$2";;
     --data) arg_data="$2";;
+    --data-file) arg_data_file="$2";;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
@@ -32,21 +40,43 @@ while true; do
     shift 2
 done
 
+if [[ ! -z ${arg_data_file} ]]; then
+    arg_data=$(cat "${arg_data_file}")
+fi
+
 box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
 echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}"
 
-while true; do
+for try in `seq 1 10`; do
     if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
     echo "Failed to download source tarball, trying again"
     sleep 5
 done
-while true; do
+
+if [[ ${try} -eq 10 ]]; then
+    echo "Release tarball download failed"
+    exit 3
+fi
+
+# ensure ownership baked into the tarball is overwritten
+chown -R root.root "${box_src_tmp_dir}"
+
+for try in `seq 1 10`; do
     # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
-    if cd "${box_src_tmp_dir}" && npm rebuild; then break; fi
+
+    # We need --unsafe-perm as we run as root and the folder is owned by root,
+    # however by default npm drops privileges for npm rebuild
+    # https://docs.npmjs.com/misc/config#unsafe-perm
+    if cd "${box_src_tmp_dir}" && npm rebuild --unsafe-perm; then break; fi
     echo "Failed to rebuild, trying again"
     sleep 5
 done
 
+if [[ ${try} -eq 10 ]]; then
+    echo "npm rebuild failed"
+    exit 4
+fi
+
 if [[ "${is_update}" == "yes" ]]; then
     echo "Setting up update splash screen"
     "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code

installer/src/installer.js

@@ -1,112 +0,0 @@
-/* jslint node: true */
-
-'use strict';
-
-var assert = require('assert'),
-    child_process = require('child_process'),
-    debug = require('debug')('installer:installer'),
-    path = require('path'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    superagent = require('superagent'),
-    util = require('util');
-
-exports = module.exports = {
-    InstallerError: InstallerError,
-
-    provision: provision,
-
-    _ensureVersion: ensureVersion
-};
-
-var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
-    SUDO = '/usr/bin/sudo';
-
-function InstallerError(reason, info) {
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info);
-}
-util.inherits(InstallerError, Error);
-InstallerError.INTERNAL_ERROR = 1;
-InstallerError.ALREADY_PROVISIONED = 2;
-
-// system until file has KillMode=control-group to bring down child processes
-function spawn(tag, cmd, args, callback) {
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof cmd, 'string');
-    assert(util.isArray(args));
-    assert.strictEqual(typeof callback, 'function');
-
-    var cp = child_process.spawn(cmd, args, { timeout: 0 });
-    cp.stdout.setEncoding('utf8');
-    cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); });
-    cp.stderr.setEncoding('utf8');
-    cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); });
-
-    cp.on('error', function (error) {
-        debug('%s : child process errored %s', tag, error.message);
-        callback(error);
-    });
-
-    cp.on('exit', function (code, signal) {
-        debug('%s : child process exited. code: %d signal: %d', tag, code, signal);
-        if (signal) return callback(new Error('Exited with signal ' + signal));
-        if (code !== 0) return callback(new Error('Exited with code ' + code));
-
-        callback(null);
-    });
-}
-
-function ensureVersion(args, callback) {
-    assert.strictEqual(typeof args, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified'));
-
-    if (args.sourceTarballUrl) return callback(null, args);
-
-    superagent.get(args.data.boxVersionsUrl).end(function (error, result) {
-        if (error && !error.response) return callback(error);
-        if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text)));
-
-        var versions = safe.JSON.parse(result.text);
-
-        if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error));
-
-        var latestVersion = Object.keys(versions).sort(semver.compare).pop();
-        debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']);
-
-        if (!versions[latestVersion]) return callback(new Error('No version available'));
-        if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified'));
-
-        args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl;
-        args.data.version = latestVersion;
-
-        callback(null, args);
-    });
-}
-
-function provision(args, callback) {
-    assert.strictEqual(typeof args, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (process.env.NODE_ENV === 'test') return callback(null);
-
-    ensureVersion(args, function (error, result) {
-        if (error) return callback(error);
-
-        var pargs = [ INSTALLER_CMD ];
-        pargs.push('--sourcetarballurl', result.sourceTarballUrl);
-        pargs.push('--data', JSON.stringify(result.data));
-
-        debug('provision: calling with args %j', pargs);
-
-        // sudo is required for update()
-        spawn('provision', SUDO, pargs, callback);
-    });
-}
-

installer/src/server.js

@@ -1,170 +0,0 @@
-#!/usr/bin/env node
-
-/* jslint node: true */
-
-'use strict';
-
-var assert = require('assert'),
-    async = require('async'),
-    debug = require('debug')('installer:server'),
-    express = require('express'),
-    fs = require('fs'),
-    http = require('http'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    installer = require('./installer.js'),
-    json = require('body-parser').json,
-    lastMile = require('connect-lastmile'),
-    morgan = require('morgan'),
-    request = require('request'),
-    superagent = require('superagent');
-
-exports = module.exports = {
-    start: start,
-    stop: stop
-};
-
-var PROVISION_CONFIG_FILE = '/root/provision.json';
-var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
-
-var gHttpServer = null; // update server; used for updates
-
-function provisionDigitalOcean(callback) {
-    superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
-        if (error || result.statusCode !== 200) {
-            console.error('Error getting metadata', error);
-            return callback(new Error('Error getting metadata'));
-        }
-
-        callback(null, JSON.parse(result.body.user_data));
-    });
-}
-
-function provisionEC2(callback) {
-    // need to use request, since octet-stream data
-    request('http://169.254.169.254/latest/user-data', function (error, response, body) {
-        if (error || response.statusCode !== 200) {
-            console.error('Error getting metadata', error);
-            return callback(new Error('Error getting metadata'));
-        }
-
-        callback(null, JSON.parse(body));
-    });
-}
-
-function provision(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
-    // try first digitalocean, then ec2
-    provisionDigitalOcean(function (error, userData) {
-        if (!error) return installer.provision(userData, callback);
-
-        provisionEC2(function (error, userData) {
-            if (!error) return installer.provision(userData, callback);
-
-            console.error('Unable to get meta data', error);
-
-            callback(new Error('Error getting metadata'));
-        });
-    });
-}
-
-function provisionLocal(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
-    if (!fs.existsSync(PROVISION_CONFIG_FILE)) {
-        console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE);
-        return callback(new Error('No provisioning data found'));
-    }
-
-    var userData = require(PROVISION_CONFIG_FILE);
-
-    installer.provision(userData, callback);
-}
-
-function update(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided'));
-    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
-
-    debug('provision: received from box %j', req.body);
-
-    installer.provision(req.body, function (error) {
-        if (error) console.error(error);
-    });
-
-    next(new HttpSuccess(202, { }));
-}
-
-function startUpdateServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Starting update server');
-
-    var app = express();
-
-    var router = new express.Router();
-
-    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
-
-    app.use(json({ strict: true }))
-       .use(router)
-       .use(lastMile());
-
-    router.post('/api/v1/installer/update', update);
-
-    gHttpServer = http.createServer(app);
-    gHttpServer.on('error', console.error);
-
-    gHttpServer.listen(2020, '127.0.0.1', callback);
-}
-
-function stopUpdateServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Stopping update server');
-
-    if (!gHttpServer) return callback(null);
-
-    gHttpServer.close(callback);
-    gHttpServer = null;
-}
-
-function start(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var actions;
-
-    if (process.env.PROVISION === 'local') {
-        debug('Starting Installer in selfhost mode');
-
-        actions = [
-            startUpdateServer,
-            provisionLocal
-        ];
-    } else {    // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh
-        debug('Starting Installer in managed mode');
-
-        actions = [
-            startUpdateServer,
-            provision
-        ];
-    }
-
-    async.series(actions, callback);
-}
-
-function stop(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        stopUpdateServer
-    ], callback);
-}
-
-if (require.main === module) {
-    start(function (error) {
-        if (error) console.error(error);
-    });
-}

installer/src/test/installer-test.js

@@ -1,179 +0,0 @@
-/* jslint node:true */
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-'use strict';
-
-var expect = require('expect.js'),
-    fs = require('fs'),
-    path = require('path'),
-    nock = require('nock'),
-    os = require('os'),
-    request = require('superagent'),
-    server = require('../server.js'),
-    installer = require('../installer.js'),
-    _ = require('lodash');
-
-var EXTERNAL_SERVER_URL = 'https://localhost:4443';
-var INTERNAL_SERVER_URL = 'http://localhost:2020';
-var APPSERVER_ORIGIN = 'http://appserver';
-var FQDN = os.hostname();
-
-describe('Server', function () {
-    this.timeout(5000);
-
-    before(function (done) {
-        var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string
-        var scope = nock('http://169.254.169.254')
-            .persist()
-            .get('/metadata/v1.json')
-            .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' });
-        done();
-    });
-
-    after(function (done) {
-        nock.cleanAll();
-        done();
-    });
-
-    describe('starts and stop', function () {
-        it('starts', function (done) {
-            server.start(done);
-        });
-
-        it('stops', function (done) {
-            server.stop(done);
-        });
-    });
-
-    describe('update (internal server)', function () {
-        before(function (done) {
-            server.start(done);
-        });
-        after(function (done) {
-            server.stop(done);
-        });
-
-        it('does not respond to provision', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(404);
-                done();
-            });
-        });
-
-        it('does not respond to restore', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(404);
-                done();
-            });
-        });
-
-        var data = {
-            sourceTarballUrl: "https://foo.tar.gz",
-
-            data: {
-                token: 'sometoken',
-                apiServerOrigin: APPSERVER_ORIGIN,
-                webServerOrigin: 'https://somethingelse.com',
-                fqdn: 'www.something.com',
-                tlsKey: 'key',
-                tlsCert: 'cert',
-                boxVersionsUrl: 'https://versions.json',
-                version: '0.1'
-            }
-        };
-
-        Object.keys(data).forEach(function (key) {
-            it('fails due to missing ' + key, function (done) {
-                var dataCopy = _.merge({ }, data);
-                delete dataCopy[key];
-
-                request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-        });
-
-        it('succeeds', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(202);
-                done();
-            });
-        });
-    });
-
-    describe('ensureVersion', function () {
-        before(function () {
-            process.env.NODE_ENV = undefined;
-        });
-
-        after(function () {
-            process.env.NODE_ENV = 'test';
-        });
-
-        it ('fails without data', function (done) {
-            installer._ensureVersion({}, function (error) {
-                expect(error).to.be.an(Error);
-                done();
-            });
-        });
-
-        it ('fails without boxVersionsUrl', function (done) {
-            installer._ensureVersion({ data: {}}, function (error) {
-                expect(error).to.be.an(Error);
-                done();
-            });
-        });
-
-        it ('succeeds with sourceTarballUrl', function (done) {
-            var data = {
-                sourceTarballUrl: 'sometarballurl',
-                data: {
-                    boxVersionsUrl: 'http://foobar/versions.json'
-                }
-            };
-
-            installer._ensureVersion(data, function (error, result) {
-                expect(error).to.equal(null);
-                expect(result).to.eql(data);
-                done();
-            });
-        });
-
-        it ('succeeds without sourceTarballUrl', function (done) {
-            var versions = {
-                '0.1.0': {
-                    sourceTarballUrl: 'sometarballurl1'
-                },
-                '0.2.0': {
-                    sourceTarballUrl: 'sometarballurl2'
-                }
-            };
-
-            var scope = nock('http://foobar')
-                .get('/versions.json')
-                .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' });
-
-            var data = {
-                data: {
-                    boxVersionsUrl: 'http://foobar/versions.json'
-                }
-            };
-
-            installer._ensureVersion(data, function (error, result) {
-                expect(error).to.equal(null);
-                expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl);
-                expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl);
-                done();
-            });
-        });
-    });
-});
-

installer/systemd/cloudron-system-setup.sh

@@ -7,23 +7,16 @@ readonly APPS_SWAP_FILE="/apps.swap"
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
-# detect device
-if [[ -b "/dev/vda1" ]]; then
-    disk_device="/dev/vda1"
-fi
+# detect device of rootfs (http://forums.fedoraforum.org/showthread.php?t=270316)
+disk_device="$(for d in $(find /dev -type b); do [ "$(mountpoint -d /)" = "$(mountpoint -x $d)" ] && echo $d && break; done)"
 
-if [[ -b "/dev/xvda1" ]]; then
-    disk_device="/dev/xvda1"
-fi
-
-# allow root access over ssh
-sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys
+existing_swap=$(cat /proc/meminfo | grep SwapTotal | awk '{ printf "%.0f", $2/1024 }')
 
 # all sizes are in mb
 readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
-readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
+readonly swap_size=$((${physical_memory} - ${existing_swap})) # if you change this, fix enoughResourcesAvailable() in client.js
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
-readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
+readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ printf "%.0f", $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
@@ -34,7 +27,7 @@ echo "Estimated app count: ${app_count}"
 echo "Disk size: ${disk_size}"
 
 # Allocate swap for general app usage
-if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
+if [[ ! -f "${APPS_SWAP_FILE}" && ${swap_size} -gt 0 ]]; then
     echo "Creating Apps swap file of size ${swap_size}M"
     fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
     chmod 600 "${APPS_SWAP_FILE}"
@@ -54,4 +47,3 @@ umount "${USER_DATA_DIR}" || true
 truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
 mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 btrfs filesystem resize max "${USER_DATA_DIR}"
-

migrations/20160831051352-settings-default-mailconfig.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.all('SELECT id FROM users', function (error, results) {
+        if (error) return callback(error);
+
+        // existing cloudrons have email enabled by default. future cloudrons will have it disabled by default
+        var enable = results.length !== 0;
+        db.runSql('INSERT settings (name, value) VALUES("mail_config", ?)', [ JSON.stringify({ enabled: enable }) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="mail_config"', [ ], callback);
+};
+

migrations/20160921205726-mailboxes-add-ownerId.js

@@ -0,0 +1,74 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerId VARCHAR(128)'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerType VARCHAR(16)'),
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addGroupMailboxes(done) {
+            console.log('Importing group mailboxes');
+
+            db.all('SELECT id, name FROM groups', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (g, next) {
+                    db.runSql('INSERT INTO mailboxes (ownerId, ownerType, name) VALUES (?, ?, ?)', [ g.id, 'group', g.name ], function (error) {
+                        if (error) console.error('Error importing group ' + JSON.stringify(g) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        function addAppMailboxes(done) {
+            console.log('Importing app mail boxes');
+
+            db.all('SELECT id, location, manifestJson FROM apps', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (a, next) {
+                    var manifest = JSON.parse(a.manifestJson);
+                    if (!manifest.addons['sendmail'] && !manifest.addons['recvmail']) return next();
+
+                    var mailboxName = (a.location ? a.location : manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+                    db.runSql('INSERT INTO mailboxes (ownerId, ownerType, name) VALUES (?, ?, ?)', [ a.id, 'app', mailboxName ], function (error) {
+                        if (error) console.error('Error importing app ' + JSON.stringify(a) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        function setUserMailboxOwnerIds(done) {
+            console.log('Setting owner id of user mailboxes and aliases');
+
+            db.all('SELECT id, username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (u,  next) {
+                    if (!u.username) return next();
+
+                    db.runSql('UPDATE mailboxes SET ownerId = ?, ownerType = ? WHERE name = ? OR aliasTarget = ?', [ u.id, 'user', u.username, u.username ], function (error) {
+                        if (error) console.error('Error setting ownerid ' + JSON.stringify(u) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'COMMIT'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerId VARCHAR(128) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerType VARCHAR(128) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerId', function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerType', function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+

migrations/20161111051640-appdb-add-sso.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN sso BOOLEAN DEFAULT 1', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN sso', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20161113014146-appdb-drop-oauthProxy.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS users(
 
 CREATE TABLE IF NOT EXISTS groups(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
+    name VARCHAR(254) NOT NULL UNIQUE,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS groupMembers(
@@ -63,11 +63,11 @@ CREATE TABLE IF NOT EXISTS apps(
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
     accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
-    oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     memoryLimit BIGINT DEFAULT 0,
     altDomain VARCHAR(256),
     xFrameOptions VARCHAR(512),
+    sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
 
     lastBackupId VARCHAR(128), // tracks last valid backup, can be removed
 
@@ -125,7 +125,9 @@ CREATE TABLE IF NOT EXISTS eventlog(
 */
 CREATE TABLE IF NOT EXISTS mailboxes(
     name VARCHAR(128) NOT NULL,
+    ownerId VARCHAR(128) NOT NULL, /* app id or user id or group id */
+    ownerType VARCHAR(16) NOT NULL, /* 'app' or 'user' or 'group' */
     aliasTarget VARCHAR(128), /* the target name type is an alias */
     creationTime TIMESTAMP,
 
-    PRIMARY KEY (id));
+    PRIMARY KEY (name));

package.json

@@ -16,7 +16,8 @@
     "async": "^1.2.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^2.4.0",
+    "checksum": "^0.1.1",
+    "cloudron-manifestformat": "^2.5.1",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "^0.1.0",
     "connect-timeout": "^1.5.0",
@@ -34,8 +35,9 @@
     "hat": "0.0.3",
     "ini": "^1.3.4",
     "json": "^9.0.3",
-    "ldapjs": "^0.7.1",
+    "ldapjs": "^1.0.0",
     "mime": "^1.3.4",
+    "moment-timezone": "^0.5.5",
     "morgan": "^1.7.0",
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
@@ -56,6 +58,7 @@
     "proxy-middleware": "^0.13.0",
     "safetydance": "^0.1.1",
     "semver": "^4.3.6",
+    "showdown": "^1.4.4",
     "split": "^1.0.0",
     "superagent": "^1.8.3",
     "supererror": "^0.7.1",

scripts/cloudron-setup

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+# change this to a hash when we make a upgrade release
+readonly INSTALLER_REVISION=master
+readonly INIT_BASESYSTEM_SCRIPT_URL="https://git.cloudron.io/cloudron/box/raw/${INSTALLER_REVISION}/baseimage/initializeBaseUbuntuImage.sh"
+readonly INSTALLER_SOURCE_DIR="/home/yellowtent/installer"
+readonly LOG_FILE="/var/log/cloudron-setup.log"
+
+domain=""
+provider=""
+encryptionKey=""
+restoreUrl=""
+tlsProvider="letsencrypt-prod"
+versionsUrl="https://s3.amazonaws.com/prod-cloudron-releases/versions.json"
+version="latest"
+
+args=$(getopt -o "" -l "domain:,help,provider:,encryption-key:,restore-url:,tls-provider:,version:,versions-url:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --domain) domain="$2"; shift 2;;
+    --help) echo "See https://cloudron.io/references/selfhosting.html on how to install Cloudron"; exit 0;;
+    --provider) provider="$2"; shift 2;;
+    --encryption-key) encryptionKey="$2"; shift 2;;
+    --restore-url) restoreUrl="$2"; shift 2;;
+    --tls-provider) tlsProvider="$2"; shift 2;;
+    --version) version="$2"; shift 2;;
+    --versions-url) versionsUrl="$2"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+if [[ -z "${domain}" ]]; then
+    echo "--domain is required"
+    exit 1
+fi
+
+if [[ -z "${provider}" ]]; then
+    echo "--provider is required (generic, scaleway, ec2, digitalocean)"
+    exit 1
+elif [[  \
+            "${provider}" != "generic" && \
+            "${provider}" != "scaleway" && \
+            "${provider}" != "ec2" && \
+            "${provider}" != "digitalocean" \
+        ]]; then
+    echo "--provider must be one of: generic, scaleway, ec2, digitalocean"
+    exit 1
+fi
+
+if [[ -z "${encryptionKey}" ]]; then
+    echo "--encryption-key for backup encryption is required"
+    exit 1
+fi
+
+echo ""
+echo "##############################################"
+echo "        Cloudron Setup (${version})           "
+echo "##############################################"
+echo ""
+echo " Follow setup logs in a second terminal with:"
+echo " $ tail -f ${LOG_FILE}"
+echo ""
+
+echo "=> Update package repositories ..."
+if ! apt-get update &>> "${LOG_FILE}"; then
+    echo "Could not update package repositories"
+    exit 1
+fi
+
+echo "=> Installing setup dependencies ..."
+if ! apt-get install curl -y &>> "${LOG_FILE}"; then
+    echo "Could not install curl"
+    exit 1
+fi
+
+echo "=> Downloading initialization script"
+if ! curl -s "${INIT_BASESYSTEM_SCRIPT_URL}" > /tmp/initializeBaseUbuntuImage.sh; then
+    echo "Could not download initialization script"
+    exit 1
+fi
+
+echo "=> Installing base dependencies ... (this takes some time)"
+if ! /bin/bash /tmp/initializeBaseUbuntuImage.sh "${INSTALLER_REVISION}" "${provider}" &>> "${LOG_FILE}"; then
+    echo "Init script failed. See ${LOG_FILE} for details"
+    exit 1
+fi
+rm /tmp/initializeBaseUbuntuImage.sh
+
+echo "=> Checking version"
+NPM_BIN=$(npm bin -g 2>/dev/null)
+if ! version=$(${NPM_BIN}/cloudron-version --out version --versions-url "${versionsUrl}" --version "${version}"); then
+    echo "No such version ${version}"
+    exit 1
+fi
+if ! sourceTarballUrl=$(${NPM_BIN}/cloudron-version --out tarballUrl --versions-url "${versionsUrl}" --version "${version}"); then
+    echo "No source code for version ${version}"
+    exit 1
+fi
+
+echo "=> Run base init service"
+systemctl start cloudron-system-setup
+
+if [[ -z "${restoreUrl}" ]]; then
+    data=$(cat <<EOF
+{
+    "boxVersionsUrl": "${versionsUrl}",
+    "fqdn": "${domain}",
+    "provider": "${provider}",
+    "tlsConfig": {
+        "provider": "${tlsProvider}"
+    },
+    "backupConfig" : {
+        "provider": "filesystem",
+        "backupFolder": "/var/backups",
+        "key": "${encryptionKey}"
+    },
+    "version": "${version}"
+}
+EOF
+)
+else
+    data=$(cat <<EOF
+{
+    "boxVersionsUrl": "${versionsUrl}",
+    "fqdn": "${domain}",
+    "provider": "${provider}",
+    "restore": {
+        "url": "${restoreUrl}",
+        "key": "${encryptionKey}"
+    },
+    "tlsConfig": {
+        "provider": "${tlsProvider}"
+    }
+    "version": "${version}"
+}
+EOF
+)
+fi
+
+echo "=> Run installer.sh for version ${version} with  ${sourceTarballUrl} ... (this takes some time)"
+if ! ${INSTALLER_SOURCE_DIR}/scripts/installer.sh --sourcetarballurl "${sourceTarballUrl}" --data "${data}" &>> "${LOG_FILE}"; then
+    echo "Failed to install cloudron. See ${LOG_FILE} for details"
+    exit 1
+fi
+
+echo -n "=> Waiting for cloudron to be ready"
+while true; do
+    echo -n "."
+    if journalctl -u box -a | grep "platformReady: configured, resuming tasks" >/dev/null; then
+        break
+    fi
+    sleep 10
+done
+
+echo ""
+echo "Visit https://my.${domain} to finish setup"
+echo ""

setup/argparser.sh

@@ -23,6 +23,7 @@ arg_dns_config=""
 arg_update_config=""
 arg_provider=""
 arg_app_bundle=""
+arg_is_demo="false"
 
 args=$(getopt -o "" -l "data:,retire-reason:,retire-info:" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -40,22 +41,34 @@ while true; do
     --data)
         # these params must be valid in all cases
         arg_fqdn=$(echo "$2" | $json fqdn)
+
         arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
+        [[ "${arg_is_custom_domain}" == "" ]] && arg_is_custom_domain="true"
 
         # only update/restore have this valid (but not migrate)
         arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
+        [[ "${arg_api_server_origin}" == "" ]] && arg_api_server_origin="https://api.cloudron.io"
         arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
+        [[ "${arg_web_server_origin}" == "" ]] && arg_web_server_origin="https://cloudron.io"
         arg_box_versions_url=$(echo "$2" | $json boxVersionsUrl)
+        [[ "${arg_box_versions_url}" == "" ]] && arg_box_versions_url="https://s3.amazonaws.com/prod-cloudron-releases/versions.json"
+
+        # TODO check if an where this is used
         arg_version=$(echo "$2" | $json version)
 
         # read possibly empty parameters here
         arg_app_bundle=$(echo "$2" | $json appBundle)
         [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"
 
+        arg_is_demo=$(echo "$2" | $json isDemo)
+        [[ "${arg_is_demo}" == "" ]] && arg_is_demo="false"
+
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
         arg_token=$(echo "$2" | $json token)
+
         arg_provider=$(echo "$2" | $json provider)
+        [[ "${arg_provider}" == "" ]] && arg_provider="generic"
 
         arg_tls_config=$(echo "$2" | $json tlsConfig)
         [[ "${arg_tls_config}" == "null" ]] && arg_tls_config=""

setup/container/mysql.cnf

@@ -5,3 +5,6 @@
 [mysqld]
 performance_schema=OFF
 max_connections=50
+# on ec2, without this we get a sporadic connection drop when doing the initial migration
+max_allowed_packet=32M
+

setup/container/sudoers

@@ -31,3 +31,8 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
 
+Defaults!/home/yellowtent/box/src/scripts/rmbackup.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmbackup.sh
+
+Defaults!/home/yellowtent/box/src/scripts/update.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/update.sh

setup/start.sh

@@ -21,7 +21,7 @@ source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used
 admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
 admin_origin="https://${admin_fqdn}"
 
-readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
+readonly is_update=$([[ -f "${CONFIG_DIR}/cloudron.conf" ]] && echo "true" || echo "false")
 
 set_progress() {
     local percent="$1"
@@ -37,11 +37,6 @@ $script_dir/container.sh
 set_progress "5" "Adjust system settings"
 hostnamectl set-hostname "${arg_fqdn}"
 
-# ec2 instances use lots of cpu for swapping, which can be significantly reduced adjusting the swappiness
-if [[ "${arg_provider}" == 'ec2' ]]; then
-    sysctl vm.swappiness=0
-fi
-
 set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
@@ -68,7 +63,13 @@ echo "Cleaning up snapshots"
 find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
 
 # restart mysql to make sure it has latest config
-service mysql restart
+# wait for all running mysql jobs
+while true; do
+    if ! systemctl list-jobs | grep mysql; then break; fi
+    echo "Waiting for mysql jobs..."
+    sleep 1
+done
+systemctl restart mysql
 
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
@@ -100,7 +101,7 @@ EOF
 
 set_progress "28" "Setup collectd"
 cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
-service collectd restart
+systemctl restart collectd
 
 set_progress "30" "Setup nginx"
 mkdir -p "${DATA_DIR}/nginx/applications"
@@ -122,12 +123,21 @@ if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key
     cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
     cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
 else
-    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
-    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+    if [[ -z "${arg_tls_cert}" || -z "${arg_tls_key}" ]]; then
+        echo "Creating fallback certs"
+        openssl req -x509 -newkey rsa:2048 -keyout "${DATA_DIR}/nginx/cert/host.key" -out "${DATA_DIR}/nginx/cert/host.cert" -days 3650 -subj "/CN=${arg_fqdn}" -nodes
+    else
+        echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
+        echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+    fi
 fi
 
 set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" -R "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+# during updates, do not trample mail ownership behind the the mail container's back
+find "${DATA_DIR}/box" -mindepth 1 -maxdepth 1 -not -path "${DATA_DIR}/box/mail" -print0 | xargs -0 chown -R "${USER}:${USER}"
+chown "${USER}:${USER}" "${DATA_DIR}/box"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box/mail/dkim" # this is owned by box currently since it generates the keys
 chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"
 
@@ -145,6 +155,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
     "provider": "${arg_provider}",
+    "isDemo": ${arg_is_demo},
     "database": {
         "hostname": "localhost",
         "username": "root",

setup/start/nginx/appconfig.ejs

@@ -36,15 +36,17 @@ server {
     proxy_set_header   X-Forwarded-For    $remote_addr;
     proxy_set_header   X-Forwarded-Host   $host;
     proxy_set_header   X-Forwarded-Proto  https;
+    proxy_set_header   X-Forwarded-Ssl    on;
 
     # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
 
     # only serve up the status page if we get proxy gateway errors
-    error_page 502 503 504 @appstatus;
-    location @appstatus {
-        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
+    root <%= sourceDir %>/webadmin/dist;
+    error_page 502 503 504 /appstatus.html;
+    location /appstatus.html {
+        internal;
     }
 
     location / {
@@ -79,9 +81,6 @@ server {
             index  index.html index.htm;
         }
 
-<% } else if ( endpoint === 'oauthproxy' ) { %>
-        proxy_pass http://127.0.0.1:3003;
-        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://127.0.0.1:<%= port %>;
 <% } else if ( endpoint === 'splash' ) { %>

src/addons.js

@@ -28,6 +28,7 @@ var appdb = require('./appdb.js'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
     infra = require('./infra_version.js'),
+    mailboxdb = require('./mailboxdb.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -253,6 +254,8 @@ function setupOauth(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!app.sso) return callback(null);
+
     var appId = app.id;
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
@@ -295,6 +298,8 @@ function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!app.sso) return callback(null);
+
     var appId = app.id;
     var scope = 'profile';
 
@@ -369,6 +374,8 @@ function setupLdap(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!app.sso) return callback(null);
+
     var env = [
         'LDAP_SERVER=172.18.0.1',
         'LDAP_PORT=' + config.get('ldapPort'),
@@ -376,7 +383,7 @@ function setupLdap(app, options, callback) {
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
         'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
         'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
-        'LDAP_BIND_PASSWORD=' + hat(8 * 128) // this is ignored
+        'LDAP_BIND_PASSWORD=' + hat(4 * 128) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -399,14 +406,21 @@ function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+    debugApp(app, 'Setting up SendMail');
 
-    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];
-
-    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+    mailboxdb.getByOwnerId(app.id, function (error, results) {
         if (error) return callback(error);
 
-        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        var mailbox = results.filter(function (r) { return !r.aliasTarget; })[0];
+
+        var env = [
+            "MAIL_SMTP_SERVER=mail",
+            "MAIL_SMTP_PORT=2525",
+            "MAIL_SMTP_USERNAME=" + mailbox.name,
+            "MAIL_SMTP_PASSWORD=" + app.id,
+            "MAIL_FROM=" + mailbox.name + '@' + config.fqdn(),
+            "MAIL_DOMAIN=" + config.fqdn()
+        ];
         debugApp(app, 'Setting sendmail addon config to %j', env);
         appdb.setAddonConfig(app.id, 'sendmail', env, callback);
     });
@@ -417,17 +431,9 @@ function teardownSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+    debugApp(app, 'Tearing down sendmail');
 
-    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
-
-    debugApp(app, 'Tearing down sendmail : %j', cmd);
-
-    docker.execContainer('mail', cmd, { }, function (error) {
-        if (error) return callback(error);
-
-        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
-    });
+    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
 }
 
 function setupRecvMail(app, options, callback) {
@@ -437,15 +443,21 @@ function setupRecvMail(app, options, callback) {
 
     debugApp(app, 'Setting up recvmail');
 
-    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
-
-    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
-
-    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+    mailboxdb.getByOwnerId(app.id, function (error, results) {
         if (error) return callback(error);
 
-        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-        debugApp(app, 'Setting recvmail addon config to %j', env);
+        var mailbox = results.filter(function (r) { return !r.aliasTarget; })[0];
+
+        var env = [
+            "MAIL_IMAP_SERVER=mail",
+            "MAIL_IMAP_PORT=9993",
+            "MAIL_IMAP_USERNAME=" + mailbox.name,
+            "MAIL_IMAP_PASSWORD=" + app.id,
+            "MAIL_TO=" + mailbox.name + '@' + config.fqdn(),
+            "MAIL_DOMAIN=" + config.fqdn()
+        ];
+
+        debugApp(app, 'Setting sendmail addon config to %j', env);
         appdb.setAddonConfig(app.id, 'recvmail', env, callback);
     });
 }
@@ -455,17 +467,9 @@ function teardownRecvMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+    debugApp(app, 'Tearing down recvmail');
 
-    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
-
-    debugApp(app, 'Tearing down recvmail: %j', cmd);
-
-    docker.execContainer('mail', cmd, { }, function (error) {
-        if (error) return callback(error);
-
-        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
-    });
+    appdb.unsetAddonConfig(app.id, 'recvmail', callback);
 }
 
 function setupMySql(app, options, callback) {

src/appdb.js

@@ -59,7 +59,8 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain', 'apps.xFrameOptions' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain',
+    'apps.xFrameOptions', 'apps.sso' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -95,6 +96,8 @@ function postProcess(result) {
 
     // TODO remove later once all apps have this attribute
     result.xFrameOptions = result.xFrameOptions || 'SAMEORIGIN';
+
+    result.sso = !!result.sso; // make it bool
 }
 
 function get(id, callback) {
@@ -181,11 +184,12 @@ function add(id, appStoreId, manifest, location, portBindings, data, callback) {
     var xFrameOptions = data.xFrameOptions || '';
     var installationState = data.installationState || exports.ISTATE_PENDING_INSTALL;
     var lastBackupId = data.lastBackupId || null; // used when cloning
+    var sso = 'sso' in data ? data.sso : null;
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId, sso) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId, sso ]
     });
 
     Object.keys(portBindings).forEach(function (env) {

src/apphealthmonitor.js

@@ -138,7 +138,7 @@ function run() {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.8.1 /bin/bash
+        docker run -ti -m 100M cloudron/base:0.9.0 /bin/bash
         apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */

src/apps.js

@@ -9,7 +9,6 @@ exports = module.exports = {
     getByIpAddress: getByIpAddress,
     getAll: getAll,
     getAllByUser: getAllByUser,
-    purchase: purchase,
     install: install,
     configure: configure,
     uninstall: uninstall,
@@ -59,15 +58,18 @@ var addons = require('./addons.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     groups = require('./groups.js'),
+    mailboxdb = require('./mailboxdb.js'),
     manifestFormat = require('cloudron-manifestformat'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
+    settings = require('./settings.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
     taskmanager = require('./taskmanager.js'),
+    updateChecker = require('./updatechecker.js'),
     url = require('url'),
     util = require('util'),
     uuid = require('node-uuid'),
@@ -104,7 +106,6 @@ AppsError.PORT_RESERVED = 'Port Reserved';
 AppsError.PORT_CONFLICT = 'Port Conflict';
 AppsError.BILLING_REQUIRED = 'Billing Required';
 AppsError.ACCESS_DENIED = 'Access denied';
-AppsError.USER_REQUIRED = 'User required';
 AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 
 // Hostname validation comes from RFC 1123 (section 2.1)
@@ -148,7 +149,6 @@ function validatePortBindings(portBindings, tcpPorts) {
         config.get('sysadminPort'), /* sysadmin app server (lo) */
         config.get('smtpPort'), /* internal smtp port (lo) */
         config.get('ldapPort'), /* ldap server (lo) */
-        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
         config.get('simpleAuthPort'), /* simple auth server (lo) */
         3306, /* mysql (lo) */
         4190, /* managesieve */
@@ -182,22 +182,16 @@ function validateAccessRestriction(accessRestriction) {
 
     if (accessRestriction === null) return null;
 
-    var noUsers = true, noGroups = true;
-
     if (accessRestriction.users) {
         if (!Array.isArray(accessRestriction.users)) return new AppsError(AppsError.BAD_FIELD, 'users array property required');
         if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All users have to be strings');
-        noUsers = accessRestriction.users.length === 0;
     }
 
     if (accessRestriction.groups) {
         if (!Array.isArray(accessRestriction.groups)) return new AppsError(AppsError.BAD_FIELD, 'groups array property required');
         if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All groups have to be strings');
-        noGroups = accessRestriction.groups.length === 0;
     }
 
-    if (noUsers && noGroups) return new AppsError(AppsError.BAD_FIELD, 'users and groups array cannot both be empty');
-
     // TODO: maybe validate if the users and groups actually exist
     return null;
 }
@@ -358,26 +352,93 @@ function getAllByUser(user, callback) {
     });
 }
 
-function purchase(appStoreId, callback) {
-    assert.strictEqual(typeof appStoreId, 'string');
+function purchase(appId, appstoreId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof appstoreId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // Skip purchase if appStoreId is empty
-    if (appStoreId === '') return callback(null);
+    if (appstoreId === '') return callback(null);
 
-    // Skip if we don't have an appstore token
-    if (config.token() === '') return callback(null);
+    function purchaseWithAppstoreConfig(appstoreConfig) {
+        assert.strictEqual(typeof appstoreConfig.userId, 'string');
+        assert.strictEqual(typeof appstoreConfig.cloudronId, 'string');
+        assert.strictEqual(typeof appstoreConfig.token, 'string');
 
-    var url = config.apiServerOrigin() + '/api/v1/apps/' + appStoreId + '/purchase';
+        var url = config.apiServerOrigin() + '/api/v1/users/' + appstoreConfig.userId + '/cloudrons/' + appstoreConfig.cloudronId + '/apps/' + appId;
+        var data = { appstoreId: appstoreId };
 
-    superagent.post(url).query({ token: config.token() }).end(function (error, res) {
-        if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
-        if (res.statusCode === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
-        if (res.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND));
-        if (res.statusCode !== 201 && res.statusCode !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
+        superagent.post(url).send(data).query({ accessToken: appstoreConfig.token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
+            if (result.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND));
+            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new AppsError(AppsError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
 
-        callback(null);
-    });
+            callback(null);
+        });
+    }
+
+    // Caas Cloudrons do not store appstore credentials in their local database
+    if (config.provider() === 'caas') {
+        var url = config.apiServerOrigin() + '/api/v1/exchangeBoxTokenWithUserToken';
+        superagent.post(url).query({ token: config.token() }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
+            if (result.statusCode !== 201) return callback(new AppsError(AppsError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+
+            purchaseWithAppstoreConfig(result.body);
+        });
+    } else {
+        settings.getAppstoreConfig(function (error, result) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+            if (!result.token) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+
+            purchaseWithAppstoreConfig(result);
+        });
+    }
+}
+
+function unpurchase(appId, appstoreId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof appstoreId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (appstoreId === '') return callback(null);
+
+    function unpurchaseWithAppstoreConfig(appstoreConfig) {
+        assert.strictEqual(typeof appstoreConfig.userId, 'string');
+        assert.strictEqual(typeof appstoreConfig.cloudronId, 'string');
+        assert.strictEqual(typeof appstoreConfig.token, 'string');
+
+        var url = config.apiServerOrigin() + '/api/v1/users/' + appstoreConfig.userId + '/cloudrons/' + appstoreConfig.cloudronId + '/apps/' + appId;
+
+        superagent.get(url).query({ accessToken: appstoreConfig.token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
+            if (result.statusCode === 404) return callback(null);   // was never purchased
+
+            superagent.del(url).query({ accessToken: appstoreConfig.token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
+                if (result.statusCode !== 204) return callback(new AppsError(AppsError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+
+                callback(null);
+            });
+        });
+    }
+
+    // Caas Cloudrons do not store appstore credentials in their local database
+    if (config.provider() === 'caas') {
+        var url = config.apiServerOrigin() + '/api/v1/exchangeBoxTokenWithUserToken';
+        superagent.post(url).query({ token: config.token() }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
+            if (result.statusCode !== 201) return callback(new AppsError(AppsError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+
+            unpurchaseWithAppstoreConfig(result.body);
+        });
+    } else {
+        settings.getAppstoreConfig(function (error, result) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+            if (!result.token) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+
+            unpurchaseWithAppstoreConfig(result);
+        });
+    }
 }
 
 function downloadManifest(appStoreId, manifest, callback) {
@@ -391,7 +452,7 @@ function downloadManifest(appStoreId, manifest, callback) {
 
     debug('downloading manifest from %s', url);
 
-    superagent.get(url).end(function (error, result) {
+    superagent.get(url).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message));
 
         if (result.statusCode !== 200) return callback(new AppsError(AppsError.BAD_FIELD, util.format('Failed to get app info from store.', result.statusCode, result.text)));
@@ -413,7 +474,8 @@ function install(data, auditSource, callback) {
         key = data.key || null,
         memoryLimit = data.memoryLimit || 0,
         altDomain = data.altDomain || null,
-        xFrameOptions = data.xFrameOptions || 'SAMEORIGIN';
+        xFrameOptions = data.xFrameOptions || 'SAMEORIGIN',
+        sso = 'sso' in data ? data.sso : null;
 
     assert(data.appStoreId || data.manifest); // atleast one of them is required
 
@@ -441,15 +503,10 @@ function install(data, auditSource, callback) {
         error = validateXFrameOptions(xFrameOptions);
         if (error) return callback(error);
 
-        // memoryLimit might come in as 0 if not specified
-        memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+        if ('sso' in data && !('optionalSso' in manifest)) return callback(new AppsError(AppsError.BAD_FIELD, 'sso can only be specified for apps with optionalSso'));
 
         if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
 
-        // singleUser mode requires accessRestriction to contain exactly one user
-        if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
-        if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
-
         var appId = uuid.v4();
 
         if (icon) {
@@ -465,31 +522,49 @@ function install(data, auditSource, callback) {
 
         debug('Will install app with id : ' + appId);
 
-        purchase(appStoreId, function (error) {
+        purchase(appId, appStoreId, function (error) {
             if (error) return callback(error);
 
+            // FIXME revise this once customAuth is gone
+            if (sso === null) {
+                if (manifest.customAuth) {
+                    sso = false;
+                } else if (manifest.addons['simpleauth'] || manifest.addons['ldap'] || manifest.addons['oauth']) {
+                    sso = true;
+                } else {
+                    sso = false;
+                }
+            }
+
             var data = {
                 accessRestriction: accessRestriction,
                 memoryLimit: memoryLimit,
                 altDomain: altDomain,
-                xFrameOptions: xFrameOptions
+                xFrameOptions: xFrameOptions,
+                sso: sso
             };
 
-            appdb.add(appId, appStoreId, manifest, location, portBindings, data, function (error) {
-                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+            var from = (location ? location : manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+            mailboxdb.add(from, appId, mailboxdb.TYPE_APP, function (error) {
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new AppsError(AppsError.ALREADY_EXISTS, 'Mailbox already exists'));
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                // save cert to data/box/certs
-                if (cert && key) {
-                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
-                }
+                appdb.add(appId, appStoreId, manifest, location, portBindings, data, function (error) {
+                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                    if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                taskmanager.restartAppTask(appId);
+                    // save cert to data/box/certs
+                    if (cert && key) {
+                        if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                        if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+                    }
 
-                eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+                    taskmanager.restartAppTask(appId);
 
-                callback(null, { id : appId });
+                    eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+
+                    callback(null, { id : appId });
+                });
             });
         });
     });
@@ -537,9 +612,6 @@ function configure(appId, data, auditSource, callback) {
             values.memoryLimit = data.memoryLimit;
             error = validateMemoryLimit(app.manifest, values.memoryLimit);
             if (error) return callback(error);
-
-            // memoryLimit might come in as 0 if not specified
-            values.memoryLimit = values.memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
         }
 
         if ('xFrameOptions' in data) {
@@ -554,11 +626,11 @@ function configure(appId, data, auditSource, callback) {
                 error = certificates.validateCertificate(data.cert, data.key, config.appFqdn(location));
                 if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
 
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), data.cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.cert'), data.cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.key'), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
             } else { // remove existing cert/key
-                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'))) debug('Error removing cert: ' + safe.error.message);
-                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'))) debug('Error removing key: ' + safe.error.message);
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.cert'))) debug('Error removing cert: ' + safe.error.message);
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.user.key'))) debug('Error removing key: ' + safe.error.message);
             }
         }
 
@@ -647,6 +719,9 @@ function update(appId, data, auditSource, callback) {
 
                 eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest, force: data.force });
 
+                // clear update indicator, if update fails, it will come back through the update checker
+                updateChecker.resetAppUpdateInfo(appId);
+
                 callback(null);
             });
         });
@@ -714,6 +789,7 @@ function restore(appId, data, auditSource, callback) {
         var func = data.backupId ? backups.getRestoreConfig.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
 
         func(function (error, restoreConfig) {
+            if (error && error.reason === BackupsError.NOT_FOUND) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
             if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -782,7 +858,7 @@ function clone(appId, data, auditSource, callback) {
 
             var newAppId = uuid.v4(), appStoreId = app.appStoreId, manifest = restoreConfig.manifest;
 
-            purchase(appStoreId, function (error) {
+            purchase(newAppId, appStoreId, function (error) {
                 if (error) return callback(error);
 
                 var data = {
@@ -790,18 +866,25 @@ function clone(appId, data, auditSource, callback) {
                     memoryLimit: app.memoryLimit,
                     accessRestriction: app.accessRestriction,
                     xFrameOptions: app.xFrameOptions,
-                    lastBackupId: backupId
+                    lastBackupId: backupId,
+                    sso: !!app.sso
                 };
 
-                appdb.add(newAppId, appStoreId, manifest, location, portBindings, data, function (error) {
-                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                var from = (location ? location : manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+                mailboxdb.add(from, newAppId, mailboxdb.TYPE_APP, function (error) {
+                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new AppsError(AppsError.ALREADY_EXISTS, 'Mailbox already exists'));
                     if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                    taskmanager.restartAppTask(newAppId);
+                    appdb.add(newAppId, appStoreId, manifest, location, portBindings, data, function (error) {
+                        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                    eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, location: location, manifest: manifest });
+                        taskmanager.restartAppTask(newAppId);
 
-                    callback(null, { id : newAppId });
+                        eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, location: location, manifest: manifest });
+
+                        callback(null, { id : newAppId });
+                    });
                 });
             });
         });
@@ -815,14 +898,26 @@ function uninstall(appId, auditSource, callback) {
 
     debug('Will uninstall app with id:%s', appId);
 
-    taskmanager.stopAppTask(appId, function () {
-        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+    get(appId, function (error, result) {
+        if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId: appId });
+        unpurchase(appId, result.appStoreId, function (error) {
+            if (error) return callback(error);
 
-            taskmanager.startAppTask(appId, callback);
+            mailboxdb.delByOwnerId(appId, function (error) {
+                if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                taskmanager.stopAppTask(appId, function () {
+                    appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
+                        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+                        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                        eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId: appId });
+
+                        taskmanager.startAppTask(appId, callback);
+                    });
+                });
+            });
         });
     });
 }

src/apptask.js

@@ -12,8 +12,6 @@ exports = module.exports = {
     _unconfigureNginx: unconfigureNginx,
     _createVolume: createVolume,
     _deleteVolume: deleteVolume,
-    _allocateOAuthProxyCredentials: allocateOAuthProxyCredentials,
-    _removeOAuthProxyCredentials: removeOAuthProxyCredentials,
     _verifyManifest: verifyManifest,
     _registerSubdomain: registerSubdomain,
     _unregisterSubdomain: unregisterSubdomain,
@@ -37,8 +35,8 @@ var addons = require('./addons.js'),
     backups = require('./backups.js'),
     certificates = require('./certificates.js'),
     clients = require('./clients.js'),
-    config = require('./config.js'),
     ClientsError = clients.ClientsError,
+    config = require('./config.js'),
     database = require('./database.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
@@ -155,32 +153,6 @@ function deleteVolume(app, callback) {
     shell.sudo('deleteVolume', [ RMAPPDIR_CMD, app.id ], callback);
 }
 
-function allocateOAuthProxyCredentials(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!nginx.requiresOAuthProxy(app)) return callback(null);
-
-    var redirectURI = 'https://' + config.appFqdn(app.location);
-    var scope = 'profile';
-
-    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
-}
-
-function removeOAuthProxyCredentials(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
-        if (error && error.reason !== ClientsError.NOT_FOUND) {
-            debugApp(app, 'Error removing OAuth client id', error);
-            return callback(error);
-        }
-
-        callback(null);
-    });
-}
-
 function addCollectdProfile(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -230,6 +202,7 @@ function downloadIcon(app, callback) {
         superagent
             .get(iconUrl)
             .buffer(true)
+            .timeout(30 * 1000)
             .end(function (error, res) {
                 if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
                 if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli
@@ -253,10 +226,19 @@ function registerSubdomain(app, callback) {
         async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
             debugApp(app, 'Registering subdomain location [%s]', app.location);
 
-            subdomains.add(app.location, 'A', [ ip ], function (error, changeId) {
-                if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+            // get the current record before updating it
+            subdomains.get(app.location, 'A', function (error, values) {
+                if (error) return retryCallback(error);
 
-                retryCallback(null, error || changeId);
+                // refuse to update any existing DNS record for custom domains that we did not create
+                // note that the appstore sets up the naked domain for non-custom domains
+                if (config.isCustomDomain() && values.length !== 0 && !app.dnsRecordId) return retryCallback(null, new Error('DNS Record already exists'));
+
+                subdomains.upsert(app.location, 'A', [ ip ], function (error, changeId) {
+                    if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                    retryCallback(null, error || changeId);
+                });
             });
         }, function (error, result) {
             if (error || result instanceof Error) return callback(error || result);
@@ -272,7 +254,7 @@ function unregisterSubdomain(app, location, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (location === '') {
+    if (!config.isCustomDomain() && location === '') {
         debugApp(app, 'Skip unregister of empty subdomain');
         return callback(null);
     }
@@ -381,7 +363,6 @@ function install(app, callback) {
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
         unregisterSubdomain.bind(null, app, app.location),
-        removeOAuthProxyCredentials.bind(null, app),
         // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
 
         reserveHttpPort.bind(null, app),
@@ -389,9 +370,6 @@ function install(app, callback) {
         updateApp.bind(null, app, { installationProgress: '20, Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '25, Creating OAuth proxy credentials' }),
-        allocateOAuthProxyCredentials.bind(null, app),
-
         updateApp.bind(null, app, { installationProgress: '30, Registering subdomain' }),
         registerSubdomain.bind(null, app),
 
@@ -485,7 +463,6 @@ function restore(app, callback) {
 
              docker.deleteImage(app.oldConfig.manifest, done);
         },
-        removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
 
         reserveHttpPort.bind(null, app),
@@ -493,9 +470,6 @@ function restore(app, callback) {
         updateApp.bind(null, app, { installationProgress: '40, Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '50, Create OAuth proxy credentials' }),
-        allocateOAuthProxyCredentials.bind(null, app),
-
         updateApp.bind(null, app, { installationProgress: '55, Registering subdomain' }), // ip might change during upgrades
         registerSubdomain.bind(null, app),
 
@@ -556,13 +530,9 @@ function configure(app, callback) {
             if (!app.oldConfig || app.oldConfig.location === app.location) return next();
             unregisterSubdomain(app, app.oldConfig.location, next);
         },
-        removeOAuthProxyCredentials.bind(null, app),
 
         reserveHttpPort.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
-        allocateOAuthProxyCredentials.bind(null, app),
-
         updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
         registerSubdomain.bind(null, app),
 
@@ -638,7 +608,7 @@ function update(app, callback) {
             if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
 
             async.series([
-                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
+                updateApp.bind(null, app, { installationProgress: '30, Backing up app' }),
                 backups.backupApp.bind(null, app, app.oldConfig.manifest)
             ], next);
         },
@@ -702,9 +672,6 @@ function uninstall(app, callback) {
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
         unregisterSubdomain.bind(null, app, app.location),
 
-        updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
-        removeOAuthProxyCredentials.bind(null, app),
-
         updateApp.bind(null, app, { installationProgress: '80, Cleanup icon' }),
         removeIcon.bind(null, app),
 

src/auth.js

@@ -32,7 +32,7 @@ function initialize(callback) {
         user.get(userId, function (error, result) {
             if (error) return callback(error);
 
-            var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
+            var md5 = crypto.createHash('md5').update(result.alternateEmail || result.email).digest('hex');
             result.gravatar = 'https://www.gravatar.com/avatar/' + md5 + '.jpg?s=24&d=mm';
 
             callback(null, result);

src/backups.js

@@ -3,6 +3,8 @@
 exports = module.exports = {
     BackupsError: BackupsError,
 
+    testConfig: testConfig,
+
     getPaged: getPaged,
     getByAppIdPaged: getByAppIdPaged,
 
@@ -15,7 +17,11 @@ exports = module.exports = {
     backupApp: backupApp,
     restoreApp: restoreApp,
 
-    backupBoxAndApps: backupBoxAndApps
+    backupBoxAndApps: backupBoxAndApps,
+
+    getLocalDownloadPath: getLocalDownloadPath,
+
+    removeBackup: removeBackup
 };
 
 var addons = require('./addons.js'),
@@ -29,7 +35,9 @@ var addons = require('./addons.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:backups'),
     eventlog = require('./eventlog.js'),
+    filesystem = require('./storage/filesystem.js'),
     locker = require('./locker.js'),
+    mailer = require('./mailer.js'),
     path = require('path'),
     paths = require('./paths.js'),
     progress = require('./progress.js'),
@@ -37,7 +45,7 @@ var addons = require('./addons.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     settings = require('./settings.js'),
-    superagent = require('superagent'),
+    SettingsError = require('./settings.js').SettingsError,
     util = require('util'),
     webhooks = require('./webhooks.js');
 
@@ -76,6 +84,7 @@ util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
 BackupsError.BAD_STATE = 'bad state';
+BackupsError.NOT_FOUND = 'not found';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
 // choose which storage backend we use for test purpose we use s3
@@ -83,10 +92,21 @@ function api(provider) {
     switch (provider) {
         case 'caas': return caas;
         case 's3': return s3;
+        case 'filesystem': return filesystem;
         default: return null;
     }
 }
 
+function testConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var func = api(backupConfig.provider);
+    if (!func) return callback(new SettingsError(SettingsError.BAD_FIELD, 'unkown storage provider'));
+
+    api(backupConfig.provider).testConfig(backupConfig, callback);
+}
+
 function getPaged(page, perPage, callback) {
     assert(typeof page === 'number' && page > 0);
     assert(typeof perPage === 'number' && perPage > 0);
@@ -112,85 +132,24 @@ function getByAppIdPaged(page, perPage, appId, callback) {
     });
 }
 
-function getBoxBackupCredentials(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    var now = new Date();
-    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
-    var filename = filebase + '.tar.gz';
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
-            if (error) return callback(error);
-
-            result.id = filename;
-            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
-            result.backupKey = backupConfig.key;
-
-            debug('getBoxBackupCredentials: %j', result);
-
-            callback(null, result);
-        });
-    });
-}
-
-function getAppBackupCredentials(app, manifest, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var now = new Date();
-    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
-    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
-            if (error) return callback(error);
-
-            result.id = dataFilename;
-            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
-            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
-            result.backupKey = backupConfig.key;
-
-            debug('getAppBackupCredentials: %j', result);
-
-            callback(null, result);
-        });
-    });
-}
-
-// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+// backupId is the filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreConfig(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
-
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
-            if (error) return callback(error);
+        api(backupConfig.provider).getAppRestoreConfig(backupConfig, backupId, function (error, result) {
+            if (error && error.reason === BackupsError.NOT_FOUND) return callback(error);
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-            superagent.get(result.url).buffer(true).end(function (error, response) {
-                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
-
-                var config = safe.JSON.parse(response.text);
-                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
-
-                return callback(null, config);
-            });
+            callback(null, result);
         });
     });
 }
 
-// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+// backupId is the filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -204,10 +163,11 @@ function getRestoreUrl(backupId, callback) {
             var obj = {
                 id: backupId,
                 url: result.url,
-                backupKey: backupConfig.key
+                backupKey: backupConfig.key,
+                sha1: result.sha1 || null       // not supported by all backends
             };
 
-            debug('getRestoreUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);
+            debug('getRestoreUrl: id:%s url:%s backupKey:%s sha1:%s', obj.id, obj.url, obj.backupKey, obj.sha1);
 
             callback(null, obj);
         });
@@ -249,26 +209,30 @@ function copyLastBackup(app, manifest, callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    getBoxBackupCredentials(appBackupIds, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';
+
+    settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        debug('backupBoxWithAppBackupIds:  %j', result);
+        api(backupConfig.provider).getBoxBackupDetails(backupConfig, filename, function (error, result) {
+            if (error) return callback(error);
 
-        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
-        if (result.sessionToken) args.push(result.sessionToken);
+            debug('backupBoxWithAppBackupIds: backup details %j', result);
 
-        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
-            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-            debug('backupBoxWithAppBackupIds: success');
-
-            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+            shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(result.backupScriptArguments), function (error) {
                 if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                    if (error) return callback(error);
-                    callback(null, result.id);
+                debug('backupBoxWithAppBackupIds: success');
+
+                backupdb.add({ id: filename, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                    webhooks.backupDone(filename, null /* app */, appBackupIds, function (error) {
+                        if (error) return callback(error);
+                        callback(null, filename);
+                    });
                 });
             });
         });
@@ -317,26 +281,31 @@ function createNewAppBackup(app, manifest, callback) {
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getAppBackupCredentials(app, manifest, function (error, result) {
-        if (error) return callback(error);
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
 
-        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
-        if (result.sessionToken) args.push(result.sessionToken);
+        api(backupConfig.provider).getAppBackupDetails(backupConfig, app.id, dataFilename, configFilename, function (error, result) {
+            if (error) return callback(error);
 
-        async.series([
-            addons.backupAddons.bind(null, app, manifest.addons),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
-        ], function (error) {
-            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+            debug('createNewAppBackup: backup details %j', result);
 
-            debugApp(app, 'createNewAppBackup: %s done', result.id);
-
-            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+            async.series([
+                addons.backupAddons.bind(null, app, manifest.addons),
+                shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(result.backupScriptArguments))
+            ], function (error) {
                 if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-                callback(null, result.id);
+                debugApp(app, 'createNewAppBackup: %s done', dataFilename);
+
+                backupdb.add({ id: dataFilename, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                    callback(null, dataFilename);
+                });
             });
         });
     });
@@ -450,7 +419,10 @@ function backup(auditSource, callback) {
     progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
 
     backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
-        if (error) debug('backup failed.', error);
+        if (error) {
+            debug('backup failed.', error);
+            mailer.backupFailed(JSON.stringify(error));
+        }
 
         locker.unlock(locker.OP_FULL_BACKUP);
     });
@@ -495,3 +467,44 @@ function restoreApp(app, addonsToRestore, backupId, callback) {
         });
     });
 }
+
+function getLocalDownloadPath(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getLocalFilePath(backupConfig, backupId, function (error, result) {
+            if (error) return callback(error);
+
+            debug('getLocalDownloadPath: id:%s path:%s', backupId, result.filePath);
+
+            callback(null, result.filePath);
+        });
+    });
+}
+
+function removeBackup(backupId, appBackupIds, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('removeBackup: %s', backupId);
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).removeBackup(backupConfig, backupId, appBackupIds, function (error) {
+            if (error) return callback(error);
+
+            backupdb.del(backupId, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                debug('removeBackup: %s done', backupId);
+
+                callback(null);
+            });
+        });
+    });
+}

src/cert/acme.js

@@ -16,7 +16,7 @@ var assert = require('assert'),
 
 var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
     CA_STAGING = 'https://acme-staging.api.letsencrypt.org',
-    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
+    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf';
 
 exports = module.exports = {
     getCertificate: getCertificate,
@@ -63,8 +63,8 @@ function Acme(options) {
 }
 
 Acme.prototype.getNonce = function (callback) {
-    superagent.get(this.caOrigin + '/directory', function (error, response) {
-        if (error) return callback(error);
+    superagent.get(this.caOrigin + '/directory').timeout(30 * 1000).end(function (error, response) {
+        if (error && !error.response) return callback(error);
         if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
 
         return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
@@ -118,7 +118,7 @@ Acme.prototype.sendSignedRequest = function (url, payload, callback) {
             signature: signature64
         };
 
-        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).end(function (error, res) {
+        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).timeout(30 * 1000).end(function (error, res) {
             if (error && !error.response) return callback(error); // network errors
 
             callback(null, res);
@@ -259,7 +259,7 @@ Acme.prototype.waitForChallenge = function (challenge, callback) {
     async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
         debug('waitingForChallenge: getting status');
 
-        superagent.get(challenge.uri).end(function (error, result) {
+        superagent.get(challenge.uri).timeout(30 * 1000).end(function (error, result) {
             if (error && !error.response) {
                 debug('waitForChallenge: network error getting uri %s', challenge.uri);
                 return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
@@ -269,7 +269,7 @@ Acme.prototype.waitForChallenge = function (challenge, callback) {
                 return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
-            debug('waitForChallenge: status is "%s"', result.body.status);
+            debug('waitForChallenge: status is "%s %j', result.body.status, result.body);
 
             if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
             else if (result.body.status === 'valid') return retryCallback();
@@ -353,7 +353,7 @@ Acme.prototype.downloadChain = function (linkHeader, callback) {
         var data = [ ];
         res.on('data', function(chunk) { data.push(chunk); });
         res.on('end', function () { res.text = Buffer.concat(data); done(); });
-    }).end(function (error, result) {
+    }).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
         if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
 
@@ -379,7 +379,7 @@ Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
         var data = [ ];
         res.on('data', function(chunk) { data.push(chunk); });
         res.on('end', function () { res.text = Buffer.concat(data); done(); });
-    }).end(function (error, result) {
+    }).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
         if (result.statusCode === 202) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, 'Retry not implemented yet'));
         if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));

src/cert/interface.js

@@ -0,0 +1,22 @@
+'use strict';
+
+// -------------------------------------------
+//  This file just describes the interface
+//
+//  New backends can start from here
+// -------------------------------------------
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+var assert = require('assert');
+
+function getCertificate(domain, options, callback) {
+	assert.strictEqual(typeof domain, 'string');
+	assert.strictEqual(typeof options, 'object');
+	assert.strictEqual(typeof callback, 'function');
+
+    return callback(new Error('Not implemented'));
+}
+

src/certificates.js

@@ -19,7 +19,6 @@ var acme = require('./cert/acme.js'),
     assert = require('assert'),
     async = require('async'),
     caas = require('./cert/caas.js'),
-    cloudron = require('./cloudron.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
@@ -37,8 +36,6 @@ var acme = require('./cert/acme.js'),
     waitForDns = require('./waitfordns.js'),
     x509 = require('x509');
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 function CertificatesError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -84,7 +81,7 @@ function getApi(app, callback) {
         // we simply update the account with the latest email we have each time when getting letsencrypt certs
         // https://github.com/ietf-wg-acme/acme/issues/30
         user.getOwner(function (error, owner) {
-            options.email = error ? 'admin@cloudron.io' : owner.email; // can error if not activated yet
+            options.email = error ? 'support@cloudron.io' : owner.email; // can error if not activated yet
 
             callback(null, api, options);
         });
@@ -92,6 +89,8 @@ function getApi(app, callback) {
 }
 
 function installAdminCertificate(callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
@@ -143,8 +142,18 @@ function renewAll(auditSource, callback) {
         var expiringApps = [ ];
         for (var i = 0; i < allApps.length; i++) {
             var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
-            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
-            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
+
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.user.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.user.key');
+
+            if (safe.fs.existsSync(certFilePath) && safe.fs.existsSync(keyFilePath)) {
+                debug('renewAll: existing user key file for %s. skipping', appDomain);
+                continue;
+            }
+
+            // check if we have an auto cert to be renewed
+            certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
             if (!safe.fs.existsSync(keyFilePath)) {
                 debug('renewAll: no existing key file for %s. skipping', appDomain);
@@ -172,11 +181,12 @@ function renewAll(auditSource, callback) {
 
                     var errorMessage = error ? error.message : '';
                     eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
-                    mailer.certificateRenewed(domain, errorMessage);
 
                     if (error) {
                         debug('renewAll: could not renew cert for %s because %s', domain, error);
 
+                        mailer.certificateRenewalError(domain, errorMessage);
+
                         // check if we should fallback if we expire in the coming day
                         if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
 
@@ -312,14 +322,21 @@ function ensureCertificate(app, callback) {
 
     var domain = app.altDomain || config.appFqdn(app.location);
 
-    // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
-    var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
-    var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.user.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.user.key');
 
-    if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
-        debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) {
+        debug('ensureCertificate: %s. user certificate already exists at %s', domain, keyFilePath);
+        return callback(null, certFilePath, keyFilePath);
+    }
 
-        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
+    certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+    keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) {
+        debug('ensureCertificate: %s. certificate already exists at %s', domain, keyFilePath);
+
+        if (!isExpiringSync(certFilePath, 24 * 1)) return callback(null, certFilePath, keyFilePath);
     }
 
     debug('ensureCertificate: %s cert require renewal', domain);

src/clientdb.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -15,10 +13,12 @@ exports = module.exports = {
     delByAppId: delByAppId,
     delByAppIdAndType: delByAppIdAndType,
 
-    _clear: clear
+    _clear: clear,
+    _addDefaultClients: addDefaultClients
 };
 
 var assert = require('assert'),
+    async = require('async'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror.js');
 
@@ -159,3 +159,10 @@ function clear(callback) {
     });
 }
 
+function addDefaultClients(callback) {
+    async.series([
+        add.bind(null, 'cid-webadmin', 'Settings', 'built-in', 'secret-webadmin', 'https://admin-localhost', 'cloudron,profile,users,apps,settings'),
+        add.bind(null, 'cid-sdk', 'SDK', 'built-in', 'secret-sdk', 'https://admin-localhost', '*,roleSdk'),
+        add.bind(null, 'cid-cli', 'Cloudron Tool', 'built-in', 'secret-cli', 'https://admin-localhost', '*,roleSdk')
+    ], callback);
+}

src/clients.js

@@ -258,10 +258,19 @@ function delByAppIdAndType(appId, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppIdAndType(appId, type, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+    getByAppIdAndType(appId, type, function (error, result) {
         if (error) return callback(error);
-        callback(null);
+
+        tokendb.delByClientId(result.id, function (error) {
+            if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            clientdb.delByAppIdAndType(appId, type, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+                if (error) return callback(error);
+
+                callback(null);
+            });
+        });
     });
 }
 

src/cloudron.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     getStatus: getStatus,
 
     sendHeartbeat: sendHeartbeat,
+    sendAliveStatus: sendAliveStatus,
 
     updateToLatest: updateToLatest,
     reboot: reboot,
@@ -23,14 +24,14 @@ exports = module.exports = {
     events: new (require('events').EventEmitter)(),
 
     EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured',
-    EVENT_FIRST_RUN: 'firstrun'
+    EVENT_CONFIGURED: 'configured'
 };
 
 var apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
+    child_process = require('child_process'),
     clients = require('./clients.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
@@ -60,7 +61,7 @@ var apps = require('./apps.js'),
     _ = require('underscore');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
+    UPDATE_CMD = path.join(__dirname, 'scripts/update.sh'),
     RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -116,26 +117,14 @@ function initialize(callback) {
     ensureDkimKeySync();
 
     exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
-    exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
 
-    // check activation state for existing cloudrons that do not have first run file
-    // can be removed once cloudrons have been updated
-    isActivated(function (error, activated) {
-        if (error) return callback(error);
+    if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
+        debug('initialize: installing app bundle on first run');
+        process.nextTick(installAppBundle);
+        fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+    }
 
-        debug('initialize: cloudron %s activated', activated ? '' : 'not');
-
-        if (activated) fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
-
-        if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
-            // EE API is sync. do not keep the server waiting
-            debug('initialize: emitting first run event');
-            process.nextTick(function () { exports.events.emit(exports.EVENT_FIRST_RUN); });
-            fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
-        }
-
-        syncConfigState(callback);
-    });
+    syncConfigState(callback);
 }
 
 function uninitialize(callback) {
@@ -151,15 +140,6 @@ function isConfiguredSync() {
     return gIsConfigured === true;
 }
 
-function isActivated(callback) {
-    user.getOwner(function (error) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
-        if (error) return callback(error);
-
-        callback(null, true);
-    });
-}
-
 function isConfigured(callback) {
     // set of rules to see if we have the configs required for cloudron to function
     // note this checks for missing configs and not invalid configs
@@ -169,7 +149,7 @@ function isConfigured(callback) {
 
         if (!dnsConfig) return callback(null, false);
 
-        var isConfigured = (config.isCustomDomain() && dnsConfig.provider === 'route53') ||
+        var isConfigured = (config.isCustomDomain() && (dnsConfig.provider === 'route53' || dnsConfig.provider === 'digitalocean' || dnsConfig.provider === 'noop')) ||
                         (!config.isCustomDomain() && dnsConfig.provider === 'caas');
 
         callback(null, isConfigured);
@@ -210,20 +190,20 @@ function setTimeZone(ip, callback) {
     // { url: 'http://ip-api.com/json/%s', jpath: 'timezone' },
     // { url: 'http://geoip.nekudo.com/api/%s', jpath: 'time_zone }
 
-    superagent.get('http://freegeoip.net/json/' + ip).end(function (error, result) {
+    superagent.get('http://ip-api.com/json/' + ip).timeout(10 * 1000).end(function (error, result) {
         if ((error && !error.response) || result.statusCode !== 200) {
             debug('Failed to get geo location: %s', error.message);
             return callback(null);
         }
 
-        if (!result.body.time_zone || typeof result.body.time_zone !== 'string') {
+        if (!result.body.timezone || typeof result.body.timezone !== 'string') {
             debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
-        debug('Setting timezone to ', result.body.time_zone);
+        debug('Setting timezone to ', result.body.timezone);
 
-        settings.setTimeZone(result.body.time_zone, callback);
+        settings.setTimeZone(result.body.timezone, callback);
     });
 }
 
@@ -250,7 +230,7 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
-            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+            var expires = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
 
             tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -298,6 +278,7 @@ function getBoxAndUserDetails(callback) {
     superagent
         .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
         .query({ token: config.token() })
+        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, 'Cannot reach appstore'));
             if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
@@ -335,6 +316,7 @@ function getConfig(callback) {
                         update: updateChecker.getUpdateInfo(),
                         progress: progress.get(),
                         isCustomDomain: config.isCustomDomain(),
+                        isDemo: config.isDemo(),
                         developerMode: developerMode,
                         region: result.box.region,
                         size: result.box.size,
@@ -355,13 +337,63 @@ function sendHeartbeat() {
     if (!config.token()) return;
 
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
-    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) debug('Network error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
     });
 }
 
+function sendAliveStatus(callback) {
+    if (typeof callback !== 'function') {
+        callback = function (error) {
+            if (error && error.reason !== CloudronError.INTERNAL_ERROR) console.error(error);
+            else if (error) debug(error);
+        };
+    }
+
+    function sendAliveStatusWithAppstoreConfig(appstoreConfig) {
+        assert.strictEqual(typeof appstoreConfig.userId, 'string');
+        assert.strictEqual(typeof appstoreConfig.cloudronId, 'string');
+        assert.strictEqual(typeof appstoreConfig.token, 'string');
+
+        var url = config.apiServerOrigin() + '/api/v1/users/' + appstoreConfig.userId + '/cloudrons/' + appstoreConfig.cloudronId;
+        var data = {
+            domain: config.fqdn(),
+            version: config.version(),
+            provider: config.provider()
+        };
+
+        superagent.post(url).send(data).query({ accessToken: appstoreConfig.token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error));
+            if (result.statusCode === 404) return callback(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.statusCode !== 201) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('Sending alive status failed. %s %j', result.status, result.body)));
+
+            callback(null);
+        });
+    }
+
+    // Caas Cloudrons do not store appstore credentials in their local database
+    if (config.provider() === 'caas') {
+        if (!config.token()) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, 'no token set'));
+
+        var url = config.apiServerOrigin() + '/api/v1/exchangeBoxTokenWithUserToken';
+        superagent.post(url).query({ token: config.token() }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error));
+            if (result.statusCode !== 201) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+
+            sendAliveStatusWithAppstoreConfig(result.body);
+        });
+    } else {
+        settings.getAppstoreConfig(function (error, result) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+            if (!result.token) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, 'not registered yet'));
+
+            sendAliveStatusWithAppstoreConfig(result);
+        });
+    }
+}
+
 function ensureDkimKeySync() {
     var dkimPrivateKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/private');
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
@@ -373,8 +405,8 @@ function ensureDkimKeySync() {
 
     debug('Generating new DKIM keys');
 
-    safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
-    safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
+    child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
+    child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
 }
 
 function readDkimPublicKeySync() {
@@ -434,7 +466,6 @@ function addDnsRecords() {
     gUpdatingDns = true;
 
     var DKIM_SELECTOR = 'cloudron';
-    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
 
     var dkimKey = readDkimPublicKeySync();
     if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
@@ -445,16 +476,11 @@ function addDnsRecords() {
         var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
-        // DMARC requires special setup if report email id is in different domain
-        var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
-
-        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
 
         var records = [ ];
         if (config.isCustomDomain()) {
             records.push(webadminRecord);
             records.push(dkimRecord);
-            records.push(mxRecord);
         } else {
             // for non-custom domains, we show a nakeddomain.html page
             var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
@@ -462,8 +488,6 @@ function addDnsRecords() {
             records.push(nakedDomainRecord);
             records.push(webadminRecord);
             records.push(dkimRecord);
-            records.push(dmarcRecord);
-            records.push(mxRecord);
         }
 
         debug('addDnsRecords: %j', records);
@@ -477,7 +501,7 @@ function addDnsRecords() {
                 debug('addDnsRecords: will update %j', records);
 
                 async.mapSeries(records, function (record, iteratorCallback) {
-                    subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
+                    subdomains.upsert(record.subdomain, record.type, record.values, iteratorCallback);
                 }, function (error, changeIds) {
                     if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
                     else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
@@ -515,12 +539,7 @@ function update(boxUpdateInfo, auditSource, callback) {
     progress.set(progress.UPDATE, 0, 'Starting');
 
     // initiate the update/upgrade but do not wait for it
-    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
-        doShortCircuitUpdate(boxUpdateInfo, function (error) {
-            if (error) debug('Short-circuit update failed', error);
-            locker.unlock(locker.OP_BOX_UPDATE);
-        });
-    } else if (boxUpdateInfo.upgrade) {
+    if (boxUpdateInfo.upgrade) {
         debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
             if (error) {
@@ -549,6 +568,15 @@ function updateToLatest(auditSource, callback) {
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
+    // check if this is just a version number change
+    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
+        doShortCircuitUpdate(boxUpdateInfo, function (error) {
+            if (error) debug('Short-circuit update failed', error);
+        });
+
+        return callback(null);
+    }
+
     if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
 
     update(boxUpdateInfo, auditSource, callback);
@@ -580,6 +608,7 @@ function doUpgrade(boxUpdateInfo, callback) {
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
           .query({ token: config.token() })
           .send({ version: boxUpdateInfo.version })
+          .timeout(30 * 1000)
           .end(function (error, result) {
             if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
             if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
@@ -606,48 +635,39 @@ function doUpdate(boxUpdateInfo, callback) {
     backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
         if (error) return updateError(error);
 
-        // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
-        var args = {
-            sourceTarballUrl: boxUpdateInfo.sourceTarballUrl,
+        // NOTE: this data is opaque and will be passed through the installer.sh
+        var data= {
+            provider: config.provider(),
+            token: config.token(),
+            apiServerOrigin: config.apiServerOrigin(),
+            webServerOrigin: config.webServerOrigin(),
+            fqdn: config.fqdn(),
+            tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+            tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+            isCustomDomain: config.isCustomDomain(),
+            isDemo: config.isDemo(),
 
-            // this data is opaque to the installer
-            data: {
-                provider: config.provider(),
+            appstore: {
+                token: config.token(),
+                apiServerOrigin: config.apiServerOrigin()
+            },
+            caas: {
                 token: config.token(),
                 apiServerOrigin: config.apiServerOrigin(),
-                webServerOrigin: config.webServerOrigin(),
-                fqdn: config.fqdn(),
-                tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
-                tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                isCustomDomain: config.isCustomDomain(),
+                webServerOrigin: config.webServerOrigin()
+            },
 
-                appstore: {
-                    token: config.token(),
-                    apiServerOrigin: config.apiServerOrigin()
-                },
-                caas: {
-                    token: config.token(),
-                    apiServerOrigin: config.apiServerOrigin(),
-                    webServerOrigin: config.webServerOrigin()
-                },
-
-                version: boxUpdateInfo.version,
-                boxVersionsUrl: config.get('boxVersionsUrl')
-            }
+            version: boxUpdateInfo.version,
+            boxVersionsUrl: config.get('boxVersionsUrl')
         };
 
-        debug('updating box %j', args);
+        debug('updating box %s %j', boxUpdateInfo.sourceTarballUrl, data);
 
-        superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
-            if (error && !error.response) return updateError(error);
-            if (result.statusCode !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
+        shell.sudo('update', [ UPDATE_CMD, boxUpdateInfo.sourceTarballUrl, JSON.stringify(data) ], function (error) {
+            if (error) return updateError(error);
 
-            progress.set(progress.UPDATE, 10, 'Updating cloudron software');
-
-            callback(null);
+            // Do not add any code here. The installer script will stop the box code any instant
         });
-
-        // Do not add any code here. The installer script will stop the box code any instant
     });
 }
 
@@ -693,7 +713,7 @@ function checkDiskSpace(callback) {
 
         var oos = entries.some(function (entry) {
             return (entry.mount === paths.DATA_DIR && entry.capacity >= 0.90) ||
-                   (entry.mount === '/' && entry.used <= (1.25 * 1024 * 1024)); // 1.5G
+                   (entry.mount === '/' && entry.available <= (1.25 * 1024 * 1024)); // 1.5G
         });
 
         debug('Disk space checked. ok: %s', !oos);
@@ -744,6 +764,7 @@ function doMigrate(options, callback) {
           .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
           .query({ token: config.token() })
           .send(options)
+          .timeout(30 * 1000)
           .end(function (error, result) {
             if (error && !error.response) return unlock(error); // network error
             if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
@@ -763,6 +784,8 @@ function migrate(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    if (config.isDemo()) return callback(new CloudronError(CloudronError.BAD_FIELD, 'Not allowed in demo mode'));
+
     if (!options.domain) return doMigrate(options, callback);
 
     var dnsConfig = _.pick(options, 'domain', 'provider', 'accessKeyId', 'secretAccessKey', 'region', 'endpoint');

src/config.js

@@ -33,6 +33,7 @@ exports = module.exports = {
     zoneName: zoneName,
 
     isDev: isDev,
+    isDemo: isDemo,
 
     // for testing resets to defaults
     _reset: _reset
@@ -80,7 +81,6 @@ function initConfig() {
     data.smtpPort = 2525; // // this value comes from mail container
     data.sysadminPort = 3001;
     data.ldapPort = 3002;
-    data.oauthProxyPort = 3003;
     data.simpleAuthPort = 3004;
     data.provider = 'caas';
     data.appBundle = [ ];
@@ -209,6 +209,11 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
-function provider() {
-    return get('provider');
+function isDemo() {
+    return get('isDemo') === true;
+}
+
+function provider() {
+    // FIXME this fallback is only there because old Cloudrons do not have the provider set till the next upgrade
+    return get('provider') || 'caas';
 }

src/constants.js

@@ -9,13 +9,29 @@ exports = module.exports = {
     MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
     POSTMAN_LOCATION: 'postman', // used in dovecot bounces
 
+    // These are combined into one array because users and groups become mailboxes
+    RESERVED_NAMES: [
+        // Reserved usernames
+        // https://github.com/gogits/gogs/blob/52c8f691630548fe091d30bcfe8164545a05d3d5/models/repo.go#L393
+        'admin', 'no-reply', 'postmaster', 'mailer-daemon', // apps like wordpress, gogs don't like these
+
+        // Reserved groups
+        'admins', 'users'         // ldap code uses 'users' pseudo group
+    ],
+
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
     ADMIN_APPID: 'admin', // admin appid (settingsdb)
 
+    ADMIN_GROUP_ID: 'admin',
+
     GHOST_USER_FILE: '/tmp/cloudron_ghost.json',
 
-    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
+    DEFAULT_TOKEN_EXPIRATION: 7 * 24 * 60 * 60 * 1000, // 1 week
+
+    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
+
+    DEMO_USERNAME: 'cloudron'
 };
 

src/cron.js

@@ -23,8 +23,10 @@ var gAutoupdaterJob = null,
     gBoxUpdateCheckerJob = null,
     gAppUpdateCheckerJob = null,
     gHeartbeatJob = null,
+    gAliveJob = null,
     gBackupJob = null,
     gCleanupTokensJob = null,
+    gCleanupBackupsJob = null,
     gDockerVolumeCleanerJob = null,
     gSchedulerSyncJob = null,
     gCertificateRenewJob = null,
@@ -52,6 +54,12 @@ function initialize(callback) {
     });
     cloudron.sendHeartbeat(); // latest unpublished version of CronJob has runOnInit
 
+    gAliveJob = new CronJob({
+        cronTime: '00 23 * * * *', // every hour on a somewhat odd 23 minute after full probably should be randomly spread out over a day?
+        onTick: cloudron.sendAliveStatus,
+        start: true
+    });
+
     if (cloudron.isConfiguredSync()) {
         recreateJobs(callback);
     } else {
@@ -106,6 +114,14 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCleanupBackupsJob) gCleanupBackupsJob.stop();
+        gCleanupBackupsJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: janitor.cleanupBackups,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
         if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
         gCleanupEventlogJob = new CronJob({
             cronTime: '00 */30 * * * *', // every 30 minutes
@@ -198,12 +214,18 @@ function uninitialize(callback) {
     if (gHeartbeatJob) gHeartbeatJob.stop();
     gHeartbeatJob = null;
 
+    if (gAliveJob) gAliveJob.stop();
+    gAliveJob = null;
+
     if (gBackupJob) gBackupJob.stop();
     gBackupJob = null;
 
     if (gCleanupTokensJob) gCleanupTokensJob.stop();
     gCleanupTokensJob = null;
 
+    if (gCleanupBackupsJob) gCleanupBackupsJob.stop();
+    gCleanupBackupsJob = null;
+
     if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
     gCleanupEventlogJob = null;
 

src/database.js

@@ -15,9 +15,10 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    once = require('once'),
+    child_process = require('child_process'),
     config = require('./config.js'),
     mysql = require('mysql'),
+    once = require('once'),
     util = require('util');
 
 var gConnectionPool = null,
@@ -47,6 +48,11 @@ function initialize(options, callback) {
         ssl: false
     });
 
+    gConnectionPool.on('connection', function (connection) {
+        connection.query('USE ' + config.database().name);
+        connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
+    });
+
     reconnect(callback);
 }
 
@@ -59,24 +65,6 @@ function uninitialize(callback) {
     }
 }
 
-function setupConnection(connection, callback) {
-    assert.strictEqual(typeof connection, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    connection.on('error', console.error);
-
-    async.series([
-        connection.query.bind(connection, 'USE ' + config.database().name),
-        connection.query.bind(connection, 'SET SESSION sql_mode = \'strict_all_tables\'')
-    ], function (error) {
-        connection.removeListener('error', console.error);
-
-        if (error) connection.release();
-
-        callback(error);
-    });
-}
-
 function reconnect(callback) {
     callback = callback ? once(callback) : function () {};
 
@@ -97,31 +85,23 @@ function reconnect(callback) {
             setTimeout(reconnect.bind(null, callback), 1000);
         });
 
-        setupConnection(connection, function (error) {
-            if (error) return setTimeout(reconnect.bind(null, callback), 1000);
+        gDefaultConnection = connection;
 
-            gDefaultConnection = connection;
-
-            callback(null);
-        });
+        callback(null);
     });
 }
 
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // the clear funcs don't completely clear the db, they leave the migration code defaults
+    var cmd = util.format('mysql --host=%s --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host=%s --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
+        config.database().hostname, config.database().username, config.database().password, config.database().name,
+        config.database().hostname, config.database().username, config.database().password, config.database().name);
+
     async.series([
-        require('./appdb.js')._clear,
-        require('./authcodedb.js')._clear,
-        require('./backupdb.js')._clear,
-        require('./clientdb.js')._clear,
-        require('./tokendb.js')._clear,
-        require('./groupdb.js')._clear,
-        require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear,
-        require('./eventlogdb.js')._clear,
-        require('./mailboxdb.js')._clear
+        child_process.exec.bind(null, cmd),
+        require('./clientdb.js')._addDefaultClients,
+        require('./groupdb.js')._addDefaultGroups
     ], callback);
 }
 
@@ -131,16 +111,15 @@ function beginTransaction(callback) {
     if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
 
     gConnectionPool.getConnection(function (error, connection) {
-        if (error) return callback(error);
+        if (error) {
+            console.error('Unable to get connection to database. Try again in a bit.', error.message);
+            return setTimeout(beginTransaction.bind(null, callback), 1000);
+        }
 
-        setupConnection(connection, function (error) {
+        connection.beginTransaction(function (error) {
             if (error) return callback(error);
 
-            connection.beginTransaction(function (error) {
-                if (error) return callback(error);
-
-                return callback(null, connection);
-            });
+            return callback(null, connection);
         });
     });
 }

src/databaseerror.js

@@ -9,13 +9,14 @@ var assert = require('assert'),
 
 function DatabaseError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined' || errorOrMessage === null);
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
+    this.name = this.constructor.name;
     this.reason = reason;
-    if (typeof errorOrMessage === 'undefined' || errorOrMessage === null) {
+    if (typeof errorOrMessage === 'undefined') {
         this.message = reason;
     } else if (typeof errorOrMessage === 'string') {
         this.message = errorOrMessage;

src/developer.js

@@ -12,8 +12,9 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    config = require('./config.js'),
     clients = require('./clients.js'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:developer'),
     eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
@@ -72,7 +73,7 @@ function issueDeveloperToken(user, auditSource, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var token = tokendb.generateToken();
-    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var expiresAt = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
     var scopes = '*,' + clients.SCOPE_ROLE_SDK;
 
     tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
@@ -88,7 +89,7 @@ function getNonApprovedApps(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
-    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
+    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
         if (result.statusCode === 401 || result.statusCode === 403) {
             debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);

src/dns/caas.js

@@ -1,13 +1,10 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
-    add: add,
+    upsert: upsert,
+    get: get,
     del: del,
-    update: update,
-    getChangeStatus: getChangeStatus,
-    get: get
+    getChangeStatus: getChangeStatus
 };
 
 var assert = require('assert'),
@@ -15,8 +12,7 @@ var assert = require('assert'),
     debug = require('debug')('box:dns/caas'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     superagent = require('superagent'),
-    util = require('util'),
-    _ = require('underscore');
+    util = require('util');
 
 function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
@@ -39,8 +35,10 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
         .post(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
         .query({ token: dnsConfig.token })
         .send(data)
+        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
+            if (result.statusCode === 400) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
             if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
             if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 
@@ -62,6 +60,7 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
     superagent
         .get(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
         .query({ token: dnsConfig.token, type: type })
+        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
@@ -70,7 +69,7 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
         });
 }
 
-function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
@@ -78,13 +77,7 @@ function update(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
-        if (error) return callback(error);
-
-        if (_.isEqual(values, result)) return callback();
-
-        add(dnsConfig, zoneName, subdomain, type, values, callback);
-    });
+    add(dnsConfig, zoneName, subdomain, type, values, callback);
 }
 
 function del(dnsConfig, zoneName, subdomain, type, values, callback) {
@@ -95,7 +88,7 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+    debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
     var data = {
         type: type,
@@ -106,8 +99,10 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
         .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
         .query({ token: dnsConfig.token })
         .send(data)
+        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
+            if (result.statusCode === 400) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
             if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
             if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
             if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
@@ -126,6 +121,7 @@ function getChangeStatus(dnsConfig, changeId, callback) {
     superagent
         .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
         .query({ token: dnsConfig.token })
+        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

src/dns/digitalocean.js

@@ -0,0 +1,171 @@
+'use strict';
+
+exports = module.exports = {
+    upsert: upsert,
+    get: get,
+    del: del,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:dns/digitalocean'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    superagent = require('superagent'),
+    util = require('util');
+
+var DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
+
+function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    superagent.get(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
+      .set('Authorization', 'Bearer ' + dnsConfig.token)
+      .timeout(30 * 1000)
+      .end(function (error, result) {
+        if (error && !error.response) return callback(error);
+        if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND, util.format('%s %j', result.statusCode, result.body)));
+        if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, util.format('%s %j', result.statusCode, result.body)));
+        if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+        var tmp = result.body.domain_records.filter(function (record) {
+            return (record.type === type && record.name === subdomain);
+        });
+
+        debug('getInternal: %j', tmp);
+
+        return callback(null, tmp);
+    });
+}
+
+function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    subdomain = subdomain || '@';
+
+    debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        // FIXME currently we only support one record!
+
+        var value = values[0];
+        var priority = null;
+
+        if (type === 'MX') {
+            priority = values[0].split(' ')[0];
+            value = values[0].split(' ')[1];
+        }
+
+        var data = {
+            type: type,
+            name: subdomain,
+            data: value,
+            priority: priority
+        };
+
+        if (result.length === 0) {
+            superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
+              .set('Authorization', 'Bearer ' + dnsConfig.token)
+              .send(data)
+              .timeout(30 * 1000)
+              .end(function (error, result) {
+                if (error && !error.response) return callback(error);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, util.format('%s %j', result.statusCode, result.body)));
+                if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+                return callback(null, 'unused');
+            });
+        } else {
+            superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + result[0].id)
+              .set('Authorization', 'Bearer ' + dnsConfig.token)
+              .send(data)
+              .timeout(30 * 1000)
+              .end(function (error, result) {
+                if (error && !error.response) return callback(error);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, util.format('%s %j', result.statusCode, result.body)));
+                if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+                return callback(null, 'unused');
+            });
+        }
+    });
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    subdomain = subdomain || '@';
+
+    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        // We only return the value string
+        var tmp = result.map(function (record) { return record.data; });
+
+        debug('get: %j', tmp);
+
+        return callback(null, tmp);
+    });
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    subdomain = subdomain || '@';
+
+    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        if (result.length === 0) return callback(null);
+
+        var tmp = result.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+
+        debug('del: %j', tmp);
+
+        if (tmp.length === 0) return callback(null);
+
+        // FIXME we only handle the first one currently
+
+        superagent.del(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + tmp[0].id)
+          .set('Authorization', 'Bearer ' + dnsConfig.token)
+          .timeout(30 * 1000)
+          .end(function (error, result) {
+            if (error && !error.response) return callback(error);
+            if (result.statusCode === 404) return callback(null);
+            if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, util.format('%s %j', result.statusCode, result.body)));
+            if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+            debug('del: done');
+
+            return callback(null);
+        });
+    });
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // Digitalocean does not have any way to check that
+    callback(null, 'INSYNC');
+}

src/dns/interface.js

@@ -0,0 +1,66 @@
+'use strict';
+
+// -------------------------------------------
+//  This file just describes the interface
+//
+//  New backends can start from here
+// -------------------------------------------
+
+exports = module.exports = {
+    upsert: upsert,
+    get: get,
+    del: del,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    util = require('util');
+
+function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    // Result: backend specific change id, to be passed into getChangeStatus()
+
+    callback(new Error('not implemented'));
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // Result: Array of matching DNS records in string format
+
+    callback(new Error('not implemented'));
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    // Result: none
+
+    callback(new Error('not implemented'));
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // Result: current change state as string. Upstream code checks for 'INSYNC'
+
+    callback(new Error('not implemented'));
+}

src/dns/noop.js

@@ -0,0 +1,56 @@
+'use strict';
+
+exports = module.exports = {
+    upsert: upsert,
+    get: get,
+    del: del,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:dns/noop'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    sysinfo = require('../sysinfo.js'),
+    util = require('util');
+
+function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    return callback(null, 'noop-record-id');
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    return callback();
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(null, 'INSYNC');
+}

src/dns/route53.js

@@ -1,10 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
+    upsert: upsert,
     get: get,
     del: del,
-    update: update,
     getChangeStatus: getChangeStatus,
 
     // not part of "dns" interface
@@ -15,8 +14,7 @@ var assert = require('assert'),
     AWS = require('aws-sdk'),
     debug = require('debug')('box:dns/route53'),
     SubdomainError = require('../subdomains.js').SubdomainError,
-    util = require('util'),
-    _ = require('underscore');
+    util = require('util');
 
 function getDnsCredentials(dnsConfig) {
     assert.strictEqual(typeof dnsConfig, 'object');
@@ -56,7 +54,7 @@ function getHostedZone(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
- 
+
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
@@ -105,6 +103,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
         route53.changeResourceRecordSets(params, function(error, result) {
             if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
             if (error && error.code === 'PriorRequestNotComplete') return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            if (error && error.code === 'InvalidChangeBatch') return callback(new SubdomainError(SubdomainError.BAD_FIELD, error.message));
             if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
 
             callback(null, result.ChangeInfo.Id);
@@ -112,7 +111,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     });
 }
 
-function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
@@ -120,13 +119,7 @@ function update(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
-        if (error) return callback(error);
-
-        if (_.isEqual(values, result)) return callback();
-
-        add(dnsConfig, zoneName, subdomain, type, values, callback);
-    });
+    add(dnsConfig, zoneName, subdomain, type, values, callback);
 }
 
 function get(dnsConfig, zoneName, subdomain, type, callback) {
@@ -151,7 +144,7 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
             if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
             if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
             if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
-            if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+            if (result.ResourceRecordSets[0].Name !== params.StartRecordName || result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
 
             var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
 

src/docker.js

@@ -66,7 +66,7 @@ function pullImage(manifest, callback) {
     var docker = exports.connection;
 
     docker.pull(manifest.dockerImage, function (err, stream) {
-        if (err) return callback(new Error('Error connecting to docker. statusCode: %s' + err.statusCode));
+        if (err) return callback(new Error('Error connecting to docker. statusCode: ' + err.statusCode));
 
         // https://github.com/dotcloud/docker/issues/1074 says each status message
         // is emitted as a chunk
@@ -91,7 +91,7 @@ function pullImage(manifest, callback) {
                 if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
                 if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
 
-                debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
+                if (data.Config.ExposedPorts) debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
 
                 callback(null);
             });
@@ -163,13 +163,17 @@ function createSubcontainer(app, name, cmd, options, callback) {
     }
 
     // first check db record, then manifest
-    var memoryLimit = app.memoryLimit || manifest.memoryLimit;
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit || 0;
 
-    // ensure we never go below minimum
-    memoryLimit = memoryLimit < constants.DEFAULT_MEMORY_LIMIT ? constants.DEFAULT_MEMORY_LIMIT : memoryLimit; // 256mb by default
+    if (developmentMode) {
+        // developerMode does not restrict memory usage
+        memoryLimit = 0;
+    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
+        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
+    }
 
-    // developerMode does not restrict memory usage
-    memoryLimit = developmentMode ? 0 : memoryLimit;
+    // apparmor is disabled on few servers
+    var enableSecurityOpt = config.CLOUDRON && safe(function () { return child_process.spawnSync('aa-enabled').status === 0; }, false);
 
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
@@ -209,7 +213,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
                 NetworkMode: isAppContainer ? 'cloudron' : ('container:' + app.containerId), // share network namespace with parent
-                SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
+                SecurityOpt: enableSecurityOpt ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };
         containerOptions = _.extend(containerOptions, options);

src/eventlog.js

@@ -105,10 +105,17 @@ function getAllPaged(action, search, page, perPage, callback) {
 function cleanup(callback) {
     callback = callback || NOOP_CALLBACK;
 
-     var d = new Date();
-     d.setDate(d.getDate() - 7); // 7 days ago
+    var d = new Date();
+    d.setDate(d.getDate() - 7); // 7 days ago
 
-    eventlogdb.delByCreationTime(d, function (error) {
+    // only cleanup high frequency events
+    var actions = [
+        exports.ACTION_USER_LOGIN,
+        exports.ACTION_BACKUP_START,
+        exports.ACTION_BACKUP_FINISH
+    ];
+
+    eventlogdb.delByCreationTime(d, actions, function (error) {
         if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
 
         callback(null);

src/eventlogdb.js

@@ -49,7 +49,7 @@ function getAllPaged(action, search, page, perPage, callback) {
     var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog';
 
     if (action || search) query += ' WHERE';
-    if (search) query += ' data LIKE ' + mysql.escape('%' + search + '%');
+    if (search) query += ' (source LIKE ' + mysql.escape('%' + search + '%') + ' OR data LIKE ' + mysql.escape('%' + search + '%') + ')';
     if (action && search) query += ' AND ';
 
     if (action) {
@@ -104,11 +104,15 @@ function clear(callback) {
     });
 }
 
-function delByCreationTime(creationTime, callback) {
+function delByCreationTime(creationTime, actions, callback) {
     assert(util.isDate(creationTime));
+    assert(Array.isArray(actions));
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM eventlog WHERE creationTime < ?', [ creationTime ], function (error) {
+    var query = 'DELETE FROM eventlog WHERE creationTime < ? ';
+    if (actions.length) query += ' AND ( ' + actions.map(function () { return 'action != ?'; }).join(' AND ') + ' ) ';
+
+    database.query(query, [ creationTime ].concat(actions), function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(error);

src/groupdb.js

@@ -12,15 +12,18 @@ exports = module.exports = {
     getMembers: getMembers,
     addMember: addMember,
     removeMember: removeMember,
+    setMembers: setMembers,
     isMember: isMember,
 
     getGroups: getGroups,
     setGroups: setGroups,
 
-    _clear: clear
+    _clear: clear,
+    _addDefaultGroups: addDefaultGroups
 };
 
 var assert = require('assert'),
+    constants = require('./constants.js'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror');
 
@@ -30,7 +33,7 @@ function get(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ?', [ groupId ], function (error, result) {
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ? ORDER BY name', [ groupId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
@@ -145,6 +148,25 @@ function getMembers(groupId, callback) {
     });
 }
 
+function setMembers(groupId, userIds, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert(Array.isArray(userIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ groupId ] });
+    for (var i = 0; i < userIds.length; i++) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', args: [ groupId, userIds[i] ] });
+    }
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+
 function getGroups(userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -214,3 +236,7 @@ function isMember(groupId, userId, callback) {
         callback(null, result.length !== 0);
     });
 }
+
+function addDefaultGroups(callback) {
+    add(constants.ADMIN_GROUP_ID, 'admin', callback);
+}

src/groups.js

@@ -1,5 +1,3 @@
-/* jshint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -14,19 +12,21 @@ exports = module.exports = {
 
     getMembers: getMembers,
     addMember: addMember,
+    setMembers: setMembers,
     removeMember: removeMember,
     isMember: isMember,
 
     getGroups: getGroups,
-    setGroups: setGroups,
-
-    ADMIN_GROUP_ID: 'admin' // see db migration code and groupdb._clear
+    setGroups: setGroups
 };
 
 var assert = require('assert'),
+    constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     groupdb = require('./groupdb.js'),
-    util = require('util');
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
 
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
@@ -56,16 +56,20 @@ GroupError.BAD_FIELD = 'Field error';
 GroupError.NOT_EMPTY = 'Not Empty';
 GroupError.NOT_ALLOWED = 'Not Allowed';
 
+// keep this in sync with validateUsername
 function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
-    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
 
-    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length < 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
     if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
 
-    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
+    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
 
-    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
+    // +/- can be tricky in emails. also need to consider valid LDAP characters here (e.g '+' is reserved)
+    if (/[^a-zA-Z0-9.]/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    // app emails are sent using the .app suffix
+    if (name.indexOf('.app') !== -1) return new GroupError(GroupError.BAD_FIELD, 'name pattern is reserved for apps');
 
     return null;
 }
@@ -74,14 +78,23 @@ function create(name, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    // we store names in lowercase
+    name = name.toLowerCase();
+
     var error = validateGroupname(name);
     if (error) return callback(error);
 
-    groupdb.add(name /* id */, name, function (error) {
+    var id = 'gid-' + uuid.v4();
+    mailboxdb.add(name, id /* owner */, mailboxdb.TYPE_GROUP, function (error) {
         if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
         if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
 
-        callback(null, { id: name, name: name });
+        groupdb.add(id, name, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
+            if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+            callback(null, { id: id, name: name });
+        });
     });
 }
 
@@ -90,13 +103,18 @@ function remove(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // never allow admin group to be deleted
-    if (id === exports.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
+    if (id === constants.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
 
-    groupdb.del(id, function (error) {
+    mailboxdb.delByOwnerId(id, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
         if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
 
-        callback(null);
+        groupdb.del(id, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+            if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
     });
 }
 
@@ -194,6 +212,19 @@ function addMember(groupId, userId, callback) {
     });
 }
 
+function setMembers(groupId, userIds, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert(Array.isArray(userIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.setMembers(groupId, userIds, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND, 'Invalid group or user id'));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
 function removeMember(groupId, userId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof userId, 'string');

src/infra_version.js

@@ -6,17 +6,19 @@
 
 exports = module.exports = {
     // a version bump means that all containers (apps and addons) are recreated
-    'version': 40,
+    'version': 42,
 
-    'baseImage': 'cloudron/base:0.8.1',
+    'baseImages': [ 'cloudron/base:0.9.0' ],
 
+    // Note that if any of the databases include an upgrade, bump the infra version above
+    // This is because we upgrade using dumps instead of mysql_upgrade, pg_upgrade etc
     'images': {
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.12.0' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.11.0' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.10.0' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.9.0' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.18.0' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.9.0' }
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.13.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.15.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.11.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.10.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.24.0' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.10.0' }
     }
 };
 

src/janitor.js

@@ -3,13 +3,16 @@
 var assert = require('assert'),
     async = require('async'),
     authcodedb = require('./authcodedb.js'),
+    backups = require('./backups.js'),
     debug = require('debug')('box:src/janitor'),
     docker = require('./docker.js').connection,
+    settings = require('./settings.js'),
     tokendb = require('./tokendb.js');
 
 exports = module.exports = {
     cleanupTokens: cleanupTokens,
-    cleanupDockerVolumes: cleanupDockerVolumes
+    cleanupDockerVolumes: cleanupDockerVolumes,
+    cleanupBackups: cleanupBackups
 };
 
 var NOOP_CALLBACK = function () { };
@@ -101,3 +104,37 @@ function cleanupDockerVolumes(callback) {
         }, callback);
     });
 }
+
+function cleanupBackups(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Cleaning backups');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(error);
+
+        // nothing to do here
+        if (backupConfig.provider !== 'filesystem') return callback();
+
+        backups.getPaged(1, 1000, function (error, result) {
+            if (error) return callback(error);
+
+            // sort with latest backups first in the array and slice 2
+            var toCleanup = result.sort(function (a, b) { return b.creationTime.getTime() - a.creationTime.getTime(); }).slice(2);
+
+            debug('cleanupBackups: about to clean: ', toCleanup);
+
+            async.each(toCleanup, function (backup, callback) {
+                backups.removeBackup(backup.id, backup.dependsOn, function (error) {
+                    if (error) console.error(error);
+
+                    debug('cleanupBackups: %s, %s done', backup.id, backup.dependsOn.join(', '));
+
+                    callback();
+                });
+            }, callback);
+        });
+    });
+}

src/ldap.js

@@ -8,27 +8,20 @@ exports = module.exports = {
 var assert = require('assert'),
     apps = require('./apps.js'),
     config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:ldap'),
     eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     ldap = require('ldapjs'),
-    mailboxes = require('./mailboxes.js'),
-    MailboxError = mailboxes.MailboxError;
+    mailboxdb = require('./mailboxdb.js'),
+    safe = require('safetydance'),
+    util = require('util');
 
 var gServer = null;
 
 var NOOP = function () {};
 
-var gLogger = {
-    trace: NOOP,
-    debug: NOOP,
-    info: debug,
-    warn: debug,
-    error: console.error,
-    fatal: console.error
-};
-
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
 var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
@@ -71,8 +64,7 @@ function userSearch(req, res, next) {
                     cn: entry.id,
                     uid: entry.id,
                     mail: entry.email,
-                    // TODO: check mailboxes before we send this
-                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    mailAlternateAddress: entry.alternateEmail,
                     displayname: displayName,
                     givenName: firstName,
                     username: entry.username,
@@ -86,7 +78,8 @@ function userSearch(req, res, next) {
             if (lastName.length !== 0) obj.attributes.sn = lastName;
 
             // ensure all filter values are also lowercase
-            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+            var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+            if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
                 res.send(obj);
@@ -125,7 +118,8 @@ function groupSearch(req, res, next) {
             };
 
             // ensure all filter values are also lowercase
-            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+            var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+            if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
                 res.send(obj);
@@ -139,31 +133,94 @@ function groupSearch(req, res, next) {
 function mailboxSearch(req, res, next) {
     debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
-    mailboxes.getAll(function (error, result) {
+    if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    var name = req.dn.rdns[0].attrs.cn.value.toLowerCase();
+    // allow login via email
+    var parts = name.split('@');
+    if (parts[1] === config.fqdn()) {
+        name = parts[0];
+    }
+
+    mailboxdb.getMailbox(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
 
-        result.forEach(function (entry) {
-            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
-
-            // TODO: send aliases
-            var obj = {
-                dn: dn.toString(),
-                attributes: {
-                    objectclass: ['mailbox'],
-                    objectcategory: 'mailbox',
-                    cn: entry.name,
-                    uid: entry.name,
-                    mail: entry.name + '@' + config.fqdn()
-                }
-            };
-
-            // ensure all filter values are also lowercase
-            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
-
-            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                res.send(obj);
+        var obj = {
+            dn: req.dn.toString(),
+            attributes: {
+                objectclass: ['mailbox'],
+                objectcategory: 'mailbox',
+                cn: mailbox.name,
+                uid: mailbox.name,
+                mail: mailbox.name + '@' + config.fqdn()
             }
-        });
+        };
+
+        // ensure all filter values are also lowercase
+        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
+
+        res.end();
+    });
+}
+
+function mailAliasSearch(req, res, next) {
+    debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    mailboxdb.getAlias(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, alias) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // https://wiki.debian.org/LDAP/MigrationTools/Examples
+        // https://docs.oracle.com/cd/E19455-01/806-5580/6jej518pp/index.html
+        var obj = {
+            dn: req.dn.toString(),
+            attributes: {
+                objectclass: ['nisMailAlias'],
+                objectcategory: 'nisMailAlias',
+                cn: alias.name,
+                rfc822MailMember: alias.aliasTarget
+            }
+        };
+
+        // ensure all filter values are also lowercase
+        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
+
+        res.end();
+    });
+}
+
+function mailingListSearch(req, res, next) {
+    debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    mailboxdb.getGroup(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, group) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // http://ldapwiki.willeke.com/wiki/Original%20Mailgroup%20Schema%20From%20Netscape
+        var obj = {
+            dn: req.dn.toString(),
+            attributes: {
+                objectclass: ['mailGroup'],
+                objectcategory: 'mailGroup',
+                cn: group.name,
+                mail: group.name + '@' + config.fqdn(),
+                mgrpRFC822MailMember: group.members
+            }
+        };
+
+        // ensure all filter values are also lowercase
+        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
 
         res.end();
     });
@@ -173,21 +230,15 @@ function authenticateUser(req, res, next) {
     debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
 
     // extract the common name which might have different attribute names
-    var attributeName = Object.keys(req.dn.rdns[0])[0];
-    var commonName = req.dn.rdns[0][attributeName];
+    var attributeName = Object.keys(req.dn.rdns[0].attrs)[0];
+    var commonName = req.dn.rdns[0].attrs[attributeName].value;
     if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
     var api;
     if (attributeName === 'mail') {
         api = user.verifyWithEmail;
     } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
-        var parts = commonName.split('@');
-        if (parts[1] === config.fqdn()) { // internal email, verify with username
-            commonName = parts[0];
-            api = user.verifyWithUsername;
-        } else { // external email
-            api = user.verifyWithEmail;
-        }
+        api = user.verifyWithEmail;
     } else if (commonName.indexOf('uid-') === 0) {
         api = user.verify;
     } else {
@@ -224,31 +275,61 @@ function authorizeUserForApp(req, res, next) {
     });
 }
 
-function authorizeUserForMailbox(req, res, next) {
-    assert(req.user);
+function authenticateMailbox(req, res, next) {
+    if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    // We simply authorize the user to access a mailbox by his own name
-    mailboxes.get(req.user.username, function (error) {
-        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    var name = req.dn.rdns[0].attrs.cn.value.toLowerCase();
+
+    // allow login via email
+    var parts = name.split('@');
+    if (parts[1] === config.fqdn()) {
+        name = parts[0];
+    }
+
+    mailboxdb.getMailbox(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.message));
 
-        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+        if (mailbox.ownerType === mailboxdb.TYPE_APP) {
+            if (req.credentials !== mailbox.ownerId) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: name }, { appId: mailbox.ownerId });
+            return res.end();
+        }
 
-        res.end();
+        assert.strictEqual(mailbox.ownerType, mailboxdb.TYPE_USER);
+
+        authenticateUser(req, res, function (error) {
+            if (error) return next(error);
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: name }, { userId: req.user.username });
+            res.end();
+        });
     });
 }
 
 function start(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    gServer = ldap.createServer({ log: gLogger });
+    var logger = {
+        trace: NOOP,
+        debug: NOOP,
+        info: debug,
+        warn: debug,
+        error: console.error,
+        fatal: console.error
+    };
+
+    gServer = ldap.createServer({ log: logger });
 
     gServer.search('ou=users,dc=cloudron', userSearch);
     gServer.search('ou=groups,dc=cloudron', groupSearch);
     gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);
 
+    // http://www.ietf.org/proceedings/43/I-D/draft-srivastava-ldap-mail-00.txt
     gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
-    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);
+    gServer.search('ou=mailaliases,dc=cloudron', mailAliasSearch);
+    gServer.search('ou=mailinglists,dc=cloudron', mailingListSearch);
+
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateMailbox);
 
     // this is the bind for addons (after bind, they might search and authenticate)
     gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
@@ -269,7 +350,7 @@ function start(callback) {
 function stop(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    gServer.close();
+    if (gServer) gServer.close();
 
     callback();
 }