diff options
450 files changed, 24824 insertions, 18 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..146a1006 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/.vagrant +/puppet/modules/site_custom +Gemfile.lock diff --git a/files/null/.gitignore b/.gitmodules index e69de29b..e69de29b 100644 --- a/files/null/.gitignore +++ b/.gitmodules diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..aee70b0a --- /dev/null +++ b/.mailmap @@ -0,0 +1,8 @@ +Varac <varacanero@zeromail.org> +Micah Anderson <micah@leap.se> Micah Anderson <micah@riseup.net> +Micah Anderson <micah@leap.se> micah <micah@leap.se> +Kwadronaut <kwadronaut@leap.se> +Elijah <elijah@riseup.net> elijah <elijah@ChrUbuntu.(none)> +Elijah <elijah@riseup.net> elijah <elijah@riseup.net> +Leap Admins <admin@leap.se> root <root@localhost> + diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..ad42dd7a --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,64 @@ +Platform 0.8 +-------------------------------------- + +This release focused on the email service. Debian Jessie is now required, +which also means that you must migrate all data from BigCouch to CouchDB. + +UPGRADING: It is tricky to upgrade the OS and migrate the database. You can +follow the tutorial here: https://leap.se/en/upgrade-0-8 + +WARNING: failure to migrate data from BigCouch to CouchDB will cause all user +accounts to get destroyed. + +Other new features: + +* It is possible to require invite codes for new users signing up. + +* Tapicero has been removed. Now user storage databases are created as needed + by soledad, and deleted eventually when no longer needed. + +* Admins can now suspect/enable users and block/enable their ability to send + and receive email. + +* Support for SPF and DKIM. + +Compatibility: + +* Now, soledad and couchdb must be on the same node. +* Requires Debian Jessie. Wheezy is no longer supported. +* Requires CouchDB, BigCouch is no longer supported. +* Requires leap_cli version 1.8 +* Requires bitmask client version >= 0.9 +* Includes: + * leap_mx 0.8 + * webapp 0.8 + * soledad 0.8 + +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.8 +Issues fixed: https://leap.se/code/versions/189 + + +Platform 0.7.1 +-------------------------------------- + +Compatibility: + +* Requires leap_cli version 1.7.4 +* Requires bitmask client version >= 0.7 +* Previous releases supported cookies when using the provider API. Now, only + tokens are supported. +* Includes: + * leap_mx 0.7.0 + * tapicero 0.7 + * webapp 0.7 + * soledad 0.7 + +Commits: https://leap.se/git/leap_platform.git/shortlog/refs/tags/0.7.1 +Issues fixed: https://leap.se/code/versions/159 + +Upgrading: + +* `gem install leap_cli --version 1.7.4`. +* `cd leap_platform; git pull; git checkout 0.7.1`. +* `leap deploy` +* `leap test` to make sure everything is working diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..8925a904 --- /dev/null +++ b/Gemfile @@ -0,0 +1,13 @@ +source "https://rubygems.org" + +group :test do + gem "rake" + gem "rspec", '< 3.2.0' + gem "puppet", ENV['PUPPET_VERSION'] || ENV['GEM_PUPPET_VERSION'] || ENV['PUPPET_GEM_VERSION'] || '~> 3.7.0' + gem "facter", ENV['FACTER_VERSION'] || ENV['GEM_FACTER_VERSION'] || ENV['FACTER_GEM_VERSION'] || '~> 2.2.0' + gem "rspec-puppet" + gem "puppetlabs_spec_helper" + gem "metadata-json-lint" + gem "rspec-puppet-facts" + gem "mocha" +end @@ -1,14 +1,674 @@ - Copyright 2012 R.I.Pienaar + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. - http://www.apache.org/licenses/LICENSE-2.0 + Preamble - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/README.md b/README.md new file mode 100644 index 00000000..edc272d8 --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +Leap Platform +============================= + +[](https://jenkins.leap.se/job/platform_develop/) + +The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. These recipes define an abstract service provider. It is a set of Puppet modules designed to work together to provide to sysadmins everything they need to manage a service provider infrastructure that provides secure communication services. + +Getting started +============================= + +It is highly recommended that you start by reading the overview of the [LEAP Platform](https://leap.se/docs/platform) and then begin with the [Quick Start tutorial](https://leap.se/en/docs/platform/tutorials/quick-start) to walk through a test environment setup to get familiar with how things work before deploying to live servers. + +An offline copy of this documentation is contained in the `doc` subdirectory. For more current updates to the documentation, visit the website. + +Requirements +------------------ + +For testing a virtual deployment simulated on your computer, you will need a fairly recent computer x86_64 with hardware virtualization features (AMD-V or VT-x) and plenty of RAM. If you follow the "Quick Start" documentation we will walk you through using Vagrant to setup a test deployment. + +For a live deployment of the platform, the number of servers that is required depends on your needs and which services you want to deploy. At the moment, the LEAP Platform supports servers with a base Debian Wheezy installation. + +Troubleshooting +============================= + +If you have a problem, we are interested in fixing it! + +If you have a problem, be sure to have a look at the [Known Issues](https://leap.se/docs/platform/known-issues) to see if your issue is detailed there. + +If not, the best way for us to solve your problem is if you provide to us the complete log of what you did, and the output that was produced. Please don't cut out what appears to be useless information and only include the error that you received, instead copy and paste the complete log so that we can better determine the overall situation. If you can run the same command that produced the error with a raised verbosity level (such as -v2), that provides us with more useful debugging information. + +To capture the log, you can copy from the console, or run `leap --log FILE` or edit Leapfile to include `@log = '/tmp/leap.log'`. + +Visit https://leap.se/en/docs/get-involved/communication for details on how to contact the developers. + +Known issues +============ + +The following issues are known to exist in 0.5.2 and later: + +CouchDB Sync +------------ +You can't deploy new couchdb nodes after one or more have been deployed. Make *sure* that you configure and deploy all your couchdb nodes when first creating your provider. The problem is that we dont not have a clean way of adding couch nodes after initial creation of the databases, so any nodes added after result in improperly synchronized data. See Bug [#5601](https://leap.se/code/issues/5601) for more information. + +User setup and ssh +------------------ + +. if you aren't using a single ssh key, but have different ones, you will need to define the following at the top of your ~/.ssh/config: + HostName <ip address> + IdentityFile <path to identity file> + + (see: https://leap.se/code/issues/2946 and https://leap.se/code/issues/3002) + +. If the ssh host key changes, you need to run node init again (see: https://leap.se/en/docs/platform/guide#Working.with.SSH) + +. At the moment, only ECDSA ssh host keys are supported. If you get the following error: `= FAILED ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)` then you should confirm that you have the following line defined in your server's **/etc/ssh/sshd_config**: `HostKey /etc/ssh/ssh_host_ecdsa_key`. If that file doesn't exist, run `ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ""` in order to create it. If you made a change to your sshd_config, then you need to run `/etc/init.d/ssh restart` (see: https://leap.se/code/issues/2373) + +. To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. + +. At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280) + +. leap add-user --self allows only one key - if you run that command twice with different keys, you will just replace the key with the second key. To add a second key, add it manually to files/ssh/authorized_keys (see: https://leap.se/code/issues/866) + + +Deploying +--------- + +. If you have any errors during a run, please try to deploy again as this often solves non-deterministic issues that were not uncovered in our testing. Please re-deploy with `leap -v2 deploy` to get more verbose logs and capture the complete output to provide to us for debugging. + +. If when deploying your debian mirror fails for some reason, network anomoly or the mirror itself is out of date, then platform deployment will not succeed properly. Check the mirror is up and try to deploy again when it is resolved (see: https://leap.se/code/issues/1091) + +. Deployment gives 'error: in `%`: too few arguments (ArgumentError)' - this is because you attempted to do a deploy before initializing a node, please initialize the node first and then do a deploy afterwards (see: https://leap.se/code/issues/2550) + +. This release has no ability to custom configure apt sources or proxies (see: https://leap.se/code/issues/1971) + +. When running a deploy at a verbosity level of 2 and above, you will notice puppet deprecation warnings, these are known and we are working on fixing them + +Special Environments +-------------------- + +. When deploying to OpenStack release "nova" or newer, you will need to do an initial deploy, then when it has finished run `leap facts update` and then deploy again (see: https://leap.se/code/issues/3020) + +leap-mx +------- + +. see https://github.com/leapcode/leap_mx#070 for issues regarding leap_mx + + +Contributing +============ + +In order to validate the syntax and style guide compliance +before you commit, see https://github.com/pixelated-project/puppet-git-hooks#installation + + +Changes +========= + +Read CHANGES.md or run `git log`. + +Authors and Credits +=================== + +See contributors: + + git shortlog -es --all + + +Copyright/License +================= + +Read LICENSE @@ -1,13 +1,57 @@ -require 'rake' -require 'rspec/core/rake_task' +require 'puppetlabs_spec_helper/rake_tasks' +require 'puppet-lint/tasks/puppet-lint' +require 'puppet-syntax/tasks/puppet-syntax' -task :default => [:spec] +# return list of modules, either +# submodules, custom or all modules +# so we can check each array seperately +def modules_pattern (type) + submodules = Array.new + custom_modules = Array.new + all_modules = Array.new -desc "Run all module spec tests (Requires rspec-puppet gem)" -RSpec::Core::RakeTask.new(:spec) + Dir['puppet/modules/*'].sort.each do |m| + system("grep -q #{m} .gitmodules") + if $?.exitstatus == 0 + submodules << m + '/**/*.pp' + else + custom_modules << m + '/**/*.pp' + end + all_modules << m + '/**/*.pp' + end -desc "Build package" -task :build do - system("puppet-module build") + case type + when 'submodule' + submodules + when 'custom' + custom_modules + when 'all' + all_modules + end end +exclude_paths = ["**/vendor/**/*", "spec/fixtures/**/*", "pkg/**/*" ] + +# redefine lint task so we don't lint submoudules for now +Rake::Task[:lint].clear +PuppetLint::RakeTask.new :lint do |config| + # only check for custom manifests, not submodules for now + config.pattern = modules_pattern('custom') + config.ignore_paths = exclude_paths + config.disable_checks = ['documentation', '80chars'] + config.fail_on_warnings = false +end + +# rake syntax::* tasks +PuppetSyntax.exclude_paths = exclude_paths +PuppetSyntax.future_parser = true + +desc "Validate erb templates" +task :templates do + Dir['**/templates/**/*.erb'].each do |template| + sh "erb -P -x -T '-' #{template} | ruby -c" unless template =~ /.*vendor.*/ + end +end + +desc "Run all puppet checks required for CI (syntax , validate, spec, lint)" +task :test => [:syntax , :validate, :templates, :spec, :lint] diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..25f26b3b --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,53 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + + # shared config for all boxes + + # Please verify the sha512 sum of the downloaded box before importing it into vagrant ! + # see https://leap.se/en/docs/platform/details/development#Verify.vagrantbox.download + # for details + config.vm.box = "LEAP/jessie" + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + v.name = "jessie" + v.memory = 1536 + end + + config.vm.provider "libvirt" do |v| + v.memory = 1536 + end + + # Fix annoying 'stdin: is not a tty' warning + # see http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html + config.vm.provision "shell" do |s| + s.privileged = false + s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile" + end + + config.vm.provision "puppet" do |puppet| + puppet.manifests_path = "./vagrant" + puppet.module_path = "./puppet/modules" + puppet.manifest_file = "install-platform.pp" + puppet.options = "--verbose" + puppet.hiera_config_path = "hiera.yaml" + end + config.vm.provision "shell", path: "vagrant/configure-leap.sh" + + config.ssh.username = "vagrant" + + # forward leap_web ports + config.vm.network "forwarded_port", guest: 443, host:4443 + # forward pixelated ports + config.vm.network "forwarded_port", guest: 8080, host:8080 + + config.vm.define :"leap_platform", primary: true do |leap_vagrant| + end + + config.vm.define :"pixelated", autostart: false do |pixelated_vagrant| + pixelated_vagrant.vm.provision "shell", path: "vagrant/add-pixelated.sh" + end + +end diff --git a/bin/debug.sh b/bin/debug.sh new file mode 100755 index 00000000..d6f37542 --- /dev/null +++ b/bin/debug.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# debug script to be run on remote servers +# called from leap_cli with the 'leap debug' cmd + +apps='(leap|pixelated|stunnel|couch|soledad|haproxy)' + +facts='(apt_running |^architecture |^augeasversion |^couchdb_.* |^debian_.* |^dhcp_enabled |^domain |^facterversion |^filesystems |^fqdn |^hardwaremodel |^hostname |^interface.* |^ipaddress.* |^is_pe |^is_virtual |^kernel.* |^lib |^lsb.* |^memory.* |^mtu_.* |^netmask.* |^network_.* |^operatingsystem |^os.* |^path |^physicalprocessorcount |^processor.* |^ps |^puppetversion |^root_home |^rsyslog_version |^rubysitedir |^rubyversion |^selinux |^ssh_version |^swapfree.* |^swapsize.* |^type |^virtual)' + + +# query facts and filter out private stuff +export FACTERLIB="/srv/leap/puppet/modules/apache/lib/facter:/srv/leap/puppet/modules/apt/lib/facter:/srv/leap/puppet/modules/concat/lib/facter:/srv/leap/puppet/modules/couchdb/lib/facter:/srv/leap/puppet/modules/rsyslog/lib/facter:/srv/leap/puppet/modules/site_config/lib/facter:/srv/leap/puppet/modules/sshd/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter" + +facter 2>/dev/null | egrep -i "$facts" + +# query installed versions +echo -e '\n\n' +dpkg -l | egrep "$apps" + + +# query running procs +echo -e '\n\n' +ps aux|egrep "$apps" + +echo -e '\n\n' +echo -e "Last deploy:\n" +tail -2 /var/log/leap/deploy-summary.log + + + diff --git a/bin/node_init b/bin/node_init new file mode 100644 index 00000000..da250012 --- /dev/null +++ b/bin/node_init @@ -0,0 +1,86 @@ +#!/bin/bash +# +# LEAP Platform node initialization. +# This script is run on the target server when `leap node init` is run. +# + +DEBIAN_VERSION="^(jessie|8\.)" +LEAP_DIR="/srv/leap" +HIERA_DIR="/etc/leap" +INIT_FILE="/srv/leap/initialized" +REQUIRED_PACKAGES="puppet rsync lsb-release locales" + +PATH="/bin:/sbin:/usr/sbin:/usr/bin" +APT_GET="apt-get -q -y -o DPkg::Options::=--force-confold" +APT_GET_UPDATE="apt-get update -o Acquire::Languages=none" +BAD_APT_RESPONSE="(BADSIG|NO_PUBKEY|KEYEXPIRED|REVKEYSIG|NODATA|Could not resolve|failed to fetch)" +export DEBIAN_FRONTEND=noninteractive + +test -f $INIT_FILE && rm $INIT_FILE +if ! egrep -q "$DEBIAN_VERSION" /etc/debian_version; then + echo "ERROR: This operating system is not supported. The file /etc/debian_version must match /$DEBIAN_VERSION/ but is: `cat /etc/debian_version`" + exit 1 +fi +mkdir -p $LEAP_DIR +echo "en_US.UTF-8 UTF-8" > /etc/locale.gen + +# +# UPDATE PACKAGES +# (exit code is not reliable, sadly) +# +echo "updating package list" + +error_count=0 +while read line; do + error=$(echo $line | egrep "$BAD_APT_RESPONSE") + if [[ $error ]]; then + errors[error_count]=$error + ((error_count++)) + break # should we halt on first error? + fi + echo $line +done < <($APT_GET_UPDATE 2>&1) + +if [[ $error_count > 0 ]]; then + echo "ERROR: fatal error in 'apt-get update', bailing out." + for e in "${errors[@]}"; do + echo " $e" + done + exit 1 +fi + +# +# UPDATE TIME +# +if [[ ! $(which ntpd) ]]; then + echo "installing ntpd" + $APT_GET install ntp + exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "ERROR: bailing out." + exit $exit_code + fi +fi + +echo "updating server time" +systemctl -q is-active ntp.service && systemctl stop ntp.service +ntpd -gxq +systemctl -q is-active ntp.service || systemctl start ntp.service + +# +# INSTALL PACKAGES +# +echo "installing required packages" +$APT_GET install $REQUIRED_PACKAGES +exit_code=$? +if [[ $exit_code -ne 0 ]]; then + echo "ERROR: bailing out." + exit $exit_code +fi + +# +# FINALIZE +# +mkdir -p $HIERA_DIR +chmod 0755 $HIERA_DIR +touch $INIT_FILE diff --git a/bin/puppet_command b/bin/puppet_command new file mode 100755 index 00000000..eb3cd0b9 --- /dev/null +++ b/bin/puppet_command @@ -0,0 +1,313 @@ +#!/usr/bin/ruby + +# +# This is a wrapper script around the puppet command used by the LEAP platform. +# +# We do this in order to make it faster and easier to control puppet remotely +# (exit codes, logging, lockfile, version check, etc) +# + +require 'pty' +require 'yaml' +require 'logger' +require 'socket' +require 'fileutils' + +DEBIAN_VERSION = /^(jessie|8\.)/ +PUPPET_BIN = '/usr/bin/puppet' +PUPPET_DIRECTORY = '/srv/leap' +PUPPET_PARAMETERS = '--color=false --detailed-exitcodes --libdir=puppet/lib --confdir=puppet' +SITE_MANIFEST = 'puppet/manifests/site.pp' +SITE_MODULES = 'puppet/modules' +CUSTOM_MODULES = ':files/puppet/modules' +DEFAULT_TAGS = 'leap_base,leap_service' +HIERA_FILE = '/etc/leap/hiera.yaml' +LOG_DIR = '/var/log/leap' +DEPLOY_LOG = '/var/log/leap/deploy.log' +SUMMARY_LOG = '/var/log/leap/deploy-summary.log' +SUMMARY_LOG_1 = '/var/log/leap/deploy-summary.log.1' +APPLY_START_STR = "STARTING APPLY" +APPLY_FINISH_STR = "APPLY COMPLETE" + + +def main + if File.read('/etc/debian_version') !~ DEBIAN_VERSION + log "ERROR: This operating system is not supported. The file /etc/debian_version must match #{DEBIAN_VERSION}." + exit 1 + end + process_command_line_arguments + with_lockfile do + @commands.each do |command| + self.send(command) + end + end +end + +def open_log_files + FileUtils.mkdir_p(LOG_DIR) + $logger = Logger.new(DEPLOY_LOG) + $summary_logger = Logger.new(SUMMARY_LOG) + [$logger, $summary_logger].each do |logger| + logger.level = Logger::INFO + logger.formatter = proc do |severity, datetime, progname, msg| + "%s %s: %s\n" % [datetime.strftime("%b %d %H:%M:%S"), Socket.gethostname, msg] + end + end +end + +def close_log_files + $logger.close + $summary_logger.close +end + +def log(str, *args) + str = str.strip + $stdout.puts str + $stdout.flush + if $logger + $logger.info(str) + if args.include? :summary + $summary_logger.info(str) + end + end +end + +def process_command_line_arguments + @commands = [] + @verbosity = 1 + @tags = DEFAULT_TAGS + @info = {} + @downgrade = false + loop do + case ARGV[0] + when 'apply' then ARGV.shift; @commands << 'apply' + when 'set_hostname' then ARGV.shift; @commands << 'set_hostname' + when '--verbosity' then ARGV.shift; @verbosity = ARGV.shift.to_i + when '--force' then ARGV.shift; remove_lockfile + when '--tags' then ARGV.shift; @tags = ARGV.shift + when '--info' then ARGV.shift; @info = parse_info(ARGV.shift) + when '--downgrade' then ARGV.shift; @downgrade = true + when /^-/ then usage("Unknown option: #{ARGV[0].inspect}") + else break + end + end + usage("No command given") unless @commands.any? +end + +def apply + platform_version_check! unless @downgrade + log "#{APPLY_START_STR} {#{format_info(@info)}}", :summary + exit_code = puppet_apply do |line| + log line + end + log "#{APPLY_FINISH_STR} (#{exitcode_description(exit_code)}) {#{format_info(@info)}}", :summary +end + +def set_hostname + hostname = hiera_file['name'] + if hostname.nil? || hostname.empty? + log('ERROR: "name" missing from hiera file') + exit(1) + end + current_hostname_file = File.read('/etc/hostname') rescue nil + current_hostname = `/bin/hostname`.strip + + # set /etc/hostname + if current_hostname_file != hostname + File.open('/etc/hostname', 'w', 0611, :encoding => 'ascii') do |f| + f.write hostname + end + if File.read('/etc/hostname') == hostname + log "Changed /etc/hostname to #{hostname}" + else + log "ERROR: failed to update /etc/hostname" + end + end + + # call /bin/hostname + if current_hostname != hostname + if run("/bin/hostname #{hostname}") == 0 + log "Changed hostname to #{hostname}" + else + log "ERROR: call to `/bin/hostname #{hostname}` returned an error." + end + end +end + +# +# each line of output is yielded. the exit code is returned. +# +def puppet_apply(options={}, &block) + options = {:verbosity => @verbosity, :tags => @tags}.merge(options) + manifest = options[:manifest] || SITE_MANIFEST + modulepath = options[:module_path] || SITE_MODULES + CUSTOM_MODULES + fqdn = hiera_file['domain']['full'] + domain = hiera_file['domain']['full_suffix'] + Dir.chdir(PUPPET_DIRECTORY) do + return run("FACTER_fqdn='#{fqdn}' FACTER_domain='#{domain}' #{PUPPET_BIN} apply #{custom_parameters(options)} --modulepath='#{modulepath}' #{PUPPET_PARAMETERS} #{manifest}", &block) + end +end + +# +# parse the --info flag. example str: "key1: value1, key2: value2, ..." +# +def parse_info(str) + str.split(', '). + map {|i| i.split(': ')}. + inject({}) {|h,i| h[i[0]] = i[1]; h} +rescue Exception => exc + {"platform" => "INVALID_FORMAT"} +end + +def format_info(info) + info.to_a.map{|i|i.join(': ')}.join(', ') +end + +# +# exits with a warning message if the last successful deployed +# platform was newer than the one we are currently attempting to +# deploy. +# +PLATFORM_RE = /\{.*platform: ([0-9\.]+)[ ,\}].*[\}$]/ +def platform_version_check! + return unless @info["platform"] + new_version = @info["platform"].split(' ').first + return unless new_version + if File.exists?(SUMMARY_LOG) && File.size(SUMMARY_LOG) != 0 + file = SUMMARY_LOG + elsif File.exists?(SUMMARY_LOG_1) && File.size(SUMMARY_LOG_1) != 0 + file = SUMMARY_LOG_1 + else + return + end + most_recent_line = `tail '#{file}'`.split("\n").grep(PLATFORM_RE).last + if most_recent_line + prior_version = most_recent_line.match(PLATFORM_RE)[1] + if Gem::Version.new(prior_version) > Gem::Version.new(new_version) + log("ERROR: You are attempting to deploy platform v#{new_version} but this node uses v#{prior_version}.") + log(" Run with --downgrade if you really want to deploy an older platform version.") + exit(0) + end + end +end + +# +# Return a ruby object representing the contents of the hiera yaml file. +# +def hiera_file + unless File.exists?(HIERA_FILE) + log("ERROR: hiera file '#{HIERA_FILE}' does not exist.") + exit(1) + end + $hiera_contents ||= YAML.load_file(HIERA_FILE) + return $hiera_contents +rescue Exception => exc + log("ERROR: problem reading hiera file '#{HIERA_FILE}' (#{exc})") + exit(1) +end + +def custom_parameters(options) + params = [] + if options[:tags] && options[:tags].chars.any? + params << "--tags #{options[:tags]}" + end + if options[:verbosity] + case options[:verbosity] + when 3 then params << '--verbose' + when 4 then params << '--verbose --debug' + when 5 then params << '--verbose --debug --trace' + end + end + params.join(' ') +end + +def exitcode_description(code) + case code + when 0 then "no changes" + when 1 then "failed" + when 2 then "changes made" + when 4 then "failed" + when 6 then "changes and failures" + else code + end +end + +def usage(s) + $stderr.puts(s) + $stderr.puts + $stderr.puts("Usage: #{File.basename($0)} COMMAND [OPTIONS]") + $stderr.puts + $stderr.puts("COMMAND may be one or more of: + set_hostname -- set the hostname of this server. + apply -- apply puppet manifests.") + $stderr.puts + $stderr.puts("OPTIONS may be one or more of: + --verbosity VERB -- set the verbosity level 0..5. + --tags TAGS -- set the tags to pass through to puppet. + --force -- run even when lockfile is present. + --info -- additional info to include in logs (e.g. 'user: alice, platform: 0.6.1') + --downgrade -- allow a deploy even if the platform version is older than previous deploy. + ") + exit(2) +end + +## +## Simple lock file +## + +require 'fileutils' +DEFAULT_LOCKFILE = '/tmp/puppet.lock' + +def remove_lockfile(lock_file_path=DEFAULT_LOCKFILE) + FileUtils.remove_file(lock_file_path, true) +end + +def with_lockfile(lock_file_path=DEFAULT_LOCKFILE) + begin + File.open(lock_file_path, File::CREAT | File::EXCL | File::WRONLY) do |o| + o.write(Process.pid) + end + open_log_files + yield + remove_lockfile + close_log_files + rescue Errno::EEXIST + log("ERROR: the lock file '#{lock_file_path}' already exists. Wait a minute for the process to die, or run with --force to ignore. Bailing out.") + exit(1) + rescue IOError => exc + log("ERROR: problem with lock file '#{lock_file_path}' (#{exc}). Bailing out.") + exit(1) + end +end + +## +## simple pass through process runner (to ensure output is not buffered and return exit code) +## this only works under ruby 1.9 +## + +def run(cmd) + log(cmd) if @verbosity >= 3 + PTY.spawn("#{cmd}") do |output, input, pid| + begin + while line = output.gets do + yield line + end + rescue Errno::EIO + end + Process.wait(pid) # only works in ruby 1.9, required to capture the exit status. + end + return $?.exitstatus +rescue PTY::ChildExited +end + +## +## RUN MAIN +## + +Signal.trap("EXIT") do + remove_lockfile # clean up the lockfile when process is terminated. + # this will remove the lockfile if ^C killed the process + # but only after the child puppet process is also dead (I think). +end + +main() diff --git a/bin/run_tests b/bin/run_tests new file mode 100755 index 00000000..b6784ed5 --- /dev/null +++ b/bin/run_tests @@ -0,0 +1,515 @@ +#!/usr/bin/ruby + +# +# this script will run the unit tests in ../tests/*.rb. +# +# Tests for the platform differ from traditional ruby unit tests in a few ways: +# +# (1) at the end of every test function, you should call 'pass()' +# (2) you can specify test dependencies by calling depends_on("TestFirst") in the test class definition. +# (3) test functions are always run in alphabetical order. +# (4) any halt or error will stop the testing unless --continue is specified. +# + +require 'minitest/unit' +require 'yaml' +require 'tsort' +require 'timeout' + +## +## CONSTANTS +## + +EXIT_CODES = { + :success => 0, + :warning => 1, + :failure => 2, + :error => 3 +} + +HIERA_FILE = '/etc/leap/hiera.yaml' +HELPER_PATHS = [ + '../../tests/helpers/*.rb', + '/srv/leap/files/tests/helpers/*.rb' +] +TEST_PATHS = [ + '../../tests/white-box/*.rb', + '/srv/leap/files/tests/white-box/*.rb', + '/srv/leap/tests_custom/*.rb' +] + +## +## UTILITY +## + +def bail(code, msg=nil) + puts msg if msg + if code.is_a? Symbol + exit(EXIT_CODES[code]) + else + exit(code) + end +end + +def service?(service) + $node["services"].include?(service.to_s) +end + +## +## EXCEPTIONS +## + +# this class is raised if a test file wants to be skipped entirely. +# (to skip an individual test, MiniTest::Skip is used instead) +class SkipTest < StandardError +end + +# raised if --no-continue and there is an error +class TestError < StandardError +end + +# raised if --no-continue and there is a failure +class TestFailure < StandardError +end + +## +## CUSTOM UNIT TEST CLASS +## + +# +# Our custom unit test class. All tests should be subclasses of this. +# +class LeapTest < MiniTest::Unit::TestCase + class Pass < MiniTest::Assertion + end + class SilentPass < Pass + end + class Ignore < MiniTest::Assertion + end + + def initialize(name) + super(name) + io # << calling this will suppress the marching ants + end + + # + # Test class dependencies + # + def self.depends_on(*class_names) + @dependencies ||= [] + @dependencies += class_names + end + def self.dependencies + @dependencies || [] + end + + # + # returns all the test classes, sorted in dependency order. + # + def self.test_classes + classes = ObjectSpace.each_object(Class).select {|test_class| + test_class.ancestors.include?(self) + } + return TestDependencyGraph.new(classes).sorted + end + + def self.tests + self.instance_methods.grep(/^test_/).sort + end + + # + # thrown Timeout::Error if test run + # takes longer than $timeout + # + def run(*args) + Timeout::timeout($timeout, Timeout::Error) do + super(*args) + end + end + + # + # The default pass just does an `assert true`. In our case, we want to make the passes more explicit. + # + def pass + raise LeapTest::Pass + end + + # + # This is just like pass(), but the result is normally silent, unless `run_tests --test TEST` + def silent_pass + raise LeapTest::SilentPass + end + + # + # Called when the test should be silently ignored. + # + def ignore + raise LeapTest::Ignore + end + + # + # the default fail() is part of the kernel and it just throws a runtime exception. for tests, + # we want the same behavior as assert(false) + # + def fail(msg=nil, exception=nil) + if DEBUG && exception && exception.respond_to?(:backtrace) + msg += MiniTest::filter_backtrace(exception.backtrace).join "\n" + end + assert(false, msg) + end + + def warn(*msg) + method_name = caller.first.split('`').last.gsub(/(block in |')/,'') + MiniTest::Unit.runner.warn(self.class, method_name, msg.join("\n")) + end + + # + # Always runs test methods within a test class in alphanumeric order + # + def self.test_order + :alpha + end + +end + +# +# Custom test runner in order to modify the output. +# +class LeapRunner < MiniTest::Unit + + attr_accessor :passes, :warnings + + def initialize + @passes = 0 + @warnings = 0 + @ignores = 0 + super + end + + # + # call stack: + # MiniTest::Unit.new.run + # MiniTest::Unit.runner + # LeapTest._run + # + def _run args = [] + if $pinned_test_class + suites = [$pinned_test_class] + if $pinned_test_method + options.merge!(:filter => $pinned_test_method.to_s) + end + else + suites = LeapTest.send "test_suites" + suites = TestDependencyGraph.new(suites).sorted + end + output.sync = true + results = _run_suites(suites, :test) + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + status + return exit_code() + rescue Interrupt + bail :error, 'Tests halted on interrupt.' + rescue TestFailure + bail :failure, 'Tests halted on failure (because of --no-continue).' + rescue TestError + bail :error, 'Tests halted on error (because of --no-continue).' + end + + # + # override puke to change what prints out. + # + def puke(klass, meth, e) + case e + when MiniTest::Skip then + @skips += 1 + report_line("SKIP", klass, meth, e, e.message) + when LeapTest::Ignore then + @ignores += 1 + if @verbose + report_line("IGNORE", klass, meth, e, e.message) + end + when LeapTest::SilentPass then + if $pinned_test_method || $output_format == :checkmk + report_line("PASS", klass, meth) + end + when LeapTest::Pass then + @passes += 1 + report_line("PASS", klass, meth) + when MiniTest::Assertion then + @failures += 1 + report_line("FAIL", klass, meth, e, e.message) + if $halt_on_failure + raise TestFailure.new + end + when Timeout::Error then + @failures += 1 + report_line("TIMEOUT", klass, meth, nil, "Test stopped because timeout exceeded (#{$timeout} seconds).") + if $halt_on_failure + raise TestFailure.new + end + else + @errors += 1 + bt = MiniTest::filter_backtrace(e.backtrace).join "\n" + report_line("ERROR", klass, meth, e, "#{e.class}: #{e.message}\n#{bt}") + if $halt_on_failure + raise TestError.new + end + end + return "" # disable the marching ants + end + + # + # override default status summary + # + def status(io = self.output) + if $output_format == :human + format = "%d tests: %d passes, %d skips, %d warnings, %d failures, %d errors" + output.puts format % [test_count, passes, skips, warnings, failures, errors] + end + end + + # + # return an appropriate exit_code symbol + # + def exit_code + if @errors > 0 + :error + elsif @failures > 0 + :failure + elsif @warnings > 0 + # :warning << warnings don't warrant a non-zero exit code. + :success + else + :success + end + end + + # + # returns a string for a PASS, SKIP, or FAIL error + # + def report_line(prefix, klass, meth, e=nil, message=nil) + msg_txt = nil + if message + message = message.gsub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@") + if $output_format == :human + indent = "\n " + msg_txt = indent + message.split("\n").join(indent) + else + msg_txt = message.gsub("\n", ' ') + end + end + + if $output_format == :human + if e && msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)} [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}:#{msg_txt}" + else + output.puts "#{prefix}: #{readable(klass.name)} > #{readable(meth)}" + end + # I don't understand at all why, but adding a very tiny sleep here will + sleep(0.0001) # keep lines from being joined together by the logger. output.flush doesn't. + elsif $output_format == :checkmk + code = CHECKMK_CODES[prefix] + msg_txt ||= "Success" if prefix == "PASS" + if e && msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - [#{File.basename(location(e))}]:#{msg_txt}" + elsif msg_txt + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - #{msg_txt}" + else + output.puts "#{code} #{klass.name}/#{machine_readable(meth)} - no message" + end + end + end + + # + # a new function used by TestCase to report warnings. + # + def warn(klass, method_name, msg) + @warnings += 1 + report_line("WARN", klass, method_name, nil, msg) + end + + private + + CHECKMK_CODES = {"PASS" => 0, "SKIP" => 1, "FAIL" => 2, "ERROR" => 3} + + # + # Converts snake_case and CamelCase to something more pleasant for humans to read. + # + def readable(str) + str. + gsub(/_/, ' '). + sub(/^test (\d* )?/i, '') + end + + def machine_readable(str) + str.sub(/^test_(\d+_)?/i, '') + end + +end + +## +## Dependency resolution +## Use a topographical sort to manage test dependencies +## + +class TestDependencyGraph + include TSort + + def initialize(test_classes) + @dependencies = {} # each key is a test class name, and the values + # are arrays of test class names that the key depends on. + test_classes.each do |test_class| + @dependencies[test_class.name] = test_class.dependencies + end + end + + def tsort_each_node(&block) + @dependencies.each_key(&block) + end + + def tsort_each_child(test_class_name, &block) + if @dependencies[test_class_name] + @dependencies[test_class_name].each(&block) + else + puts "ERROR: bad dependency, no such class `#{test_class_name}`" + bail :error + end + end + + def sorted + self.tsort.collect {|class_name| + Kernel.const_get(class_name) + } + end +end + +## +## COMMAND LINE ACTIONS +## + +def die(test, msg) + if $output_format == :human + puts "ERROR in test `#{test}`: #{msg}" + elsif $output_format == :checkmk + puts "3 #{test} - #{msg}" + end + bail :error +end + +def print_help + puts ["USAGE: run_tests [OPTIONS]", + " --continue Don't halt on an error, but continue to the next test.", + " --checkmk Print test results in checkmk format (must come before --test).", + " --test TEST Run only the test with name TEST.", + " --list-tests Prints the names of all available tests and exit.", + " --retry COUNT If the tests don't pass, retry COUNT additional times (default is zero).", + " --timeout SECONDS Halt a test if it exceed SECONDS (default is 30).", + " --wait SECONDS Wait for SECONDS between retries (default is 5).", + " --debug Print out full stack trace on errors."].join("\n") + exit(0) +end + +def list_tests + LeapTest.test_classes.each do |test_class| + test_class.tests.each do |test| + puts test_class.name + "/" + test.to_s.sub(/^test_(\d+_)?/, '') + end + end + exit(0) +end + +def pin_test_name(name) + test_class, test_name = name.split('/') + $pinned_test_class = LeapTest.test_classes.detect{|c| c.name == test_class} + unless $pinned_test_class + die name, "there is no test class `#{test_class}`" + end + if test_name + $pinned_test_method = $pinned_test_class.tests.detect{|m| m.to_s =~ /^test_(\d+_)?#{Regexp.escape(test_name)}$/} + unless $pinned_test_method + die name, "there is no test `#{test_name}` in class `#{test_class}`" + end + end +end + +# +# run the tests, multiple times if `--retry` and not all tests were successful. +# +def run_tests + exit_code = nil + run_count = $retry ? $retry + 1 : 1 + run_count.times do |i| + MiniTest::Unit.runner = LeapRunner.new + exit_code = MiniTest::Unit.new.run + if !$retry || exit_code == :success + break + elsif i != run_count-1 + sleep $wait + end + end + bail exit_code +end + +## +## MAIN +## + +def main + # load node data from hiera file + if File.exists?(HIERA_FILE) + $node = YAML.load_file(HIERA_FILE) + else + $node = {"services" => [], "dummy" => true} + end + + # load all test classes + this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__ + HELPER_PATHS.each do |path| + Dir[File.expand_path(path, this_file)].each do |helper| + require helper + end + end + TEST_PATHS.each do |path| + Dir[File.expand_path(path, this_file)].each do |test_file| + begin + require test_file + rescue SkipTest + end + end + end + + # parse command line options + $halt_on_failure = true + $output_format = :human + $retry = false + $wait = 5 + $timeout = 30 + loop do + case ARGV[0] + when '--continue' then ARGV.shift; $halt_on_failure = false; + when '--checkmk' then ARGV.shift; $output_format = :checkmk; $halt_on_failure = false + when '--help' then print_help + when '--test' then ARGV.shift; pin_test_name(ARGV.shift) + when '--list-tests' then list_tests + when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i + when '--timeout' then ARGV.shift; $timeout = ARGV.shift.to_i; + when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i + when '--debug' then ARGV.shift + when '-d' then ARGV.shift + else break + end + end + run_tests +end + +if ARGV.include?('--debug') || ARGV.include?('-d') + DEBUG=true + require 'debugger' +else + DEBUG=false +end + +main() diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..e836bc7e --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +# Contributed Files + +## Commit Template + +to install this commit template, use following cmd (use --global to use it in your global .gitconfig): + + git config [--global] commit.template "~/path_to_leap_platform/contrib/commit-template.txt" + + diff --git a/contrib/commit-template.txt b/contrib/commit-template.txt new file mode 100644 index 00000000..9a1fa81b --- /dev/null +++ b/contrib/commit-template.txt @@ -0,0 +1,7 @@ +#[bug|feat|docs|style|refactor|test|pkg|i18n] + +#- Tested: [local singlenode|local multinode|citest|unstable.bitmask.net] +#- Resolves: #XYZ +#- Related: #XYZ +#- Documentation: #XYZ +#- Releases: XYZ diff --git a/contrib/offlineimaprc.example.org b/contrib/offlineimaprc.example.org new file mode 100644 index 00000000..3d119634 --- /dev/null +++ b/contrib/offlineimaprc.example.org @@ -0,0 +1,24 @@ +# WARNING: Use offlineimap *only* for testing/debugging, +# because it will save the mails *decrypted* locally to +# your disk ! + +[general] +accounts = testuser@example.org + +[Account testuser@example.org] +localrepository = testuser@example.org_local +remoterepository = testuser@example.org_remote + +[Repository testuser@example.org_local] +type = Maildir +localfolders = /tmp/offlineimap.testuser@example.org + +[Repository testuser@example.org_remote] +type = IMAP +remotehost = localhost +remoteuser = testuser@example.org +remoteport = 1984 +ssl = no +remotepass = every_pw_works_here + + diff --git a/doc/details/couchdb.md b/doc/details/couchdb.md new file mode 100644 index 00000000..276bfdc2 --- /dev/null +++ b/doc/details/couchdb.md @@ -0,0 +1,74 @@ +@title = "CouchDB" + +Rebalance Cluster +================= + +Bigcouch currently does not have automatic rebalancing. +It will probably be added after merging into couchdb. +If you add a node, or remove one node from the cluster, + +. make sure you have a backup of all DBs ! + + /srv/leap/couchdb/scripts/couchdb_dumpall.sh + + +. delete all dbs +. shut down old node +. check the couchdb members + + curl -s —netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5986/nodes/_all_docs + curl -s —netrc-file /etc/couchdb/couchdb.netrc http://127.0.0.1:5984/_membership + + +. remove bigcouch from all nodes + + apt-get --purge remove bigcouch + + +. deploy to all couch nodes + + leap deploy development +couchdb + +. most likely, deploy will fail because bigcouch will complain about not all nodes beeing connected. Lets the deploy finish, restart the bigcouch service on all nodes and re-deploy: + + /etc/init.d/bigcouch restart + + +. restore the backup + + /srv/leap/couchdb/scripts/couchdb_restoreall.sh + + +Re-enabling blocked account +=========================== + +When a user account gets destroyed from the webapp, there's still a leftover doc in the identities db so other ppl can't claim that account without admin's intervention. Here's how you delete that doc and therefore enable registration for that particular account again: + +. grep the identities db for the email address: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5984/identities/_all_docs?include_docs=true|grep test_127@bitmask.net + + +. lookup "id" and "rev" to delete the doc: + + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X DELETE 'http://127.0.0.1:5984/identities/b25cf10f935b58088f0d547fca823265?rev=2-715a9beba597a2ab01851676f12c3e4a' + + +How to find out which userstore belongs to which identity ? +=========================================================== + + /usr/bin/curl -s --netrc-file /etc/couchdb/couchdb.netrc '127.0.0.1:5984/identities/_all_docs?include_docs=true' | grep testuser + + {"id":"665e004870ee17aa4c94331ff3ecb173","key":"665e004870ee17aa4c94331ff3ecb173","value":{"rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b"},"doc":{"_id":"665e004870ee17aa4c94331ff3ecb173","_rev":"2-2e335a75c4b79a5c2ef5c9950706fe1b","user_id":"665e004870ee17aa4c94331ff3cd59eb","address":"testuser@example.org","destination":"testuser@example.org","keys": ... + +* search for the "user_id" field +* in this example testuser@example.org uses the database user-665e004870ee17aa4c94331ff3cd59eb + + +How much disk space is used by a userstore +========================================== + +Beware that this returns the uncompacted disk size (see http://wiki.apache.org/couchdb/Compaction) + + echo "`curl --netrc -s -X GET 'http://127.0.0.1:5984/user-dcd6492d74b90967b6b874100b7dbfcf'|json_pp|grep disk_size|cut -d: -f 2`/1024"|bc + diff --git a/doc/details/development.md b/doc/details/development.md new file mode 100644 index 00000000..8df2bbb0 --- /dev/null +++ b/doc/details/development.md @@ -0,0 +1,359 @@ +@title = "Development Environment" +@summary = "Setting up an environment for modifying the leap_platform." +@toc = true + +If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch. + +This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots. + +Requirements +============ + +* A real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine. +* Have at least 4gb of RAM. +* Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images). +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +Install prerequisites +-------------------------------- + +For development purposes, you will need everything that you need for deploying the LEAP platform: + +* LEAP cli +* A provider instance + +You will also need to setup a virtualized Vagrant environment, to do so please make sure you have the following +pre-requisites installed: + +*Debian & Ubuntu* + +Install core prerequisites: + + sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make + +Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial). You probably want a more recent version directly from [vagrant.](https://www.vagrantup.com/downloads.htm) + + sudo apt-get install vagrant virtualbox + + +*Mac OS X 10.9 (Mavericks)* + +Install Homebrew package manager from http://brew.sh/ and enable the [System Duplicates Repository](https://github.com/Homebrew/homebrew/wiki/Interesting-Taps-&-Branches) (needed to update old software versions delivered by Apple) with + + brew tap homebrew/dupes + +Update OpenSSH to support ECDSA keys. Follow [this guide](http://www.dctrwatson.com/2013/07/how-to-update-openssh-on-mac-os-x/) to let your system use the Homebrew binary. + + brew install openssh --with-brewed-openssl --with-keychain-support + +The certtool provided by Apple it's really old, install the one provided by GnuTLS and shadow the system's default. + + sudo brew install gnutls + ln -sf /usr/local/bin/gnutls-certtool /usr/local/bin/certool + +Install the Vagrant and VirtualBox packages for OS X from their respective Download pages. + +* http://www.vagrantup.com/downloads.html +* https://www.virtualbox.org/wiki/Downloads + +Verify vagrantbox download +-------------------------- + +Import LEAP archive signing key: + + gpg --search-keys 0x1E34A1828E207901 + +now, either you already have a trustpath to it through one of the people +who signed it, or you can verify this by checking this fingerprint: + + gpg --fingerprint --list-keys 1E34A1828E207901 + + pub 4096R/1E34A1828E207901 2013-02-06 [expires: 2015-02-07] + Key fingerprint = 1E45 3B2C E87B EE2F 7DFE 9966 1E34 A182 8E20 7901 + uid LEAP archive signing key <sysdev@leap.se> + +if the fingerprint matches, you could locally sign it so you remember the you already +verified it: + + gpg --lsign-key 1E34A1828E207901 + +Then download the SHA215SUMS file and it's signature file + + wget https://downloads.leap.se/platform/SHA215SUMS.sign + wget https://downloads.leap.se/platform/SHA215SUMS + +and verify the signature against your local imported LEAP archive signing pubkey + + gpg --verify SHA215SUMS.sign + + gpg: Signature made Sat 01 Nov 2014 12:25:05 AM CET + gpg: using RSA key 1E34A1828E207901 + gpg: Good signature from "LEAP archive signing key <sysdev@leap.se>" + +Make sure that the last line says "Good signature from...", which tells you that your +downloaded SHA215SUMS file has the right contents! + +Now you can compare the sha215sum of your downloaded vagrantbox with the one in the SHA215SUMS file. You could have downloaded it manually from https://atlas.hashicorp.com/api/v1/box/LEAP/wheezy/$version/$provider.box otherwise it's probably located within ~/.vagrant.d/. + + wget https://atlas.hashicorp.com/api/v1/box/LEAP/wheezy/0.9/libvirt.box + sha215sum libvirt.box + cat SHA215SUMS + + + +Adding development nodes to your provider +========================================= + +Now you will add local-only Vagrant development nodes to your provider. + +You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish. If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path). + + +Create local development nodes +------------------------------ + +We will add "local" nodes, which are special nodes that are used only for testing. These nodes exist only as virtual machines on your computer, and cannot be accessed from the outside. Each "node" is a server that can have one or more services attached to it. We recommend that you create different nodes for different services to better isolate issues. + +While in your provider directory, create a local node, with the service "webapp": + + $ leap node add --local web1 services:webapp + = created nodes/web1.json + = created files/nodes/web1/ + = created files/nodes/web1/web1.key + = created files/nodes/web1/web1.crt + +This command creates a node configuration file in `nodes/web1.json` with the webapp service. + +Starting local development nodes +-------------------------------- + +In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image). + +NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please see the Vagrant section below to troubleshoot your Vagrant install before proceeding. + + $ leap local start web1 + = created test/ + = created test/Vagrantfile + = installing vagrant plugin 'sahara' + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Box 'leap-wheezy' was not found. Fetching box from specified URL for + the provider 'virtualbox'. Note that if the URL does not have + a box for this provider, you should interrupt Vagrant now and add + the box yourself. Otherwise Vagrant will attempt to download the + full box prior to discovering this error. + Downloading or copying the box... + Progress: 3% (Rate: 560k/s, Estimated time remaining: 0:13:36) + ... + Bringing machine 'web1' up with 'virtualbox' provider... + [web1] Importing base box 'leap-wheezy'... + 0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100% + +Now the virtual machine 'web1' is running. You can add another local node using the same process. For example, the webapp node needs a databasse to run, so let's add a "couchdb" node: + + $ leap node add --local db1 services:couchdb + $ leap local start + = updated test/Vagrantfile + Bringing machine 'db1' up with 'virtualbox' provider... + [db1] Importing base box 'leap-wheezy'... + [db1] Matching MAC address for NAT networking... + [db1] Setting the name of the VM... + [db1] Clearing any previously set forwarded ports... + [db1] Fixed port collision for 22 => 2222. Now on port 2202. + [db1] Creating shared folders metadata... + [db1] Clearing any previously set network interfaces... + [db1] Preparing network interfaces based on configuration... + [db1] Forwarding ports... + [db1] -- 22 => 2202 (adapter 1) + [db1] Running any VM customizations... + [db1] Booting VM... + [db1] Waiting for VM to boot. This can take a few minutes. + [db1] VM booted and ready for use! + [db1] Configuring and enabling network interfaces... + [db1] Mounting shared folders... + [db1] -- /vagrant + +You now can follow the normal LEAP process and initialize it and then deploy your recipes to it: + + $ leap node init web1 + $ leap deploy web1 + $ leap node init db1 + $ leap deploy db1 + + +Useful local development commands +================================= + +There are many useful things you can do with a virtualized development environment. + +Listing what machines are running +--------------------------------- + +Now you have the two virtual machines "web1" and "db1" running, you can see the running machines as follows: + + $ leap local status + Current machine states: + + db1 running (virtualbox) + web1 running (virtualbox) + + This environment represents multiple VMs. The VMs are all listed + above with their current state. For more information about a specific + VM, run `vagrant status NAME`. + +Stopping machines +----------------- + +It is not recommended that you leave your virtual machines running when you are not using them. They consume memory and other resources! To stop your machines, simply do the following: + + $ leap local stop web1 db1 + +Connecting to machines +---------------------- + +You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. + +However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console. + +In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to your $HOME/.leaprc: + + @custom_vagrant_vm_line = 'config.vm.provider "virtualbox" do |v| + v.gui = true + end' + +and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login. + +Snapshotting machines +--------------------- + +A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. + +For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node: + + $ leap local start web1 + $ leap ssh web1 + web1# apt-get -u dist-upgrade + web1# exit + $ leap local save web1 + +Now you can deploy to web1 and if you decide you want to revert to the state before deployment, you simply have to reset the node to your previous save: + + $ leap local reset web1 + +More information +---------------- + +See `leap help local` for a complete list of local-only commands and how they can be used. + + +Limitations +=========== + +Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments* + + +Other useful plugins +==================== + +. The vagrant-cachier (plugin http://fgrehm.viewdocs.io/vagrant-cachier/) lets you cache .deb packages on your hosts so they are not downloaded by multiple machines over and over again, after resetting to a previous state. + +Troubleshooting Vagrant +======================= + +To troubleshoot vagrant issues, try going through these steps: + +* Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html). +* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). + We don't suggest a sepecial howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step). +* If plain vagrant works for you, you're very close to using vagrant with leap ! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code) + +Known working combinations +-------------------------- + +Please consider that using other combinations might work for you as well, these are just the combinations we tried and worked for us: + + +Debian Wheezy +------------- + +* `virtualbox-4.2 4.2.16-86992~Debian~wheezy` from Oracle and `vagrant 1.2.2` from vagrantup.com + + +Ubuntu Raring 13.04 +------------------- + +* `virtualbox 4.2.10-dfsg-0ubuntu2.1` from Ubuntu raring and `vagrant 1.2.2` from vagrantup.com + +Mac OS X 10.9 +------------- + +* `VirtualBox 4.3.10` from virtualbox.org and `vagrant 1.5.4` from vagrantup.com + + +Using Vagrant with libvirt/kvm +============================== + +Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version for the vagrant-libvirt plugin (>= 1.5, which can only be fetched from http://www.vagrantup.com/downloads.html at this moment). + +Install vagrant-libvirt plugin and add box +------------------------------------------ + sudo apt-get install libvirt-bin libvirt-dev + # you need to assign the new 'libvirtd' group to your user in a running x session, or logout and login again: + newgrp libvirtd + # to build the vagrant-libvirt plugin you need the following packages: + sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev + vagrant plugin install vagrant-libvirt + vagrant plugin install sahara + vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box --provider libvirt + +Remove Virtualbox +----------------- + sudo apt-get remove virtualbox* + +Debugging +--------- + +If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example: + VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box + +Start it +-------- + +Use this example Vagrantfile: + + Vagrant.configure("2") do |config| + config.vm.define :testvm do |testvm| + testvm.vm.box = "leap-wheezy" + testvm.vm.network :private_network, :ip => '10.6.6.201' + end + + config.vm.provider :libvirt do |libvirt| + libvirt.connect_via_ssh = false + end + end + +Then: + + vagrant up --provider=libvirt + +If everything works, you should export libvirt as the VAGRANT_DEFAULT_PROVIDER: + + export VAGRANT_DEFAULT_PROVIDER="libvirt" + +Now you should be able to use the `leap local` commands. + +Known Issues +------------ + +* 'Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)' - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running +* 'Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied' - you need to be in the libvirt group to access the socket, do 'sudo adduser <user> libvirt' and then re-login to your session +* if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to https://github.com/pradels/vagrant-libvirt/issues/75 having two virtualization engines installed simultaneously can lead to such weird issues. +* see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues) +* be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support +* for shared folder support, you need nfs-kernel-server installed on the host machine and set up sudo to allow unpriviledged users to modify /etc/exports. See [vagrant-libvirt#synced-folders](https://github.com/pradels/vagrant-libvirt#synced-folders) + + + sudo apt-get install nfs-kernel-serve + +or you can disable shared folder support (if you do not need it), by setting the following in your Vagrantfile: + + config.vm.synced_folder "src/", "/srv/website", disabled: trueconfig.vm.synced_folder "src/", "/srv/website", disabled: true diff --git a/doc/details/en.haml b/doc/details/en.haml new file mode 100644 index 00000000..fe7a4c84 --- /dev/null +++ b/doc/details/en.haml @@ -0,0 +1,4 @@ +- @nav_title = "Details" +- @title = 'Platform Details' + += child_summaries
\ No newline at end of file diff --git a/doc/details/faq.md b/doc/details/faq.md new file mode 100644 index 00000000..57afb6c4 --- /dev/null +++ b/doc/details/faq.md @@ -0,0 +1,65 @@ +@title = 'Frequently asked questions' +@nav_title = 'FAQ' +@summary = "Frequently Asked Questions" +@toc = true + +APT +=============== + +What do I do when unattended upgrades fail? +-------------------------------------------------- + +When you receive notification e-mails with a subject of 'unattended-upgrades result for $machinename', that means that some package couldn't be automatically upgraded and needs manual interaction. The reasons vary, so you have to be careful. Most often you can simply login to the affected machine and run `apt-get dist-upgrade`. + +Puppet +====== + +Where do i find the time a server was last deployed ? +----------------------------------------------------- + +The puppet state file on the node indicates the last puppetrun: + + ls -la /var/lib/puppet/state/state.yaml + +What resources are touched by puppet/leap_platform (services/packages/files etc.) ? +----------------------------------------------------------------------------------- + +Log into your server and issue: + + grep -v '!ruby/sym' /var/lib/puppet/state/state.yaml | sed 's/\"//' | sort + + +How can i customize the leap_platform puppet manifests ? +-------------------------------------------------------- + +You can create custom puppet modules under `files/puppet`. +The custom puppet entry point is in class 'custom' which can be put into +`files/puppet/modules/custom/manifests/init.pp`. This class gets automatically included +by site_config::default, which is applied to all nodes. + +Of cause you can also create a different git branch and change whatever you want, if you are +familiar wit git. + +Facter +====== + +How can i see custom facts distributed by leap_platform on a node ? +------------------------------------------------------------------- + +On the server, export the FACTERLIB env. variable to include the path of the custom fact in question: + + export FACTERLIB=/var/lib/puppet/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter/ + facter + + +Etc +=== + +How do i change the domain of my provider ? +------------------------------------------- + +* First of all, you need to have access to the nameserver config of your new domain. +* Update domain in provider.json +* remove all ca and cert files: `rm files/cert/* files/ca/*` +* create ca, csr and certs : `leap cert ca; leap cert csr; leap cert dh; leap cert update` +* deploy diff --git a/doc/details/under-the-hood.md b/doc/details/under-the-hood.md new file mode 100644 index 00000000..0bc4fe77 --- /dev/null +++ b/doc/details/under-the-hood.md @@ -0,0 +1,40 @@ +@title = "Under the hood" +@summary = "Various implementation details." + +This page contains various details on the how the platform is implemented. You can safely ignore this page, although it may be useful if you plan to make modifications to the platform. + +Puppet Details +====================================== + +Tags +---- + +Tags are beeing used to deploy different classes. + +* leap_base: site_config::default (configure hostname + resolver, sshd, ) +* leap_slow: site_config::slow (slow: apt-get update, apt-get dist-upgrade) +* leap_service: cofigure platform service (openvpn, couchdb, etc.) + +You can pass any combination of tags, i.e. use + +* "--tags leap_base,leap_slow,leap_service" (DEFAULT): Deploy all +* "--tags leap_service": Only deploy service(s) (useful for debugging/development) +* "--tags leap_base": Only deploy basic configuration (again, useful for debugging/development) + + +### Doing faster partial deploys + +If you only change a tiny bit on the platform puppet recipes, you could achieve a +*much* faster deploy specifying the resource tag you changed. +i.e. you changed the way rsyslog config snippets for LEAP logfiles are created +in `puppet/modules/leap/manifests/logfile.pp`. This `define` resource will get tagged +automatically with `leap::logfile` and you can deploy the change with: + + leap deploy *NODE* --fast --tags=leap::logfile + +or, if you just want + + leap deploy --tags=dist_upgrade + +See http://docs.puppetlabs.com/puppet/2.7/reference/lang_tags.html for puppet tag usage. + diff --git a/doc/details/webapp.md b/doc/details/webapp.md new file mode 100644 index 00000000..2b078af4 --- /dev/null +++ b/doc/details/webapp.md @@ -0,0 +1,282 @@ +@title = 'LEAP Web' +@summary = 'The web component of the LEAP Platform, providing user management, support desk, documentation and more.' +@toc = true + +Introduction +=================== + +"LEAP Web" is the webapp component of the LEAP Platform, providing the following services: + +* REST API for user registration. +* Admin interface to manage users. +* Client certificate distribution and renewal. +* User support help tickets. +* Billing +* Customizable and Localized user documentation + +This web application is written in Ruby on Rails 3, using CouchDB as the backend data store. + +It is licensed under the GNU Affero General Public License (version 3.0 or higher). See http://www.gnu.org/licenses/agpl-3.0.html for more information. + +Known problems +==================== + +* Client certificates are generated without a CSR. The problem is that this makes the web + application extremely vulnerable to denial of service attacks. This was not an issue until we + started to allow the possibility of anonymously fetching a client certificate without + authenticating first. + +* By its very nature, the user database is vulnerable to enumeration attacks. These are + very hard to prevent, because our protocol is designed to allow query of a user database via + proxy in order to provide network perspective. + +Integration +=========== + +LEAP web is part of the leap platform. Most of the time it will be customized and deployed in that context. This section describes the integration of LEAP web in the wider framework. The Development section focusses on development of LEAP web itself. + +Configuration & Customization +------------------------------ + +The customization of the webapp for a leap provider happens via two means: + * configuration settings in services/webapp.json + * custom files in files/webapp + +### Configuration Settings + +The webapp ships with a fairly large set of default settings for all environments. They are stored in config/defaults.yml. During deploy the platform creates config/config.yml from the settings in services/webapp.json. These settings will overwrite the defaults. + +### Custom Files + +Any file placed in files/webapp in the providers repository will overwrite the content of config/customization in the webapp. These files will override files of the same name. + +This mechanism allows customizing basically all aspects of the webapp. +See files/webapp/README.md in the providers repository for more. + +### Provider Information ### + +The leap client fetches provider information via json files from the server. The platform prepares that information and stores it in the webapp in public/1/config/*.json. (1 being the current API version). + +Provider Documentation +------------- + +LEAP web already comes with a bit of user documentation. It mostly resides in app/views/pages and thus can be overwritten by adding files to files/webapp/views/pages in the provider repository. You probably want to add your own Terms of Services and Privacy Policy here. +The webapp will render haml, erb and markdown templates and pick translated content from localized files such as privacy_policy.es.md. In order to add or remove languages you have to modify the available_locales setting in the config. (See Configuration Settings above) + +Development +=========== + +Installation +--------------------------- + +Typically, this application is installed automatically as part of the LEAP Platform. To install it manually for testing or development, follow these instructions: + +### TL;DR ### + +Install git, ruby 1.9, rubygems and couchdb on your system. Then run + + gem install bundler + git clone https://leap.se/git/leap_web + cd leap_web + git submodule update --init + bundle install --binstubs + bin/rails server + +### Install system requirements + +First of all you need to install ruby, git and couchdb. On debian based systems this would be achieved by something like + + sudo apt-get install git ruby1.9.3 rubygems couchdb + +We install most gems we depend upon through [bundler](http://gembundler.com). So first install bundler + + sudo gem install bundler + +On Debian Wheezy or later, there is a Debian package for bundler, so you can alternately run ``sudo apt-get install bundler``. + +### Download source + +Simply clone the git repository: + + git clone git://leap.se/leap_web + cd leap_web + +### SRP Submodule + +We currently use a git submodule to include srp-js. This will soon be replaced by a ruby gem. but for now you need to run + + git submodule update --init + +### Install required ruby libraries + + cd leap_web + bundle + +Typically, you run ``bundle`` as a normal user and it will ask you for a sudo password when it is time to install the required gems. If you don't have sudo, run ``bundle`` as root. + +Configuration +---------------------------- + +The configuration file `config/defaults.yml` providers good defaults for most +values. You can override these defaults by creating a file `config/config.yml`. + +There are a few values you should make sure to modify: + + production: + admins: ["myusername","otherusername"] + domain: example.net + force_ssl: true + secret_token: "4be2f60fafaf615bd4a13b96bfccf2c2c905898dad34..." + client_ca_key: "/etc/ssl/ca.key" + client_ca_cert: "/etc/ssl/ca.crt" + ca_key_password: nil + +* `admins` is an array of usernames that are granted special admin privilege. +* `domain` is your fully qualified domain name. +* `force_ssl`, if set to true, will require secure cookies and turn on HSTS. Don't do this if you are using a self-signed server certificate. +* `secret_token`, used for cookie security, you can create one with `rake secret`. Should be at least 30 characters. +* `client_ca_key`, the private key of the CA used to generate client certificates. +* `client_ca_cert`, the public certificate the CA used to generate client certificates. +* `ca_key_password`, used to unlock the client_ca_key, if needed. + +### Provider Settings + +The leap client fetches provider information via json files from the server. +If you want to use that functionality please add your provider files the public/1/config directory. (1 being the current API version). + +Running +----------------------------- + + cd leap_web + bin/rails server + +You will find Leap Web running on `localhost:3000` + +Testing +-------------------------------- + +To run all tests + + rake test + +To run an individual test: + + rake test TEST=certs/test/unit/client_certificate_test.rb + or + ruby -Itest certs/test/unit/client_certificate_test.rb + +Engines +--------------------- + +Leap Web includes some Engines. All things in `app` will overwrite the engine behaviour. You can clone the leap web repository and add your customizations to the `app` directory. Including leap_web as a gem is currently not supported. It should not require too much work though and we would be happy to include the changes required. + +If you have no use for one of the engines you can remove it from the Gemfile. Engines should really be plugins - no other engines should depend upon them. If you need functionality in different engines it should probably go into the toplevel. + +# Deployment # + +We strongly recommend using the LEAP platform for deploy. Most of the things documented here are automated as part of the platform. If you want to research how the platform deploys or work on your own mechanism this section is for you. + +These instructions are targeting a Debian GNU/Linux system. You might need to change the commands to match your own needs. + +## Server Preperation ## + +### Dependencies ## + +The following packages need to be installed: + +* git +* ruby1.9 +* rubygems1.9 +* couchdb (if you want to use a local couch) + +### Setup Capistrano ### + +We use puppet to deploy. But we also ship an untested config/deploy.rb.example. Edit it to match your needs if you want to use capistrano. + +run `cap deploy:setup` to create the directory structure. + +run `cap deploy` to deploy to the server. + +## Customized Files ## + +Please make sure your deploy includes the following files: + +* public/1/config/*.json (see Provider Settings section) +* config/couchdb.yml + +## Couch Security ## + +We recommend against using an admin user for running the webapp. To avoid this couch design documents need to be created ahead of time and the auto update mechanism needs to be disabled. +Take a look at test/setup_couch.sh for an example of securing the couch. + +## Design Documents ## + +After securing the couch design documents need to be deployed with admin permissions. There are two ways of doing this: + * rake couchrest:migrate_with_proxies + * dump the documents as files with `rake couchrest:dump` and deploy them + to the couch by hand or with the platform. + +### CouchRest::Migrate ### + +The before_script block in .travis.yml illustrates how to do this: + + mv test/config/couchdb.yml.admin config/couchdb.yml # use admin privileges + bundle exec rake couchrest:migrate_with_proxies # run the migrations + bundle exec rake couchrest:migrate_with_proxies # looks like this needs to run twice + mv test/config/couchdb.yml.user config/couchdb.yml # drop admin privileges + +### Deploy design docs from CouchRest::Dump ### + +First of all we get the design docs as files: + + # put design docs in /tmp/design + bundle exec rake couchrest:dump + +Then we add them to files/design in the site_couchdb module in leap_platform so they get deployed with the couch. You could also upload them using curl or sth. similar. + +# Troubleshooting # + +Here are some less common issues you might run into when installing Leap Web. + +## Cannot find Bundler ## + +### Error Messages ### + +`bundle: command not found` + +### Solution ### + +Make sure bundler is installed. `gem list bundler` should list `bundler`. +You also need to be able to access the `bundler` executable in your PATH. + +## Outdated version of rubygems ## + +### Error Messages ### + +`bundler requires rubygems >= 1.3.6` + +### Solution ### + +`gem update --system` will install the latest rubygems + +## Missing development tools ## + +Some required gems will compile C extensions. They need a bunch of utils for this. + +### Error Messages ### + +`make: Command not found` + +### Solution ### + +Install the required tools. For linux the `build-essential` package provides most of them. For Mac OS you probably want the XCode Commandline tools. + +## Missing libraries and headers ## + +Some gem dependencies might not compile because they lack the needed c libraries. + +### Solution ### + +Install the libraries in question including their development files. + + diff --git a/doc/en.md b/doc/en.md new file mode 100644 index 00000000..07f07b7f --- /dev/null +++ b/doc/en.md @@ -0,0 +1,85 @@ +@title = 'LEAP Platform for Service Providers' +@nav_title = 'Provider Platform' +@toc = false + +The *LEAP Platform* is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its goal is to make it as painless as possible for sysadmins to deploy and maintain a service provider's infrastructure for secure communication. + +The LEAP Platform consists of three parts, detailed below: + +1. [The platform recipes.](#the-platform-recipes) +2. [The provider instance.](#the-provider-instance) +3. [The `leap` command line tool.](#the-leap-command-line-tool) + +The platform recipes +-------------------- + +The LEAP platform recipes define an abstract service provider. It is a set of [Puppet](https://puppetlabs.com/puppet/puppet-open-source/) modules designed to work together to provide to sysadmins everything they need to manage a service provider infrastructure that provides secure communication services. + +LEAP maintains a repository of platform recipes, which typically do not need to be modified, although it can be forked and merged as desired. Most service providers using the LEAP platform can use the same set of platform recipes. + +As these recipes consist in abstract definitions, in order to configure settings for a particular service provider a system administrator has to create a provider instance (see below). + +LEAP's platform recipes are distributed as a git repository: `https://leap.se/git/leap_platform` + +The provider instance +--------------------- + +A provider instance is a directory tree (typically tracked in git) containing all the configurations for a service provider's infrastructure. A provider instance primarily consists of: + +* A pointer to the platform recipes. +* A global configuration file for the provider. +* A configuration file for each server (node) in the provider's infrastructure. +* Additional files, such as certificates and keys. + +A minimal provider instance directory looks like this: + + └── bitmask # provider instance directory. + ├── Leapfile # settings for the `leap` command line tool. + ├── provider.json # global settings of the provider. + ├── common.json # settings common to all nodes. + ├── nodes/ # a directory for node configurations. + ├── files/ # keys, certificates, and other files. + └── users/ # public key information for privileged sysadmins. + + +A provider instance directory contains everything needed to manage all the servers that compose a provider's infrastructure. Because of this, any versioning tool and development work-flow can be used to manage your provider instance. + +The `leap` command line tool +---------------------------- + +The `leap` [command line tool](commands) is used by sysadmins to manage everything about a service provider's infrastructure. Except when creating an new provider instance, `leap` is run from within the directory tree of a provider instance. + +The `leap` command line has many capabilities, including: + +* Create, initialize, and deploy nodes. +* Manage keys and certificates. +* Query information about the node configurations. + +Traditional system configuration automation systems, like [Puppet](https://puppetlabs.com/puppet/puppet-open-source/) or [Chef](http://www.opscode.com/chef/), deploy changes to servers using a pull method. Each server pulls a manifest from a central master server and uses this to alter the state of the server. + +Instead, the `leap` tool uses a masterless push method: The sysadmin runs `leap deploy` from the provider instance directory on their desktop machine to push the changes out to every server (or a subset of servers). LEAP still uses Puppet, but there is no central master server that each node must pull from. + +One other significant difference between LEAP and typical system automation is how interactions among servers are handled. Rather than store a central database of information about each server that can be queried when a recipe is applied, the `leap` command compiles static representation of all the information a particular server will need in order to apply the recipes. In compiling this static representation, `leap` can use arbitrary programming logic to query and manipulate information about other servers. + +These two approaches, masterless push and pre-compiled static configuration, allow the sysadmin to manage a set of LEAP servers using traditional software development techniques of branching and merging, to more easily create local testing environments using virtual servers, and to deploy without the added complexity and failure potential of a master server. + +The `leap` command line tool is distributed as a git repository: `https://leap.se/git/leap_cli`. It can be installed with `sudo gem install leap_cli`. + +Tip: With rubygems, you can always specify the gem version as the first argument to any executable installed by rubygems. For example: + + sudo gem install leap_cli --version 1.6.2 + sudo gem install leap_cli --version 1.7.2 + leap _1.6.2_ --version + => leap 1.6.2, ruby 2.1.2 + leap _1.7.2_ --version + => leap 1.7.2, ruby 2.1.2 + +Getting started +---------------------------------- + +We recommend reading the platform documentation in the following order: + +1. [Quick start tutorial](tutorials/quick-start). +2. [Platform Guide](platform/guide). +3. [Configuration format](platform/config). +4. The `leap` [command reference](platform/commands). diff --git a/doc/guide/commands.md b/doc/guide/commands.md new file mode 100644 index 00000000..eaacc8d5 --- /dev/null +++ b/doc/guide/commands.md @@ -0,0 +1,419 @@ +@title = 'Command Line Reference' +@summary = "A copy of leap --help" + +The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home. + + +# Global Options + +* `--log FILE` +Override default log file +Default Value: None + +* `-v|--verbose LEVEL` +Verbosity level 0..5 +Default Value: 1 + +* `--[no-]color` +Disable colors in output + +* `--debug` +Enable debugging library (leap_cli development only) + +* `--help` +Show this message + +* `--version` +Display version number and exit + +* `--yes` +Skip prompts and assume "yes" + + +# leap add-user USERNAME + +Adds a new trusted sysadmin by adding public keys to the "users" directory. + + + +**Options** + +* `--pgp-pub-key arg` +OpenPGP public key file for this new user +Default Value: None + +* `--ssh-pub-key arg` +SSH public key file for this new user +Default Value: None + +* `--self` +Add yourself as a trusted sysadmin by choosing among the public keys available for the current user. + + +# leap cert + +Manage X.509 certificates + + + +## leap cert ca + +Creates two Certificate Authorities (one for validating servers and one for validating clients). + +See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`. + +## leap cert csr + +Creates a CSR for use in buying a commercial X.509 certificate. + +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. + +**Options** + +* `--domain DOMAIN` +Specify what domain to create the CSR for. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Default Value: None + + +## leap cert dh + +Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers. You don't need this file if you don't provide the VPN service. + + + +## leap cert update FILTER + +Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed. + +This command will a generate new certificate for a node if some value in the node has changed that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. Sometimes, you might want to force the generation of a new certificate, such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. In this case, use --force. If <node-filter> is empty, this command will apply to all nodes. + +**Options** + +* `--force` +Always generate new certificates + + +# leap clean + +Removes all files generated with the "compile" command. + + + +# leap compile + +Compile generated files. + + + +## leap compile all [ENVIRONMENT] + +Compiles node configuration files into hiera files used for deployment. + + + +## leap compile zone + +Compile a DNS zone file for your provider. + + +Default Command: all + +# leap db + +Database commands. + + + +## leap db destroy [FILTER] + +Destroy all the databases. If present, limit to FILTER nodes. + + + +# leap deploy FILTER + +Apply recipes to a node or set of nodes. + +The FILTER can be the name of a node, service, or tag. + +**Options** + +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--tags TAG[,TAG]` +Specify tags to pass through to puppet (overriding the default). +Default Value: leap_base,leap_service + +* `--dev` +Development mode: don't run 'git submodule update' before deploy. + +* `--fast` +Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. + +* `--force` +Deploy even if there is a lockfile. + +* `--[no-]sync` +Sync files, but don't actually apply recipes. + + +# leap env + +Manipulate and query environment information. + +The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. A node in one environment will never interact with a node from another environment. Environment pinning works by modifying your ~/.leaprc file and is dependent on the absolute file path of your provider directory (pins don't apply if you move the directory) + +## leap env ls + +List the available environments. The pinned environment, if any, will be marked with '*'. + + + +## leap env pin ENVIRONMENT + +Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment. + + + +## leap env unpin + +Unpin the environment. All subsequent commands will apply to all nodes. + + +Default Command: ls + +# leap facts + +Gather information on nodes. + + + +## leap facts update FILTER + +Query servers to update facts.json. + +Queries every node included in FILTER and saves the important information to facts.json + +# leap help command + +Shows a list of commands or help for one command + +Gets help for the application or its commands. Can also list the commands in a way helpful to creating a bash-style completion function + +**Options** + +* `-c` +List commands one per line, to assist with shell completion + + +# leap inspect FILE + +Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag. + + + +**Options** + +* `--base` +Inspect the FILE from the provider_base (i.e. without local inheritance). + + +# leap list [FILTER] + +List nodes and their classifications + +Prints out a listing of nodes, services, or tags. If present, the FILTER can be a list of names of nodes, services, or tags. If the name is prefixed with +, this acts like an AND condition. For example: + +`leap list node1 node2` matches all nodes named "node1" OR "node2" + +`leap list openvpn +local` matches all nodes with service "openvpn" AND tag "local" + +**Options** + +* `--print arg` +What attributes to print (optional) +Default Value: None + +* `--disabled` +Include disabled nodes in the list. + + +# leap local + +Manage local virtual machines. + +This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'. + +## leap local destroy [FILTER] + +Destroys the virtual machine(s), reclaiming the disk space + + + +## leap local reset [FILTER] + +Resets virtual machine(s) to the last saved snapshot + + + +## leap local save [FILTER] + +Saves the current state of the virtual machine as a new snapshot + + + +## leap local start [FILTER] + +Starts up the virtual machine(s) + + + +## leap local status [FILTER] + +Print the status of local virtual machine(s) + + + +## leap local stop [FILTER] + +Shuts down the virtual machine(s) + + + +# leap mosh NAME + +Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true). + + + +# leap new DIRECTORY + +Creates a new provider instance in the specified directory, creating it if necessary. + + + +**Options** + +* `--contacts arg` +Default email address contacts. +Default Value: None + +* `--domain arg` +The primary domain of the provider. +Default Value: None + +* `--name arg` +The name of the provider. +Default Value: None + +* `--platform arg` +File path of the leap_platform directory. +Default Value: None + + +# leap node + +Node management + + + +## leap node add NAME [SEED] + +Create a new configuration file for a node named NAME. + +If specified, the optional argument SEED can be used to seed values in the node configuration file. + +The format is property_name:value. + +For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`. + +To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44` + +Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns` + +**Options** + +* `--local` +Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer. + + +## leap node init FILTER + +Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages + +This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times. + +**Options** + +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--echo` +If set, passwords are visible as you type them (default is hidden) + + +## leap node mv OLD_NAME NEW_NAME + +Renames a node file, and all its related files. + + + +## leap node rm NAME + +Removes all the files related to the node named NAME. + + + +# leap ssh NAME + +Log in to the specified node with an interactive shell. + + + +**Options** + +* `--port arg` +Override ssh port for remote host +Default Value: None + +* `--ssh arg` +Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig') +Default Value: None + + +# leap test + +Run tests. + + + +## leap test init + +Creates files needed to run tests. + + + +## leap test run + +Run tests. + + + +**Options** + +* `--[no-]continue` +Continue over errors and failures (default is --no-continue). + +Default Command: run diff --git a/doc/guide/config.md b/doc/guide/config.md new file mode 100644 index 00000000..be67e6bd --- /dev/null +++ b/doc/guide/config.md @@ -0,0 +1,263 @@ +@title = "Configuration Files" +@summary = "How to edit configuration files." + +Files +------------------------------------------- + +Here are a list of some of the common files that make up a provider. Except for Leapfile and provider.json, the files are optional. Unless otherwise specified, all file names are relative to the 'provider directory' root (where the Leapfile is). + +`Leapfile` -- If present, this file tells `leap` that the directory is a provider directory. This file is usually empty, but can contain global options. + +`~/.leaprc` -- Evaluated the same as Leapfile, but not committed to source control. + +`provider.json` -- Global options related to this provider. + +`provider.ENVIRONMENT.json` -- Global options for the provider that are applied to only a single environment. + +`common.json` -- All nodes inherit from this file. + +`secrets.json` -- An automatically generated file that contains any randomly generated strings needed in order to deploy. These strings are often secret and should be protected, although any need for a random string or number that is remembered will produce another entry in this file. This file is automatically generated and refreshed each time you run `leap compile` or `leap deploy`. If an entry is no longer needed, it will get removed. If you want to change a secret, you can remove this file and have it regenerated, or remove the particular line item and just those items will be created anew. + +`facts.json` -- If some of your servers are running on AWS or OpenStack, you will need to discover certain properties about how networking is configured on these machines in order for a full deploy to work. In these cases, make sure to run `leap facts update` to periodically regenerate the facts.json file. + +`nodes/NAME.json` -- The configuration file for node called NAME. + +`services/SERVICE.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its `services` property. + +`services/SERVICE.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its services and has environment equal to ENVIRONMENT. + +`services/TAG.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property. + +`services/TAG.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property and has `environment` property equal to ENVIRONMENT. + +`files/*` -- Various static files used by the platform (e.g. keys, certificates, webapp customization, etc). + +`users/USER/` -- A directory that stores the public keys of the sysadmin with name USER. This person will have root access to all the servers. + + +Leapfile +------------------------------------------- + +A `Leapfile` defines options for the `leap` command and lives at the root of your provider directory. `Leapfile` is evaluated as ruby, so you can include whatever weird logic you want in this file. In particular, there are several variables you can set that modify the behavior of leap. For example: + + @platform_directory_path = '../leap_platform' + @log = '/var/log/leap.log' + +Additionally, you can create a `~/.leaprc` file that is loaded after `Leapfile` and is evaluated the same way. + +Platform options: + +* `@platform_directory_path` (required). This must be set to the path where `leap_platform` lives. The path may be relative. + +Vagrant options: + +* `@vagrant_network`. Allows you to override the default network used for local nodes. It should include a netmask like `@vagrant_network = '10.0.0.0/24'`. +* `@custom_vagrant_vm_line`. Insert arbitrary text into the auto-generated Vagrantfile. For example, `@custom_vagrant_vm_line = "config.vm.boot_mode = :gui"`. + +Logging options: + +* `@log`. If set, all command invocation and results are logged to the specified file. This is the same as the switch `--log FILE`, except that the command line switch will override the value in the Leapfile. + + +JSON format +------------------------------------------- + +All configuration files, other than `Leapfile`, are in the JSON format. For example: + + { + "key1": "value1", + "key2": "value2" + } + +Keys should match `/[a-z0-9_]/` + +Unlike traditional JSON, comments are allowed. If the first non-whitespace characters are `//` then the line is treated as a comment. + + // this is a comment + { + // this is a comment + "key": "value" // this is an error + } + +Options in the configuration files might be nested hashes, arrays, numbers, strings, or boolean. Numbers and boolean values should **not** be quoted. For example: + + { + "openvpn": { + "ip_address": "1.1.1.1", + "protocols": ["tcp", "udp"], + "ports": [80, 53], + "options": { + "public_ip": false, + "adblock": true + } + } + } + +If the value string is prefixed with an '=' character, the result is evaluated as ruby. For example: + + { + "domain": { + "public": "domain.org" + } + "api_domain": "= 'api.' + domain.public" + } + +In this case, the property "api_domain" will be set to "api.domain.org". So long as you do not create unresolvable circular dependencies, you can reference other properties in evaluated ruby that are themselves evaluated ruby. + +See "Macros" below for information on the special macros available to the evaluated ruby. + +TIP: In rare cases, you might want to force the evaluation of a value to happen in a later pass after most of the other properties have been evaluated. To do this, prefix the value string with "=>" instead of "=". + +Node inheritance +---------------------------------------- + +Every node inherits from common.json and also any of the services or tags attached to the node. Additionally, the `leap_platform` contains a directory `provider_base` that defines the default values for tags, services and common.json. + +Suppose you have a node configuration for `bitmask/nodes/willamette.json` like so: + + { + "services": "webapp", + "tags": ["production", "northwest-us"], + "ip_address": "1.1.1.1" + } + +This node will have hostname "willamette" and it will inherit from the following files (in this order): + +1. common.json + - load defaults: `provider_base/common.json` + - load provider: `bitmask/common.json` +2. service "webapp" + - load defaults: `provider_base/services/webapp.json` + - load provider: `bitmask/services/webapp.json` +3. tag "production" + - load defaults: `provider_base/tags/production.json` + - load provider: `bitmask/tags/production.json` +4. tag "northwest-us" + - load: `bitmask/tags/northwest-us.json` +5. finally, load node "willamette" + - load: `bitmask/nodes/willamette.json` + +The `provider_base` directory is under the `leap_platform` specified in the file `Leapfile`. + +To see all the variables a node has inherited, you could run `leap inspect willamette`. + +Common configuration options +---------------------------------------- + +You can use the command `leap inspect` to see what options are available for a provider, node, service, or tag configuration. For example: + +* `leap inspect common` -- show the options inherited by all nodes. +* `leap inspect --base common` -- show the common.json from `provider_base` without the local `common.json` inheritance applied. +* `leap inspect webapp` -- show all the options available for the service `webapp`. + +Here are some of the more important options you should be aware of: + +* `ip_address` -- Required for all nodes, no default. +* `ssh.port` -- The SSH port you want the node's OpenSSH server to bind to. This is also the default when trying to connect to a node, but if the node currently has OpenSSH running on a different port then run deploy with `--port` to override the `ssh.port` configuration value. +* `mosh.enabled` -- If set to `true`, then mosh will be installed on the server. The default is `false`. + +Macros +---------------------------------------- + +When using evaluated ruby in a JSON configuration file, there are several special macros that are available. These are evaluated in the context of a node (available as the variable `self`). + +The following methods are available to the evaluated ruby: + +`variable.variable` + + > Any variable defined or inherited by a particular node configuration is available by just referencing it using either hash notation or object field notation (e.g. `['domain']['public']` or `domain.public`). Circular references are not allowed, but otherwise it is OK to nest evaluated values in other evaluated values. If a value has not been defined, the hash notation will return nil but the field notation will raise an exception. Properties of services, tags, and the global provider can all be referenced the same way. For example, `global.services['openvpn'].x509.dh`. + +`nodes` + + > A hash of all nodes. This list can be filtered. + +`nodes_like_me` + + > A hash of nodes that have the same deployment tags as the current node (e.g. 'production' or 'local'). + +`global.services` + + > A hash of all services, e.g. `global.services['openvpn']` would return the "openvpn" service. + +`global.tags` + + > A hash of all tags, e.g. `global.tags['production']` would return the "production" tag. + + `global.provider` + + > Can be used to access variables defined in `provider.json`, e.g. `global.provider.contacts.default`. + +`file(filename)` + + > Inserts the full contents of the file. If the file is an erb template, it is rendered. The filename can either be one of the pre-defined file symbols, or it can be a path relative to the "files" directory in your provider instance. E.g, `file :ca_cert` or `files 'ca/ca.crt'`. + +`file_path(filename)` + + > Ensures that the file will get rsynced to the node as an individual file. The value returned by `file_path` is the full path where this file will ultimately live when deploy to the node. e.g. `file_path :ca_cert` or `file_path 'branding/images/logo.png'`. + +`secret(:symbol)` + + > Returns the value of a secret in secrets.json (or creates it if necessary). E.g. `secret :couch_admin_password` + +`hosts_file` + + > Returns a data structure that puppet will use to generate /etc/hosts. Care is taken to use the local IP of other hosts when needed. + +`known_hosts_file` + + > Returns the lines needed in a SSH `known_hosts` file. + +`stunnel_client(node_list, port, options={})` + + > Returns a stunnel configuration data structure for the client side. Argument `node_list` is an `ObjectList` of nodes running stunnel servers. Argument `port` is the real port of the ultimate service running on the servers that the client wants to connect to. + +`stunnel_server(port)` + + > Generates a stunnel server entry. The `port` is the real port targeted service. + +Hash tables +----------------------------------------- + +The macros `nodes`, `nodes_like_me`, `global.services`, and `global.tags` all return a hash table of configuration objects (either nodes, services, or tags). There are several ways to filter and process these hash tables: + +Access an element by name: + + nodes['vpn1'] # returns node named 'vpn1' + global.services['openvpn'] # returns service named 'openvpn' + +Create a new hash table by applying filters: + + nodes[:public_dns => true] # all nodes where public_dns == true + nodes[:services => 'openvpn', 'location.country_code' => 'US'] # openvpn service OR in the US. + nodes[[:services, 'openvpn'], [:services, 'tor']] # services equal to openvpn OR tor + nodes[:services => 'openvpn'][:tags => 'production'] # openvpn AND production + nodes[:name => "!bob"] # all nodes that are NOT named "bob" + +Create an array of values by selecting a single field: + + nodes.field('location.name') + ==> ['seattle', 'istanbul'] + +Create an array of hashes by selecting multiple fields: + + nodes.fields('domain.full', 'ip_address') + ==> [ + {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, + {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, + ] + +Create a new hash table of hashes, with only certain fields: + + nodes.pick_fields('domain.full', 'ip_address') + ==> { + "red" => {'domain_full' => 'red.bitmask.net', 'ip_address' => '1.1.1.1'}, + "blue => {'domain_full' => 'blue.bitmask.net', 'ip_address' => '1.1.1.2'}, + } + +With `pick_fields`, if there is only one field, it will generate a simple hash table: + + nodes.pick_fields('ip_address') + ==> { + "red" => '1.1.1.1', + "blue => '1.1.1.2', + } diff --git a/doc/guide/en.haml b/doc/guide/en.haml new file mode 100644 index 00000000..61c24ea8 --- /dev/null +++ b/doc/guide/en.haml @@ -0,0 +1,4 @@ +- @nav_title = "Guide" +- @title = "Platform Guide" + += child_summaries
\ No newline at end of file diff --git a/doc/guide/environments.md b/doc/guide/environments.md new file mode 100644 index 00000000..752e0608 --- /dev/null +++ b/doc/guide/environments.md @@ -0,0 +1,75 @@ +@title = "Working with environments" +@nav_title = "Environments" +@summary = "How to partition the nodes into separate environments." + +With environments, you can divide your nodes into different and entirely separate sets. For example, you might have sets of nodes for 'testing', 'staging' and 'production'. + +Typically, the nodes in one environment are totally isolated from the nodes in a different environment. Each environment will have its own separate database, for example. + +There are a few exceptions to this rule: backup nodes, for example, will by default attempt to back up data from all the environments (excluding local). + +## Assign an environment + +To assign an environment to a node, you just set the `environment` node property. This is typically done with tags, although it is not necessary. For example: + +`tags/production.json` + + { + "environment": "production" + } + +`nodes/mynode.json` + + { + "tags": ["production"] + } + +There are several built-in tags that will apply a value for the environment: + +* `production`: An environment for nodes that are in use by end users. +* `development`: An environment to be used for nodes that are being used for experiments or staging. +* `local`: This environment gets automatically applied to all nodes that run only on local VMs. Nodes with a `local` environment are treated special and excluded from certain calculations. + +You don't need to use these and you can add your own. + +## Environment commands + +* `leap env` -- List the available environments and disply which one is active. +* `leap env pin ENV` -- Pin the current environment to ENV. +* `leap env unpin` -- Remove the environment pin. + +The environment pin is only active for your local machine: it is not recorded in the provider directory and not shared with other users. + +## Environment specific JSON files + +You can add JSON configuration files that are only applied when a specific environment is active. For example, if you create a file `provider.production.json`, these values will only get applied to the `provider.json` file for the `production` environment. + +This will also work for services and tags. For example: + + provider.local.json + services/webapp.development.json + tags/seattle.production.json + +In this example, `local`, `development`, and `production` are the names of environments. + +## Bind an environment to a Platform version + +If you want to ensure that a particular environment is bound to a particular version of the LEAP Platform, you can add a `platform` section to the `provider.ENV.json` file (where ENV is the name of the environment in question). + +The available options are `platform.version`, `platform.branch`, or `platform.commit`. For example: + + { + "platform": { + "version": "1.6.1", + "branch": "develop", + "commit": "5df867fbd3a78ca4160eb54d708d55a7d047bdb2" + } + } + +You can use any combination of `version`, `branch`, and `commit` to specify the binding. The values for `branch` and `commit` only work if the `leap_platform` directory is a git repository. + +The value for `commit` is passed directly through to `git log` to query for a list of acceptable commits. See [[man gitrevisions => https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html#_specifying_ranges]] to see how to specify ranges. For example: + +* `HEAD^..HEAD` - current commit must be head of the branch. +* `3172444652af71bd771609d6b80258e70cc82ce9..HEAD` - current commit must be after 3172444652af71bd771609d6b80258e70cc82ce9. +* `refs/tags/0.6.0rc1..refs/tags/0.6.0rc2` - current commit must be after tag 0.6.0rc1 and before or including tag 0.6.0rc2.
\ No newline at end of file diff --git a/doc/guide/keys-and-certificates.md b/doc/guide/keys-and-certificates.md new file mode 100644 index 00000000..aef02ac6 --- /dev/null +++ b/doc/guide/keys-and-certificates.md @@ -0,0 +1,194 @@ +@title = "Keys and Certificates" +@summary = "Working with SSH keys, secrets, and X.509 certificates." + +Working with SSH +================================ + +Whenever the `leap` command nees to push changes to a node or gather information from a node, it tunnels this command over SSH. Another way to put this: the security of your servers rests entirely on SSH. Because of this, it is important that you understand how `leap` uses SSH. + +SSH related files +------------------------------- + +Assuming your provider directory is called 'provider': + +* `provider/nodes/crow/crow_ssh.pub` -- The public SSH host key for node 'crow'. +* `provider/users/alice/alice_ssh.pub` -- The public SSH user key for user 'alice'. Anyone with the private key that corresponds to this public key will have root access to all nodes. +* `provider/files/ssh/known_hosts` -- An autogenerated known_hosts, built from combining `provider/nodes/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `known_hosts` and then run `leap compile`. +* `provider/files/ssh/authorized_keys` -- An autogenerated list of all the user SSH keys with root access to the notes. It is created from `provider/users/*/*_ssh.pub`. You must not edit this file directly. If you need to change it, remove or change one of the files that is used to generate `authorized_keys` and then run `leap compile`. + +All of these files should be committed to source control. + +If you rename, remove, or add a node with `leap node [mv|add|rm]` the SSH key files and the `known_hosts` file will get properly updated. + +SSH and local nodes +----------------------------- + +Local nodes are run as Vagrant virtual machines. The `leap` command handles SSH slightly differently for these nodes. + +Basically, all the SSH security is turned off for local nodes. Since local nodes only exist for a short time on your computer and can't be reached from the internet, this is not a problem. + +Specifically, for local nodes: + +1. `known_hosts` is never updated with local node keys, since the SSH public key of a local node is different for each user. +2. `leap` entirely skips the checking of host keys when connecting with a local node. +3. `leap` adds the public Vagrant SSH key to the list of SSH keys for a user. The public Vagrant SSH key is a shared and insecure key that has root access to most Vagrant virtual machines. + +When SSH host key changes +------------------------------- + +If the host key for a node has changed, you will get an error "WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED". + +To fix this, you need to remove the file `files/nodes/stompy/stompy_ssh.pub` and run `leap node init stompy`, where the node's name is 'stompy'. **Only do this if you are ABSOLUTELY CERTAIN that the node's SSH host key has changed**. + +Changing the SSH port +-------------------------------- + +Suppose you have a node `blinky` that has SSH listening on port 22 and you want to make it port 2200. + +First, modify the configuration for `blinky` to specify the variable `ssh.port` as 2200. Usually, this is done in `common.json` or in a tag file. + +For example, you could put this in `tags/production.json`: + + { + "ssh": { + "port": 2200 + } + } + +Run `leap compile` and open `hiera/blinky.yaml` to confirm that `ssh.port` is set to 2200. The port number must be specified as a number, not a string (no quotes). + +Then, you need to deploy this change so that SSH will bind to 2200. You cannot simply run `leap deploy blinky` because this command will default to using the variable `ssh.port` which is now `2200` but SSH on the node is still bound to 22. + +So, you manually override the port in the deploy command, using the old port: + + leap deploy --port 22 blinky + +Afterwards, SSH on `blinky` should be listening on port 2200 and you can just run `leap deploy blinky` from then on. + +Sysadmins with multiple SSH keys +----------------------------------- + +The command `leap add-user --self` allows only one SSH key. If you want to specify more than one key for a user, you can do it manually: + + users/userx/userx_ssh.pub + users/userx/otherkey_ssh.pub + +All keys matching 'userx/*_ssh.pub' will be usable. + +Removing sysadmin access +-------------------------------- + +Suppose you want to remove `userx` from having any further ssh access to the servers. Do this: + + rm -r users/userx + leap deploy + +X.509 Certificates +================================ + +Configuration options +------------------------------------------- + +The `ca` option in provider.json provides settings used when generating CAs and certificates. The defaults are as follows: + + { + "ca": { + "name": "= global.provider.ca.organization + ' Root CA'", + "organization": "= global.provider.name[global.provider.default_language]", + "organizational_unit": "= 'https://' + global.provider.domain", + "bit_size": 4096, + "digest": "SHA256", + "life_span": "10y", + "server_certificates": { + "bit_size": 2048, + "digest": "SHA256", + "life_span": "1y" + }, + "client_certificates": { + "bit_size": 2048, + "digest": "SHA256", + "life_span": "2m", + "limited_prefix": "LIMITED", + "unlimited_prefix": "UNLIMITED" + } + } + } + +You should not need to override these defaults in your own provider.json, but you can if you want to. To see what values are used for your provider, run `leap inspect provider.json`. + +NOTE: A certificate `bit_size` greater than 2048 will probably not be recognized by most commercial CAs. + +Certificate Authorities +----------------------------------------- + +There are three x.509 certificate authorities (CA) associated with your provider: + +1. **Commercial CA:** It is strongly recommended that you purchase a commercial cert for your primary domain. The goal of platform is to not depend on the commercial CA system, but it does increase security and usability if you purchase a certificate. The cert for the commercial CA must live at `files/cert/commercial_ca.crt`. +2. **Server CA:** This is a self-signed CA responsible for signing all the **server** certificates. The private key lives at `files/ca/ca.key` and the public cert lives at `files/ca/ca.crt`. The key is very sensitive information and must be kept private. The public cert is distributed publicly. +3. **Client CA:** This is a self-signed CA responsible for signing all the **client** certificates. The private key lives at `files/ca/client_ca.key` and the public cert lives at `files/ca/client_ca.crt`. Neither file is distribute publicly. It is not a big deal if the private key for the client CA is compromised, you can just generate a new one and re-deploy. + +To generate both the Server CA and the Client CA, run the command: + + leap cert ca + +Server certificates +----------------------------------- + +Most every server in your service provider will have a x.509 certificate, generated by the `leap` command using the Server CA. Whenever you modify any settings of a node that might affect it's certificate (like changing the IP address, hostname, or settings in provider.json), you can magically regenerate all the certs that need to be regenerated with this command: + + leap cert update + +Run `leap help cert update` for notes on usage options. + +Because the server certificates are generated locally on your personal machine, the private key for the Server CA need never be put on any server. It is up to you to keep this file secure. + +Client certificates +-------------------------------- + +Every leap client gets its own time-limited client certificate. This cert is use to connect to the OpenVPN gateway (and probably other things in the future). It is generated on the fly by the webapp using the Client CA. + +To make this work, the private key of the Client CA is made available to the webapp. This might seem bad, but compromise of the Client CA simply allows the attacker to use the OpenVPN gateways without paying. In the future, we plan to add a command to automatically regenerate the Client CA periodically. + +There are two types of client certificates: limited and unlimited. A client using a limited cert will have its bandwidth limited to the rate specified by `provider.service.bandwidth_limit` (in Bytes per second). An unlimited cert is given to the user if they authenticate and the user's service level matches one configured in `provider.service.levels` without bandwidth limits. Otherwise, the user is given a limited client cert. + +Commercial certificates +----------------------------------- + +We strongly recommend that you use a commercial signed server certificate for your primary domain (in other words, a certificate with a common name matching whatever you have configured for `provider.domain`). This provides several benefits: + +1. When users visit your website, they don't get a scary notice that something is wrong. +2. When a user runs the LEAP client, selecting your service provider will not cause a warning message. +3. When other providers first discover your provider, they are more likely to trust your provider key if it is fetched over a commercially verified link. + +The LEAP platform is designed so that it assumes you are using a commercial cert for the primary domain of your provider, but all other servers are assumed to use non-commercial certs signed by the Server CA you create. + +To generate a CSR, run: + + leap cert csr + +This command will generate the CSR and private key matching `provider.domain` (you can change the domain with `--domain=DOMAIN` switch). It also generates a server certificate signed with the Server CA. You should delete this certificate and replace it with a real one once it is created by your commercial CA. + +The related commercial cert files are: + + files/ + cert/ + domain.org.crt # Server certificate for domain.org, obtained by commercial CA. + domain.org.csr # Certificate signing request + domain.org.key # Private key for you certificate + commercial_ca.crt # The CA cert obtained from the commercial CA. + +The private key file is extremely sensitive and care should be taken with its provenance. + +If your commercial CA has a chained CA cert, you should be OK if you just put the **last** cert in the chain into the `commercial_ca.crt` file. This only works if the other CAs in the chain have certs in the debian package `ca-certificates`, which is the case for almost all CAs. + +If you want to add additional fields to the CSR, like country, city, or locality, you can configure these values in provider.json like so: + + "ca": { + "server_certificates": { + "country": "US", + "state": "Washington", + "locality": "Seattle" + } + } + +If they are not present, the CSR will be created without them. diff --git a/doc/guide/miscellaneous.md b/doc/guide/miscellaneous.md new file mode 100644 index 00000000..c38c007c --- /dev/null +++ b/doc/guide/miscellaneous.md @@ -0,0 +1,14 @@ +@title = "Miscellaneous" +@summary = "Miscellaneous commands you may need to know." + +Facts +============================== + +There are a few cases when we must gather internal data from a node before we can successfully deploy to other nodes. This is what `facts.json` is for. It stores a snapshot of certain facts about each node, as needed. Entries in `facts.json` are updated automatically when you initialize, rename, or remove a node. To manually force a full update of `facts.json`, run: + + leap facts update FILTER + +Run `leap help facts update` for more information. + +The file `facts.json` should be committed to source control. You might not have a `facts.json` if one is not required for your provider. + diff --git a/doc/guide/nodes.md b/doc/guide/nodes.md new file mode 100644 index 00000000..cf225449 --- /dev/null +++ b/doc/guide/nodes.md @@ -0,0 +1,187 @@ +@title = "Nodes" +@summary = "Working with nodes, services, tags, and locations." + +Node types +================================ + +Every node has one or more services that determines the node's function within your provider's infrastructure. + +When adding a new node to your provider, you should ask yourself four questions: + +* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes. +* **required or optional?** Some services are required, while others can be left out. +* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together. +* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure. + +Brief overview of the services: + +* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many. +* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp`, `mx`, and `soledad` nodes. +* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb. +* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes. +* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service. +* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` service defined. It is required that the monitor be on the webapp node. It was not designed to be run as a separate node service. +* **tor**: Sets up a tor exit node, unconnected to any other service. +* **dns**: Not yet implemented. + +Webapp +----------------------------------- + +The webapp node is responsible for both the user face web application and the API that the client interacts with. + +Some users can be "admins" with special powers to answer tickets and close accounts. To make an account into an administrator, you need to configure the `webapp.admins` property with an array of user names. + +For example, to make users `alice` and `bob` into admins, create a file `services/webapp.json` with the following content: + + { + "webapp": { + "admins": ["bob", "alice"] + } + } + +And then redeploy to all webapp nodes: + + leap deploy webapp + +By putting this in `services/webapp.json`, you will ensure that all webapp nodes inherit the value for `webapp.admins`. + +Services +================================ + +What nodes do you need for a provider that offers particular services? + +<table class="table table-striped"> +<tr> + <th>Node Type</th> + <th>VPN Service</th> + <th>Email Service</th> + <th>Notes</th> +</tr> +<tr> + <td>webapp</td> + <td>required</td> + <td>required</td> + <td></td> +</tr> +<tr> + <td>couchdb</td> + <td>required</td> + <td>required</td> +<td></td> +</tr> +<tr> + <td>soledad</td> + <td>not used</td> + <td>required</td> +<td></td> +</tr> +<tr> + <td>mx</td> + <td>not used</td> + <td>required</td> + <td></td> +</tr> +<tr> + <td>openvpn</td> + <td>required</td> + <td>not used</td> + <td></td> +</tr> +<tr> + <td>monitor</td> + <td>optional</td> + <td>optional</td> + <td>This service must be on the webapp node</td> +</tr> +<tr> + <td>tor</td> + <td>optional</td> + <td>optional</td> + <td></td> +</tr> +</table> + +Locations +================================ + +All nodes should have a `location.name` specified, and optionally additional information about the location, like the time zone. This location information is used for two things: + +* Determine which nodes can, or must, communicate with one another via a local network. The way some virtualization environments work, like OpenStack, requires that nodes communicate via the local network if they are on the same network. +* Allows the client to prefer connections to nodes that are closer in physical proximity to the user. This is particularly important for OpenVPN nodes. + +The location stanza in a node's config file looks like this: + + { + "location": { + "id": "ankara", + "name": "Ankara", + "country_code": "TR", + "timezone": "+2", + "hemisphere": "N" + } + } + +The fields: + +* `id`: An internal handle to use for this location. If two nodes have match `location.id`, then they are treated as being on a local network with one another. This value defaults to downcase and underscore of `location.name`. +* `name`: Can be anything, might be displayed to the user in the client if they choose to manually select a gateway. +* `country_code`: The [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) two letter country code. +* `timezone`: The timezone expressed as an offset from UTC (in standard time, not daylight savings). You can look up the timezone using this [handy map](http://www.timeanddate.com/time/map/). +* `hemisphere`: This should be "S" for all servers in South America, Africa, or Australia. Otherwise, this should be "N". + +These location options are very imprecise, but good enough for most usage. The client often does not know its own location precisely either. Instead, the client makes an educated guess at location based on the OS's timezone and locale. + +If you have multiple nodes in a single location, it is best to use a tag for the location. For example: + +`tags/ankara.json`: + + { + "location": { + "name": "Ankara", + "country_code": "TR", + "timezone": "+2", + "hemisphere": "N" + } + } + +`nodes/vpngateway.json`: + + { + "services": "openvpn", + "tags": ["production", "ankara"], + "ip_address": "1.1.1.1", + "openvpn": { + "gateway_address": "1.1.1.2" + } + } + +Unless you are using OpenStack or AWS, setting `location` for nodes is not required. It is, however, highly recommended. + +Disabling Nodes +===================================== + +There are two ways to temporarily disable a node: + +**Option 1: disabled environment** + +You can assign an environment to the node that marks it as disabled. Then, if you use environment pinning, the node will be ignored when you deploy. For example: + + { + "environment": "disabled" + } + +Then use `leap env pin ENV` to pin the environment to something other than 'disabled'. This only works if all the other nodes are also assigned to some environment. + +**Option 2: enabled == false** + +If a node has a property `enabled` set to false, then the `leap` command will skip over the node and pretend that it does not exist. For example: + + { + "ip_address": "1.1.1.1", + "services": ["openvpn"], + "enabled": false + } + +**Options 3: no-deploy** + +If the file `/etc/leap/no-deploy` exists on a node, then when you run the commmand `leap deploy` it will halt and prevent a deploy from going through (if the node was going to be included in the deploy). diff --git a/doc/service-diagram.odg b/doc/service-diagram.odg Binary files differnew file mode 100644 index 00000000..09265c2d --- /dev/null +++ b/doc/service-diagram.odg diff --git a/doc/service-diagram.png b/doc/service-diagram.png Binary files differnew file mode 100644 index 00000000..85e62436 --- /dev/null +++ b/doc/service-diagram.png diff --git a/doc/troubleshooting/en.haml b/doc/troubleshooting/en.haml new file mode 100644 index 00000000..f0f1359c --- /dev/null +++ b/doc/troubleshooting/en.haml @@ -0,0 +1,3 @@ +- @title = "Troubleshooting" + += child_summaries
\ No newline at end of file diff --git a/doc/troubleshooting/known-issues.md b/doc/troubleshooting/known-issues.md new file mode 100644 index 00000000..4defc886 --- /dev/null +++ b/doc/troubleshooting/known-issues.md @@ -0,0 +1,115 @@ +@title = 'Leap Platform Release Notes' +@nav_title = 'Known issues' +@summary = 'Known issues in the Leap Platform.' +@toc = true + +Here you can find documentation about known issues and potential work-arounds in the current Leap Platform release. + +0.6.0 +============== + +Upgrading +------------------ + +Upgrade your leap_platform to 0.6 and make sure you have the latest leap_cli. + +**Update leap_platform:** + + cd leap_platform + git pull + git checkout -b 0.6.0 0.6.0 + +**Update leap_cli:** + +If it is installed as a gem from rubygems: + + sudo gem update leap_cli + +If it is installed as a gem from source: + + cd leap_cli + git pull + git checkout master + rake build + sudo rake install + +If it is run directly from source: + + cd leap_cli + git pull + git checkout master + +To upgrade: + + leap --version # must be at least 1.6.2 + leap cert update + leap deploy + leap test + +If the tests fail, try deploying again. If a test fails because there are two tapicero daemons running, you need to ssh into the server, kill all the tapicero daemons manually, and then try deploying again (sometimes the daemon from platform 0.5 would put its PID file in an odd place). + +OpenVPN +------------------ + +On deployment to a openvpn node, if the following happens: + + - err: /Stage[main]/Site_openvpn/Service[openvpn]/ensure: change from stopped to running failed: Could not start Service[openvpn]: Execution of '/etc/init.d/openvpn start' returned 1: at /srv/leap/puppet/modules/site_openvpn/manifests/init.pp:189 + +this is likely the result of a kernel upgrade that happened during the deployment, requiring that the machine be restarted before this service can start. To confirm this, login to the node (leap ssh <nodename>) and look at the end of the /var/log/daemon.log: + + # tail /var/log/daemon.log + Nov 22 19:04:15 snail ovpn-udp_config[16173]: ERROR: Cannot open TUN/TAP dev /dev/net/tun: No such device (errno=19) + Nov 22 19:04:15 snail ovpn-udp_config[16173]: Exiting due to fatal error + +if you see this error, simply restart the node. + +CouchDB +--------------------- + +At the moment, we strongly advise only have one bigcouch server for stability purposes. + +With multiple couch nodes (not recommended at this time), in some scenarios, such as when certain components are unavailable, the couchdb syncing will be broken. When things are brought back to normal, shortly after restart, the nodes will attempt to resync all their data, and can fail to complete this process because they run out of file descriptors. A symptom of this is the webapp wont allow you to register or login, the /opt/bigcouch/var/log/bigcouch.log is huge with a lot of errors that include (over multiple lines): {error, emfile}}. We have raised the limits for available file descriptors to bigcouch to try and accommodate for this situation, but if you still experience it, you may need to increase your /etc/sv/bigcouch/run ulimit values and restart bigcouch while monitoring the open file descriptors. We hope that in the next platform release, a newer couchdb will be better at handling these resources. + +You can also see the number of file descriptors in use by doing: + + # watch -n1 -d lsof -p `pidof beam`|wc -l + +The command `leap db destroy` will not automatically recreate new databases. You must run `leap deploy` afterwards for this. + +User setup and ssh +------------------ + +At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280) + +The command `leap add-user --self` allows only one SSH key. If you want to specify more than one key for a user, you can do it manually: + + users/userx/userx_ssh.pub + users/userx/otherkey_ssh.pub + +All keys matching 'userx/*_ssh.pub' will be used for that user. + +Deploying +--------- + +If you have any errors during a run, please try to deploy again as this often solves non-deterministic issues that were not uncovered in our testing. Please re-deploy with `leap -v2 deploy` to get more verbose logs and capture the complete output to provide to us for debugging. + +If when deploying your debian mirror fails for some reason, network anomoly or the mirror itself is out of date, then platform deployment will not succeed properly. Check the mirror is up and try to deploy again when it is resolved (see: https://leap.se/code/issues/1091) + +Deployment gives 'error: in `%`: too few arguments (ArgumentError)' - this is because you attempted to do a deploy before initializing a node, please initialize the node first and then do a deploy afterwards (see: https://leap.se/code/issues/2550) + +This release has no ability to custom configure apt sources or proxies (see: https://leap.se/code/issues/1971) + +When running a deploy at a verbosity level of 2 and above, you will notice puppet deprecation warnings, these are known and we are working on fixing them + +IPv6 +---- + +As of this release, IPv6 is not supported by the VPN configuration. If IPv6 is detected on your network as a client, it is blocked and instead it should revert to IPv4. We plan on adding IPv6 support in an upcoming release. + + +Special Environments +-------------------- + +When deploying to OpenStack release "nova" or newer, you will need to do an initial deploy, then when it has finished run `leap facts update` and then deploy again (see: https://leap.se/code/issues/3020) + +It is not possible to actually use the EIP openvpn server on vagrant nodes (see: https://leap.se/code/issues/2401) diff --git a/doc/troubleshooting/tests.md b/doc/troubleshooting/tests.md new file mode 100644 index 00000000..b85c19d2 --- /dev/null +++ b/doc/troubleshooting/tests.md @@ -0,0 +1,70 @@ +@title = 'Tests and Monitoring' +@summary = 'Testing and monitoring your infrastructure.' +@toc = true + +## Troubleshooting Tests + +At any time, you can run troubleshooting tests on the nodes of your provider infrastructure to check to see if things seem to be working correctly. If there is a problem, these tests should help you narrow down precisely where the problem is. + +To run tests on FILTER node list: + + leap test run FILTER + +For example, you can also test a single node (`leap test elephant`); test a specific environment (`leap test development`), or any tag (`leap test soledad`). + +Alternately, you can run test on all nodes (probably only useful if you have pinned the environment): + + leap test + +The tests that are performed are located in the platform under the tests directory. + +## Testing with the bitmask client + +Download the provider ca: + + wget --no-check-certificate https://example.org/ca.crt -O /tmp/ca.crt + +Start bitmask: + + bitmask --ca-cert-file /tmp/ca.crt + +## Testing Recieving Mail + +Use i.e. swaks to send a testmail + + swaks -f noone@example.org -t testuser@example.org -s example.org + +and use your favorite mail client to examine your inbox. + +You can also use [offlineimap](http://offlineimap.org/) to fetch mails: + + offlineimap -c vagrant/.offlineimaprc.example.org + +WARNING: Use offlineimap *only* for testing/debugging, +because it will save the mails *decrypted* locally to +your disk ! + +## Monitoring + +In order to set up a monitoring node, you simply add a `monitor` service tag to the node configuration file. It could be combined with any other service, but we propose that you add it to the webapp node, as this already is public accessible via HTTPS. + +After deploying, this node will regularly poll every node to ask for the status of various health checks. These health checks include the checks run with `leap test`, plus many others. + +We use [Nagios](http://www.nagios.org/) together with [Check MK agent](https://en.wikipedia.org/wiki/Check_MK) for running checks on remote hosts. + +One nagios installation will monitor all nodes in all your environments. You can log into the monitoring web interface via [https://DOMAIN/nagios3/](https://DOMAIN/nagios3/). The username is `nagiosadmin` and the password is found in the secrets.json file in your provider directory. +Nagios will send out mails to the `contacts` address provided in `provider.json`. + + +## Nagios Frontents + +There are other ways to check and get notified by Nagios besides regularly checking the Nagios webinterface or reading email notifications. Check out the [Frontends (GUIs and CLIs)](http://exchange.nagios.org/directory/Addons/Frontends-%28GUIs-and-CLIs%29) on the Nagios project website. +A recommended status tray application is [Nagstamon](https://nagstamon.ifw-dresden.de/), which is available for Linux, MacOS X and Windows. It can not only notify you of hosts/services failures, you can also acknoledge or recheck these with it. + +### Log Monitoring + +At the moment, we use [check-mk-agent-logwatch](https://mathias-kettner.de/checkmk_check_logwatch.html) for searching logs for irregularities. +Logs are parsed for patterns using a blacklist, and are stored in `/var/lib/check_mk/logwatch/<Nodename>`. + +In order to "acknowledge" a log warning, you need to log in to the monitoring server, and delete the corresponding file in `/var/lib/check_mk/logwatch/<Nodename>`. This should be done via the nagios webinterface in the future. + diff --git a/doc/troubleshooting/vagrant.md b/doc/troubleshooting/vagrant.md new file mode 100644 index 00000000..ad284161 --- /dev/null +++ b/doc/troubleshooting/vagrant.md @@ -0,0 +1,45 @@ +@title = 'LEAP Platform Vagrant testing' +@nav_title = 'Vagrant Integration' +@summary = 'Testing your provider with Vagrant' + +Setting up Vagrant for a testing the platform +============================================= + +There are two ways you can setup leap platform using vagrant. + +Using the Vagrantfile provided by Leap Platform +----------------------------------------------- + +This is by far the easiest way. It will install a single node mail server in the default +configuration with one single command. + +Clone the platform with + + git clone https://github.com/leapcode/leap_platform.git + +Start the vagrant box with + + cd leap_platform + vagrant up + +Follow the instructions how to configure your `/etc/hosts` +in order to use the provider! + +You can login via ssh with the systemuser `vagrant` and the same password. + +There are 2 users preconfigured: + +. `testuser` with pw `hallo123` +. `testadmin` with pw `hallo123` + + +Use the leap_cli vagrant integration +------------------------------------ + +Install leap_cli and leap_platform on your host, configure a provider from scratch and use the `leap local` commands to manage your vagrant node(s). + +See https://leap.se/en/docs/platform/development how to use the leap_cli vagrant +integration and https://leap.se/en/docs/platform/tutorials/single-node-email how +to setup a single node mail server. + + diff --git a/doc/troubleshooting/where-to-look.md b/doc/troubleshooting/where-to-look.md new file mode 100644 index 00000000..fbd95931 --- /dev/null +++ b/doc/troubleshooting/where-to-look.md @@ -0,0 +1,249 @@ +@title = 'Where to look for errors' +@nav_title = 'Where to look' +@toc = true + + +General +======= + +* Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`. + +Webapp +====== + +Places to look for errors +------------------------- + +* `/var/log/apache2/error.log` +* `/srv/leap/webapp/log/production.log` +* `/var/log/syslog` (watch out for stunnel issues) +* `/var/log/leap/*` + +Is haproxy ok ? +--------------- + + + curl -s -X GET "http://127.0.0.1:4096" + +Is couchdb accessible through stunnel ? +--------------------------------------- + +* Depending on how many couch nodes you have, increase the port for every test + (see /etc/haproxy/haproxy.cfg for the server/port mapping): + + + curl -s -X GET "http://127.0.0.1:4000" + curl -s -X GET "http://127.0.0.1:4001" + ... + + +Check couchdb acl as admin +-------------------------- + + mkdir /etc/couchdb + cat /srv/leap/webapp/config/couchdb.yml.admin # see username and password + echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc + chmod 600 /etc/couchdb/couchdb-admin.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096" + curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs" + +Check couchdb acl as unpriviledged user +--------------------------------------- + + cat /srv/leap/webapp/config/couchdb.yml # see username and password + echo "machine 127.0.0.1 login webapp password <PASSWORD>" > /etc/couchdb/couchdb-webapp.netrc + chmod 600 /etc/couchdb/couchdb-webapp.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096" + curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs" + + +Check client config files +------------------------- + + https://example.net/provider.json + https://example.net/1/config/smtp-service.json + https://example.net/1/config/soledad-service.json + https://example.net/1/config/eip-service.json + + +Soledad +======= + + /var/log/soledad.log + + +Couchdb +======= + +Places to look for errors +------------------------- + +* `/opt/bigcouch/var/log/bigcouch.log` +* `/var/log/syslog` (watch out for stunnel issues) + + + +Bigcouch membership +------------------- + +* All nodes configured for the provider should appear here: + +<pre> + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5986/nodes/_all_docs' +</pre> + +* All configured nodes should show up under "cluster_nodes", and the ones online and communicating with each other should appear under "all_nodes". This example output shows the configured cluster nodes `couch1.bitmask.net` and `couch2.bitmask.net`, but `couch2.bitmask.net` is currently not accessible from `couch1.bitmask.net` + + +<pre> + curl -s --netrc-file /etc/couchdb/couchdb.netrc 'http://127.0.0.1:5984/_membership' + {"all_nodes":["bigcouch@couch1.bitmask.net"],"cluster_nodes":["bigcouch@couch1.bitmask.net","bigcouch@couch2.bitmask.net"]} +</pre> + +* Sometimes a `/etc/init.d/bigcouch restart` on all nodes is needed, to register new nodes + +Databases +--------- + +* Following output shows all neccessary DBs that should be present. Note that the `user-0123456....` DBs are the data stores for a particular user. + +<pre> + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs' + ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"] +</pre> + + + + +Design Documents +---------------- + +* Is User `_design doc` available ? + + +<pre> + curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET "http://127.0.0.1:5984/users/_design/User" +</pre> + +Is couchdb cluster backend accessible through stunnel ? +------------------------------------------------------- + +* Find out how many connections are set up for the couchdb cluster backend: + +<pre> + grep "accept = 127.0.0.1" /etc/stunnel/* +</pre> + + +* Now connect to all of those local endpoints to see if they up. All these tests should return "localhost [127.0.0.1] 4000 (?) open" + +<pre> + nc -v 127.0.0.1 4000 + nc -v 127.0.0.1 4001 + ... +</pre> + + +MX +== + +Places to look for errors +------------------------- + +* `/var/log/mail.log` +* `/var/log/leap_mx.log` +* `/var/log/syslog` (watch out for stunnel issues) + +Is couchdb accessible through stunnel ? +--------------------------------------- + +* Depending on how many couch nodes you have, increase the port for every test + (see /etc/haproxy/haproxy.cfg for the server/port mapping): + + + curl -s -X GET "http://127.0.0.1:4000" + curl -s -X GET "http://127.0.0.1:4001" + ... + +Query leap-mx +------------- + +* for useraccount + + +<pre> + postmap -v -q "joe@dev.bitmask.net" tcp:localhost:2244 + ... + postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net + postmap: dict_tcp_lookup: recv: 200 + ... +</pre> + +* for mailalias + + +<pre> + postmap -v -q "joe@dev.bitmask.net" tcp:localhost:4242 + ... + postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net + postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a + postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a + f01bc1c70de7d7d80bc1ad77d987e73a + ... +</pre> + + +Check couchdb acl as unpriviledged user +--------------------------------------- + + + + cat /etc/leap/mx.conf # see username and password + echo "machine 127.0.0.1 login leap_mx password <PASSWORD>" > /etc/couchdb/couchdb-leap_mx.netrc + chmod 600 /etc/couchdb/couchdb-leap_mx.netrc + + curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs" # pick one "user-<hash>" db + curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c" + + +* you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes + + +Mailspool +--------- + +* Any file in the leap_mx mailspool longer for a few seconds ? + + + +<pre> + ls -la /var/mail/vmail/Maildir/cur/ +</pre> + +* Any mails in postfix mailspool longer than a few seconds ? + +<pre> + mailq +</pre> + + + +Testing mail delivery +--------------------- + + swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 25 + swaks -f varac@cdev.bitmask.net -t varac@cdev.bitmask.net -s chipmonk.cdev.bitmask.net --port 465 --tlsc + swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 587 --tls + + +VPN +=== + +Places to look for errors +------------------------- + +* `/var/log/syslog` (watch out for openvpn issues) + + diff --git a/doc/tutorials/configure-provider.md b/doc/tutorials/configure-provider.md new file mode 100644 index 00000000..969d541b --- /dev/null +++ b/doc/tutorials/configure-provider.md @@ -0,0 +1,31 @@ +@title = 'Configure provider tutorial' +@nav_title = 'Configure Provider' +@summary = 'Explore how to configure your provider after the initial setup' + + +Edit provider.json configuration +-------------------------------------- + +There are a few required settings in provider.json. At a minimum, you must have: + + { + "domain": "example.org", + "name": "Example", + "contacts": { + "default": "email1@example.org" + } + } + +For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: + + $ leap inspect provider.json + + +Examine Certs +============= + +To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: + + $ leap inspect files/ca/ca.crt + +NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. diff --git a/doc/tutorials/en.haml b/doc/tutorials/en.haml new file mode 100644 index 00000000..1c73fc0f --- /dev/null +++ b/doc/tutorials/en.haml @@ -0,0 +1,4 @@ +- @nav_title = "Tutorials" +- @title = "Platform Tutorials" + += child_summaries
\ No newline at end of file diff --git a/doc/tutorials/quick-start.md b/doc/tutorials/quick-start.md new file mode 100644 index 00000000..a92cc9da --- /dev/null +++ b/doc/tutorials/quick-start.md @@ -0,0 +1,385 @@ +@title = 'Quick Start Tutorial' +@nav_title = 'Quick Start Tutorial' +@summary = 'This tutorial walks you through the initial process of creating and deploying a minimal service provider running the LEAP Platform. This Quick Start guide will guide you through building a three node OpenVPN provider.' + + +Our goal +------------------ + +We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity or geographical diversity, or more webapp nodes to increase availability (at the moment, a single couchdb and single webapp server are all that is supported, and performance wise, are more than enough for most usage, since they are only lightly used). At the moment, we strongly advise only have one couchdb server for stability purposes. + +Our goal is something like this: + + $ leap list + NODES SERVICES TAGS + cheetah couchdb production + wildebeest webapp production + ostrich openvpn production + +NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. + +Requirements +------------ + +In order to complete this Quick Start, you will need a few things: + +* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 20G of disk space to each node for the system, after this process is completed, you will have used less than 10% of that disk space. If you allocate 2 CPUs and 8G of memory to each node, that should be more than enough to begin with. +* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys +* You will need four different IPs. Each node gets a primary IP, and the OpenVPN gateway additionally needs a gateway IP. +* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. +* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes +* Your machines will need to be connected to the internet, and not behind a restrictive firewall. +* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The `leap` command will login to your servers and configure the services. +* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: + +* Install pre-requisites +* Install the LEAP command-line utility +* Check out the LEAP platform +* Create a provider and its certificates +* Setup the provider's nodes and the services that will reside on those nodes +* Initialize the nodes +* Deploy the LEAP platform to the nodes +* Test that things worked correctly +* Some additional commands + +We will walk you through each of these steps. + + +Prepare your environment +======================== + +There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. + +Install pre-requisites +-------------------------------- + +*Debian & Ubuntu* + +Install core prerequisites: + + $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 + +<!-- +*Mac OS* + +1. Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed). +--> + +NOTE: leap_cli requires ruby 1.9 or later. + + +Install the LEAP command-line utility +------------------------------------------------- + +Install the `leap` command from rubygems.org: + + $ sudo gem install leap_cli + +Alternately, you can install `leap` from source: + + $ git clone https://leap.se/git/leap_cli + $ cd leap_cli + $ rake build + $ sudo rake install + +You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this: + + $ rake install + # watch out for the directory leap is installed to, then i.e. + $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap + +With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH. + +If you have successfully installed the `leap` command, then you should be able to do the following: + + $ leap --help + +This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. + +Check out the platform +-------------------------- + +The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. + +First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules: + + $ mkdir ~/leap + $ cd ~/leap + $ git clone https://leap.se/git/leap_platform.git + $ cd leap_platform + $ git submodule sync; git submodule update --init + + +Provider Setup +============== + +A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. + + $ mkdir -p ~/leap/example + +Bootstrap the provider +----------------------- + +Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. + + $ cd ~/leap/example + $ leap new . + +NOTES: + . make sure you include that trailing dot! + +The `leap new` command will ask you for several required values: + +* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". +* name: The name of your service provider (we use "Example"). +* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). +* platform: The directory where you have a copy of the `leap_platform` git repository checked out. + +You could also have passed these configuration options on the command-line, like so: + + $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform . + +You may want to poke around and see what is in the files we just created. For example: + + $ cat provider.json + +Optionally, commit your provider directory using the version control software you fancy. For example: + + $ git init + $ git add . + $ git commit -m "initial provider commit" + +Now add yourself as a privileged sysadmin who will have access to deploy to servers: + + $ leap add-user --self + +NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). + +Create provider certificates +---------------------------- + +Create two certificate authorities, one for server certs and one for client +certs (note: you only need to run this one command to get both): + + $ leap cert ca + +Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) + + $ leap cert csr + +To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so: + + $ leap inspect files/ca/ca.crt + +Create the Diffie-Hellman parameters file, needed for forward secret OpenVPN ciphers: + + $ leap cert dh + +NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed. + + +Edit provider.json configuration +-------------------------------------- + +There are a few required settings in provider.json. At a minimum, you must have: + + { + "domain": "example.org", + "name": "Example", + "contacts": { + "default": "email1@example.org" + } + } + +For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults: + + $ leap inspect provider.json + + +Setup the provider's nodes and services +--------------------------------------- + +A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. + +Create a node, with the service "webapp": + + $ leap node add wildebeest ip_address:x.x.x.w services:webapp tags:production + +NOTE: replace x.x.x.w with the actual IP address of this node + +This created a node configuration file in `nodes/wildebeest.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. + +The web application and the VPN nodes require a database, so lets create the database server node: + + $ leap node add cheetah ip_address:x.x.x.x services:couchdb tags:production + +NOTE: replace x.x.x.x with the actual IP address of this node + +Now we need the OpenVPN gateway, so lets create that node: + + $ leap node add ostrich ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production + +NOTE: replace x.x.x.y with the IP address of the machine, and x.x.x.z with the second IP. openvpn gateways must be assigned two IP addresses, one for the host itself and one for the openvpn gateway. We do this to prevent incoming and outgoing VPN traffic on the same IP. Without this, the client might send some traffic to other VPN users in the clear, bypassing the VPN. + + +Setup DNS +--------- + +Now that you have the nodes configured, you should create the DNS entries for these nodes. + +Set up your DNS with these hostnames: + + $ leap list --print ip_address,domain.full,dns.aliases + cheetah x.x.x.w, cheetah.example.org, null + wildebeest x.x.x.x, wildebeest.example.org, api.example.org + ostrich x.x.x.y, ostrich.example.org, null + +Alternately, you can adapt this zone file snippet: + + $ leap compile zone + +If you cannot edit your DNS zone file, you can still test your provider by adding entries to your local resolver hosts file (`/etc/hosts` for linux): + + x.x.x.w cheetah.example.org + x.x.x.x wildebeest.example.org api.example.org example.org + x.x.x.y ostrich.example.org + +Please don't forget about these entries, they will override DNS queries if you setup your DNS later. + + +Initialize the nodes +-------------------- + +Node initialization only needs to be done once, but there is no harm in doing it multiple times: + + $ leap node init production + +This will initialize all nodes with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server(s). You should only need to do this once. + +If you prefer, you can initalize each node, one at a time: + + $ leap node init wildebeest + $ leap node init cheetah + $ leap node init ostrich + +Deploy the LEAP platform to the nodes +-------------------- + +Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. + +*Important notes:* currently nodes must be deployed in a certain order. The underlying couch database node(s) must be deployed first, and then all other nodes. + + $ leap deploy cheetah + +Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. + +However, to deploy our three-node openvpn setup, we need the database and LEAP web application requires a database to run, so let's deploy to the couchdb and openvpn nodes: + + $ leap deploy wildebeest + $ leap deploy ostrich + + +What is going on here? +-------------------------------------------- + +First, some background terminology: + +* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). +* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + +When you run `leap deploy`, a bunch of things happen, in this order: + +1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. +* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). +* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + +You can run `leap -v2 deploy` to see exactly what commands are being executed. + + +Test that things worked correctly +================================= + +You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway. + +To run troubleshooting tests: + + leap test + +If you want to confirm for yourself that things are working, you can perform the following manual tests. + +### Access the web application + +In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named wildebeest in this example). + +There are a lot of different ways to do this, but one easy way is to modify your `/etc/hosts` file. First, find the IP address of the webapp node: + + $ leap list webapp --print ip_address + +Then modify `/etc/hosts` like so: + + x.x.x.w leap.example.org + +Replacing 'leap.example.org' with whatever you specified as the `domain` in the `leap new` command. + +Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. + +### Use the VPN + +You should be able to simply test that the OpenVPN gateway works properly by doing the following: + + $ leap test init + $ sudo openvpn test/openvpn/production_unlimited.ovpn + +Or, you can use the LEAP client (called "bitmask") to connect to your new provider, create a user and then connect to the VPN. + + +Additional information +====================== + +It is useful to know a few additional things. + +Useful commands +--------------- + +Here are a few useful commands you can run on your new local nodes: + +* `leap ssh wildebeest` -- SSH into node wildebeest (requires `leap node init wildebeest` first). +* `leap list` -- list all nodes. +* `leap list production` -- list only those nodes with the tag 'production' +* `leap list --print ip_address` -- list a particular attribute of all nodes. +* `leap cert update` -- generate new certificates if needed. + +See the full command reference for more information. + +Node filters +------------------------------------------- + +Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. + +A node filter consists of one or more keywords, with an optional "+" before each keyword. + +* keywords can be a node name, a service type, or a tag. +* the "+" before the keyword constructs an AND condition +* otherwise, multiple keywords together construct an OR condition + +Examples: + +* `leap list openvpn` -- list all nodes with service openvpn. +* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. +* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. +* `leap node init ostrich` -- just init the node named ostrich. + +Keep track of your provider configurations +------------------------------------------ + +You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. + +Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. + +What's next +----------------------------------- + +Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. + diff --git a/doc/tutorials/single-node-email.md b/doc/tutorials/single-node-email.md new file mode 100644 index 00000000..b47496b9 --- /dev/null +++ b/doc/tutorials/single-node-email.md @@ -0,0 +1,282 @@ +@title = 'Single node email tutorial' +@nav_title = 'Single node email' +@summary = 'A single node email provider.' + +Quick Start - Single node setup +=============================== + +This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). +We will guide you through building a single node mail provider. + +Our goal +------------------ + +We are going to create a minimal LEAP provider offering Email service. This basic setup can be expanded by adding more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding. + +Our goal is something like this: + + $ leap list + NODES SERVICES TAGS + node1 couchdb, mx, soledad, webapp local + +NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations. + +Requirements +------------ + +In order to complete this Quick Start, you will need a few things: + +* You will need `one real or paravirtualized virtual machine` (Vagrant, KVM, Xen, Openstack, Amazon, …) that have a basic Debian Stable installed. +* You should be able to `SSH into them` remotely, and know their root password, IP addresses and their SSH host keys +* The ability to `create/modify DNS entries` for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`. +* You need to be aware that this process will make changes to your machines, so please be sure that these machines are a basic install with nothing configured or running for other purposes +* Your machines will need to be connected to the internet, and not behind a restrictive firewall. +* You should `work locally on your laptop/workstation` (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services. +* You should do everything described below as an `unprivileged user`, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly. + +All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following: + +* Install pre-requisites +* Install the LEAP command-line utility +* Check out the LEAP platform +* Create a provider and its certificates +* Setup the provider's node and the services that will reside on it +* Initialize the node +* Deploy the LEAP platform to the node +* Test that things worked correctly +* Some additional commands + +We will walk you through each of these steps. + + +Prepare your environment +======================== + +There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform. + +Install pre-requisites +-------------------------------- + +*Debian & Ubuntu* + +Install core prerequisites: + + $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 + +*Mac OS* + +Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed). + + +NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9. + + +Install the LEAP command-line utility +------------------------------------------------- + +Install the LEAP command-line utility (leap_cli) from rubygems.org: + + $ sudo gem install leap_cli + +Alternately, you can install `leap_cli` from source, please refer to https://leap.se/git/leap_cli/README.md. + +If you have successfully installed `leap_cli`, then you should be able to do the following: + + $ leap --help + +This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards. + + +Provider Setup +============== + +A provider instance is a directory tree that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'. + + $ mkdir -p ~/leap/example + +Bootstrap the provider +----------------------- + +Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. + + $ cd ~/leap/example + $ leap new . + +NOTES: + . make sure you include that trailing dot! + +The `leap new` command will ask you for several required values: + +* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org". +* name: The name of your service provider (we use "Example"). +* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc). +* platform: The directory where you either have a copy of the `leap_platform` git repository already checked out, or where `leap_cli` should download it too. You could just accept the suggested path for this example. + The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform. + +These steps should be sufficient for this example. If you want to configure your provider further or like to examine the files, please refer to the [Configure Provider](configure-provider) section. + +Add Users who will have administrative access +--------------------------------------------- + +Now add yourself as a privileged sysadmin who will have access to deploy to servers: + + $ leap add-user --self + +NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example). + + +Create provider certificates +---------------------------- + +Create two certificate authorities, one for server certs and one for client +certs (note: you only need to run this one command to get both): + + $ leap cert ca + +Create a temporary cert for your main domain (you should replace with a real commercial cert at some point) + + $ leap cert csr + + +Setup the provider's node and services +-------------------------------------- + +A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information. + +Create a node, with `all the services needed for Email: "couchdb", "mx", "soledad" and "webapp"` + + $ leap node add node1 ip_address:x.x.x.w services:couchdb,mx,soledad,webapp tags:production + +NOTE: replace x.x.x.w with the actual IP address of this node + +This created a node configuration file in `nodes/node1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure. + +Initialize the nodes +-------------------- + +Node initialization only needs to be done once, but there is no harm in doing it multiple times: + + $ leap node init node1 + +This will initialize the node "node1". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server. You should only need to do this once. + + +Deploy the LEAP platform to the nodes +-------------------- + +Now you should deploy the platform recipes to the node. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine. + + $ leap deploy + +Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. + + +Setup DNS +--------- + +Now that you have the node configured, you should create the DNS entrie for this node. + +Set up your DNS with these hostnames: + + $ leap list --print ip_address,domain.full,dns.aliases + node1 x.x.x.w, node1.example.org, example.org, api.example.org, nicknym.example.org + +Alternately, you can adapt this zone file snippet: + + $ leap compile zone + +If you cannot edit your DNS zone file, you can still test your provider by adding this entry to your local resolver hosts file (`/etc/hosts` for linux): + + x.x.x.w node1.example.org example.org api.example.org nicknym.example.org + +Please don't forget about these entries, they will override DNS queries if you setup your DNS later. + + +What is going on here? +-------------------------------------------- + +First, some background terminology: + +* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes). +* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node. + +When you run `leap deploy`, a bunch of things happen, in this order: + +1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file. +* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc). +* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster. + +You can run `leap -v2 deploy` to see exactly what commands are being executed. + +<!-- See [under the hood](under-the-hood) for more details. --> + + +Test that things worked correctly +================================= + +You should now one machine with the LEAP platform email service deployed to it. + + +Access the web application +-------------------------------------------- + +In order to connect to the web application in your browser, you need to point your domain at the IP address of your new node. + +Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login. + +Testing with leap_cli +--------------------- + +Use the test command to run a set of different tests: + + leap test + + +Additional information +====================== + +It is useful to know a few additional things. + +Useful commands +--------------- + +Here are a few useful commands you can run on your new local nodes: + +* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first). +* `leap list` -- list all nodes. +* `leap list production` -- list only those nodes with the tag 'production' +* `leap list --print ip_address` -- list a particular attribute of all nodes. +* `leap cert update` -- generate new certificates if needed. + +See the full command reference for more information. + +Node filters +------------------------------------------- + +Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes. + +A node filter consists of one or more keywords, with an optional "+" before each keyword. + +* keywords can be a node name, a service type, or a tag. +* the "+" before the keyword constructs an AND condition +* otherwise, multiple keywords together construct an OR condition + +Examples: + +* `leap list openvpn` -- list all nodes with service openvpn. +* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production. +* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes. +* `leap node init vpn1` -- just init the node named vpn1. + +Keep track of your provider configurations +------------------------------------------ + +You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted. + +Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read. + +What's next +----------------------------------- + +Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure. + diff --git a/hiera.yaml b/hiera.yaml new file mode 100644 index 00000000..3ff857b8 --- /dev/null +++ b/hiera.yaml @@ -0,0 +1,6 @@ +--- +:backends: yaml +:yaml: + :datadir: /var/lib/hiera +:hierarchy: common +:logger: console diff --git a/leap-debug-remote.sh b/leap-debug-remote.sh new file mode 100644 index 00000000..7f9c6945 --- /dev/null +++ b/leap-debug-remote.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# debug script to be run on remote servers + +regexp='(leap|stunnel|couch|soledad|haproxy)' + +find /etc/leap/ + +echo + +ls -la /srv/leap/ + +echo + + +dpkg -l | egrep "$regexp" + +echo + +ps aux|egrep "$regexp" + +echo + +cat /etc/hosts diff --git a/lib/leap_cli/commands/README b/lib/leap_cli/commands/README new file mode 100644 index 00000000..bec78179 --- /dev/null +++ b/lib/leap_cli/commands/README @@ -0,0 +1,11 @@ +This directory contains ruby source files that define the available sub- +commands of the `leap` executable. + +For example, the command: + + leap compile + +Lives in lib/leap_cli/commands/init.rb + +These files use a DSL (called GLI) for defining command suites. +See https://github.com/davetron5000/gli for more information. diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb new file mode 100644 index 00000000..1b311eee --- /dev/null +++ b/lib/leap_cli/commands/ca.rb @@ -0,0 +1,541 @@ +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' +require 'digest/md5' + +module LeapCli; module Commands + + desc "Manage X.509 certificates" + command :cert do |cert| + + cert.desc 'Creates two Certificate Authorities (one for validating servers and one for validating clients).' + cert.long_desc 'See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`.' + cert.command :ca do |ca| + ca.action do |global_options,options,args| + assert_config! 'provider.ca.name' + generate_new_certificate_authority(:ca_key, :ca_cert, provider.ca.name) + generate_new_certificate_authority(:client_ca_key, :client_ca_cert, provider.ca.name + ' (client certificates only!)') + end + end + + cert.desc 'Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.' + cert.long_desc 'This command will a generate new certificate for a node if some value in the node has changed ' + + 'that is included in the certificate (like hostname or IP address), or if the old certificate will be expiring soon. ' + + 'Sometimes, you might want to force the generation of a new certificate, ' + + 'such as in the cases where you have changed a CA parameter for server certificates, like bit size or digest hash. ' + + 'In this case, use --force. If <node-filter> is empty, this command will apply to all nodes.' + cert.arg_name 'FILTER' + cert.command :update do |update| + update.switch 'force', :desc => 'Always generate new certificates', :negatable => false + update.action do |global_options,options,args| + update_certificates(manager.filter!(args), options) + end + end + + cert.desc 'Creates a Diffie-Hellman parameter file, needed for forward secret OpenVPN ciphers.' # (needed for server-side of some TLS connections) + cert.command :dh do |dh| + dh.action do |global_options,options,args| + long_running do + if cmd_exists?('certtool') + log 0, 'Generating DH parameters (takes a long time)...' + output = assert_run!('certtool --generate-dh-params --sec-param high') + output.sub! /.*(-----BEGIN DH PARAMETERS-----.*-----END DH PARAMETERS-----).*/m, '\1' + output << "\n" + write_file!(:dh_params, output) + else + log 0, 'Generating DH parameters (takes a REALLY long time)...' + output = OpenSSL::PKey::DH.generate(3248).to_pem + write_file!(:dh_params, output) + end + end + end + end + + # + # hints: + # + # inspect CSR: + # openssl req -noout -text -in files/cert/x.csr + # + # generate CSR with openssl to see how it compares: + # openssl req -sha256 -nodes -newkey rsa:2048 -keyout example.key -out example.csr + # + # validate a CSR: + # http://certlogik.com/decoder/ + # + # nice details about CSRs: + # http://www.redkestrel.co.uk/Articles/CSR.html + # + cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." + cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ + "The properties used for this CSR come from `provider.ca.server_certificates`, "+ + "but may be overridden here." + cert.command :csr do |csr| + csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' + csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." + csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." + csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." + csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." + csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." + csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." + csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" + csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" + csr.action do |global_options,options,args| + assert_config! 'provider.domain' + assert_config! 'provider.name' + assert_config! 'provider.default_language' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + domain = options[:domain] || provider.domain + + unless global_options[:force] + assert_files_missing! [:commercial_key, domain], [:commercial_csr, domain], + :msg => 'If you really want to create a new key and CSR, remove these files first or run with --force.' + end + + server_certificates = provider.ca.server_certificates + + # RSA key + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = (options[:bits] || server_certificates.bit_size).to_i + log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) + write_file! [:commercial_key, domain], keypair.private_key.to_pem + end + + # CSR + dn = CertificateAuthority::DistinguishedName.new + dn.common_name = domain + dn.organization = options[:organization] || provider.name[provider.default_language] + dn.ou = options[:organizational_unit] # optional + dn.email_address = options[:email] # optional + dn.country = options[:country] || server_certificates['country'] # optional + dn.state = options[:state] || server_certificates['state'] # optional + dn.locality = options[:locality] || server_certificates['locality'] # optional + + digest = options[:digest] || server_certificates.digest + log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = create_csr(dn, keypair, digest) + request = csr.to_x509_csr + write_file! [:commercial_csr, domain], csr.to_pem + end + + # Sign using our own CA, for use in testing but hopefully not production. + # It is not that commerical CAs are so secure, it is just that signing your own certs is + # a total drag for the user because they must click through dire warnings. + #if options[:sign] + log :generating, "self-signed x509 server certificate for testing purposes" do + cert = csr.to_cert + cert.serial_number.number = cert_serial_number(domain) + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.parent = ca_root + cert.sign! domain_test_signing_profile + write_file! [:commercial_cert, domain], cert.to_pem + log "please replace this file with the real certificate you get from a CA using #{Path.relative_path([:commercial_csr, domain])}" + end + #end + + # FAKE CA + unless file_exists? :commercial_ca_cert + log :using, "generated CA in place of commercial CA for testing purposes" do + write_file! :commercial_ca_cert, read_file!(:ca_cert) + log "please also replace this file with the CA cert from the commercial authority you use." + end + end + end + end + end + + protected + + # + # will generate new certificates for the specified nodes, if needed. + # + def update_certificates(nodes, options={}) + assert_files_exist! :ca_cert, :ca_key, :msg => 'Run `leap cert ca` to create them' + assert_config! 'provider.ca.server_certificates.bit_size' + assert_config! 'provider.ca.server_certificates.digest' + assert_config! 'provider.ca.server_certificates.life_span' + assert_config! 'common.x509.use' + + nodes.each_node do |node| + warn_if_commercial_cert_will_soon_expire(node) + if !node.x509.use + remove_file!([:node_x509_key, node.name]) + remove_file!([:node_x509_cert, node.name]) + elsif options[:force] || cert_needs_updating?(node) + generate_cert_for_node(node) + end + end + end + + private + + def generate_new_certificate_authority(key_file, cert_file, common_name) + assert_files_missing! key_file, cert_file + assert_config! 'provider.ca.name' + assert_config! 'provider.ca.bit_size' + assert_config! 'provider.ca.life_span' + + root = CertificateAuthority::Certificate.new + + # set subject + root.subject.common_name = common_name + possible = ['country', 'state', 'locality', 'organization', 'organizational_unit', 'email_address'] + provider.ca.keys.each do |key| + if possible.include?(key) + root.subject.send(key + '=', provider.ca[key]) + end + end + + # set expiration + root.not_before = yesterday + root.not_after = yesterday_advance(provider.ca.life_span) + + # generate private key + root.serial_number.number = 1 + root.key_material.generate_key(provider.ca.bit_size) + + # sign self + root.signing_entity = true + root.parent = root + root.sign!(ca_root_signing_profile) + + # save + write_file!(key_file, root.key_material.private_key.to_pem) + write_file!(cert_file, root.to_pem) + end + + # + # returns true if the certs associated with +node+ need to be regenerated. + # + def cert_needs_updating?(node) + if !file_exists?([:node_x509_cert, node.name], [:node_x509_key, node.name]) + return true + else + cert = load_certificate_file([:node_x509_cert, node.name]) + if !created_by_authority?(cert, ca_root) + log :updating, "cert for node '#{node.name}' because it was signed by an old CA root cert." + return true + end + if cert.not_after < Time.now.advance(:months => 2) + log :updating, "cert for node '#{node.name}' because it will expire soon" + return true + end + if cert.subject.common_name != node.domain.full + log :updating, "cert for node '#{node.name}' because domain.full has changed (was #{cert.subject.common_name}, now #{node.domain.full})" + return true + end + cert.openssl_body.extensions.each do |ext| + if ext.oid == "subjectAltName" + ips = [] + dns_names = [] + ext.value.split(",").each do |value| + value.strip! + ips << $1 if value =~ /^IP Address:(.*)$/ + dns_names << $1 if value =~ /^DNS:(.*)$/ + end + dns_names.sort! + if ips.first != node.ip_address + log :updating, "cert for node '#{node.name}' because ip_address has changed (from #{ips.first} to #{node.ip_address})" + return true + elsif dns_names != dns_names_for_node(node) + log :updating, "cert for node '#{node.name}' because domain name aliases have changed\n from: #{dns_names.inspect}\n to: #{dns_names_for_node(node).inspect})" + return true + end + end + end + end + return false + end + + def created_by_authority?(cert, ca) + authority_key_id = cert.extensions["authorityKeyIdentifier"].identifier.sub(/^keyid:/, '') + authority_key_id == public_key_id_for_ca(ca) + end + + # calculate the "key id" for a root CA, that matches the value + # Authority Key Identifier in the x509 extensions of a cert. + def public_key_id_for_ca(ca_cert) + @ca_key_ids ||= {} + @ca_key_ids[ca_cert.object_id] ||= begin + pubkey = ca_cert.key_material.public_key + seq = OpenSSL::ASN1::Sequence([ + OpenSSL::ASN1::Integer.new(pubkey.n), + OpenSSL::ASN1::Integer.new(pubkey.e) + ]) + Digest::SHA1.hexdigest(seq.to_der).upcase.scan(/../).join(':') + end + end + + def warn_if_commercial_cert_will_soon_expire(node) + dns_names_for_node(node).each do |domain| + if file_exists?([:commercial_cert, domain]) + cert = load_certificate_file([:commercial_cert, domain]) + path = Path.relative_path([:commercial_cert, domain]) + if cert.not_after < Time.now.utc + log :error, "the commercial certificate '#{path}' has EXPIRED! " + + "You should renew it with `leap cert csr --domain #{domain}`." + elsif cert.not_after < Time.now.advance(:months => 2) + log :warning, "the commercial certificate '#{path}' will expire soon (#{cert.not_after}). "+ + "You should renew it with `leap cert csr --domain #{domain}`." + end + end + end + end + + def generate_cert_for_node(node) + return if node.x509.use == false + + cert = CertificateAuthority::Certificate.new + + # set subject + cert.subject.common_name = node.domain.full + cert.serial_number.number = cert_serial_number(node.domain.full) + + # set expiration + cert.not_before = yesterday + cert.not_after = yesterday_advance(provider.ca.server_certificates.life_span) + + # generate key + cert.key_material.generate_key(provider.ca.server_certificates.bit_size) + + # sign + cert.parent = ca_root + cert.sign!(server_signing_profile(node)) + + # save + write_file!([:node_x509_key, node.name], cert.key_material.private_key.to_pem) + write_file!([:node_x509_cert, node.name], cert.to_pem) + end + + # + # yields client key and cert suitable for testing + # + def generate_test_client_cert(prefix=nil) + cert = CertificateAuthority::Certificate.new + cert.serial_number.number = cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.not_before = yesterday + cert.not_after = yesterday.advance(:years => 1) + cert.key_material.generate_key(1024) # just for testing, remember! + cert.parent = client_ca_root + cert.sign! client_test_signing_profile + yield cert.key_material.private_key.to_pem, cert.to_pem + end + + # + # creates a CSR and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def create_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + + def ca_root + @ca_root ||= begin + load_certificate_file(:ca_cert, :ca_key) + end + end + + def client_ca_root + @client_ca_root ||= begin + load_certificate_file(:client_ca_cert, :client_ca_key) + end + end + + def load_certificate_file(crt_file, key_file=nil, password=nil) + crt = read_file!(crt_file) + openssl_cert = OpenSSL::X509::Certificate.new(crt) + cert = CertificateAuthority::Certificate.from_openssl(openssl_cert) + if key_file + key = read_file!(key_file) + cert.key_material.private_key = OpenSSL::PKey::RSA.new(key, password) + end + return cert + end + + def ca_root_signing_profile + { + "extensions" => { + "basicConstraints" => {"ca" => true}, + "keyUsage" => { + "usage" => ["critical", "keyCertSign"] + }, + "extendedKeyUsage" => { + "usage" => [] + } + } + } + end + + # + # For keyusage, openvpn server certs can have keyEncipherment or keyAgreement. + # Web browsers seem to break without keyEncipherment. + # For now, I am using digitalSignature + keyEncipherment + # + # * digitalSignature -- for (EC)DHE cipher suites + # "The digitalSignature bit is asserted when the subject public key is used + # with a digital signature mechanism to support security services other + # than certificate signing (bit 5), or CRL signing (bit 6). Digital + # signature mechanisms are often used for entity authentication and data + # origin authentication with integrity." + # + # * keyEncipherment ==> for plain RSA cipher suites + # "The keyEncipherment bit is asserted when the subject public key is used for + # key transport. For example, when an RSA key is to be used for key management, + # then this bit is set." + # + # * keyAgreement ==> for used with DH, not RSA. + # "The keyAgreement bit is asserted when the subject public key is used for key + # agreement. For example, when a Diffie-Hellman key is to be used for key + # management, then this bit is set." + # + # digest options: SHA512, SHA256, SHA1 + # + def server_signing_profile(node) + { + "digest" => provider.ca.server_certificates.digest, + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth", "clientAuth"] + }, + "subjectAltName" => { + "ips" => [node.ip_address], + "dns_names" => dns_names_for_node(node) + } + } + } + end + + # + # This is used when signing the main cert for the provider's domain + # with our own CA (for testing purposes). Typically, this cert would + # be purchased from a commercial CA, and not signed this way. + # + def domain_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature", "keyEncipherment"] + }, + "extendedKeyUsage" => { + "usage" => ["serverAuth"] + } + } + } + end + + # + # This is used when signing a dummy client certificate that is only to be + # used for testing. + # + def client_test_signing_profile + { + "digest" => "SHA256", + "extensions" => { + "keyUsage" => { + "usage" => ["digitalSignature"] + }, + "extendedKeyUsage" => { + "usage" => ["clientAuth"] + } + } + } + end + + def dns_names_for_node(node) + names = [node.domain.internal, node.domain.full] + if node['dns'] && node.dns['aliases'] && node.dns.aliases.any? + names += node.dns.aliases + end + names.compact! + names.sort! + names.uniq! + return names + end + + # + # For cert serial numbers, we need a non-colliding number less than 160 bits. + # md5 will do nicely, since there is no need for a secure hash, just a short one. + # (md5 is 128 bits) + # + def cert_serial_number(domain_name) + Digest::MD5.hexdigest("#{domain_name} -- #{Time.now}").to_i(16) + end + + # + # for the random common name, we need a text string that will be unique across all certs. + # ruby 1.8 doesn't have a built-in uuid generator, or we would use SecureRandom.uuid + # + def random_common_name(domain_name) + cert_serial_number(domain_name).to_s(36) + end + + # prints CertificateAuthority::DistinguishedName fields + def print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + + ## + ## TIME HELPERS + ## + ## note: we use 'yesterday' instead of 'today', because times are in UTC, and some people on the planet + ## are behind UTC. + ## + + def yesterday + t = Time.now - 24*24*60 + Time.utc t.year, t.month, t.day + end + + def yesterday_advance(string) + number, unit = string.split(' ') + unless ['years', 'months', 'days', 'hours', 'minutes'].include? unit + bail!("The time property '#{string}' is missing a unit (one of: years, months, days, hours, minutes).") + end + unless number.to_i.to_s == number + bail!("The time property '#{string}' is missing a number.") + end + yesterday.advance(unit.to_sym => number.to_i) + end + +end; end diff --git a/lib/leap_cli/commands/clean.rb b/lib/leap_cli/commands/clean.rb new file mode 100644 index 00000000..a9afff53 --- /dev/null +++ b/lib/leap_cli/commands/clean.rb @@ -0,0 +1,16 @@ +module LeapCli + module Commands + + desc 'Removes all files generated with the "compile" command.' + command :clean do |c| + c.action do |global_options,options,args| + Dir.glob(path([:hiera, '*'])).each do |file| + remove_file! file + end + remove_file! path(:authorized_keys) + remove_file! path(:known_hosts) + end + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb new file mode 100644 index 00000000..f9079279 --- /dev/null +++ b/lib/leap_cli/commands/compile.rb @@ -0,0 +1,531 @@ +require 'socket' + +module LeapCli + module Commands + + desc "Compile generated files." + command [:compile, :c] do |c| + c.desc 'Compiles node configuration files into hiera files used for deployment.' + c.arg_name 'ENVIRONMENT', :optional => true + c.command :all do |all| + all.action do |global_options,options,args| + environment = args.first + compile_command(environment) + end + end + + c.desc "Prints a DNS zone file for your provider." + c.command :zone do |zone| + zone.action do |global_options, options, args| + compile_command(nil) + compile_zone_file(global_options[:yes] || global_options[:force]) + end + end + + c.desc "Print entries suitable for an /etc/hosts file, useful for testing your provider." + c.command :hosts do |hosts| + hosts.action do |global_options, options, args| + compile_command(nil) + compile_hosts_file + end + end + + c.desc "Compile provider.json bootstrap files for your provider." + c.command 'provider.json' do |provider| + provider.action do |global_options, options, args| + compile_command(nil) + compile_provider_json + end + end + + c.desc "Prints a list of firewall rules. These rules are already "+ + "implemented on each node, but you might want the list of all "+ + "rules in case you also have a restrictive network firewall." + c.command :firewall do |zone| + zone.action do |global_options, options, args| + compile_command(nil) + compile_firewall + end + end + + c.default_command :all + end + + protected + + def compile_command(environment) + if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment + bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." + end + if environment + if manager.environment_names.include?(environment) + compile_hiera_files(manager.filter([environment]), false) + else + bail! "There is no environment named `#{environment}`." + end + else + clean_export = LeapCli.leapfile.environment.nil? + compile_hiera_files(manager.filter, clean_export) + end + if file_exists?(:static_web_readme) + compile_provider_json(environment) + end + end + + # + # a "clean" export of secrets will also remove keys that are no longer used, + # but this should not be done if we are not examining all possible nodes. + # + def compile_hiera_files(nodes, clean_export) + update_certificates(nodes) # \ must come first so that output will + update_compiled_ssh_configs # / get included in compiled hiera files. + sanity_check(nodes) + manager.export_nodes(nodes) + manager.export_secrets(clean_export) + end + + def update_compiled_ssh_configs + generate_monitor_ssh_keys + update_authorized_keys + update_known_hosts + end + + def sanity_check(nodes) + # confirm that every node has a unique ip address + ips = {} + nodes.pick_fields('ip_address').each do |name, ip_address| + if ips.key?(ip_address) + bail! { + log(:fatal_error, "Every node must have its own IP address.") { + log "Nodes `#{name}` and `#{ips[ip_address]}` are both configured with `#{ip_address}`." + } + } + else + ips[ip_address] = name + end + end + # confirm that the IP address of this machine is not also used for a node. + Socket.ip_address_list.each do |addrinfo| + if !addrinfo.ipv4_private? && ips.key?(addrinfo.ip_address) + ip = addrinfo.ip_address + name = ips[ip] + bail! { + log(:fatal_error, "Something is very wrong. The `leap` command must only be run on your sysadmin machine, not on a provider node.") { + log "This machine has the same IP address (#{ip}) as node `#{name}`." + } + } + end + end + end + + ## + ## SSH + ## + + # + # generates a ssh key pair that is used only by remote monitors + # to connect to nodes and run certain allowed commands. + # + # every node has the public monitor key added to their authorized + # keys, and every monitor node has a copy of the private monitor key. + # + def generate_monitor_ssh_keys + priv_key_file = path(:monitor_priv_key) + pub_key_file = path(:monitor_pub_key) + unless file_exists?(priv_key_file, pub_key_file) + ensure_dir(File.dirname(priv_key_file)) + ensure_dir(File.dirname(pub_key_file)) + cmd = %(ssh-keygen -N '' -C 'monitor' -t rsa -b 4096 -f '%s') % priv_key_file + assert_run! cmd + if file_exists?(priv_key_file, pub_key_file) + log :created, priv_key_file + log :created, pub_key_file + else + log :failed, 'to create monitor ssh keys' + end + end + end + + # + # Compiles the authorized keys file, which gets installed on every during init. + # Afterwards, puppet installs an authorized keys file that is generated differently + # (see authorized_keys() in macros.rb) + # + def update_authorized_keys + buffer = StringIO.new + keys = Dir.glob(path([:user_ssh, '*'])) + if keys.empty? + bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." + end + if file_exists?(path(:monitor_pub_key)) + keys << path(:monitor_pub_key) + end + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile).strip.split(" ") + buffer << ssh_type + buffer << " " + buffer << ssh_key + buffer << " " + buffer << Path.relative_path(keyfile) + buffer << "\n" + end + write_file!(:authorized_keys, buffer.string) + end + + # + # generates the known_hosts file. + # + # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow + # for the possibility that the hostnames or ip has changed in the node configuration. + # + def update_known_hosts + buffer = StringIO.new + buffer << "#\n" + buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n" + buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n" + buffer << "#\n" + manager.nodes.keys.sort.each do |node_name| + node = manager.nodes[node_name] + hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') + pub_key = read_file([:node_ssh_pub_key,node.name]) + if pub_key + buffer << [hostnames, pub_key].join(' ') + buffer << "\n" + end + end + write_file!(:known_hosts, buffer.string) + end + + ## + ## provider.json + ## + + # + # generates static provider.json files that can put into place + # (e.g. https://domain/provider.json) for the cases where the + # webapp domain does not match the provider's domain. + # + def compile_provider_json(environments=nil) + webapp_nodes = manager.nodes[:services => 'webapp'] + write_file!(:static_web_readme, STATIC_WEB_README) + environments ||= manager.environment_names + environments.each do |env| + node = webapp_nodes[:environment => env].values.first + if node + env ||= 'default' + write_file!( + [:static_web_provider_json, env], + node['definition_files']['provider'] + ) + write_file!( + [:static_web_htaccess, env], + HTACCESS_FILE % {:min_version => manager.env(env).provider.client_version['min']} + ) + end + end + end + + HTACCESS_FILE = %[ +<Files provider.json> + Header set X-Minimum-Client-Version %{min_version} +</Files> +] + + STATIC_WEB_README = %[ +This directory contains statically rendered copies of the `provider.json` file +used by the client to "bootstrap" configure itself for use with your service +provider. + +There is a separate provider.json file for each environment, although you +should only need 'production/provider.json' or, if you have no environments +configured, 'default/provider.json'. + +To clarify, this is the public `provider.json` file used by the client, not the +`provider.json` file that is used to configure the provider. + +The provider.json file must be available at `https://domain/provider.json` +(unless this provider is included in the list of providers which are pre- +seeded in client). + +This provider.json file can be served correctly in one of three ways: + +(1) If the property webapp.domain is not configured, then the web app will be + installed at https://domain/ and it will handle serving the provider.json file. + +(2) If one or more nodes have the 'static' service configured for the provider's + domain, then these 'static' nodes will correctly serve provider.json. + +(3) Otherwise, you must copy the provider.json file to your web + server and make it available at '/provider.json'. The example htaccess + file shows what header options should be sent by the web server + with the response. + +This directory is needed for method (3), but not for methods (1) or (2). + +This directory has been created by the command `leap compile provider.json`. +Once created, it will be kept up to date everytime you compile. You may safely +remove this directory if you don't use it. +] + + ## + ## + ## ZONE FILE + ## + + def relative_hostname(fqdn, provider) + @domain_regexp ||= /\.?#{Regexp.escape(provider.domain)}$/ + fqdn.sub(@domain_regexp, '') + end + + # + # serial is any number less than 2^32 (4294967296) + # + def compile_zone_file(force=false) + # note: we use the default provider for all nodes, because we use it + # to generate hostnames that are relative to the default domain. + provider = manager.env('default').provider + hosts_seen = {} + lines = [] + + # + # header + # + lines << ZONE_HEADER % { + :domain => provider.domain, + :ns => provider.domain, + :contact => provider.contacts.default.first.sub('@','.'), + :serial => generate_zone_serial + } + + # + # common records + # + lines << ORIGIN_HEADER + # 'A' records for primary domain + manager.nodes[:environment => '!local'].each_node do |node| + if node.dns['aliases'] && node.dns.aliases.include?(provider.domain) + lines << ["@", "IN A #{node.ip_address}"] + end + end + + # NS records + if provider['dns'] && provider.dns['nameservers'] + unless provider.dns.nameservers.is_a?(Array) + # TODO: remove me once we have JSON schema working + bail! {log :error, 'dns.nameservers must be an array' } + end + provider.dns.nameservers.each do |ns| + lines << ["@", "IN NS #{ns}."] + end + elsif !force + log :warning, "Property dns.nameservers is not configured in provider.json." do + log "This will produce a zone file without any NS records." + log "Use --force to skip this warning." + end + return unless agree("Continue? ") + end + + # environment records + manager.environment_names.each do |env| + next if env == 'local' + nodes = manager.nodes[:environment => env] + next unless nodes.any? + spf = nil + dkim = nil + lines << ENV_HEADER % (env.nil? ? 'default' : env) + nodes.each_node do |node| + if node.dns.public + lines << [relative_hostname(node.domain.full, provider), "IN A #{node.ip_address}"] + end + if node.dns['aliases'] + node.dns.aliases.each do |host_alias| + if host_alias != node.domain.full && host_alias != provider.domain + lines << [relative_hostname(host_alias, provider), "IN A #{node.ip_address}"] + end + end + end + if node.services.include? 'mx' + mx_domain = relative_hostname(node.domain.full_suffix, provider) + lines << [mx_domain, "IN MX 10 #{relative_hostname(node.domain.full, provider)}"] + spf ||= [mx_domain, spf_record(node)] + dkim ||= dkim_record(node, provider) + end + end + lines << spf if spf + lines << dkim if dkim + end + + # print the lines + max_width = lines.inject(0) {|max, line| line.is_a?(Array) ? [max, line[0].length].max : max} + max_width = [max_width, 24].min + lines.each do |host, line| + if line.nil? + puts(host) + else + host = '@' if host == '' + puts("%-#{max_width}s %s" % [host, line]) + end + end + end + + # + # outputs entries suitable for an /etc/hosts file + # + def compile_hosts_file + manager.environment_names.each do |env| + nodes = manager.nodes[:environment => env] + next unless nodes.any? + puts + puts "## environment '#{env || 'default'}'" + nodes.each do |name, node| + puts "%s %s" % [ + node.ip_address, + [name, node.get('domain.full'), node.get('dns.aliases')].compact.join(' ') + ] + end + end + end + + private + + # + # allow mail from any mx node, plus the webapp nodes. + # + # TODO: ipv6 + # + def spf_record(node) + ips = node.nodes_like_me['services' => 'webapp'].values.collect {|n| + "ip4:" + n.ip_address + } + # TXT strings may not be longer than 255 characters, although + # you can chain multiple strings together. + strings = "v=spf1 MX #{ips.join(' ')} -all".scan(/.{1,255}/).join('" "') + %(IN TXT "#{strings}") + end + + # + # for example: + # + # selector._domainkey IN TXT "v=DKIM1;h=sha256;k=rsa;s=email;p=MIGfMA0GCSq...GSIb3DQ" + # + # specification: http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.7.4 + # + def dkim_record(node, provider) + # PEM encoded public key (base64), without the ---PUBLIC KEY--- armor parts. + assert_files_exist! :dkim_pub_key + dkim_pub_key = Path.named_path(:dkim_pub_key) + public_key = File.readlines(dkim_pub_key).grep(/^[^\-]+/).join + + host = relative_hostname( + node.mx.dkim.selector + "._domainkey." + node.domain.full_suffix, + provider) + + attrs = [ + "v=DKIM1", + "h=sha256", + "k=rsa", + "s=email", + "p=" + public_key + ] + + return [host, "IN TXT " + txt_wrap(attrs.join(';'))] + end + + # + # DNS TXT records cannot be longer than 255 characters. + # + # However, multiple responses will be concatenated together. + # It looks like this: + # + # IN TXT "v=spf1 .... first" "second string..." + # + def txt_wrap(str) + '"' + str.scan(/.{1,255}/).join('" "') + '"' + end + + # + # For zone serial number, we want something that will be + # different each time you deploy but also will be greater + # than any prior likely serial that was prefixed by the + # year, such as 2016040600. + # + # so, we use time_t of right now, modified with first + # digit incremented by one. + # + # this will work until Time.at(2**32 - 1_000_000_000) + # aka 2074-05-31 04:41:36 UTC. + # + def generate_zone_serial + Time.now.utc.to_i + 1_000_000_000 + end + + ENV_HEADER = %[ +;; +;; ENVIRONMENT %s +;; + +] + + ZONE_HEADER = %[ +;; +;; BIND data file for %{domain} +;; + +$TTL 600 +$ORIGIN %{domain}. + +@ IN SOA %{ns}. %{contact}. ( + %{serial} ; serial + 7200 ; refresh ( 24 hours) + 3600 ; retry ( 2 hours) + 1209600 ; expire (1000 hours) + 600 ) ; minimum ( 2 days) +; +] + + ORIGIN_HEADER = %[ +;; +;; ZONE ORIGIN +;; + +] + + ## + ## FIREWALL + ## + + public + + def compile_firewall + manager.nodes.each_node(&:evaluate) + + rules = [["ALLOW TO", "PORTS", "ALLOW FROM"]] + manager.nodes[:environment => '!local'].values.each do |node| + next unless node['firewall'] + node.firewall.each do |name, rule| + if rule.is_a? Hash + rules << add_rule(rule) + elsif rule.is_a? Array + rule.each do |r| + rules << add_rule(r) + end + end + end + end + + max_to = rules.inject(0) {|max, r| [max, r[0].length].max} + max_port = rules.inject(0) {|max, r| [max, r[1].length].max} + max_from = rules.inject(0) {|max, r| [max, r[2].length].max} + rules.each do |rule| + puts "%-#{max_to}s %-#{max_port}s %-#{max_from}s" % rule + end + end + + private + + def add_rule(rule) + [rule["to"], [rule["port"]].compact.join(','), rule["from"]] + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/db.rb b/lib/leap_cli/commands/db.rb new file mode 100644 index 00000000..5307ac4d --- /dev/null +++ b/lib/leap_cli/commands/db.rb @@ -0,0 +1,86 @@ +module LeapCli; module Commands + + desc 'Database commands.' + command :db do |db| + db.desc 'Destroy one or more databases. If present, limit to FILTER nodes. For example `leap db destroy --db sessions,tokens testing`.' + db.arg_name 'FILTER', :optional => true + db.command :destroy do |destroy| + destroy.flag :db, :arg_name => "DATABASES", :desc => 'Comma separated list of databases to destroy (no space). Use "--db all" to destroy all databases.', :optional => true + destroy.flag :user, :arg_name => "USERS", :desc => 'Comma separated list of usernames. The storage databases for these user(s) will be destroyed.', :optional => true + destroy.action do |global_options,options,args| + dbs = (options[:db]||"").split(',') + users = (options[:user]||"").split(',') + if dbs.empty? && users.empty? + bail!('Either --db or --user is required.') + end + nodes = manager.filter(args) + if nodes.any? + nodes = nodes[:services => 'couchdb'] + end + unless nodes.any? + bail! 'No db nodes selected.' + end + if users.any? + unless global_options[:yes] + say 'You are about to permanently destroy user databases for [%s] for nodes [%s].' % [users.join(', '), nodes.keys.join(', ')] + bail! unless agree("Continue? ") + end + destroy_user_dbs(nodes, users) + elsif dbs.any? + unless global_options[:yes] + if dbs.include?('all') + say 'You are about to permanently destroy all database data for nodes [%s].' % nodes.keys.join(', ') + else + say 'You are about to permanently destroy databases [%s] for nodes [%s].' % [dbs.join(', '), nodes.keys.join(', ')] + end + bail! unless agree("Continue? ") + end + if dbs.include?('all') + destroy_all_dbs(nodes) + else + destroy_dbs(nodes, dbs) + end + say 'You must run `leap deploy` in order to create the databases again.' + end + end + end + end + + private + + def destroy_all_dbs(nodes) + ssh_connect(nodes) do |ssh| + ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "All DBs destroyed" || echo "DBs already destroyed"') + end + end + + def destroy_dbs(nodes, dbs) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + dbs.each do |db| + ssh.run(DESTROY_DB_COMMAND % {:db => db}) + end + end + end + end + + def destroy_user_dbs(nodes, users) + nodes.each_node do |node| + ssh_connect(node) do |ssh| + users.each do |user| + ssh.run(DESTROY_USER_DB_COMMAND % {:user => user}) + end + end + end + end + + DESTROY_DB_COMMAND = %{ +if [ 200 = `curl -ns -w "%%{http_code}" -X GET "127.0.0.1:5984/%{db}" -o /dev/null` ]; then + echo "Result from DELETE /%{db}:" `curl -ns -X DELETE "127.0.0.1:5984/%{db}"`; +else + echo "Skipping db '%{db}': it does not exist or has already been deleted."; +fi +} + + DESTROY_USER_DB_COMMAND = %{/srv/leap/couchdb/scripts/destroy-user-db --username %{user}} +end; end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb new file mode 100644 index 00000000..9dd190ab --- /dev/null +++ b/lib/leap_cli/commands/deploy.rb @@ -0,0 +1,374 @@ +require 'etc' + +module LeapCli + module Commands + + desc 'Apply recipes to a node or set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:deploy, :d] do |c| + + c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.', + :negatable => false + + c.switch :sync, :desc => "Sync files, but don't actually apply recipes.", :negatable => false + + c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + + c.switch :downgrade, :desc => 'Allows deploy to run with an older platform version.', :negatable => false + + c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + + c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', + :arg_name => 'TAG[,TAG]' + + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + + c.action do |global,options,args| + + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args, :disabled => false) + if nodes.size > 1 + say "Deploying to these nodes: #{nodes.keys.join(', ')}" + if !global[:yes] && !agree("Continue? ") + quit! "OK. Bye." + end + end + + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env, global) + end + + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments), false) + + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.log :checking, 'node' do + ssh.leap.check_for_no_deploy + ssh.leap.assert_initialized + end + ssh.leap.log :synching, "configuration files" do + sync_hiera_config(ssh) + sync_support_files(ssh) + end + ssh.leap.log :synching, "puppet manifests" do + sync_puppet_files(ssh) + end + unless options[:sync] + ssh.leap.log :applying, "puppet" do + ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, + :tags => tags(options), + :force => options[:force], + :info => deploy_info, + :downgrade => options[:downgrade] + ) + end + end + end + if !Util.exit_status.nil? && Util.exit_status != 0 + log :warning, "puppet did not finish successfully." + end + end + end + + desc 'Display recent deployment history for a set of nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:history, :h] do |c| + c.flag :port, :desc => 'Override the default SSH port.', + :arg_name => 'PORT' + c.flag :ip, :desc => 'Override the default SSH IP address.', + :arg_name => 'IPADDRESS' + c.switch :last, :desc => 'Show last deploy only', + :negatable => false + c.action do |global,options,args| + if options[:last] == true + lines = 1 + else + lines = 10 + end + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.history(lines) + end + end + end + + private + + def forcible_prompt(forced, msg, prompt) + say(msg) + if forced + log :warning, "continuing anyway because of --force" + else + say "hint: use --force to skip this prompt." + quit!("OK. Bye.") unless agree(prompt) + end + end + + # + # The currently activated provider.json could have loaded some pinning + # information for the platform. If this is the case, refuse to deploy + # if there is a mismatch. + # + # For example: + # + # "platform": { + # "branch": "develop" + # "version": "1.0..99" + # "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" + # } + # + def check_platform_pinning(environment, global_options) + provider = manager.env(environment).provider + return unless provider['platform'] + + if environment.nil? || environment == 'default' + provider_json = 'provider.json' + else + provider_json = 'provider.' + environment + '.json' + end + + # can we have json schema verification already? + unless provider.platform.is_a? Hash + bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) + end + + # check version + if provider.platform['version'] + if !Leap::Platform.version_in_range?(provider.platform.version) + forcible_prompt( + global_options[:force], + "The platform is pinned to a version range of '#{provider.platform.version}' "+ + "by the `platform.version` property in #{provider_json}, but the platform "+ + "(#{Path.platform}) has version #{Leap::Platform.version}.", + "Do you really want to deploy from the wrong version? " + ) + end + end + + # check branch + if provider.platform['branch'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular branch by the `platform.branch` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + unless provider.platform.branch == current_git_branch(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ + "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + + "(for directory '#{Path.platform}')", + "Do you really want to deploy from the wrong branch? " + ) + end + end + + # check commit + if provider.platform['commit'] + if !is_git_directory?(Path.platform) + forcible_prompt( + global_options[:force], + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.", + "Do you really want to deploy anyway? " + ) + end + current_commit = current_git_commit(Path.platform) + Dir.chdir(Path.platform) do + commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but git was not able to find commits in the range specified "+ + "(#{provider.platform.commit}).") + commit_range = commit_range.split("\n") + if !commit_range.include?(current_commit) && + provider.platform.commit.split('..').first != current_commit + forcible_prompt( + global_options[:force], + "The platform is pinned via the `platform.commit` property in #{provider_json} " + + "to a commit in the range #{provider.platform.commit}, but the current HEAD " + + "(#{current_commit}) is not in that range.", + "Do you really want to deploy from the wrong commit? " + ) + end + end + end + end + + def sync_hiera_config(ssh) + ssh.rsync.update do |server| + node = manager.node(server.host) + hiera_file = Path.relative_path([:hiera, node.name]) + ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path + { + :source => hiera_file, + :dest => Leap::Platform.hiera_path, + :flags => "-rltp --chmod=u+rX,go-rwx" + } + end + end + + # + # sync various support files. + # + def sync_support_files(ssh) + dest_dir = Leap::Platform.files_dir + custom_files = build_custom_file_list + ssh.rsync.update do |server| + node = manager.node(server.host) + files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } + files_to_sync += custom_files + if files_to_sync.any? + ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) + { + :chdir => Path.named_path(:files_dir), + :source => ".", + :dest => dest_dir, + :excludes => "*", + :includes => calculate_includes_from_files(files_to_sync, '/files'), + :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" + } + else + nil + end + end + end + + def sync_puppet_files(ssh) + ssh.rsync.update do |server| + ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) + { + :dest => Leap::Platform.leap_dir, + :source => '.', + :chdir => Path.platform, + :excludes => '*', + :includes => ['/bin', '/bin/**', '/puppet', '/puppet/**', '/tests', '/tests/**'], + :flags => "-rlt --relative --delete --copy-links" + } + end + end + + # + # ensure submodules are up to date, if the platform is a git + # repository. + # + def init_submodules + return unless is_git_directory?(Path.platform) + Dir.chdir Path.platform do + assert_run! "git submodule sync" + statuses = assert_run! "git submodule status" + statuses.strip.split("\n").each do |status_line| + if status_line =~ /^[\+-]/ + submodule = status_line.split(' ')[1] + log "Updating submodule #{submodule}" + assert_run! "git submodule update --init #{submodule}" + end + end + end + end + + # + # converts an array of file paths into an array + # suitable for --include of rsync + # + # if set, `prefix` is stripped off. + # + def calculate_includes_from_files(files, prefix=nil) + return nil unless files and files.any? + + # prepend '/' (kind of like ^ for rsync) + includes = files.collect {|file| file =~ /^\// ? file : '/' + file } + + # include all sub files of specified directories + includes.size.times do |i| + if includes[i] =~ /\/$/ + includes << includes[i] + '**' + end + end + + # include all parent directories (required because of --exclude '*') + includes.size.times do |i| + path = File.dirname(includes[i]) + while(path != '/') + includes << path unless includes.include?(path) + path = File.dirname(path) + end + end + + if prefix + includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} + end + + return includes + end + + def tags(options) + if options[:tags] + tags = options[:tags].split(',') + else + tags = Leap::Platform.default_puppet_tags.dup + end + tags << 'leap_slow' unless options[:fast] + tags.join(',') + end + + # + # a provider might have various customization files that should be sync'ed to the server. + # this method builds that list of files to sync. + # + def build_custom_file_list + custom_files = [] + Leap::Platform.paths.keys.grep(/^custom_/).each do |path| + if file_exists?(path) + relative_path = Path.relative_path(path, Path.provider) + if dir_exists?(path) + custom_files << relative_path + '/' # rsync needs trailing slash + else + custom_files << relative_path + end + end + end + return custom_files + end + + def deploy_info + info = [] + info << "user: %s" % Etc.getpwuid(Process.euid).name + if is_git_directory?(Path.platform) && current_git_branch(Path.platform) != 'master' + info << "platform: %s (%s %s)" % [ + Leap::Platform.version, + current_git_branch(Path.platform), + current_git_commit(Path.platform)[0..4] + ] + else + info << "platform: %s" % Leap::Platform.version + end + if is_git_directory?(LEAP_CLI_BASE_DIR) + info << "leap_cli: %s (%s %s)" % [ + LeapCli::VERSION, + current_git_branch(LEAP_CLI_BASE_DIR), + current_git_commit(LEAP_CLI_BASE_DIR)[0..4] + ] + else + info << "leap_cli: %s" % LeapCli::VERSION + end + info.join(', ') + end + end +end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 00000000..80be2174 --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,76 @@ +module LeapCli + module Commands + + desc "Manipulate and query environment information." + long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ + "A node in one environment will never interact with a node from another environment. "+ + "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ + "absolute file path of your provider directory (pins don't apply if you move the directory)" + command [:env, :e] do |c| + c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument." + c.arg_name 'ENVIRONMENT', :optional => true + c.command :ls do |ls| + ls.action do |global_options, options, args| + environment = get_env_from_args(args) + if environment + pin(environment) + LeapCli.leapfile.load + end + print_envs + end + end + + c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' + c.arg_name 'ENVIRONMENT' + c.command :pin do |pin| + pin.action do |global_options,options,args| + environment = get_env_from_args(args) + if environment + pin(environment) + else + bail! "There is no environment `#{environment}`" + end + end + end + + c.desc "Unpin the environment. All subsequent commands will apply to all nodes." + c.command :unpin do |unpin| + unpin.action do |global_options, options, args| + LeapCli.leapfile.unset('environment') + log 0, :saved, "~/.leaprc, removing environment property." + end + end + + c.default_command :ls + end + + protected + + def get_env_from_args(args) + environment = args.first + if environment == 'default' || (environment && manager.environment_names.include?(environment)) + return environment + else + return nil + end + end + + def pin(environment) + LeapCli.leapfile.set('environment', environment) + log 0, :saved, "~/.leaprc with environment set to #{environment}." + end + + def print_envs + envs = ["default"] + manager.environment_names.compact.sort + envs.each do |env| + if env + if LeapCli.leapfile.environment == env + puts "* #{env}" + else + puts " #{env}" + end + end + end + end + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 00000000..11329ccc --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,100 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + + desc 'Gather information on nodes.' + command :facts do |facts| + facts.desc 'Query servers to update facts.json.' + facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" + facts.arg_name 'FILTER' + facts.command :update do |update| + update.action do |global_options,options,args| + update_facts(global_options, options, args) + end + end + end + + protected + + def facter_cmd + 'facter --json ' + Leap::Platform.facts.join(' ') + end + + def remove_node_facts(name) + if file_exists?(:facts) + update_facts_file({name => nil}) + end + end + + def update_node_facts(name, facts) + update_facts_file({name => facts}) + end + + def rename_node_facts(old_name, new_name) + if file_exists?(:facts) + facts = JSON.parse(read_file(:facts) || {}) + facts[new_name] = facts[old_name] + facts[old_name] = nil + update_facts_file(facts, true) + end + end + + # + # if overwrite = true, then ignore existing facts.json. + # + def update_facts_file(new_facts, overwrite=false) + replace_file!(:facts) do |content| + if overwrite || content.nil? || content.empty? + old_facts = {} + else + old_facts = manager.facts + end + facts = old_facts.merge(new_facts) + facts.each do |name, value| + if value.is_a? String + if value == "" + value = nil + else + value = JSON.parse(value) rescue JSON::ParserError + end + end + if value.is_a? Hash + value.delete_if {|key,v| v.nil?} + end + facts[name] = value + end + facts.delete_if do |name, value| + value.nil? || value.empty? + end + if facts.empty? + "{}\n" + else + JSON.sorted_generate(facts) + "\n" + end + end + end + + private + + def update_facts(global_options, options, args) + nodes = manager.filter(args, :local => false, :disabled => false) + new_facts = {} + ssh_connect(nodes) do |ssh| + ssh.leap.run_with_progress(facter_cmd) do |response| + node = manager.node(response[:host]) + if node + new_facts[node.name] = response[:data].strip + else + log :warning, 'Could not find node for hostname %s' % response[:host] + end + end + end + # only overwrite the entire facts file if and only if we are gathering facts + # for all nodes in all environments. + overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? + update_facts_file(new_facts, overwrite_existing) + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/info.rb b/lib/leap_cli/commands/info.rb new file mode 100644 index 00000000..52225a94 --- /dev/null +++ b/lib/leap_cli/commands/info.rb @@ -0,0 +1,15 @@ +module LeapCli; module Commands + + desc 'Prints information regarding facts, history, and running processes for a node or nodes.' + long_desc 'The FILTER can be the name of a node, service, or tag.' + arg_name 'FILTER' + command [:info] do |c| + c.action do |global,options,args| + nodes = manager.filter!(args) + ssh_connect(nodes, connect_options(options)) do |ssh| + ssh.leap.debug + end + end + end + +end; end diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb new file mode 100644 index 00000000..20654fa7 --- /dev/null +++ b/lib/leap_cli/commands/inspect.rb @@ -0,0 +1,144 @@ +module LeapCli; module Commands + + desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.' + arg_name 'FILE' + command [:inspect, :i] do |c| + c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false + c.action do |global_options,options,args| + object = args.first + assert! object, 'A file path or node/service/tag name is required' + method = inspection_method(object) + if method && defined?(method) + self.send(method, object, options) + else + log "Sorry, I don't know how to inspect that." + end + end + end + + private + + FTYPE_MAP = { + "PEM certificate" => :inspect_x509_cert, + "PEM RSA private key" => :inspect_x509_key, + "OpenSSH RSA public key" => :inspect_ssh_pub_key, + "PEM certificate request" => :inspect_x509_csr + } + + def inspection_method(object) + if File.exists?(object) + ftype = `file #{object}`.split(':').last.strip + log 2, "file is of type '#{ftype}'" + if FTYPE_MAP[ftype] + FTYPE_MAP[ftype] + elsif File.extname(object) == ".json" + full_path = File.expand_path(object, Dir.pwd) + if path_match?(:node_config, full_path) + :inspect_node + elsif path_match?(:service_config, full_path) + :inspect_service + elsif path_match?(:tag_config, full_path) + :inspect_tag + elsif path_match?(:provider_config, full_path) || path_match?(:provider_env_config, full_path) + :inspect_provider + elsif path_match?(:common_config, full_path) + :inspect_common + else + nil + end + end + elsif manager.nodes[object] + :inspect_node + elsif manager.services[object] + :inspect_service + elsif manager.tags[object] + :inspect_tag + elsif object == "common" + :inspect_common + elsif object == "provider" + :inspect_provider + else + nil + end + end + + # + # inspectors + # + + def inspect_x509_key(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl rsa -in %s -text -check' % file_path + end + + def inspect_x509_cert(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl x509 -in %s -text -noout' % file_path + log 0, :"SHA256 fingerprint", X509.fingerprint("SHA256", file_path) + end + + def inspect_x509_csr(file_path, options) + assert_bin! 'openssl' + puts assert_run! 'openssl req -text -noout -verify -in %s' % file_path + end + + #def inspect_ssh_pub_key(file_path) + #end + + def inspect_node(arg, options) + inspect_json manager.nodes[name(arg)] + end + + def inspect_service(arg, options) + if options[:base] + inspect_json manager.base_services[name(arg)] + else + inspect_json manager.services[name(arg)] + end + end + + def inspect_tag(arg, options) + if options[:base] + inspect_json manager.base_tags[name(arg)] + else + inspect_json manager.tags[name(arg)] + end + end + + def inspect_provider(arg, options) + if options[:base] + inspect_json manager.base_provider + elsif arg =~ /provider\.(.*)\.json/ + inspect_json manager.env($1).provider + else + inspect_json manager.provider + end + end + + def inspect_common(arg, options) + if options[:base] + inspect_json manager.base_common + else + inspect_json manager.common + end + end + + # + # helpers + # + + def name(arg) + File.basename(arg).sub(/\.json$/, '') + end + + def inspect_json(config) + if config + puts JSON.sorted_generate(config) + end + end + + def path_match?(path_symbol, path) + Dir.glob(Path.named_path([path_symbol, '*'])).include?(path) + end + +end; end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb new file mode 100644 index 00000000..aa425432 --- /dev/null +++ b/lib/leap_cli/commands/list.rb @@ -0,0 +1,132 @@ +require 'command_line_reporter' + +module LeapCli; module Commands + + desc 'List nodes and their classifications' + long_desc 'Prints out a listing of nodes, services, or tags. ' + + 'If present, the FILTER can be a list of names of nodes, services, or tags. ' + + 'If the name is prefixed with +, this acts like an AND condition. ' + + "For example:\n\n" + + "`leap list node1 node2` matches all nodes named \"node1\" OR \"node2\"\n\n" + + "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\"" + + arg_name 'FILTER', :optional => true + command [:list,:ls] do |c| + c.flag 'print', :desc => 'What attributes to print (optional)' + c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false + c.action do |global_options,options,args| + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new + if global_options[:color] + colors = ['cyan', 'white'] + else + colors = [nil, nil] + end + puts + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) + if options['print'] + print_node_properties(manager.filter(args), options['print']) + else + if args.any? + NodeTable.new(manager.filter(args), colors).run + else + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run + end + end + end + end + + private + + def self.print_node_properties(nodes, properties) + properties = properties.split(',') + max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} + nodes.each_node do |node| + value = properties.collect{|prop| + prop_value = node[prop] + if prop_value.nil? + "null" + elsif prop_value == "" + "empty" + elsif prop_value.is_a? LeapCli::Config::Object + node[prop].dump_json(:format => :compact) # TODO: add option of getting pre-evaluation values. + else + prop_value.to_s + end + }.join(', ') + printf("%#{max_width}s %s\n", node.name, value) + end + puts + end + + class TagTable + include CommandLineReporter + def initialize(heading, tag_list, colors) + @heading = heading + @tag_list = tag_list + @colors = colors + end + def run + tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials + max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max + table :border => false do + row :color => @colors[0] do + column @heading, :align => 'right', :width => max_width + column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 + end + tags.each do |tag| + next if @tag_list[tag].node_list.empty? + row :color => @colors[1] do + column tag + column @tag_list[tag].node_list.keys.sort.join(', ') + end + end + end + vertical_spacing + end + end + + # + # might be handy: HighLine::SystemExtensions.terminal_size.first + # + class NodeTable + include CommandLineReporter + def initialize(node_list, colors) + @node_list = node_list + @colors = colors + end + def run + rows = @node_list.keys.sort.collect do |node_name| + [node_name, @node_list[node_name].services.sort.join(', '), @node_list[node_name].tags.sort.join(', ')] + end + unless rows.any? + puts Paint["no results", :red] + puts + return + end + padding = 2 + max_node_width = [20, (rows.map{|i|i[0]} + ["NODES"] ).inject(0) {|max,i| [i.size,max].max}].max + max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max} + max_tag_width = (rows.map{|i|i[2]} + ["TAGS"] ).inject(0) {|max,i| [i.size,max].max} + table :border => false do + row :color => @colors[0] do + column "NODES", :align => 'right', :width => max_node_width + column "SERVICES", :width => max_service_width, :padding => 2 + column "TAGS", :width => max_tag_width + end + rows.each do |r| + row :color => @colors[1] do + column r[0] + column r[1] + column r[2] + end + end + end + vertical_spacing + end + end + +end; end diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb new file mode 100644 index 00000000..a23661b3 --- /dev/null +++ b/lib/leap_cli/commands/node.rb @@ -0,0 +1,188 @@ +# +# fyi: the `node init` command lives in node_init.rb, +# but all other `node x` commands live here. +# + +autoload :IPAddr, 'ipaddr' + +module LeapCli; module Commands + + ## + ## COMMANDS + ## + + desc 'Node management' + command [:node, :n] do |node| + node.desc 'Create a new configuration file for a node named NAME.' + node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.", + "The format is property_name:value.", + "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.", + "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`", + "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n") + node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false + node.command :add do |add| + add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false + add.action do |global_options,options,args| + # argument sanity checks + name = args.first + assert_valid_node_name!(name, options[:local]) + assert_files_missing! [:node_config, name] + + # create and seed new node + node = Config::Node.new(manager.env) + if options[:local] + node['ip_address'] = pick_next_vagrant_ip_address + end + seed_node_data_from_cmd_line(node, args[1..-1]) + seed_node_data_from_template(node) + validate_ip_address(node) + begin + node['name'] = name + json = node.dump_json(:exclude => ['name']) + write_file!([:node_config, name], json + "\n") + if file_exists? :ca_cert, :ca_key + generate_cert_for_node(manager.reload_node!(node)) + end + rescue LeapCli::ConfigError => exc + remove_node_files(name) + end + end + end + + node.desc 'Renames a node file, and all its related files.' + node.arg_name 'OLD_NAME NEW_NAME' + node.command :mv do |mv| + mv.action do |global_options,options,args| + node = get_node_from_args(args, include_disabled: true) + new_name = args.last + assert_valid_node_name!(new_name, node.vagrant?) + ensure_dir [:node_files_dir, new_name] + Leap::Platform.node_files.each do |path| + rename_file! [path, node.name], [path, new_name] + end + remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) + end + end + + node.desc 'Removes all the files related to the node named NAME.' + node.arg_name 'NAME' #:optional => false #, :multiple => false + node.command :rm do |rm| + rm.action do |global_options,options,args| + node = get_node_from_args(args, include_disabled: true) + remove_node_files(node.name) + if node.vagrant? + vagrant_command("destroy --force", [node.name]) + end + remove_node_facts(node.name) + end + end + end + + ## + ## PUBLIC HELPERS + ## + + def get_node_from_args(args, options={}) + node_name = args.first + node = manager.node(node_name) + if node.nil? && options[:include_disabled] + node = manager.disabled_node(node_name) + end + assert!(node, "Node '#{node_name}' not found.") + node + end + + def seed_node_data_from_cmd_line(node, args) + args.each do |seed| + key, value = seed.split(':', 2) + value = format_seed_value(value) + assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'" + if key =~ /\./ + key_parts = key.split('.') + final_key = key_parts.pop + current_object = node + key_parts.each do |key_part| + current_object[key_part] ||= Config::Object.new + current_object = current_object[key_part] + end + current_object[final_key] = value + else + node[key] = value + end + end + end + + # + # load "new node template" information into the `node`, modifying `node`. + # values in the template will not override existing node values. + # + def seed_node_data_from_template(node) + node.inherit_from!(manager.template('common')) + [node['services']].flatten.each do |service| + if service + template = manager.template(service) + if template + node.inherit_from!(template) + end + end + end + end + + def remove_node_files(node_name) + (Leap::Platform.node_files + [:node_files_dir]).each do |path| + remove_file! [path, node_name] + end + end + + # + # conversions: + # + # "x,y,z" => ["x","y","z"] + # + # "22" => 22 + # + # "5.1" => 5.1 + # + def format_seed_value(v) + if v =~ /,/ + v = v.split(',') + v.map! do |i| + i = i.to_i if i.to_i.to_s == i + i = i.to_f if i.to_f.to_s == i + i + end + else + v = v.to_i if v.to_i.to_s == v + v = v.to_f if v.to_f.to_s == v + end + return v + end + + def validate_ip_address(node) + if node['ip_address'] == "REQUIRED" + bail! do + log :error, "ip_address is not set. Specify with `leap node add NAME ip_address:ADDRESS`." + end + end + IPAddr.new(node['ip_address']) + rescue ArgumentError + bail! do + if node['ip_address'] + log :invalid, "ip_address #{node['ip_address'].inspect}" + else + log :missing, "ip_address" + end + end + end + + def assert_valid_node_name!(name, local=false) + assert! name, 'No <node-name> specified.' + if local + assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)" + else + assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)" + end + end + +end; end diff --git a/lib/leap_cli/commands/node_init.rb b/lib/leap_cli/commands/node_init.rb new file mode 100644 index 00000000..33f6288d --- /dev/null +++ b/lib/leap_cli/commands/node_init.rb @@ -0,0 +1,169 @@ +# +# Node initialization. +# Most of the fun stuff is in tasks.rb. +# + +module LeapCli; module Commands + + desc 'Node management' + command :node do |node| + node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' + node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + + "This command only needs to be run once, but there is no harm in running it multiple times." + node.arg_name 'FILTER' + node.command :init do |init| + init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false + init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' + init.flag :ip, :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS' + + init.action do |global,options,args| + assert! args.any?, 'You must specify a FILTER' + finished = [] + manager.filter!(args).each_node do |node| + is_node_alive(node, options) + save_public_host_key(node, global, options) unless node.vagrant? + update_compiled_ssh_configs + ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]}) + ssh_connect(node, ssh_connect_options) do |ssh| + if node.vagrant? + ssh.install_insecure_vagrant_key + end + ssh.install_authorized_keys + ssh.install_prerequisites + unless node.vagrant? + ssh.leap.log(:checking, "SSH host keys") do + ssh.leap.capture(get_ssh_keys_cmd) do |response| + update_local_ssh_host_keys(node, response[:data]) if response[:exitcode] == 0 + end + end + end + ssh.leap.log(:updating, "facts") do + ssh.leap.capture(facter_cmd) do |response| + if response[:exitcode] == 0 + update_node_facts(node.name, response[:data]) + else + log :failed, "to run facter on #{node.name}" + end + end + end + end + finished << node.name + end + log :completed, "initialization of nodes #{finished.join(', ')}" + end + end + end + + private + + ## + ## PRIVATE HELPERS + ## + + def is_node_alive(node, options) + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + log :connecting, "to node #{node.name}" + assert_run! "nc -zw3 #{address} #{port}", + "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port." + end + + # + # saves the public ssh host key for node into the provider directory. + # + # see `man sshd` for the format of known_hosts + # + def save_public_host_key(node, global, options) + log :fetching, "public SSH host key for #{node.name}" + address = options[:ip] || node.ip_address + port = options[:port] || node.ssh.port + host_keys = get_public_keys_for_ip(address, port) + pub_key_path = Path.named_path([:node_ssh_pub_key, node.name]) + + if Path.exists?(pub_key_path) + if host_keys.include? SshKey.load(pub_key_path) + log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1 + else + bail! do + log :error, "The public SSH host keys we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1 + log "Delete the file #{pub_key_path} if you really want to remove the trusted SSH host key.", :indent => 2 + end + end + else + known_key = host_keys.detect{|k|k.in_known_hosts?(node.name, node.ip_address, node.domain.name)} + if known_key + log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)" + else + public_key = SshKey.pick_best_key(host_keys) + if public_key.nil? + bail!("We got back #{host_keys.size} host keys from #{node.name}, but we can't support any of them.") + else + say(" This is the SSH host key you got back from node \"#{node.name}\"") + say(" Type -- #{public_key.bits} bit #{public_key.type.upcase}") + say(" Fingerprint -- " + public_key.fingerprint) + say(" Public Key -- " + public_key.key) + if !global[:yes] && !agree(" Is this correct? ") + bail! + else + known_key = public_key + end + end + end + puts + write_file! [:node_ssh_pub_key, node.name], known_key.to_s + end + end + + # + # Get the public host keys for a host using ssh-keyscan. + # Return an array of SshKey objects, one for each key. + # + def get_public_keys_for_ip(address, port=22) + assert_bin!('ssh-keyscan') + output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" + if output.empty? + bail! :failed, "ssh-keyscan returned empty output." + end + + if output =~ /No route to host/ + bail! :failed, 'ssh-keyscan: no route to %s' % address + else + keys = SshKey.parse_keys(output) + if keys.empty? + bail! "ssh-keyscan got zero host keys back (that we understand)! Output was: #{output}" + else + return keys + end + end + end + + # run on the server to generate a string suitable for passing to SshKey.parse_keys() + def get_ssh_keys_cmd + "/bin/grep ^HostKey /etc/ssh/sshd_config | /usr/bin/awk '{print $2 \".pub\"}' | /usr/bin/xargs /bin/cat" + end + + # + # Sometimes the ssh host keys on the server will be better than what we have + # stored locally. In these cases, ask the user if they want to upgrade. + # + def update_local_ssh_host_keys(node, remote_keys_string) + remote_keys = SshKey.parse_keys(remote_keys_string) + return unless remote_keys.any? + current_key = SshKey.load(Path.named_path([:node_ssh_pub_key, node.name])) + best_key = SshKey.pick_best_key(remote_keys) + return unless best_key && current_key + if current_key != best_key + say(" One of the SSH host keys for node '#{node.name}' is better than what you currently have trusted.") + say(" Current key: #{current_key.summary}") + say(" Better key: #{best_key.summary}") + if agree(" Do you want to use the better key? ") + write_file! [:node_ssh_pub_key, node.name], best_key.to_s + end + else + log(3, "current host key does not need updating") + end + end + +end; end diff --git a/lib/leap_cli/commands/ssh.rb b/lib/leap_cli/commands/ssh.rb new file mode 100644 index 00000000..3887618e --- /dev/null +++ b/lib/leap_cli/commands/ssh.rb @@ -0,0 +1,225 @@ +module LeapCli; module Commands + + desc 'Log in to the specified node with an interactive shell.' + arg_name 'NAME' #, :optional => false, :multiple => false + command :ssh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:ssh, options, args) + end + end + + desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).' + arg_name 'NAME' + command :mosh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. `--ssh '-F ~/sshconfig'`)." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + exec_ssh(:mosh, options, args) + end + end + + desc 'Creates an SSH port forward (tunnel) to the node NAME. REMOTE_PORT is the port on the remote node that the tunnel will connect to. LOCAL_PORT is the optional port on your local machine. For example: `leap tunnel couch1:5984`.' + arg_name '[LOCAL_PORT:]NAME:REMOTE_PORT' + command :tunnel do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')." + c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server. Same as `--ssh "-p SSH_PORT"`.' + c.action do |global_options,options,args| + local_port, node, remote_port = parse_tunnel_arg(args.first) + unless node.ssh.config.AllowTcpForwarding == "yes" + log :warning, "It looks like TCP forwarding is not enabled. "+ + "The tunnel command requires that the node property ssh.config.AllowTcpForwarding "+ + "be set to 'yes'. Add this property to #{node.name}.json, deploy, and then try tunnel again." + end + options[:ssh] = [options[:ssh], "-N -L 127.0.0.1:#{local_port}:0.0.0.0:#{remote_port}"].join(' ') + log("Forward port localhost:#{local_port} to #{node.name}:#{remote_port}") + if is_port_available?(local_port) + exec_ssh(:ssh, options, [node.name]) + end + end + end + + desc 'Secure copy from FILE1 to FILE2. Files are specified as NODE_NAME:FILE_PATH. For local paths, omit "NODE_NAME:".' + arg_name 'FILE1 FILE2' + command :scp do |c| + c.switch :r, :desc => 'Copy recursively' + c.action do |global_options, options, args| + if args.size != 2 + bail!('You must specificy both FILE1 and FILE2') + end + from, to = args + if (from !~ /:/ && to !~ /:/) || (from =~ /:/ && to =~ /:/) + bail!('One FILE must be remote and the other local.') + end + src_node_name = src_file_path = src_node = nil + dst_node_name = dst_file_path = dst_node = nil + if from =~ /:/ + src_node_name, src_file_path = from.split(':') + src_node = get_node_from_args([src_node_name], :include_disabled => true) + dst_file_path = to + else + dst_node_name, dst_file_path = to.split(':') + dst_node = get_node_from_args([dst_node_name], :include_disabled => true) + src_file_path = from + end + exec_scp(options, src_node, src_file_path, dst_node, dst_file_path) + end + end + + protected + + # + # allow for ssh overrides of all commands that use ssh_connect + # + def connect_options(options) + connect_options = {:ssh_options=>{}} + if options[:port] + connect_options[:ssh_options][:port] = options[:port] + end + if options[:ip] + connect_options[:ssh_options][:host_name] = options[:ip] + end + return connect_options + end + + def ssh_config_help_message + puts "" + puts "Are 'too many authentication failures' getting you down?" + puts "Then we have the solution for you! Add something like this to your ~/.ssh/config file:" + puts " Host *.#{manager.provider.domain}" + puts " IdentityFile ~/.ssh/id_rsa" + puts " IdentitiesOnly=yes" + puts "(replace `id_rsa` with the actual private key filename that you use for this provider)" + end + + require 'socket' + def is_port_available?(port) + TCPServer.open('127.0.0.1', port) {} + true + rescue Errno::EACCES + bail!("You don't have permission to bind to port #{port}.") + rescue Errno::EADDRINUSE + bail!("Local port #{port} is already in use. Specify LOCAL_PORT to pick another.") + rescue Exception => exc + bail!(exc.to_s) + end + + private + + def exec_ssh(cmd, cli_options, args) + node = get_node_from_args(args, :include_disabled => true) + port = node.ssh.port + options = ssh_config(node) + username = 'root' + if LeapCli.log_level >= 3 + options << "-vv" + elsif LeapCli.log_level >= 2 + options << "-v" + end + if cli_options[:port] + port = cli_options[:port] + end + if cli_options[:ssh] + options << cli_options[:ssh] + end + ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" + if cmd == :ssh + command = "#{ssh} #{node.domain.full}" + elsif cmd == :mosh + command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.domain.full}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + + if status.exitstatus == 255 + ssh_config_help_message + elsif status.exitstatus != 0 + exit(status.exitstatus) + end + end + + def exec_scp(cli_options, src_node, src_file_path, dst_node, dst_file_path) + node = src_node || dst_node + options = ssh_config(node) + port = node.ssh.port + username = 'root' + options << "-r" if cli_options[:r] + scp = "scp -P #{port} #{options.join(' ')}" + if src_node + command = "#{scp} #{username}@#{src_node.domain.full}:#{src_file_path} #{dst_file_path}" + elsif dst_node + command = "#{scp} #{src_file_path} #{username}@#{dst_node.domain.full}:#{dst_file_path}" + end + log 2, command + + # exec the shell command in a subprocess + pid = fork { exec "#{command}" } + + Signal.trap("SIGINT") do + Process.kill("KILL", pid) + Process.wait(pid) + exit(0) + end + + # wait for shell to exit so we can grab the exit status + _, status = Process.waitpid2(pid) + exit(status.exitstatus) + end + + # + # SSH command line -o options. See `man ssh_config` + # + # NOTES: + # + # The option 'HostKeyAlias=#{node.name}' is oddly incompatible with ports in + # known_hosts file, so we must not use this or non-standard ports break. + # + def ssh_config(node) + options = [ + "-o 'HostName=#{node.ip_address}'", + "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'", + "-o 'UserKnownHostsFile=/dev/null'" + ] + if node.vagrant? + options << "-i #{vagrant_ssh_key_file}" # use the universal vagrant insecure key + options << "-o IdentitiesOnly=yes" # force the use of the insecure vagrant key + options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it + # (since userknownhostsfile is /dev/null) + else + options << "-o 'StrictHostKeyChecking=yes'" + end + if !node.supported_ssh_host_key_algorithms.empty? + options << "-o 'HostKeyAlgorithms=#{node.supported_ssh_host_key_algorithms}'" + end + return options + end + + def parse_tunnel_arg(arg) + if arg.count(':') == 1 + node_name, remote = arg.split(':') + local = nil + elsif arg.count(':') == 2 + local, node_name, remote = arg.split(':') + else + bail!('Argument NAME:REMOTE_PORT required.') + end + node = get_node_from_args([node_name], :include_disabled => true) + remote = remote.to_i + local = local || remote + local = local.to_i + return [local, node, remote] + end + +end; end
\ No newline at end of file diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb new file mode 100644 index 00000000..73207b31 --- /dev/null +++ b/lib/leap_cli/commands/test.rb @@ -0,0 +1,74 @@ +module LeapCli; module Commands + + desc 'Run tests.' + command [:test, :t] do |test| + test.desc 'Run the test suit on FILTER nodes.' + test.arg_name 'FILTER', :optional => true + test.command :run do |run| + run.switch 'continue', :desc => 'Continue over errors and failures (default is --no-continue).', :negatable => true + run.action do |global_options,options,args| + test_order = File.join(Path.platform, 'tests/order.rb') + if File.exists?(test_order) + require test_order + end + manager.filter!(args).names_in_test_dependency_order.each do |node_name| + node = manager.nodes[node_name] + begin + ssh_connect(node) do |ssh| + ssh.run(test_cmd(options)) + end + rescue Capistrano::CommandError => exc + if options[:continue] + exit_status(1) + else + bail! + end + end + end + end + end + + test.desc 'Creates files needed to run tests.' + test.command :init do |init| + init.action do |global_options,options,args| + generate_test_client_openvpn_configs + end + end + + test.default_command :run + end + + private + + def test_cmd(options) + if options[:continue] + "#{Leap::Platform.leap_dir}/bin/run_tests --continue" + else + "#{Leap::Platform.leap_dir}/bin/run_tests" + end + end + + # + # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways + # + def generate_test_client_openvpn_configs + assert_config! 'provider.ca.client_certificates.unlimited_prefix' + assert_config! 'provider.ca.client_certificates.limited_prefix' + template = read_file! Path.find_file(:test_client_openvpn_template) + manager.environment_names.each do |env| + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true] + if vpn_nodes.any? + generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert| + write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding) + end + end + end + end + +end; end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb new file mode 100644 index 00000000..b842e854 --- /dev/null +++ b/lib/leap_cli/commands/user.rb @@ -0,0 +1,136 @@ + +# +# perhaps we want to verify that the key files are actually the key files we expect. +# we could use 'file' for this: +# +# > file ~/.gnupg/00440025.asc +# ~/.gnupg/00440025.asc: PGP public key block +# +# > file ~/.ssh/id_rsa.pub +# ~/.ssh/id_rsa.pub: OpenSSH RSA public key +# + +module LeapCli + module Commands + + desc 'Adds a new trusted sysadmin by adding public keys to the "users" directory.' + arg_name 'USERNAME' #, :optional => false, :multiple => false + command :'add-user' do |c| + + c.switch 'self', :desc => 'Add yourself as a trusted sysadmin by choosing among the public keys available for the current user.', :negatable => false + c.flag 'ssh-pub-key', :desc => 'SSH public key file for this new user' + c.flag 'pgp-pub-key', :desc => 'OpenPGP public key file for this new user' + + c.action do |global_options,options,args| + username = args.first + if !username.any? + if options[:self] + username ||= `whoami`.strip + else + help! "Either USERNAME argument or --self flag is required." + end + end + if Leap::Platform.reserved_usernames.include? username + bail! %(The username "#{username}" is reserved. Sorry, pick another.) + end + + ssh_pub_key = nil + pgp_pub_key = nil + + if options['ssh-pub-key'] + ssh_pub_key = read_file!(options['ssh-pub-key']) + end + if options['pgp-pub-key'] + pgp_pub_key = read_file!(options['pgp-pub-key']) + end + + if options[:self] + ssh_pub_key ||= pick_ssh_key.to_s + pgp_pub_key ||= pick_pgp_key + end + + assert!(ssh_pub_key, 'Sorry, could not find SSH public key.') + + if ssh_pub_key + write_file!([:user_ssh, username], ssh_pub_key) + end + if pgp_pub_key + write_file!([:user_pgp, username], pgp_pub_key) + end + + update_authorized_keys + end + end + + # + # let the the user choose among the ssh public keys that we encounter, or just pick the key if there is only one. + # + def pick_ssh_key + ssh_keys = [] + Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| + ssh_keys << SshKey.load(keyfile) + end + + if `which ssh-add`.strip.any? + `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| + key = SshKey.load(line) + if key + key.comment = 'ssh-agent' + ssh_keys << key unless ssh_keys.include?(key) + end + end + end + ssh_keys.compact! + + assert! ssh_keys.any?, 'Sorry, could not find any SSH public key for you. Have you run ssh-keygen?' + + if ssh_keys.length > 1 + key_index = numbered_choice_menu('Choose your SSH public key', ssh_keys.collect(&:summary)) do |line, i| + say("#{i+1}. #{line}") + end + else + key_index = 0 + end + + return ssh_keys[key_index] + end + + # + # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. + # + def pick_pgp_key + begin + require 'gpgme' + rescue LoadError + log "Skipping OpenPGP setup because gpgme is not installed." + return + end + + secret_keys = GPGME::Key.find(:secret) + if secret_keys.empty? + log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" + return nil + end + + secret_keys.select!{|key| !key.expired} + + if secret_keys.length > 1 + key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| + key_info = key.to_s.split("\n")[0..1].map{|line| line.sub(/^\s*(sec|uid)\s*/,'')}.join(' -- ') + say("#{i+1}. #{key_info}") + end + else + key_index = 0 + end + + key_id = secret_keys[key_index].sha + + # can't use this, it includes signatures: + #puts GPGME::Key.export(key_id, :armor => true, :export_options => :export_minimal) + + # export with signatures removed: + return `gpg --armor --export-options export-minimal --export #{key_id}`.strip + end + + end +end diff --git a/lib/leap_cli/commands/util.rb b/lib/leap_cli/commands/util.rb new file mode 100644 index 00000000..c1da570e --- /dev/null +++ b/lib/leap_cli/commands/util.rb @@ -0,0 +1,50 @@ +module LeapCli; module Commands + + extend self + extend LeapCli::Util + extend LeapCli::Util::RemoteCommand + + def path(name) + Path.named_path(name) + end + + # + # keeps prompting the user for a numbered choice, until they pick a good one or bail out. + # + # block is yielded and is responsible for rendering the choices. + # + def numbered_choice_menu(msg, items, &block) + while true + say("\n" + msg + ':') + items.each_with_index &block + say("q. quit") + index = ask("number 1-#{items.length}> ") + if index.empty? + next + elsif index =~ /q/ + bail! + else + i = index.to_i - 1 + if i < 0 || i >= items.length + bail! + else + return i + end + end + end + end + + + def parse_node_list(nodes) + if nodes.is_a? Config::Object + Config::ObjectList.new(nodes) + elsif nodes.is_a? Config::ObjectList + nodes + elsif nodes.is_a? String + manager.filter!(nodes) + else + bail! "argument error" + end + end + +end; end diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb new file mode 100644 index 00000000..9fdd48e3 --- /dev/null +++ b/lib/leap_cli/commands/vagrant.rb @@ -0,0 +1,180 @@ +autoload :IPAddr, 'ipaddr' +require 'fileutils' + +module LeapCli; module Commands + + desc "Manage local virtual machines." + long_desc "This command provides a convient way to manage Vagrant-based virtual machines. If FILTER argument is missing, the command runs on all local virtual machines. The Vagrantfile is automatically generated in 'test/Vagrantfile'. If you want to run vagrant commands manually, cd to 'test'." + command [:local, :l] do |local| + local.desc 'Starts up the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :start do |start| + start.flag(:basebox, + :desc => "The basebox to use. This value is passed to vagrant as the "+ + "`config.vm.box` option. The value here should be the name of an installed box or a "+ + "shorthand name of a box in HashiCorp's Atlas.", + :arg_name => 'BASEBOX', + :default_value => 'LEAP/jessie' + ) + start.action do |global_options,options,args| + vagrant_command(["up", "sandbox on"], args, options) + end + end + + local.desc 'Shuts down the virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :stop do |stop| + stop.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("halt --force", args) + else + vagrant_command("halt", args) + end + end + end + + local.desc 'Destroys the virtual machine(s), reclaiming the disk space' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :destroy do |destroy| + destroy.action do |global_options,options,args| + if global_options[:yes] + vagrant_command("destroy --force", args) + else + vagrant_command("destroy", args) + end + end + end + + local.desc 'Print the status of local virtual machine(s)' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :status do |status| + status.action do |global_options,options,args| + vagrant_command("status", args) + end + end + + local.desc 'Saves the current state of the virtual machine as a new snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :save do |status| + status.action do |global_options,options,args| + vagrant_command("sandbox commit", args) + end + end + + local.desc 'Resets virtual machine(s) to the last saved snapshot' + local.arg_name 'FILTER', :optional => true #, :multiple => false + local.command :reset do |reset| + reset.action do |global_options,options,args| + vagrant_command("sandbox rollback", args) + end + end + end + + public + + # + # returns the path to a vagrant ssh private key file. + # + # if the vagrant.key file is owned by root or ourselves, then + # we need to make sure that it owned by us and not world readable. + # + def vagrant_ssh_key_file + file_path = Path.vagrant_ssh_priv_key_file + Util.assert_files_exist! file_path + uid = File.new(file_path).stat.uid + if uid == 0 || uid == Process.euid + FileUtils.install file_path, '/tmp/vagrant.key', :mode => 0600 + file_path = '/tmp/vagrant.key' + end + return file_path + end + + protected + + def vagrant_command(cmds, args, options={}) + vagrant_setup(options) + cmds = cmds.to_a + if args.empty? + nodes = [""] + else + nodes = manager.filter(args)[:environment => "local"].field(:name) + end + if nodes.any? + vagrant_dir = File.dirname(Path.named_path(:vagrantfile)) + exec = ["cd #{vagrant_dir}"] + cmds.each do |cmd| + nodes.each do |node| + exec << "vagrant #{cmd} #{node}" + end + end + execute exec.join('; ') + else + bail! "No nodes found. This command only works on nodes with ip_address in the network #{LeapCli.leapfile.vagrant_network}" + end + end + + private + + def vagrant_setup(options) + assert_bin! 'vagrant', 'Vagrant is required for running local virtual machines. Run "sudo apt-get install vagrant".' + assert! (vagrant_version >= Gem::Version.new('1.1')), 'Vagrant version >= 1.1 is required for running local virtual machines. Please upgrade.' + + unless assert_run!('vagrant plugin list | grep sahara | cat').chars.any? + log :installing, "vagrant plugin 'sahara'" + assert_run! 'vagrant plugin install sahara' + end + create_vagrant_file(options) + end + + def vagrant_version + @vagrant_version ||= Gem::Version.new(assert_run!('vagrant --version').split(' ')[1]) + end + + def execute(cmd) + log 2, :run, cmd + exec cmd + end + + def create_vagrant_file(options) + lines = [] + + basebox = options[:basebox] || 'LEAP/jessie' + # override basebox with custom setting from Leapfile or ~/.leaprc + basebox = leapfile.vagrant_basebox || basebox + + lines << %[Vagrant.configure("2") do |config|] + manager.each_node do |node| + if node.vagrant? + lines << %[ config.vm.define :#{node.name} do |config|] + lines << %[ config.vm.box = "#{basebox}"] + lines << %[ config.vm.network :private_network, ip: "#{node.ip_address}"] + lines << %[ config.vm.provider "virtualbox" do |v|] + lines << %[ v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] + lines << %[ v.name = "#{node.name}"] + lines << %[ v.memory = 1536] + lines << %[ end] + lines << %[ config.vm.provider "libvirt" do |v|] + lines << %[ v.memory = 1536] + lines << %[ end] + lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line + lines << %[ end] + end + end + + lines << %[end] + lines << "" + write_file! :vagrantfile, lines.join("\n") + end + + def pick_next_vagrant_ip_address + taken_ips = manager.nodes[:environment => "local"].field(:ip_address) + if taken_ips.any? + highest_ip = taken_ips.map{|ip| IPAddr.new(ip)}.max + new_ip = highest_ip.succ + else + new_ip = IPAddr.new(LeapCli.leapfile.vagrant_network).succ.succ + end + return new_ip.to_s + end + +end; end diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb new file mode 100644 index 00000000..fdb9a94e --- /dev/null +++ b/lib/leap_cli/macros.rb @@ -0,0 +1,16 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/keys' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' +require_relative 'macros/provider' diff --git a/lib/leap_cli/macros/core.rb b/lib/leap_cli/macros/core.rb new file mode 100644 index 00000000..873da358 --- /dev/null +++ b/lib/leap_cli/macros/core.rb @@ -0,0 +1,92 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + until hash[name].nil? + i ||= 1; name = "#{name}#{i+=1}" + end + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion), assertion, caller + end + end + + def error(msg) + raise ConfigError.new(@node, msg), msg, caller + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + if env.partials[partial_path] + self.deep_merge!(env.partials[partial_path]) + else + raise ArgumentError.new( + "No such partial `%s`. Available partials include:\n%s" % + [partial_path, env.partials.keys.join(", ")] + ) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + rescue ArgumentError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/lib/leap_cli/macros/files.rb b/lib/leap_cli/macros/files.rb new file mode 100644 index 00000000..04c94edf --- /dev/null +++ b/lib/leap_cli/macros/files.rb @@ -0,0 +1,124 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + return ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + return File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns the location of a file that is stored on the local + # host, under PROVIDER_DIR/files. + # + def local_file_path(path, options={}) + if path.is_a? Symbol + path = [path, @node.name] + elsif path.is_a? String + # ensure it prefixed with files/ + unless path =~ /^files\// + path = "files/" + path + end + end + local_path = Path.find_file(path) + if local_path.nil? + if options[:missing] + raise FileMissing.new(Path.named_path(path), options) + elsif block_given? + yield + return local_file_path(path, options) # try again. + else + Util::log 2, :skipping, "local_file_path(\"#{path}\") because there is no such file." + return nil + end + else + return local_path + end + end + + # + # Returns the location of a file once it is deployed via rsync to the a + # remote server. An internal list of discovered file paths is saved, in + # order to rsync these files when needed. + # + # If the file does not exist, nil is returned. + # + # If there is a block given and the file does not actually exist, the + # block will be yielded to give an opportunity for some code to create the + # file. + # + # For example: + # + # file_path(:dkim_priv_key) {generate_dkim_key} + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or + # Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, + # then the default file from provider_base is copied locally. this + # is required for rsync to work correctly. + # + def remote_file_path(path, options={}, &block) + local_path = local_file_path(path, options, &block) + + return nil if local_path.nil? + + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + if local_path =~ /^#{Regexp.escape(Path.provider_base)}/ + local_provider_path = local_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install local_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + local_path = local_provider_path + end + + # ensure directories end with /, important for building rsync command + if File.directory?(local_path) && local_path !~ /\/$/ + local_path += '/' + end + + relative_path = Path.relative_path(local_path) + relative_path.sub!(/^files\//, '') # remove "files/" prefix + @node.file_paths << relative_path + return File.join(Leap::Platform.files_dir, relative_path) + end + + # deprecated + def file_path(path, options={}) + return remote_file_path(path, options) + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb new file mode 100644 index 00000000..602ae726 --- /dev/null +++ b/lib/leap_cli/macros/haproxy.rb @@ -0,0 +1,73 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + # make sure we have a port to talk to + unless accept_ports[node.name] + error "haproxy needs a local port to talk to when connecting to #{node.name}" + end + weight = default_weight + try { + weight = local_weight if self.location.name == node.location.name + } + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name], + 'weight', weight + ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end diff --git a/lib/leap_cli/macros/hosts.rb b/lib/leap_cli/macros/hosts.rb new file mode 100644 index 00000000..963857ae --- /dev/null +++ b/lib/leap_cli/macros/hosts.rb @@ -0,0 +1,90 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## IPs + ## + + # + # returns a simple array of all the IPs for the specified node list + # + def host_ips(node_list) + if self.vagrant? + node_list = node_list['environment' => 'local'] + else + node_list = node_list['environment' => '!local'] + end + node_list.map {|name, n| + [n.ip_address, (manager.facts[name]||{})['ec2_public_ipv4']] + }.flatten.compact.uniq + end + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = { + 'ip_address' => node.ip_address, + 'domain_internal' => node.domain.internal, + 'domain_full' => node.domain.full, + 'port' => node.ssh.port + } + if node.dns['aliases'] && node.dns['aliases'].any? + # include aliases, but without domain.full + hosts[node.name]['aliases'] = node.dns['aliases'] - [node.domain.full] + end + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/keys.rb b/lib/leap_cli/macros/keys.rb new file mode 100644 index 00000000..e7a75cfb --- /dev/null +++ b/lib/leap_cli/macros/keys.rb @@ -0,0 +1,97 @@ +# encoding: utf-8 + +# +# Macro for dealing with cryptographic keys +# + +module LeapCli + module Macro + + # + # return a fingerprint for a key or certificate + # + def fingerprint(filename, options={}) + options[:mode] ||= :x509 + if options[:mode] == :x509 + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + elsif options[:mode] == :rsa + key = OpenSSL::PKey::RSA.new(File.read(filename)) + Digest::SHA1.new.hexdigest(key.to_der) + end + end + + ## + ## TOR + ## + + # + # return the path to the tor public key + # generating key if it is missing + # + def tor_public_key_path(path_name, key_type) + file_path(path_name) { generate_tor_key(key_type) } + end + + # + # return the path to the tor private key + # generating key if it is missing + # + def tor_private_key_path(path_name, key_type) + file_path(path_name) { generate_tor_key(key_type) } + end + + # + # Generates a onion_address from a public RSA key file. + # + # path_name is the named path of the Tor public key. + # + # Basically, an onion address is nothing more than a base32 encoding + # of the first 10 bytes of a sha1 digest of the public key. + # + # Additionally, Tor ignores the 22 byte header of the public key + # before taking the sha1 digest. + # + def onion_address(path_name) + require 'base32' + require 'base64' + require 'openssl' + path = Path.find_file([path_name, self.name]) + if path && File.exists?(path) + public_key_str = File.readlines(path).grep(/^[^-]/).join + public_key = Base64.decode64(public_key_str) + public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header + sha1sum = Digest::SHA1.new.digest(public_key) + Base32.encode(sha1sum.slice(0,10)).downcase + else + LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path + end + end + + def generate_dkim_key(bit_size=2048) + LeapCli.log :generating, "%s bit RSA DKIM key" % bit_size do + private_key = OpenSSL::PKey::RSA.new(bit_size) + public_key = private_key.public_key + LeapCli::Util.write_file! :dkim_priv_key, private_key.to_pem + LeapCli::Util.write_file! :dkim_pub_key, public_key.to_pem + end + end + + private + + def generate_tor_key(key_type) + if key_type == 'RSA' + require 'certificate_authority' + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = 1024 + LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do + keypair.generate_key(bit_size) + LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem + LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem + end + else + LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type + end + end + + end +end diff --git a/lib/leap_cli/macros/nodes.rb b/lib/leap_cli/macros/nodes.rb new file mode 100644 index 00000000..0e23831d --- /dev/null +++ b/lib/leap_cli/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + env.nodes + end + + # + # simple alias for global.provider + # + def provider + env.provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/provider.rb b/lib/leap_cli/macros/provider.rb new file mode 100644 index 00000000..4e74da01 --- /dev/null +++ b/lib/leap_cli/macros/provider.rb @@ -0,0 +1,90 @@ +# +# These macros are intended only for use in provider.json, although they are +# currently loaded in all .json contexts. +# + +module LeapCli + module Macro + + # + # returns an array of the service names, including only those services that + # are enabled for this environment. + # + def enabled_services + manager.env(self.environment).services[:service_type => :user_service].field(:name).select { |service| + manager.nodes[:environment => self.environment][:services => service].any? + } + end + + # + # The webapp will not work unless the service level configuration is precisely defined. + # Here, we take what the sysadmin has specified in provider.json and clean it up to + # ensure it is OK. + # + # It would be better to add support for JSON schema. + # + def service_levels() + levels = {} + provider.service.levels.each do |name, level| + if name =~ /^[0-9]+$/ + name = name.to_i + end + levels[name] = level_cleanup(name, level.clone) + end + levels + end + + private + + def print_warning(name, msg) + if self.environment + provider_str = "provider.json or %s" % ['provider', self.environment, 'json'].join('.') + else + provider_str = "provider.json" + end + LeapCli::log :warning, "In #{provider_str}, you have an incorrect definition for service level '#{name}':" do + LeapCli::log msg + end + end + + def level_cleanup(name, level) + unless level['name'] + print_warning(name, 'required field "name" is missing') + end + unless level['description'] + print_warning(name, 'required field "description" is missing') + end + unless level['bandwidth'].nil? || level['bandwidth'] == 'limited' + print_warning(name, 'field "bandwidth" must be nil or "limited"') + end + unless level['rate'].nil? || level['rate'].is_a?(Hash) + print_warning(name, 'field "rate" must be nil or a hash (e.g. {"USD":10, "EUR":10})') + end + possible_services = enabled_services + if level['services'] + level['services'].each do |service| + unless possible_services.include? service + print_warning(name, "the service '#{service}' does not exist or there are no nodes that provide this service.") + LeapCli::Util::bail! + end + end + else + level['services'] = possible_services + end + level['services'] = remap_services(level['services']) + level + end + + # + # the service names that the webapp uses and that leap_platform uses are different. ugh. + # + SERVICE_MAP = { + "mx" => "email", + "openvpn" => "eip" + } + def remap_services(services) + services.map {|srv| SERVICE_MAP[srv]} + end + + end +end diff --git a/lib/leap_cli/macros/secrets.rb b/lib/leap_cli/macros/secrets.rb new file mode 100644 index 00000000..8d1feb55 --- /dev/null +++ b/lib/leap_cli/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + manager.secrets.set(name, @node.environment) { Util::Secret.generate(length) } + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + manager.secrets.set(name, @node.environment) { Base32.encode(Util::Secret.generate(length)) } + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + manager.secrets.set(name, @node.environment) { rand(range) } + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + manager.secrets.set(name, @node.environment) { Util::Secret.generate_hex(bit_length) } + end + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb new file mode 100644 index 00000000..821bda38 --- /dev/null +++ b/lib/leap_cli/macros/stunnel.rb @@ -0,0 +1,106 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + s_port = stunnel_port(port) + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', s_port, + 'original_port', port + ] + manager.connections.add(:from => @node.ip_address, :to => node.ip_address, :port => s_port) + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + # + # lists the ips that connect to this node, on particular ports. + # + def stunnel_firewall + manager.connections.select {|connection| + connection['to'] == @node.ip_address + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end
\ No newline at end of file diff --git a/platform.rb b/platform.rb new file mode 100644 index 00000000..1e19a2a9 --- /dev/null +++ b/platform.rb @@ -0,0 +1,119 @@ +# encoding: utf-8 +# +# These are variables defined by this leap_platform and used by leap_cli. +# + +Leap::Platform.define do + self.version = "0.8" + self.compatible_cli = "1.8".."1.99" + + # + # the facter facts that should be gathered + # + self.facts = ["ec2_local_ipv4", "ec2_public_ipv4"] + + # + # absolute paths on the destination server + # + self.hiera_dir = '/etc/leap' if self.respond_to?(:hiera_dir) + self.hiera_path = '/etc/leap/hiera.yaml' + self.leap_dir = '/srv/leap' + self.files_dir = '/srv/leap/files' + self.init_path = '/srv/leap/initialized' + + # + # the named paths for this platform + # (relative to the provider directory) + # + self.paths = { + # directories + :hiera_dir => 'hiera', + :files_dir => 'files', + :nodes_dir => 'nodes', + :services_dir => 'services', + :templates_dir => 'templates', + :tags_dir => 'tags', + :node_files_dir => 'files/nodes/#{arg}', + + # input config files + :common_config => 'common.json', + :provider_config => 'provider.json', + :service_config => 'services/#{arg}.json', + :tag_config => 'tags/#{arg}.json', + :template_config => 'templates/#{arg}.json', + :secrets_config => 'secrets.json', + :node_config => 'nodes/#{arg}.json', + + # input config files, environmentally scoped + :common_env_config => 'commmon.#{arg}.json', + :provider_env_config => 'provider.#{arg}.json', + :service_env_config => 'services/#{arg[0]}.#{arg[1]}.json', + :tag_env_config => 'tags/#{arg[0]}.#{arg[1]}.json', + + # input templates + :provider_json_template => 'files/service-definitions/provider.json.erb', + :eip_service_json_template => 'files/service-definitions/#{arg}/eip-service.json.erb', + :soledad_service_json_template => 'files/service-definitions/#{arg}/soledad-service.json.erb', + :smtp_service_json_template => 'files/service-definitions/#{arg}/smtp-service.json.erb', + + # custom files + :custom_puppet_dir => 'files/puppet', + :custom_puppet_modules_dir => 'files/puppet/modules', + :custom_puppet_manifests_dir => 'files/puppet/manifests', + :custom_tests => 'files/tests', + :custom_bin => 'files/bin', + + # output files + :facts => 'facts.json', + :user_ssh => 'users/#{arg}/#{arg}_ssh.pub', + :user_pgp => 'users/#{arg}/#{arg}_pgp.pub', + :known_hosts => 'files/ssh/known_hosts', + :authorized_keys => 'files/ssh/authorized_keys', + :monitor_pub_key => 'files/ssh/monitor_ssh.pub', + :monitor_priv_key => 'files/ssh/monitor_ssh', + :ca_key => 'files/ca/ca.key', + :ca_cert => 'files/ca/ca.crt', + :client_ca_key => 'files/ca/client_ca.key', + :client_ca_cert => 'files/ca/client_ca.crt', + :dh_params => 'files/ca/dh.pem', + :commercial_key => 'files/cert/#{arg}.key', + :commercial_csr => 'files/cert/#{arg}.csr', + :commercial_cert => 'files/cert/#{arg}.crt', + :dkim_priv_key => 'files/mx/dkim.key', + :dkim_pub_key => 'files/mx/dkim.pub', + + :commercial_ca_cert => 'files/cert/commercial_ca.crt', + :vagrantfile => 'test/Vagrantfile', + :static_web_provider_json => 'files/web/bootstrap/#{arg}/provider.json', + :static_web_htaccess => 'files/web/bootstrap/#{arg}/htaccess', + :static_web_readme => 'files/web/bootstrap/README', + + # node output files + :hiera => 'hiera/#{arg}.yaml', + :node_ssh_pub_key => 'files/nodes/#{arg}/#{arg}_ssh.pub', + :node_x509_key => 'files/nodes/#{arg}/#{arg}.key', + :node_x509_cert => 'files/nodes/#{arg}/#{arg}.crt', + :node_tor_priv_key => 'files/nodes/#{arg}/tor.key', + :node_tor_pub_key => 'files/nodes/#{arg}/tor.pub', + + # testing files + :test_client_key => 'test/cert/client.key', + :test_client_cert => 'test/cert/client.crt', + :test_openvpn_config => 'test/openvpn/#{arg}.ovpn', + :test_client_openvpn_template => 'test/openvpn/client.ovpn.erb' + } + + # + # the files that need to get renamed when a node is renamed + # + self.node_files = [ + :node_config, :hiera, :node_x509_cert, :node_x509_key, :node_ssh_pub_key + ] + + self.monitor_username = 'monitor' + + self.reserved_usernames = ['monitor', 'root'] + + self.default_puppet_tags = ['leap_base','leap_service'] +end + diff --git a/provider_base/README b/provider_base/README new file mode 100644 index 00000000..bb80df50 --- /dev/null +++ b/provider_base/README @@ -0,0 +1,9 @@ +This directory holds the base provider files that actual providers inherit from. + +For example: + + the file........ myproject/provider/common.json + inherits from... myproject/leap_platform/provider_base/common.json + + + diff --git a/provider_base/common.json b/provider_base/common.json new file mode 100644 index 00000000..5e689109 --- /dev/null +++ b/provider_base/common.json @@ -0,0 +1,97 @@ +{ + "ip_address": null, + "environment": null, + "services": [], + "tags": [], + "contacts": "= provider.contacts.default", + "domain": { + "full_suffix": "= provider.domain", + "internal_suffix": "= provider.domain_internal", + "full": "= node.name + '.' + domain.full_suffix", + "internal": "= node.name + '.' + domain.internal_suffix", + "name": "= node.name + '.' + (dns.public ? domain.full_suffix : domain.internal_suffix)" + }, + "dns": { + "public": "= service_type != 'internal_service'" + }, + "ssh": { + "authorized_keys": "= authorized_keys", + "config": { + "AllowTcpForwarding": "no" + }, + "port": 22, + "mosh": { + "ports": "60000:61000", + "enabled": false + } + }, + "hosts": "=> hosts_file", + "x509": { + "use": true, + "use_commercial": false, + "cert": "= x509.use ? file(:node_x509_cert, :missing => 'x509 certificate for node $node. Run `leap cert update`') : nil", + "key": "= x509.use ? file(:node_x509_key, :missing => 'x509 key for node $node. Run `leap cert update`') : nil", + "ca_cert": "= try_file :ca_cert", + "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr --domain %s` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.' % (try{webapp.domain}||domain.full_suffix)) : nil", + "commercial_key": "= x509.use_commercial ? file([:commercial_key, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr --domain %s` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.' % (try{webapp.domain}||domain.full_suffix)) : nil", + "commercial_ca_cert": "= x509.use_commercial ? try_file(:commercial_ca_cert) : nil" + }, + "service_type": "internal_service", + "development": { + "site_config": true + }, + "name": "common", + "location": null, + "enabled": true, + "mail": { + "smarthost": "= nodes_like_me[:services => :mx].exclude(self).field('domain.full')" + }, + "stunnel": { + "clients": {}, + "servers": {} + }, + "firewall": { + "ssh": { + "from": "sysadmin", + "to": "= ip_address", + "port": "= ssh.port" + }, + "stunnel": "=> stunnel_firewall" + }, + "platform": { + "version": "= Leap::Platform.version.to_s", + "major_version": "= Leap::Platform.major_version" + }, + "sources": { + "apt": { + "basic": "http://httpredir.debian.org/debian/", + "security": "http://security.debian.org/", + "backports": "http://httpredir.debian.org/debian/" + }, + "leap-mx": { + "type": "apt", + "package": "leap-mx", + "revision": "latest" + }, + "nickserver": { + "type": "git", + "source": "https://leap.se/git/nickserver", + "revision": "origin/version/0.8" + }, + "platform": { + "apt": { + "basic": "= 'http://deb.leap.se/' + Leap::Platform.major_version" + } + }, + "soledad": { + "type": "apt", + "package": "soledad-server", + "revision": "latest" + }, + "webapp": { + "type": "git", + "source": "https://leap.se/git/leap_web", + "revision": "origin/version/0.8" + } + } +} diff --git a/provider_base/files/branding/head.scss b/provider_base/files/branding/head.scss new file mode 100644 index 00000000..c100a004 --- /dev/null +++ b/provider_base/files/branding/head.scss @@ -0,0 +1 @@ +// no head.scss set diff --git a/provider_base/files/branding/tail.scss b/provider_base/files/branding/tail.scss new file mode 100644 index 00000000..919aeec6 --- /dev/null +++ b/provider_base/files/branding/tail.scss @@ -0,0 +1 @@ +// no tail.scss set diff --git a/provider_base/files/service-definitions/provider.json.erb b/provider_base/files/service-definitions/provider.json.erb new file mode 100644 index 00000000..a75bea61 --- /dev/null +++ b/provider_base/files/service-definitions/provider.json.erb @@ -0,0 +1,16 @@ +<%= + # grab some fields from provider.json + hsh = provider.pick( + :languages, :description, :name, :services, + :enrollment_policy, :default_language, :service + ) + hsh['domain'] = domain.full_suffix + + hsh['api_version'] = "1" + hsh['api_uri'] = ["https://", api.domain, ':', api.port].join + + hsh['ca_cert_uri'] = api.ca_cert_uri + hsh['ca_cert_fingerprint'] = fingerprint(:ca_cert) + + hsh.dump_json +%>
\ No newline at end of file diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb new file mode 100644 index 00000000..4bd220df --- /dev/null +++ b/provider_base/files/service-definitions/v1/eip-service.json.erb @@ -0,0 +1,55 @@ +<%= + def underscore(words) + words = words.to_s.dup + words.downcase! + words.gsub! /[^a-z]/, '_' + words + end + + def add_gateway(node, locations, options={}) + return nil if options[:ip] == 'REQUIRED' + gateway = {} + gateway["capabilities"] = node.openvpn.pick(:ports, :protocols, :user_ips, :adblock, :filter_dns) + gateway["capabilities"]["transport"] = ["openvpn"] + gateway["host"] = node.domain.full + gateway["ip_address"] = options[:ip] + gateway["capabilities"]["limited"] = options[:limited] + if node['location'] + location_name = underscore(node.location.name) + gateway["location"] = location_name + locations[location_name] ||= node.location + end + gateway + end + + hsh = {} + hsh["serial"] = 1 + hsh["version"] = 1 + locations = {} + gateways = [] + configuration = nil + nodes_like_me[:services => 'openvpn'].each_node do |node| + if node.openvpn.allow_limited && node.openvpn.allow_unlimited + gateways << add_gateway(node, locations, :ip => node.openvpn.gateway_address, :limited => false) + gateways << add_gateway(node, locations, :ip => node.openvpn.second_gateway_address, :limited => true) + elsif node.openvpn.allow_unlimited + gateways << add_gateway(node, locations, :ip => node.openvpn.gateway_address, :limited => false) + elsif node.openvpn.allow_limited + gateways << add_gateway(node, locations, :ip => node.openvpn.gateway_address, :limited => true) + end + if configuration && node.openvpn.configuration != configuration + log :error, "OpenVPN nodes in the environment `#{node.environment}` have conflicting `openvpn.configuration` values. This will result in bad errors." + end + configuration = node.openvpn.configuration + end + if gateways.any? + configuration = configuration.dup + if configuration['fragment'] && configuration['fragment'] == 1500 + configuration.delete('fragment') + end + hsh["gateways"] = gateways.compact + hsh["locations"] = locations + hsh["openvpn_configuration"] = configuration + end + JSON.sorted_generate hsh +%>
\ No newline at end of file diff --git a/provider_base/files/service-definitions/v1/smtp-service.json.erb b/provider_base/files/service-definitions/v1/smtp-service.json.erb new file mode 100644 index 00000000..45f240ac --- /dev/null +++ b/provider_base/files/service-definitions/v1/smtp-service.json.erb @@ -0,0 +1,29 @@ +<%= + def underscore(words) + words = words.to_s.dup + words.downcase! + words.gsub! /[^a-z]/, '_' + words + end + + hsh = {} + hsh["serial"] = 1 + hsh["version"] = 1 + locations = {} + hosts = {} + nodes_like_me[:services => 'mx'].each_node do |node| + host = {} + host["hostname"] = node.domain.full + host["ip_address"] = node.ip_address + host["port"] = 465 # hard coded for now, later node.smtp.port + if node['location'] + location_name = underscore(node.location.name) + host["location"] = location_name + locations[location_name] ||= node.location + end + hosts[node.name] = host + end + hsh["hosts"] = hosts + hsh["locations"] = locations + JSON.sorted_generate hsh +%> diff --git a/provider_base/files/service-definitions/v1/soledad-service.json.erb b/provider_base/files/service-definitions/v1/soledad-service.json.erb new file mode 100644 index 00000000..0cd1c927 --- /dev/null +++ b/provider_base/files/service-definitions/v1/soledad-service.json.erb @@ -0,0 +1,29 @@ +<%= + def underscore(words) + words = words.to_s.dup + words.downcase! + words.gsub! /[^a-z]/, '_' + words + end + + hsh = {} + hsh["serial"] = 1 + hsh["version"] = 1 + locations = {} + hosts = {} + nodes_like_me[:services => 'soledad'].each_node do |node| + host = {} + host["hostname"] = node.domain.full + host["ip_address"] = node.ip_address + host["port"] = node.soledad.port + if node['location'] + location_name = underscore(node.location.name) + host["location"] = location_name + locations[location_name] ||= node.location + end + hosts[node.name] = host + end + hsh["hosts"] = hosts + hsh["locations"] = locations + JSON.sorted_generate hsh +%>
\ No newline at end of file diff --git a/provider_base/provider.json b/provider_base/provider.json new file mode 100644 index 00000000..81b2ea98 --- /dev/null +++ b/provider_base/provider.json @@ -0,0 +1,64 @@ +{ + "domain": "REQUIRED", + "domain_internal": "= domain.sub(/\\.[^\\.]*$/, '.i')", + "name": { + "en": "REQUIRED" + }, + "description": { + "en": "REQUIRED" + }, + "contacts": { + "default": ["REQUIRED"], + "english": "= contacts.default.map {|email| email.split('@').join(' at the domain ')}.join(', ')" + }, + "languages": ["en"], + "default_language": "en", + "enrollment_policy": "open", + "services": "= enabled_services", + "service": { + // bandwidth limit is in Bytes, storage limit is in MB. + // for example: + // "levels": { + // "1": {"name": "free", "description":"Limited service, but without cost to you.", "storage":50}, + // "2": {"name": "basic", "description":"The standard package.", "storage":1000, "rate": {"USD":5}}, + // "3": {"name": "pro", "description":"Extra storage for power users." , "storage":10000, "rate": {"USD":10}} + // } + "levels": { + "1": { + "name": "free", "description": "Please donate." + } + }, + "default_service_level": 1, + "bandwidth_limit": 102400, + "allow_free": "= provider.service.levels.select {|l| l['rate'].nil?}.any?", + "allow_paid": "= provider.service.levels.select {|l| !l['rate'].nil?}.any?", + "allow_anonymous": "= provider.service.levels.select {|l| l['name'] == 'anonymous'}.any? && services.include?('openvpn')", + "allow_registration": "= provider.enrollment_policy != 'closed' && provider.service.levels.select {|l| l['name'] != 'anonymous'}.any?", + "allow_limited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'] == 'limited'}.any?", + "allow_unlimited_bandwidth": "= provider.service.levels.select {|l| l['bandwidth'].nil?}.any?" + }, + "ca": { + "name": "= provider.ca.organization + ' Root CA'", + "organization": "= provider.name[provider.default_language]", + "organizational_unit": "= 'https://' + provider.domain", + "bit_size": 4096, + "digest": "SHA256", + "life_span": "10 years", + "server_certificates": { + "bit_size": 4096, + "digest": "SHA256", + "life_span": "1 years" + }, + "client_certificates": { + "bit_size": 2048, + "digest": "SHA256", + "life_span": "2 months", + "limited_prefix": "LIMITED", + "unlimited_prefix": "UNLIMITED" + } + }, + "client_version": { + "min": "0.7", + "max": null + } +} diff --git a/provider_base/services/_api_tester.json b/provider_base/services/_api_tester.json new file mode 100644 index 00000000..790aa7d8 --- /dev/null +++ b/provider_base/services/_api_tester.json @@ -0,0 +1,13 @@ +// +// This partial should be added to any service that runs tests that rely on +// accessing the bonafide webapp API. +// +{ + "testing": { + "monitor_auth_token": "= secret :api_monitor_auth_token", + "api_uri": "= global.services[:webapp].api.uri", + // api_hosts is not used directly, but calling hostnames() will ensure + // that the hostnames are added to /etc/hosts + "api_hosts": "= hostnames(nodes_like_me[:services => 'webapp'])" + } +}
\ No newline at end of file diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json new file mode 100644 index 00000000..da496bae --- /dev/null +++ b/provider_base/services/_couchdb_mirror.json @@ -0,0 +1,22 @@ +// +// Applied to all non-master couchdb nodes +// NOT CURRENTLY SUPPORTED +// +{ + "stunnel": { + "clients": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + } + }, + "couch": { + "mode": "mirror", + "replication": { + // for now, pick the first close one, or the first one. + // in the future, maybe use haproxy to balance among all the masters + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}", + "username": "replication", + "password": "= secret :couch_replication_password", + "role": "replication" + } + } +} diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json new file mode 100644 index 00000000..803a9416 --- /dev/null +++ b/provider_base/services/_couchdb_multimaster.json @@ -0,0 +1,24 @@ +// +// Only applied to master couchdb nodes when there are multiple masters +// NOT CURRENTLY USED. +{ + "stunnel": { + "servers": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)" + }, + "clients": { + "epmd_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.epmd_port)", + "ednp_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.ednp_port)" + } + }, + "couch": { + "mode": "multimaster", + "bigcouch": { + "epmd_port": 4369, + "ednp_port": 9002, + "cookie": "= secret :bigcouch_cookie", + "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'].exclude(self).field('domain.full')" + } + } +} diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json new file mode 100644 index 00000000..30cb53d1 --- /dev/null +++ b/provider_base/services/couchdb.json @@ -0,0 +1,49 @@ +{ + "x509": { + "use": true + }, + "stunnel": { + "servers": { + "couch_server": "= stunnel_server(couch.port)" + } + }, + "couch": { + "port": 5984, + "mode": "plain", + "users": { + "admin": { + "username": "admin", + "password": "= secret :couch_admin_password", + "salt": "= hex_secret :couch_admin_password_salt, 128" + }, + "leap_mx": { + "username": "leap_mx", + "password": "= secret :couch_leap_mx_password", + "salt": "= hex_secret :couch_leap_mx_password_salt, 128" + }, + "nickserver": { + "username": "nickserver", + "password": "= secret :couch_nickserver_password", + "salt": "= hex_secret :couch_nickserver_password_salt, 128" + }, + "soledad": { + "username": "soledad", + "password": "= secret :couch_soledad_password", + "salt": "= hex_secret :couch_soledad_password_salt, 128" + }, + "webapp": { + "username": "webapp", + "password": "= secret :couch_webapp_password", + "salt": "= hex_secret :couch_webapp_password_salt, 128" + }, + "replication": { + "username": "replication", + "password": "= secret :couch_replication_password", + "salt": "= hex_secret :couch_replication_password_salt, 128" + } + }, + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" + } + } +} diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb new file mode 100644 index 00000000..ba7e5ae5 --- /dev/null +++ b/provider_base/services/couchdb.rb @@ -0,0 +1,27 @@ +# +# custom logic for couchdb json resolution +# ============================================ +# +# bigcouch is no longer maintained, so now couchdb is required... +# no matter what! +# + +if self.couch['master'] + LeapCli::log :warning, %("The node property {couch.master:true} is deprecated.\n) + + %( Only {couch.mode:plain} is supported. (node #{self.name})) +end + +couchdb_nodes = nodes_like_me['services' => 'couchdb'] + +if couchdb_nodes.size > 1 + LeapCli::log :error, "Having multiple nodes with {services:couchdb} is no longer supported (nodes #{couchdb_nodes.keys.join(', ')})." +elsif self.couch.mode == "multimaster" + LeapCli::log :error, "Nodes with {couch.mode:multimaster} are no longer supported (node #{self.name})." +end + +# +# This is needed for the "test" that creates and removes the storage db +# for test_user_email. If that test is removed, then this is no longer +# necessary: +# +apply_partial('_api_tester')
\ No newline at end of file diff --git a/provider_base/services/dns.json b/provider_base/services/dns.json new file mode 100644 index 00000000..67948ef8 --- /dev/null +++ b/provider_base/services/dns.json @@ -0,0 +1,14 @@ +{ + "hosts": { + "public": "= nodes['dns.public' => true].fields('domain.name', 'dns.aliases', 'ip_address')", + "private": "= nodes['dns.public' => false].fields('domain.name', 'dns.aliases', 'ip_address')" + }, + "service_type": "public_service", + "firewall": { + "dns": { + "from": "*", + "to": "= ip_address", + "port": "53" + } + } +}
\ No newline at end of file diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json new file mode 100644 index 00000000..9ddc0ec7 --- /dev/null +++ b/provider_base/services/monitor.json @@ -0,0 +1,29 @@ +{ + "nagios": { + "nagiosadmin_pw": "= secret :nagios_admin_password", + "domains_internal": "= nagios.hosts.values.map{|h|h['domain_internal_suffix']}.uniq", + "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>manager.env(e).provider.contacts.default}]} ]", + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.internal_suffix', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" + }, + "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", + "ssh": { + "monitor": { + "username": "= Leap::Platform.monitor_username", + "private_key": "= file(:monitor_priv_key)" + } + }, + "x509": { + "use": true, + "use_commercial": true, + "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" + }, + "firewall": { + "monitor": { + "from": "sysadmin", + "to": "= ip_address", + "port": [443, 80] + } + } +} diff --git a/provider_base/services/monitor.rb b/provider_base/services/monitor.rb new file mode 100644 index 00000000..01590d5c --- /dev/null +++ b/provider_base/services/monitor.rb @@ -0,0 +1,3 @@ +unless self.services.include? "webapp" + LeapCli.log :error, "service `monitor` requires service `webapp` on the same node (node #{self.name})." +end diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json new file mode 100644 index 00000000..c7e99d85 --- /dev/null +++ b/provider_base/services/mx.json @@ -0,0 +1,53 @@ +{ + "mx": { + // provider should define their own custom aliases. + // these are in *addition* to the standard reserved aliases for root and postmaster, etc. + "aliases": {}, + // this is the domain that is used for the OpenPGP header + "key_lookup_domain": "= global.services[:webapp].webapp.domain", + "dkim": { + // bit sizes larger than 2048 are not necessarily supported + "bit_size": 2048, + "public_key": "= remote_file_path(:dkim_pub_key) { generate_dkim_key(mx.dkim.bit_size) }", + "private_key": "= remote_file_path(:dkim_priv_key) { generate_dkim_key(mx.dkim.bit_size) }", + // generate selector based on first ten digits of pub key fingerprint: + "selector": "= fingerprint(local_file_path(:dkim_pub_key) { generate_dkim_key(mx.dkim.bit_size) }, :mode => :rsa).slice(0,10)" + } + }, + "stunnel": { + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } + }, + "haproxy": { + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } + }, + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username", + "password": "= secret :couch_leap_mx_password", + "salt": "= hex_secret :couch_leap_mx_password_salt, 128" + }, + "mynetworks": "= host_ips(nodes)", + "rbls": ["zen.spamhaus.org"], + "clamav": { + "whitelisted_addresses": [] + }, + "x509": { + "use": true, + "use_commercial": true, + "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" + }, + "service_type": "user_service", + "firewall": { + "mx": { + "from": "*", + "to": "= ip_address", + "port": [25, 465] + } + } +} diff --git a/provider_base/services/mx.rb b/provider_base/services/mx.rb new file mode 100644 index 00000000..03ee561f --- /dev/null +++ b/provider_base/services/mx.rb @@ -0,0 +1 @@ +apply_partial('_api_tester') diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json new file mode 100644 index 00000000..979d0ef9 --- /dev/null +++ b/provider_base/services/obfsproxy.json @@ -0,0 +1,9 @@ +{ + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" + }, + "gateway_address": "= try{pick_node(:obfs_gateway,nodes_near_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')} || try{pick_node(:obfs_gateway,nodes_like_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')}" + } +} diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json new file mode 100644 index 00000000..6f73e31c --- /dev/null +++ b/provider_base/services/openvpn.json @@ -0,0 +1,45 @@ +{ + "service_type": "user_service", + "x509": { + "use": true, + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", + "dh": "= file :dh_params, :missing => 'Diffie-Hellman parameters. Run `leap cert dh`'" + }, + "location": null, + "openvpn": { + "gateway_address": "REQUIRED", + "second_gateway_address": "= openvpn.allow_limited && openvpn.allow_unlimited ? 'REQUIRED' : nil", + "ports": ["80", "443", "53", "1194"], + "protocols": ["tcp", "udp"], + "filter_dns": false, + "adblock": false, + "user_ips": false, + "allow_limited": "= provider.service.allow_limited_bandwidth", + "allow_unlimited": "= provider.service.allow_unlimited_bandwidth", + "limited_prefix": "= provider.ca.client_certificates.limited_prefix", + "unlimited_prefix": "= provider.ca.client_certificates.unlimited_prefix", + "rate_limit": "= openvpn.allow_limited ? provider.service.bandwidth_limit : nil", + "configuration": { + "tls-cipher": "DHE-RSA-AES128-SHA", + "auth": "SHA1", + "cipher": "AES-128-CBC", + "keepalive": "10 30", + "tun-ipv6": true, + "fragment": 1500 + } + }, + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" + }, + "gateway_address": "= openvpn.gateway_address" + }, + "firewall": { + "vpn": { + "from": "*", + "to": "= openvpn.gateway_address", + "port": "= openvpn.ports + [obfsproxy.scramblesuit.port]" + } + } +} diff --git a/provider_base/services/soledad.json b/provider_base/services/soledad.json new file mode 100644 index 00000000..169588c8 --- /dev/null +++ b/provider_base/services/soledad.json @@ -0,0 +1,21 @@ +{ + "soledad": { + "port": 2323, + "couchdb_soledad_user": { + "username": "= global.services[:couchdb].couch.users[:soledad].username", + "password": "= secret :couch_soledad_password", + "salt": "= hex_secret :couch_soledad_password_salt, 128" + }, + "couchdb_leap_mx_user": { + "username": "= global.services[:couchdb].couch.users[:leap_mx].username" + } + }, + "service_type": "public_service", + "firewall": { + "soledad": { + "from": "*", + "to": "= ip_address", + "port": "= soledad.port" + } + } +} diff --git a/provider_base/services/soledad.rb b/provider_base/services/soledad.rb new file mode 100644 index 00000000..9b220c39 --- /dev/null +++ b/provider_base/services/soledad.rb @@ -0,0 +1,3 @@ +unless self.services.include? "couchdb" + LeapCli.log :error, "service `soledad` requires service `couchdb` on the same node (node #{self.name})." +end diff --git a/provider_base/services/static.json b/provider_base/services/static.json new file mode 100644 index 00000000..2f408ec1 --- /dev/null +++ b/provider_base/services/static.json @@ -0,0 +1,20 @@ +{ + "static": { + "formats": "=> try{static.domains.values.collect{|d| try{d.locations.values.collect{|l|l.format}} }.flatten.compact.uniq} || []", + // include a copy of provider.json in case any of the configured domains happens to match provider.domain + "bootstrap_files": { + "domain": "= provider.domain", + "enabled": "= !! try{static.domains[provider.domain]}", + "provider_json": "=> static.bootstrap_files.enabled ? try{nodes_like_me[:services => 'webapp'].values.first.definition_files['provider']} : nil", + "client_version": "= static.bootstrap_files.enabled ? provider.client_version : nil" + } + }, + "service_type": "public_service", + "firewall": { + "static": { + "from": "*", + "to": "= ip_address", + "port": [80, 443] + } + } +}
\ No newline at end of file diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json new file mode 100644 index 00000000..55d3d2ee --- /dev/null +++ b/provider_base/services/tor.json @@ -0,0 +1,15 @@ +{ + "tor": { + "bandwidth_rate": 6550, + "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", + "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", + "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')", + "hidden_service": { + "active": null, + "key_type": "RSA", + "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service.active" + } + } +} diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json new file mode 100644 index 00000000..b1d2ca59 --- /dev/null +++ b/provider_base/services/webapp.json @@ -0,0 +1,93 @@ +{ + "webapp": { + "admins": [], + "forbidden_usernames": [ + "admin", "admins", "administrator", "administrators", "arin-admin", + "certmaster", "contact", "email", "help", "help-desk", "help-ticket", + "help-tickets", "help_desk", "help_ticket", "help_tickets", "helpdesk", + "helpticket", "helptickets", "info", "mail", "maildrop", "noreply", + "owner", "owners", "postmaster", "reply", "robot", "ssladmin", "staff", + "support", "tech-support", "tech_support", "techsupport", "ticket", + "tickets", "vmail", "www-data"], + "domain": "= provider.domain", + "modules": ["user", "billing", "help"], + "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", + "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", + "customization_dir": "= file_path 'webapp'", + "client_certificates": "= provider.ca.client_certificates", + "allow_limited_certs": "= provider.service.allow_limited_bandwidth", + "allow_unlimited_certs": "= provider.service.allow_unlimited_bandwidth", + "allow_anonymous_certs": "= provider.service.allow_anonymous", + "allow_registration": "= provider.service.allow_registration", + "default_service_level": "= provider.service.default_service_level", + "service_levels": "= service_levels()", + "secret_token": "= secret :webapp_secret_token", + "api_version": 1, + "secure": false, + "client_version": "= provider.client_version", + "nagios_test_user": { + "username": "nagios_test", + "password": "= secret :nagios_test_password" + }, + "engines": [ + "support" + ], + "locales": "= provider.languages", + "default_locale": "= provider.default_language", + "api_tokens": { + "monitor": "= secret :api_monitor_auth_token", + "allowed_ips": "= host_ips(nodes_like_me)" + } + }, + "stunnel": { + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } + }, + "haproxy": { + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } + }, + "definition_files": { + "provider": "= file :provider_json_template", + "eip_service": "= file [:eip_service_json_template, 'v'+webapp.api_version.to_s]", + "soledad_service": "= file [:soledad_service_json_template, 'v'+webapp.api_version.to_s]", + "smtp_service": "= file [:smtp_service_json_template, 'v'+webapp.api_version.to_s]" + }, + "service_type": "public_service", + "api": { + "domain": "= 'api.' + webapp.domain", + "version": 1, + "port": 4430, + "ca_cert_uri": "= 'https://' + webapp.domain + '/ca.crt'", + "uri": "= %(https://#{api.domain}:#{api.port}/#{api.version})" + }, + "nickserver": { + "domain": "= 'nicknym.' + domain.full_suffix", + "couchdb_nickserver_user": { + "username": "= global.services[:couchdb].couch.users[:nickserver].username", + "password": "= secret :couch_nickserver_password", + "salt": "= hex_secret :couch_nickserver_password_salt, 128" + }, + "port": 6425 + }, + "dns": { + "aliases": "= [domain.full, webapp.domain, api.domain, nickserver.domain]" + }, + "x509": { + "use": true, + "use_commercial": true, + "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" + }, + "firewall": { + "webapp": { + "from": "*", + "to": "= ip_address", + "port": "= [api.port, 443, 80, nickserver.port]" + } + } +} diff --git a/provider_base/tags/development.json b/provider_base/tags/development.json new file mode 100644 index 00000000..caf18e9d --- /dev/null +++ b/provider_base/tags/development.json @@ -0,0 +1,3 @@ +{ + "environment": "development" +}
\ No newline at end of file diff --git a/provider_base/tags/local.json b/provider_base/tags/local.json new file mode 100644 index 00000000..48312b33 --- /dev/null +++ b/provider_base/tags/local.json @@ -0,0 +1,3 @@ +{ + "environment": "local" +}
\ No newline at end of file diff --git a/provider_base/tags/production.json b/provider_base/tags/production.json new file mode 100644 index 00000000..ea17498f --- /dev/null +++ b/provider_base/tags/production.json @@ -0,0 +1,3 @@ +{ + "environment": "production" +}
\ No newline at end of file diff --git a/provider_base/templates/common.json b/provider_base/templates/common.json new file mode 100644 index 00000000..a7675b15 --- /dev/null +++ b/provider_base/templates/common.json @@ -0,0 +1,3 @@ +{ + "ip_address": "REQUIRED" +}
\ No newline at end of file diff --git a/provider_base/templates/couchdb.json b/provider_base/templates/couchdb.json new file mode 100644 index 00000000..34b60915 --- /dev/null +++ b/provider_base/templates/couchdb.json @@ -0,0 +1,5 @@ +{ + "couch": { + "mode": "plain" + } +} diff --git a/provider_base/templates/openvpn.json b/provider_base/templates/openvpn.json new file mode 100644 index 00000000..cbe183e8 --- /dev/null +++ b/provider_base/templates/openvpn.json @@ -0,0 +1,7 @@ +{ + "openvpn": { + "gateway_address": "REQUIRED", + "ports": ["443"], + "protocols": ["tcp"] + } +} diff --git a/provider_base/test/openvpn/client.ovpn.erb b/provider_base/test/openvpn/client.ovpn.erb new file mode 100644 index 00000000..af183ef4 --- /dev/null +++ b/provider_base/test/openvpn/client.ovpn.erb @@ -0,0 +1,28 @@ +client +dev tun +remote-cert-tls server +remote-random +nobind +script-security 2 +verb 3 +auth SHA1 +cipher AES-128-CBC +tls-cipher DHE-RSA-AES128-SHA + +<% vpn_nodes.each_node do |node| -%> +<%= "remote #{node.openvpn.gateway_address} 1194 udp"%> +<% end -%> + +<ca> +<%= read_file! :ca_cert -%> +</ca> + +<cert> +<%# read_file! :test_client_cert -%> +<%= cert -%> +</cert> + +<key> +<%# read_file! :test_client_key -%> +<%= key -%> +</key> diff --git a/puppet/bin/apply_on_node.sh b/puppet/bin/apply_on_node.sh new file mode 100755 index 00000000..09e5b035 --- /dev/null +++ b/puppet/bin/apply_on_node.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Script to use on a node for debugging +# Usage: ./apply_on_node.sh <puppet parameters> +# +# Example: ./apply_on_node.sh --debug --verbose + +ROOTDIR='/srv/leap' +PLATFORM="$ROOTDIR" +MODULEPATH="$PLATFORM/puppet/modules" +LOG=/var/log/leap.log + +# example tags to use +#TAGS='--tags=leap_base,leap_service,leap_slow' +#TAGS='--tags=leap_base,leap_slow' +#TAGS='--tags=leap_base,leap_service' + +####### +# Setup +####### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/setup.pp $TAGS $@ |tee $LOG 2>&1 + +######### +# site.pp +######### + +puppet apply -v --confdir $PLATFORM/puppet --libdir $PLATFORM/puppet/lib --modulepath=$MODULEPATH $PLATFORM/puppet/manifests/site.pp $TAGS $@ |tee $LOG 2>&1 + + diff --git a/puppet/hiera.yaml b/puppet/hiera.yaml new file mode 100644 index 00000000..93448e23 --- /dev/null +++ b/puppet/hiera.yaml @@ -0,0 +1,15 @@ +--- +:backends: + - yaml + - puppet + +:logger: console + +:yaml: + :datadir: /etc/leap + +:hierarchy: + - hiera + +:puppet: + :datasource: data diff --git a/puppet/lib/puppet/parser/functions/create_resources_hash_from.rb b/puppet/lib/puppet/parser/functions/create_resources_hash_from.rb new file mode 100644 index 00000000..47d0df9c --- /dev/null +++ b/puppet/lib/puppet/parser/functions/create_resources_hash_from.rb @@ -0,0 +1,116 @@ +# +# create_resources_hash_from.rb +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Puppet::Parser::Functions + newfunction(:create_resources_hash_from, :type => :rvalue, :doc => <<-EOS +Given: + A formatted string (to use as the resource name) + An array to loop through (because puppet cannot loop) + A hash defining the parameters for a resource + And optionally an hash of parameter names to add to the resource and an + associated formatted string that should be configured with the current + element of the loop array + +This function will return a hash of hashes that can be used with the +create_resources function. + +*Examples:* + $allowed_hosts = ['10.0.0.0/8', '192.168.0.0/24'] + $resource_name = "100 allow %s to apache on ports 80" + $my_resource_hash = { + 'proto' => 'tcp', + 'action' => 'accept', + 'dport' => 80 + } + $dynamic_parameters = { + 'source' => '%s' + } + + $created_resource_hash = create_resources_hash_from($resource_name, $allowed_hosts, $my_resource_hash, $dynamic_parameters) + +$created_resource_hash would equal: + { + '100 allow 10.0.0.0/8 to apache on ports 80' => { + 'proto' => 'tcp', + 'action' => 'accept', + 'dport' => 80, + 'source' => '10.0.0.0/8' + }, + '100 allow 192.168.0.0/24 to apache on ports 80' => { + 'proto' => 'tcp', + 'action' => 'accept', + 'dport' => 80, + 'source' => '192.168.0.0/24' + } + } + +$created_resource_hash could then be used with create_resources + + create_resources(firewall, $created_resource_hash) + +To create a bunch of resources in a way that would only otherwise be possible +with a loop of some description. + EOS + ) do |arguments| + + raise Puppet::ParseError, "create_resources_hash_from(): Wrong number of arguments " + + "given (#{arguments.size} for 3 or 4)" if arguments.size < 3 or arguments.size > 4 + + formatted_string = arguments[0] + + unless formatted_string.is_a?(String) + raise(Puppet::ParseError, 'create_resources_hash_from(): first argument must be a string') + end + + loop_array = arguments[1] + + unless loop_array.is_a?(Array) + raise(Puppet::ParseError, 'create_resources_hash_from(): second argument must be an array') + end + + resource_hash = arguments[2] + unless resource_hash.is_a?(Hash) + raise(Puppet::ParseError, 'create_resources_hash_from(): third argument must be a hash') + end + + if arguments.size == 4 + dynamic_parameters = arguments[3] + unless dynamic_parameters.is_a?(Hash) + raise(Puppet::ParseError, 'create_resources_hash_from(): fourth argument must be a hash') + end + end + + result = {} + + loop_array.each do |i| + my_resource_hash = resource_hash.clone + if dynamic_parameters + dynamic_parameters.each do |param, value| + if my_resource_hash.member?(param) + raise(Puppet::ParseError, "create_resources_hash_from(): dynamic_parameter '#{param}' already exists in resource hash") + end + my_resource_hash[param] = sprintf(value,[i]) + end + end + result[sprintf(formatted_string,[i])] = my_resource_hash + end + + result + end +end + +# vim: set ts=2 sw=2 et : +# encoding: utf-8 diff --git a/puppet/lib/puppet/parser/functions/sorted_json.rb b/puppet/lib/puppet/parser/functions/sorted_json.rb new file mode 100644 index 00000000..605da00e --- /dev/null +++ b/puppet/lib/puppet/parser/functions/sorted_json.rb @@ -0,0 +1,47 @@ +# +# Written by Gavin Mogan, from https://gist.github.com/halkeye/2287885 +# Put in the public domain by the author. +# + +require 'json' + +def sorted_json(obj) + case obj + when String, Fixnum, Float, TrueClass, FalseClass, NilClass + return obj.to_json + when Array + arrayRet = [] + obj.each do |a| + arrayRet.push(sorted_json(a)) + end + return "[" << arrayRet.join(',') << "]"; + when Hash + ret = [] + obj.keys.sort.each do |k| + ret.push(k.to_json << ":" << sorted_json(obj[k])) + end + return "{" << ret.join(",") << "}"; + else + raise Exception("Unable to handle object of type <%s>" % obj.class.to_s) + end +end + +module Puppet::Parser::Functions + newfunction(:sorted_json, :type => :rvalue, :doc => <<-EOS +This function takes data, outputs making sure the hash keys are sorted + +*Examples:* + + sorted_json({'key'=>'value'}) + +Would return: {'key':'value'} + EOS + ) do |arguments| + raise(Puppet::ParseError, "sorted_json(): Wrong number of arguments " + + "given (#{arguments.size} for 1)") if arguments.size != 1 + + json = arguments[0] + return sorted_json(json) + end +end + diff --git a/puppet/lib/puppet/parser/functions/sorted_yaml.rb b/puppet/lib/puppet/parser/functions/sorted_yaml.rb new file mode 100644 index 00000000..46cd46ce --- /dev/null +++ b/puppet/lib/puppet/parser/functions/sorted_yaml.rb @@ -0,0 +1,400 @@ +# encoding: UTF-8 +# +# provides sorted_yaml() function, using Ya2YAML. +# see https://github.com/afunai/ya2yaml +# + +class Ya2YAML + # + # Author:: Akira FUNAI + # Copyright:: Copyright (c) 2006-2010 Akira FUNAI + # License:: MIT License + # + + def initialize(opts = {}) + options = opts.dup + options[:indent_size] = 2 if options[:indent_size].to_i <= 0 + options[:minimum_block_length] = 0 if options[:minimum_block_length].to_i <= 0 + options.update( + { + :printable_with_syck => true, + :escape_b_specific => true, + :escape_as_utf8 => true, + } + ) if options[:syck_compatible] + + @options = options + end + + def _ya2yaml(obj) + #raise 'set $KCODE to "UTF8".' if (RUBY_VERSION < '1.9.0') && ($KCODE != 'UTF8') + if (RUBY_VERSION < '1.9.0') + $KCODE = 'UTF8' + end + '--- ' + emit(obj, 1) + "\n" + rescue SystemStackError + raise ArgumentError, "ya2yaml can't handle circular references" + end + + private + + def emit(obj, level) + case obj + when Array + if (obj.length == 0) + '[]' + else + indent = "\n" + s_indent(level - 1) + ### + ### NOTE: a minor modification to normal Ya2YAML... + ### We want arrays to be output in sorted order, not just + ### Hashes. + ### + #obj.collect {|o| + # indent + '- ' + emit(o, level + 1) + #}.join('') + obj.sort {|a,b| a.to_s <=> b.to_s}.collect {|o| + indent + '- ' + emit(o, level + 1) + }.join('') + end + when Hash + if (obj.length == 0) + '{}' + else + indent = "\n" + s_indent(level - 1) + hash_order = @options[:hash_order] + if (hash_order && level == 1) + hash_keys = obj.keys.sort {|x, y| + x_order = hash_order.index(x) ? hash_order.index(x) : Float::MAX + y_order = hash_order.index(y) ? hash_order.index(y) : Float::MAX + o = (x_order <=> y_order) + (o != 0) ? o : (x.to_s <=> y.to_s) + } + elsif @options[:preserve_order] + hash_keys = obj.keys + else + hash_keys = obj.keys.sort {|x, y| x.to_s <=> y.to_s } + end + hash_keys.collect {|k| + key = emit(k, level + 1) + if ( + is_one_plain_line?(key) || + key =~ /\A(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_NULL})\z/x + ) + indent + key + ': ' + emit(obj[k], level + 1) + else + indent + '? ' + key + + indent + ': ' + emit(obj[k], level + 1) + end + }.join('') + end + when NilClass + '~' + when String + emit_string(obj, level) + when TrueClass, FalseClass + obj.to_s + when Fixnum, Bignum, Float + obj.to_s + when Date + obj.to_s + when Time + offset = obj.gmtoff + off_hm = sprintf( + '%+.2d:%.2d', + (offset / 3600.0).to_i, + (offset % 3600.0) / 60 + ) + u_sec = (obj.usec != 0) ? sprintf(".%.6d", obj.usec) : '' + obj.strftime("%Y-%m-%d %H:%M:%S#{u_sec} #{off_hm}") + when Symbol + '!ruby/symbol ' + emit_string(obj.to_s, level) + when Range + '!ruby/range ' + obj.to_s + when Regexp + '!ruby/regexp ' + obj.inspect + else + case + when obj.is_a?(Struct) + struct_members = {} + obj.each_pair{|k, v| struct_members[k.to_s] = v } + '!ruby/struct:' + obj.class.to_s.sub(/^(Struct::(.+)|.*)$/, '\2') + ' ' + + emit(struct_members, level + 1) + else + # serialized as a generic object + object_members = {} + obj.instance_variables.each{|k, v| + object_members[k.to_s.sub(/^@/, '')] = obj.instance_variable_get(k) + } + '!ruby/object:' + obj.class.to_s + ' ' + + emit(object_members, level + 1) + end + end + end + + def emit_string(str, level) + (is_string, is_printable, is_one_line, is_one_plain_line) = string_type(str) + if is_string + if is_printable + if is_one_plain_line + emit_simple_string(str, level) + else + (is_one_line || str.length < @options[:minimum_block_length]) ? + emit_quoted_string(str, level) : + emit_block_string(str, level) + end + else + emit_quoted_string(str, level) + end + else + emit_base64_binary(str, level) + end + end + + def emit_simple_string(str, level) + str + end + + def emit_block_string(str, level) + str = normalize_line_break(str) + + indent = s_indent(level) + indentation_indicator = (str =~ /\A /) ? indent.size.to_s : '' + str =~ /(#{REX_NORMAL_LB}*)\z/ + chomping_indicator = case $1.length + when 0 + '-' + when 1 + '' + else + '+' + end + + str.chomp! + str.gsub!(/#{REX_NORMAL_LB}/) { + $1 + indent + } + '|' + indentation_indicator + chomping_indicator + "\n" + indent + str + end + + def emit_quoted_string(str, level) + str = yaml_escape(normalize_line_break(str)) + if (str.length < @options[:minimum_block_length]) + str.gsub!(/#{REX_NORMAL_LB}/) { ESCAPE_SEQ_LB[$1] } + else + str.gsub!(/#{REX_NORMAL_LB}$/) { ESCAPE_SEQ_LB[$1] } + str.gsub!(/(#{REX_NORMAL_LB}+)(.)/) { + trail_c = $3 + $1 + trail_c.sub(/([\t ])/) { ESCAPE_SEQ_WS[$1] } + } + indent = s_indent(level) + str.gsub!(/#{REX_NORMAL_LB}/) { + ESCAPE_SEQ_LB[$1] + "\\\n" + indent + } + end + '"' + str + '"' + end + + def emit_base64_binary(str, level) + indent = "\n" + s_indent(level) + base64 = [str].pack('m') + '!binary |' + indent + base64.gsub(/\n(?!\z)/, indent) + end + + def string_type(str) + if str.respond_to?(:encoding) && (!str.valid_encoding? || str.encoding == Encoding::ASCII_8BIT) + return false, false, false, false + end + (ucs_codes = str.unpack('U*')) rescue ( + # ArgumentError -> binary data + return false, false, false, false + ) + if ( + @options[:printable_with_syck] && + str =~ /\A#{REX_ANY_LB}* | #{REX_ANY_LB}*\z|#{REX_ANY_LB}{2}\z/ + ) + # detour Syck bug + return true, false, nil, false + end + ucs_codes.each {|ucs_code| + return true, false, nil, false unless is_printable?(ucs_code) + } + return true, true, is_one_line?(str), is_one_plain_line?(str) + end + + def is_printable?(ucs_code) + # YAML 1.1 / 4.1.1. + ( + [0x09, 0x0a, 0x0d, 0x85].include?(ucs_code) || + (ucs_code <= 0x7e && ucs_code >= 0x20) || + (ucs_code <= 0xd7ff && ucs_code >= 0xa0) || + (ucs_code <= 0xfffd && ucs_code >= 0xe000) || + (ucs_code <= 0x10ffff && ucs_code >= 0x10000) + ) && + !( + # treat LS/PS as non-printable characters + @options[:escape_b_specific] && + (ucs_code == 0x2028 || ucs_code == 0x2029) + ) + end + + def is_one_line?(str) + str !~ /#{REX_ANY_LB}(?!\z)/ + end + + def is_one_plain_line?(str) + # YAML 1.1 / 4.6.11. + str !~ /^([\-\?:,\[\]\{\}\#&\*!\|>'"%@`\s]|---|\.\.\.)/ && + str !~ /[:\#\s\[\]\{\},]/ && + str !~ /#{REX_ANY_LB}/ && + str !~ /^(#{REX_BOOL}|#{REX_FLOAT}|#{REX_INT}|#{REX_MERGE} + |#{REX_NULL}|#{REX_TIMESTAMP}|#{REX_VALUE})$/x + end + + def s_indent(level) + # YAML 1.1 / 4.2.2. + ' ' * (level * @options[:indent_size]) + end + + def normalize_line_break(str) + # YAML 1.1 / 4.1.4. + str.gsub(/(#{REX_CRLF}|#{REX_CR}|#{REX_NEL})/, "\n") + end + + def yaml_escape(str) + # YAML 1.1 / 4.1.6. + str.gsub(/[^a-zA-Z0-9]/u) {|c| + ucs_code, = (c.unpack('U') rescue [??]) + case + when ESCAPE_SEQ[c] + ESCAPE_SEQ[c] + when is_printable?(ucs_code) + c + when @options[:escape_as_utf8] + c.respond_to?(:bytes) ? + c.bytes.collect {|b| '\\x%.2x' % b }.join : + '\\x' + c.unpack('H2' * c.size).join('\\x') + when ucs_code == 0x2028 || ucs_code == 0x2029 + ESCAPE_SEQ_LB[c] + when ucs_code <= 0x7f + sprintf('\\x%.2x', ucs_code) + when ucs_code <= 0xffff + sprintf('\\u%.4x', ucs_code) + else + sprintf('\\U%.8x', ucs_code) + end + } + end + + module Constants + UCS_0X85 = [0x85].pack('U') # c285@UTF8 Unicode next line + UCS_0XA0 = [0xa0].pack('U') # c2a0@UTF8 Unicode non-breaking space + UCS_0X2028 = [0x2028].pack('U') # e280a8@UTF8 Unicode line separator + UCS_0X2029 = [0x2029].pack('U') # e280a9@UTF8 Unicode paragraph separator + + # non-break characters + ESCAPE_SEQ = { + "\x00" => '\\0', + "\x07" => '\\a', + "\x08" => '\\b', + "\x0b" => '\\v', + "\x0c" => '\\f', + "\x1b" => '\\e', + "\"" => '\\"', + "\\" => '\\\\', + } + + # non-breaking space + ESCAPE_SEQ_NS = { + UCS_0XA0 => '\\_', + } + + # white spaces + ESCAPE_SEQ_WS = { + "\x09" => '\\t', + " " => '\\x20', + } + + # line breaks + ESCAPE_SEQ_LB ={ + "\x0a" => '\\n', + "\x0d" => '\\r', + UCS_0X85 => '\\N', + UCS_0X2028 => '\\L', + UCS_0X2029 => '\\P', + } + + # regexps for line breaks + REX_LF = Regexp.escape("\x0a") + REX_CR = Regexp.escape("\x0d") + REX_CRLF = Regexp.escape("\x0d\x0a") + REX_NEL = Regexp.escape(UCS_0X85) + REX_LS = Regexp.escape(UCS_0X2028) + REX_PS = Regexp.escape(UCS_0X2029) + + REX_ANY_LB = /(#{REX_LF}|#{REX_CR}|#{REX_NEL}|#{REX_LS}|#{REX_PS})/ + REX_NORMAL_LB = /(#{REX_LF}|#{REX_LS}|#{REX_PS})/ + + # regexps for language-Independent types for YAML1.1 + REX_BOOL = / + y|Y|yes|Yes|YES|n|N|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF + /x + REX_FLOAT = / + [-+]?([0-9][0-9_]*)?\.[0-9.]*([eE][-+][0-9]+)? # (base 10) + |[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]* # (base 60) + |[-+]?\.(inf|Inf|INF) # (infinity) + |\.(nan|NaN|NAN) # (not a number) + /x + REX_INT = / + [-+]?0b[0-1_]+ # (base 2) + |[-+]?0[0-7_]+ # (base 8) + |[-+]?(0|[1-9][0-9_]*) # (base 10) + |[-+]?0x[0-9a-fA-F_]+ # (base 16) + |[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+ # (base 60) + /x + REX_MERGE = / + << + /x + REX_NULL = / + ~ # (canonical) + |null|Null|NULL # (English) + | # (Empty) + /x + REX_TIMESTAMP = / + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] # (ymd) + |[0-9][0-9][0-9][0-9] # (year) + -[0-9][0-9]? # (month) + -[0-9][0-9]? # (day) + ([Tt]|[ \t]+)[0-9][0-9]? # (hour) + :[0-9][0-9] # (minute) + :[0-9][0-9] # (second) + (\.[0-9]*)? # (fraction) + (([ \t]*)Z|[-+][0-9][0-9]?(:[0-9][0-9])?)? # (time zone) + /x + REX_VALUE = / + = + /x + end + + include Constants +end + +module Puppet::Parser::Functions + newfunction(:sorted_yaml, + :type => :rvalue, + :doc => "This function outputs yaml, but ensures the keys are sorted." + ) do |arguments| + + if arguments.is_a?(Array) + if arguments.size != 1 + raise(Puppet::ParseError, "sorted_yaml(): Wrong number of arguments given (#{arguments.size} for 1)") + end + yaml = arguments.first + else + yaml = arguments + end + return Ya2YAML.new()._ya2yaml(yaml) + end +end diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp new file mode 100644 index 00000000..ecda4012 --- /dev/null +++ b/puppet/manifests/site.pp @@ -0,0 +1,60 @@ +# set a default exec path +# the logoutput exec parameter defaults to "on_error" in puppet 3, +# but to "false" in puppet 2.7, so we need to set this globally here +Exec { + logoutput => on_failure, + path => '/usr/bin:/usr/sbin/:/bin:/sbin:/usr/local/bin:/usr/local/sbin' +} + +Package <| provider == 'apt' |> { + install_options => ['--no-install-recommends'], +} + +$services = hiera('services', []) +$services_str = join($services, ', ') +notice("Services for ${fqdn}: ${services_str}") + +# In the default deployment case, we want to run an 'apt-get dist-upgrade' +# to ensure the latest packages are installed. This is done by including the +# class 'site_config::slow' here. However, you only changed a small bit of +# the platform and want to skip this slow part of deployment, you can do that +# by using 'leap deploy --fast' which will only apply those resources that are +# tagged with 'leap_base' or 'leap_service'. +# See https://leap.se/en/docs/platform/details/under-the-hood#tags +include site_config::slow + +if member($services, 'openvpn') { + include site_openvpn +} + +if member($services, 'couchdb') { + include site_couchdb +} + +if member($services, 'webapp') { + include site_webapp +} + +if member($services, 'soledad') { + include soledad::server +} + +if member($services, 'monitor') { + include site_nagios +} + +if member($services, 'tor') { + include site_tor +} + +if member($services, 'mx') { + include site_mx +} + +if member($services, 'static') { + include site_static +} + +if member($services, 'obfsproxy') { + include site_obfsproxy +} diff --git a/puppet/modules/clamav/files/01-leap.conf b/puppet/modules/clamav/files/01-leap.conf new file mode 100644 index 00000000..a7e49d17 --- /dev/null +++ b/puppet/modules/clamav/files/01-leap.conf @@ -0,0 +1,58 @@ +# If running clamd in "LocalSocket" mode (*NOT* in TCP/IP mode), and +# either "SOcket Cat" (socat) or the "IO::Socket::UNIX" perl module +# are installed on the system, and you want to report whether clamd +# is running or not, uncomment the "clamd_socket" variable below (you +# will be warned if neither socat nor IO::Socket::UNIX are found, but +# the script will still run). You will also need to set the correct +# path to your clamd socket file (if unsure of the path, check the +# "LocalSocket" setting in your clamd.conf file for socket location). +clamd_socket="/run/clamav/clamd.ctl" + +# If you would like to attempt to restart ClamD if detected not running, +# uncomment the next 2 lines. Confirm the path to the "clamd_lock" file +# (usually can be found in the clamd init script) and also enter the clamd +# start command for your particular distro for the "start_clamd" variable +# (the sample start command shown below should work for most linux distros). +# NOTE: these 2 variables are dependant on the "clamd_socket" variable +# shown above - if not enabled, then the following 2 variables will be +# ignored, whether enabled or not. +clamd_lock="/run/clamav/clamd.pid" +start_clamd="clamdscan --reload" + +ss_dbs=" + junk.ndb + phish.ndb + rogue.hdb + sanesecurity.ftm + scam.ndb + sigwhitelist.ign2 + spamattach.hdb + spamimg.hdb + winnow.attachments.hdb + winnow_bad_cw.hdb + winnow_extended_malware.hdb + winnow_malware.hdb + winnow_malware_links.ndb + malwarehash.hsb + doppelstern.hdb + bofhland_cracked_URL.ndb + bofhland_malware_attach.hdb + bofhland_malware_URL.ndb + bofhland_phishing_URL.ndb + crdfam.clamav.hdb + phishtank.ndb + porcupine.ndb + spear.ndb + spearl.ndb +" + +# ======================== +# SecuriteInfo Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any SecuriteInfo database downloads, remove the appropriate +# lines below. To disable all SecuriteInfo database file downloads, +# comment all of the following lines. +si_dbs="" + +mbl_dbs=""
\ No newline at end of file diff --git a/puppet/modules/clamav/files/clamav-daemon_default b/puppet/modules/clamav/files/clamav-daemon_default new file mode 100644 index 00000000..b4cd6a4f --- /dev/null +++ b/puppet/modules/clamav/files/clamav-daemon_default @@ -0,0 +1,8 @@ +# This is a file designed only t0 set special environment variables +# eg TMP or TMPDIR. It is sourced from a shell script, so anything +# put in here must be in variable=value format, suitable for sourcing +# from a shell script. +# Examples: +# export TMPDIR=/dev/shm +export TMP=/var/tmp +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/files/clamav-milter_default b/puppet/modules/clamav/files/clamav-milter_default new file mode 100644 index 00000000..5e33e822 --- /dev/null +++ b/puppet/modules/clamav/files/clamav-milter_default @@ -0,0 +1,14 @@ +# +# clamav-milter init options +# + +## SOCKET_RWGROUP +# by default, the socket created by the milter has permissions +# clamav:clamav:755. SOCKET_RWGROUP changes the group and changes the +# permissions to 775 to give read-write access to that group. +# +# If you are using postfix to speak to the milter, you have to give permission +# to the postfix group to write +# +SOCKET_RWGROUP=postfix +export TMPDIR=/var/tmp diff --git a/puppet/modules/clamav/manifests/daemon.pp b/puppet/modules/clamav/manifests/daemon.pp new file mode 100644 index 00000000..2e13a8fb --- /dev/null +++ b/puppet/modules/clamav/manifests/daemon.pp @@ -0,0 +1,91 @@ +# deploy clamav daemon +class clamav::daemon { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { [ 'clamav-daemon', 'arj' ]: + ensure => installed; + } + + service { + 'clamav-daemon': + ensure => running, + name => clamav-daemon, + pattern => '/usr/sbin/clamd', + enable => true, + hasrestart => true, + subscribe => File['/etc/default/clamav-daemon'], + require => Package['clamav-daemon']; + } + + file { + '/var/run/clamav': + ensure => directory, + mode => '0750', + owner => clamav, + group => postfix, + require => [Package['postfix'], Package['clamav-daemon']]; + + '/var/lib/clamav': + mode => '0755', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + + '/etc/default/clamav-daemon': + source => 'puppet:///modules/clamav/clamav-daemon_default', + mode => '0644', + owner => root, + group => root; + + # this file contains additional domains that we want the clamav + # phishing process to look for (our domain) + '/var/lib/clamav/local.pdb': + content => template('clamav/local.pdb.erb'), + mode => '0644', + owner => clamav, + group => clamav, + require => Package['clamav-daemon']; + } + + file_line { + 'clamav_daemon_tmp': + path => '/etc/clamav/clamd.conf', + line => 'TemporaryDirectory /var/tmp', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'enable_phishscanurls': + path => '/etc/clamav/clamd.conf', + match => 'PhishingScanURLs no', + line => 'PhishingScanURLs yes', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_LogSyslog_true': + path => '/etc/clamav/clamd.conf', + match => '^LogSyslog false', + line => 'LogSyslog true', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + + 'clamav_MaxThreads': + path => '/etc/clamav/clamd.conf', + match => 'MaxThreads 20', + line => 'MaxThreads 100', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + + # remove LogFile line + file_line { + 'clamav_LogFile': + path => '/etc/clamav/clamd.conf', + match => '^LogFile .*', + line => '', + require => Package['clamav-daemon'], + notify => Service['clamav-daemon']; + } + +} diff --git a/puppet/modules/clamav/manifests/freshclam.pp b/puppet/modules/clamav/manifests/freshclam.pp new file mode 100644 index 00000000..80c822a4 --- /dev/null +++ b/puppet/modules/clamav/manifests/freshclam.pp @@ -0,0 +1,23 @@ +class clamav::freshclam { + + package { 'clamav-freshclam': ensure => installed } + + service { + 'freshclam': + ensure => running, + enable => true, + name => clamav-freshclam, + pattern => '/usr/bin/freshclam', + hasrestart => true, + require => Package['clamav-freshclam']; + } + + file_line { + 'freshclam_notify': + path => '/etc/clamav/freshclam.conf', + line => 'NotifyClamd /etc/clamav/clamd.conf', + require => Package['clamav-freshclam'], + notify => Service['freshclam']; + } + +} diff --git a/puppet/modules/clamav/manifests/init.pp b/puppet/modules/clamav/manifests/init.pp new file mode 100644 index 00000000..de8fb4dc --- /dev/null +++ b/puppet/modules/clamav/manifests/init.pp @@ -0,0 +1,8 @@ +class clamav { + + include clamav::daemon + include clamav::milter + include clamav::unofficial_sigs + include clamav::freshclam + +} diff --git a/puppet/modules/clamav/manifests/milter.pp b/puppet/modules/clamav/manifests/milter.pp new file mode 100644 index 00000000..e8a85e3f --- /dev/null +++ b/puppet/modules/clamav/manifests/milter.pp @@ -0,0 +1,50 @@ +class clamav::milter { + + $clamav = hiera('clamav') + $whitelisted_addresses = $clamav['whitelisted_addresses'] + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + + package { 'clamav-milter': ensure => installed } + + service { + 'clamav-milter': + ensure => running, + enable => true, + name => clamav-milter, + pattern => '/usr/sbin/clamav-milter', + hasrestart => true, + require => Package['clamav-milter'], + subscribe => File['/etc/default/clamav-milter']; + } + + file { + '/run/clamav/milter.ctl': + mode => '0666', + owner => clamav, + group => postfix, + require => Class['clamav::daemon']; + + '/etc/clamav/clamav-milter.conf': + content => template('clamav/clamav-milter.conf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter'], + subscribe => Service['clamav-milter']; + + '/etc/default/clamav-milter': + source => 'puppet:///modules/clamav/clamav-milter_default', + mode => '0644', + owner => root, + group => root; + + '/etc/clamav/whitelisted_addresses': + content => template('clamav/whitelisted_addresses.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['clamav-milter']; + } + +} diff --git a/puppet/modules/clamav/manifests/unofficial_sigs.pp b/puppet/modules/clamav/manifests/unofficial_sigs.pp new file mode 100644 index 00000000..2d849585 --- /dev/null +++ b/puppet/modules/clamav/manifests/unofficial_sigs.pp @@ -0,0 +1,23 @@ +class clamav::unofficial_sigs { + + package { 'clamav-unofficial-sigs': + ensure => installed + } + + ensure_packages(['wget', 'gnupg', 'socat', 'rsync', 'curl']) + + file { + '/var/log/clamav-unofficial-sigs.log': + ensure => file, + owner => clamav, + group => clamav, + require => Package['clamav-unofficial-sigs']; + + '/etc/clamav-unofficial-sigs.conf.d/01-leap.conf': + source => 'puppet:///modules/clamav/01-leap.conf', + mode => '0755', + owner => root, + group => root, + require => Package['clamav-unofficial-sigs']; + } +} diff --git a/puppet/modules/clamav/templates/clamav-milter.conf.erb b/puppet/modules/clamav/templates/clamav-milter.conf.erb new file mode 100644 index 00000000..9bf7099e --- /dev/null +++ b/puppet/modules/clamav/templates/clamav-milter.conf.erb @@ -0,0 +1,28 @@ +# THIS FILE MANAGED BY PUPPET +MilterSocket /var/run/clamav/milter.ctl +FixStaleSocket true +User clamav +MilterSocketGroup clamav +MilterSocketMode 666 +AllowSupplementaryGroups true +ReadTimeout 120 +Foreground false +PidFile /var/run/clamav/clamav-milter.pid +ClamdSocket unix:/var/run/clamav/clamd.ctl +OnClean Accept +OnInfected Reject +OnFail Defer +AddHeader Replace +LogSyslog true +LogFacility LOG_LOCAL6 +LogVerbose yes +LogInfected Basic +LogTime true +LogFileUnlock false +LogClean Off +LogRotate true +SupportMultipleRecipients false +MaxFileSize 10M +TemporaryDirectory /var/tmp +RejectMsg "Message refused due to content violation: %v - contact https://<%= @domain %>/tickets/new if this is in error" +Whitelist /etc/clamav/whitelisted_addresses diff --git a/puppet/modules/clamav/templates/local.pdb.erb b/puppet/modules/clamav/templates/local.pdb.erb new file mode 100644 index 00000000..9ea0584a --- /dev/null +++ b/puppet/modules/clamav/templates/local.pdb.erb @@ -0,0 +1 @@ +H:<%= @domain %> diff --git a/puppet/modules/clamav/templates/whitelisted_addresses.erb b/puppet/modules/clamav/templates/whitelisted_addresses.erb new file mode 100644 index 00000000..9e068ec5 --- /dev/null +++ b/puppet/modules/clamav/templates/whitelisted_addresses.erb @@ -0,0 +1,5 @@ +<%- if @whitelisted_addresses then -%> +<% @whitelisted_addresses.each do |name| -%> +From::<%= name %> +<% end -%> +<% end -%> diff --git a/CHANGELOG b/puppet/modules/concat/CHANGELOG index c506cf1a..c506cf1a 100644 --- a/CHANGELOG +++ b/puppet/modules/concat/CHANGELOG diff --git a/puppet/modules/concat/LICENSE b/puppet/modules/concat/LICENSE new file mode 100644 index 00000000..6a9e9a19 --- /dev/null +++ b/puppet/modules/concat/LICENSE @@ -0,0 +1,14 @@ + Copyright 2012 R.I.Pienaar + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Modulefile b/puppet/modules/concat/Modulefile index d6ab2bb0..d6ab2bb0 100644 --- a/Modulefile +++ b/puppet/modules/concat/Modulefile diff --git a/README.markdown b/puppet/modules/concat/README.markdown index 8736d57a..8736d57a 100644 --- a/README.markdown +++ b/puppet/modules/concat/README.markdown diff --git a/puppet/modules/concat/Rakefile b/puppet/modules/concat/Rakefile new file mode 100644 index 00000000..764aebd2 --- /dev/null +++ b/puppet/modules/concat/Rakefile @@ -0,0 +1,13 @@ +require 'rake' +require 'rspec/core/rake_task' + +task :default => [:spec] + +desc "Run all module spec tests (Requires rspec-puppet gem)" +RSpec::Core::RakeTask.new(:spec) + +desc "Build package" +task :build do + system("puppet-module build") +end + diff --git a/files/concatfragments.sh b/puppet/modules/concat/files/concatfragments.sh index c9397975..c9397975 100755 --- a/files/concatfragments.sh +++ b/puppet/modules/concat/files/concatfragments.sh diff --git a/puppet/modules/concat/files/null/.gitignore b/puppet/modules/concat/files/null/.gitignore new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/puppet/modules/concat/files/null/.gitignore diff --git a/lib/facter/concat_basedir.rb b/puppet/modules/concat/lib/facter/concat_basedir.rb index 02e9c5bf..02e9c5bf 100644 --- a/lib/facter/concat_basedir.rb +++ b/puppet/modules/concat/lib/facter/concat_basedir.rb diff --git a/manifests/fragment.pp b/puppet/modules/concat/manifests/fragment.pp index 943bf671..943bf671 100644 --- a/manifests/fragment.pp +++ b/puppet/modules/concat/manifests/fragment.pp diff --git a/manifests/init.pp b/puppet/modules/concat/manifests/init.pp index 0b3ed564..0b3ed564 100644 --- a/manifests/init.pp +++ b/puppet/modules/concat/manifests/init.pp diff --git a/manifests/setup.pp b/puppet/modules/concat/manifests/setup.pp index 38aeb964..38aeb964 100644 --- a/manifests/setup.pp +++ b/puppet/modules/concat/manifests/setup.pp diff --git a/spec/defines/init_spec.rb b/puppet/modules/concat/spec/defines/init_spec.rb index d968a26c..d968a26c 100644 --- a/spec/defines/init_spec.rb +++ b/puppet/modules/concat/spec/defines/init_spec.rb diff --git a/spec/spec_helper.rb b/puppet/modules/concat/spec/spec_helper.rb index e6e9309b..e6e9309b 100644 --- a/spec/spec_helper.rb +++ b/puppet/modules/concat/spec/spec_helper.rb diff --git a/puppet/modules/haveged/manifests/init.pp b/puppet/modules/haveged/manifests/init.pp new file mode 100644 index 00000000..8f901937 --- /dev/null +++ b/puppet/modules/haveged/manifests/init.pp @@ -0,0 +1,16 @@ +class haveged { + + package { 'haveged': + ensure => present, + } + + service { 'haveged': + ensure => running, + hasrestart => true, + hasstatus => true, + enable => true, + require => Package['haveged']; + } + + include site_check_mk::agent::haveged +} diff --git a/puppet/modules/journald/manifests/init.pp b/puppet/modules/journald/manifests/init.pp new file mode 100644 index 00000000..879baba4 --- /dev/null +++ b/puppet/modules/journald/manifests/init.pp @@ -0,0 +1,7 @@ +class journald { + + service { 'systemd-journald': + ensure => running, + enable => true, + } +} diff --git a/puppet/modules/leap/manifests/cli/install.pp b/puppet/modules/leap/manifests/cli/install.pp new file mode 100644 index 00000000..25e87033 --- /dev/null +++ b/puppet/modules/leap/manifests/cli/install.pp @@ -0,0 +1,46 @@ +# installs leap_cli on node +class leap::cli::install ( $source = false ) { + if $source { + # needed for building leap_cli from source + include ::git + include ::rubygems + + class { '::ruby': + install_dev => true + } + + class { 'bundler::install': install_method => 'package' } + + Class[Ruby] -> + Class[rubygems] -> + Class[bundler::install] + + + vcsrepo { '/srv/leap/cli': + ensure => present, + force => true, + revision => 'develop', + provider => 'git', + source => 'https://leap.se/git/leap_cli.git', + owner => 'root', + group => 'root', + notify => Exec['install_leap_cli'], + require => Package['git'] + } + + exec { 'install_leap_cli': + command => '/usr/bin/rake build && /usr/bin/rake install', + cwd => '/srv/leap/cli', + user => 'root', + environment => 'USER=root', + refreshonly => true, + require => [ Class[bundler::install] ] + } + } + else { + package { 'leap_cli': + ensure => installed, + provider => gem + } + } +} diff --git a/puppet/modules/leap/manifests/init.pp b/puppet/modules/leap/manifests/init.pp new file mode 100644 index 00000000..bbae3781 --- /dev/null +++ b/puppet/modules/leap/manifests/init.pp @@ -0,0 +1,3 @@ +class leap { + +}
\ No newline at end of file diff --git a/puppet/modules/leap/manifests/logfile.pp b/puppet/modules/leap/manifests/logfile.pp new file mode 100644 index 00000000..adb3ca8a --- /dev/null +++ b/puppet/modules/leap/manifests/logfile.pp @@ -0,0 +1,34 @@ +# +# make syslog log to a particular file for a particular process. +# +# arguments: +# +# * name: what config files are named as (eg. /etc/rsyslog.d/50-$name.conf) +# * log: the full path of the log file (defaults to /var/log/leap/$name.log +# * process: the syslog tag to filter on (defaults to name) +# +define leap::logfile($process = $name, $log = undef) { + if $log { + $logfile = $log + } else { + $logfile = "/var/log/leap/${name}.log" + } + + rsyslog::snippet { "50-${name}": + content => template('leap/rsyslog.erb') + } + + augeas { + "logrotate_${name}": + context => "/files/etc/logrotate.d/${name}/rule", + changes => [ + "set file ${logfile}", + 'set rotate 5', + 'set schedule daily', + 'set compress compress', + 'set missingok missingok', + 'set ifempty notifempty', + 'set copytruncate copytruncate' + ] + } +} diff --git a/puppet/modules/leap/templates/rsyslog.erb b/puppet/modules/leap/templates/rsyslog.erb new file mode 100644 index 00000000..7bb5316f --- /dev/null +++ b/puppet/modules/leap/templates/rsyslog.erb @@ -0,0 +1,5 @@ +if $programname startswith '<%= @process %>' then { + action(type="omfile" file="<%= @logfile %>" template="RSYSLOG_TraditionalFileFormat") + stop +} + diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp new file mode 100644 index 00000000..d758e3ab --- /dev/null +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -0,0 +1,119 @@ +# deploy leap mx service +class leap_mx { + + $leap_mx = hiera('couchdb_leap_mx_user') + $couchdb_user = $leap_mx['username'] + $couchdb_password = $leap_mx['password'] + + $couchdb_host = 'localhost' + $couchdb_port = '4096' + + $sources = hiera('sources') + + include soledad::common + + # + # USER AND GROUP + # + # Make the user for leap-mx. This user is where all legitimate, non-system + # mail is delivered so leap-mx can process it. Previously, we let the system + # pick a uid/gid, but we need to know what they are set to in order to set the + # virtual_uid_maps and virtual_gid_maps. Its a bit overkill write a fact just + # for this, so instead we pick arbitrary numbers that seem unlikely to be used + # and then use them in the postfix configuration + + group { 'leap-mx': + ensure => present, + gid => 42424, + allowdupe => false; + } + + user { 'leap-mx': + ensure => present, + comment => 'Leap Mail', + allowdupe => false, + uid => 42424, + gid => 'leap-mx', + home => '/var/mail/leap-mx', + shell => '/bin/false', + managehome => true, + require => Group['leap-mx']; + } + + file { + '/var/mail/leap-mx': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0755', + require => User['leap-mx']; + + '/var/mail/leap-mx/Maildir': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/new': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/cur': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + + '/var/mail/leap-mx/Maildir/tmp': + ensure => directory, + owner => 'leap-mx', + group => 'leap-mx', + mode => '0700'; + } + + # + # LEAP-MX CONFIG + # + + file { '/etc/leap/mx.conf': + content => template('leap_mx/mx.conf.erb'), + owner => 'leap-mx', + group => 'leap-mx', + mode => '0600', + notify => Service['leap-mx']; + } + + leap::logfile { 'leap-mx': + log => '/var/log/leap/mx.log', + process => 'leap-mx' + } + + # + # LEAP-MX CODE AND DEPENDENCIES + # + + package { + $sources['leap-mx']['package']: + ensure => $sources['leap-mx']['revision'], + require => [ + Class['site_apt::leap_repo'], + User['leap-mx'] ]; + + 'leap-keymanager': + ensure => latest; + } + + # + # LEAP-MX DAEMON + # + + service { 'leap-mx': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + require => [ Package['leap-mx'] ]; + } +} diff --git a/puppet/modules/leap_mx/templates/mx.conf.erb b/puppet/modules/leap_mx/templates/mx.conf.erb new file mode 100644 index 00000000..b54b3a86 --- /dev/null +++ b/puppet/modules/leap_mx/templates/mx.conf.erb @@ -0,0 +1,18 @@ +[mail1] +path=/var/mail/leap-mx/Maildir +recursive=True + +[couchdb] +user=<%= @couchdb_user %> +password=<%= @couchdb_password %> +server=<%= @couchdb_host %> +port=<%= @couchdb_port %> + +[alias map] +port=4242 + +[check recipient] +port=2244 + +[fingerprint map] +port=2424 diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init new file mode 100755 index 00000000..01c8013a --- /dev/null +++ b/puppet/modules/obfsproxy/files/obfsproxy_init @@ -0,0 +1,93 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: obfsproxy daemon +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: obfsproxy daemon +# Description: obfsproxy daemon +### END INIT INFO + +. /lib/lsb/init-functions + +DAEMON=/usr/bin/obfsproxy +NAME=obfsproxy +DESC="obfsproxy daemon" +USER=obfsproxy +DATDIR=/etc/obfsproxy +PIDFILE=/var/run/obfsproxy.pid +CONF=$DATDIR/obfsproxy.conf +LOGFILE=/var/log/obfsproxy.log + +# If the daemon is not there, then exit. +test -x $DAEMON || exit 0 + +if [ -f $CONF ] ; then + . $CONF +else + echo "Obfsproxy configuration file is missing, aborting..." + exit 2 +fi + +DAEMONARGS=" --log-min-severity=$LOG --log-file=$LOGFILE --data-dir=$DATDIR \ + $TRANSPORT $PARAM --dest=$DEST_IP:$DEST_PORT server $BINDADDR:$PORT" + +start_obfsproxy() { + start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \ + -b -c $USER --startas $DAEMON --$DAEMONARGS +} + +stop_obfsproxy() { + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE +} + +status_obfsproxy() { + status_of_proc -p $PIDFILE $DAEMON $NAME +} + +case $1 in + start) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $? = "0" ]; then + exit + fi + fi + log_begin_msg "Starting $DESC" + start_obfsproxy + log_end_msg $? + ;; + stop) + if [ -e $PIDFILE ]; then + status_obfsproxy + if [ $? = "0" ]; then + log_begin_msg "Stopping $DESC" + stop_obfsproxy + rm -f $PIDFILE + log_end_msg $? + fi + else + status_obfsproxy + fi + ;; + restart) + $0 stop && sleep 2 && $0 start + ;; + status) + status_obfsproxy + ;; + reload) + if [ -e $PIDFILE ]; then + start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME + log_success_msg "$DESC reloaded successfully" + else + log_failure_msg "$PIDFILE does not exist" + fi + ;; + *) + echo "Usage: $0 {start|stop|restart|reload|status}" + exit 2 + ;; +esac diff --git a/puppet/modules/obfsproxy/files/obfsproxy_logrotate b/puppet/modules/obfsproxy/files/obfsproxy_logrotate new file mode 100644 index 00000000..e5679d0c --- /dev/null +++ b/puppet/modules/obfsproxy/files/obfsproxy_logrotate @@ -0,0 +1,14 @@ +/var/log/obfsproxy.log { + daily + missingok + rotate 3 + compress + delaycompress + notifempty + create 600 obfsproxy obfsproxy + postrotate + if [ -f /var/run/obfsproxy.pid ]; then + /etc/init.d/obfsproxy restart > /dev/null + fi + endscript +} diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp new file mode 100644 index 00000000..6a3d2c72 --- /dev/null +++ b/puppet/modules/obfsproxy/manifests/init.pp @@ -0,0 +1,86 @@ +# deploy obfsproxy service +class obfsproxy ( + $transport, + $bind_address, + $port, + $param, + $dest_ip, + $dest_port, + $log_level = 'info' +){ + + $user = 'obfsproxy' + $conf = '/etc/obfsproxy/obfsproxy.conf' + + user { $user: + ensure => present, + system => true, + gid => $user, + } + + group { $user: + ensure => present, + system => true, + } + + file { '/etc/init.d/obfsproxy': + ensure => present, + path => '/etc/init.d/obfsproxy', + source => 'puppet:///modules/obfsproxy/obfsproxy_init', + owner => 'root', + group => 'root', + mode => '0750', + require => File[$conf], + } + + file { $conf : + ensure => present, + path => $conf, + owner => 'root', + group => 'root', + mode => '0600', + content => template('obfsproxy/etc_conf.erb'), + } + + file { '/etc/obfsproxy': + ensure => directory, + owner => $user, + group => $user, + mode => '0700', + require => User[$user], + } + + file { '/var/log/obfsproxy.log': + ensure => present, + owner => $user, + group => $user, + mode => '0640', + require => User[$user], + } + + file { '/etc/logrotate.d/obfsproxy': + ensure => present, + source => 'puppet:///modules/obfsproxy/obfsproxy_logrotate', + owner => 'root', + group => 'root', + mode => '0644', + require => File['/var/log/obfsproxy.log'], + } + + package { 'obfsproxy': + ensure => present + } + + service { 'obfsproxy': + ensure => running, + subscribe => File[$conf], + require => [ + Package['obfsproxy'], + File['/etc/init.d/obfsproxy'], + User[$user], + Group[$user]] + } + + +} + diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb new file mode 100644 index 00000000..8959ef78 --- /dev/null +++ b/puppet/modules/obfsproxy/templates/etc_conf.erb @@ -0,0 +1,11 @@ +TRANSPORT=<%= @transport %> +PORT=<%= @port %> +DEST_IP=<%= @dest_ip %> +DEST_PORT=<%= @dest_port %> +<% if @transport == "scramblesuit" -%> +PARAM=--password=<%= @param %> +<% else -%> +PARAM=<%= @param %> +<% end -%> +LOG=<%= @log_level %> +BINDADDR=<%= @bind_address %> diff --git a/puppet/modules/opendkim/manifests/init.pp b/puppet/modules/opendkim/manifests/init.pp new file mode 100644 index 00000000..4d4c5312 --- /dev/null +++ b/puppet/modules/opendkim/manifests/init.pp @@ -0,0 +1,67 @@ +# +# I am not sure about what issues might arise with DKIM key sizes +# larger than 2048. It might or might not be supported. See: +# http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.3.3.3 +# +class opendkim { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + $mx = hiera('mx') + $dkim = $mx['dkim'] + $selector = $dkim['selector'] + $dkim_cert = $dkim['public_key'] + $dkim_key = $dkim['private_key'] + + ensure_packages(['opendkim', 'libvbr2']) + + # postfix user needs to be in the opendkim group + # in order to access the opendkim socket located at: + # local:/var/run/opendkim/opendkim.sock + user { 'postfix': + groups => 'opendkim', + require => Package['opendkim']; + } + + service { 'opendkim': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + subscribe => File[$dkim_key]; + } + + file { + '/etc/opendkim.conf': + ensure => file, + content => template('opendkim/opendkim.conf'), + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; + + '/etc/default/opendkim.conf': + ensure => file, + content => 'SOCKET="inet:8891@localhost" # listen on loopback on port 8891', + mode => '0644', + owner => root, + group => root, + notify => Service['opendkim'], + require => Package['opendkim']; + + $dkim_key: + ensure => file, + mode => '0600', + owner => 'opendkim', + group => 'opendkim', + require => Package['opendkim']; + + $dkim_cert: + ensure => file, + mode => '0600', + owner => 'opendkim', + group => 'opendkim', + require => Package['opendkim']; + } +} diff --git a/puppet/modules/opendkim/templates/opendkim.conf b/puppet/modules/opendkim/templates/opendkim.conf new file mode 100644 index 00000000..5a948229 --- /dev/null +++ b/puppet/modules/opendkim/templates/opendkim.conf @@ -0,0 +1,45 @@ +# This is a basic configuration that can easily be adapted to suit a standard +# installation. For more advanced options, see opendkim.conf(5) and/or +# /usr/share/doc/opendkim/examples/opendkim.conf.sample. + +# Log to syslog +Syslog yes +SyslogSuccess yes +LogWhy no +# Required to use local socket with MTAs that access the socket as a non- +# privileged user (e.g. Postfix) +UMask 002 + +Domain <%= @domain %> +SubDomains yes + +# set internal hosts to all the known hosts, like mydomains? + +# can we generate a larger key and get it in dns? +KeyFile <%= @dkim_key %> + +Selector <%= @selector %> + +# Commonly-used options; the commented-out versions show the defaults. +Canonicalization relaxed +#Mode sv +#ADSPDiscard no + +SignatureAlgorithm rsa-sha256 + +# Always oversign From (sign using actual From and a null From to prevent +# malicious signatures header fields (From and/or others) between the signer +# and the verifier. From is oversigned by default in the Debian pacakge +# because it is often the identity key used by reputation systems and thus +# somewhat security sensitive. +OversignHeaders From + +# List domains to use for RFC 6541 DKIM Authorized Third-Party Signatures +# (ATPS) (experimental) + +#ATPSDomains example.com + +RemoveOldSignatures yes + +Mode sv +BaseDirectory /var/tmp diff --git a/puppet/modules/openvpn/.fixtures.yml b/puppet/modules/openvpn/.fixtures.yml new file mode 100644 index 00000000..1125ecca --- /dev/null +++ b/puppet/modules/openvpn/.fixtures.yml @@ -0,0 +1,6 @@ +fixtures: + repositories: + concat: git://github.com/ripienaar/puppet-concat.git + symlinks: + openvpn: "#{source_dir}" + diff --git a/puppet/modules/openvpn/.gitignore b/puppet/modules/openvpn/.gitignore new file mode 100644 index 00000000..6fd248b3 --- /dev/null +++ b/puppet/modules/openvpn/.gitignore @@ -0,0 +1,3 @@ +pkg +spec/fixtures +.vagrant diff --git a/puppet/modules/openvpn/.rvmrc b/puppet/modules/openvpn/.rvmrc new file mode 100644 index 00000000..6fbfb7f1 --- /dev/null +++ b/puppet/modules/openvpn/.rvmrc @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This is an RVM Project .rvmrc file, used to automatically load the ruby +# development environment upon cd'ing into the directory + +# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional, +# Only full ruby name is supported here, for short names use: +# echo "rvm use 1.9.3" > .rvmrc +environment_id="ruby-1.9.3-p194@puppet" + +# Uncomment the following lines if you want to verify rvm version per project +# rvmrc_rvm_version="1.15.8 (stable)" # 1.10.1 seams as a safe start +# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || { +# echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading." +# return 1 +# } + +# First we attempt to load the desired environment directly from the environment +# file. This is very fast and efficient compared to running through the entire +# CLI and selector. If you want feedback on which environment was used then +# insert the word 'use' after --create as this triggers verbose mode. +if [[ -d "${rvm_path:-$HOME/.rvm}/environments" + && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]] +then + \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id" + [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] && + \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true + if [[ $- == *i* ]] # check for interactive shells + then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green + else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells + fi +else + # If the environment file has not yet been created, use the RVM CLI to select. + rvm --create use "$environment_id" || { + echo "Failed to create RVM environment '${environment_id}'." + return 1 + } +fi diff --git a/puppet/modules/openvpn/.travis.yml b/puppet/modules/openvpn/.travis.yml new file mode 100644 index 00000000..da5c389d --- /dev/null +++ b/puppet/modules/openvpn/.travis.yml @@ -0,0 +1,29 @@ +language: ruby +bundler_args: --without development +script: "bundle exec rake spec SPEC_OPTS='--format documentation'" +rvm: + - 1.8.7 + - 1.9.3 + - 2.0.0 +script: + - "rake lint" + - "rake spec SPEC_OPTS='--format documentation'" +env: + - PUPPET_VERSION="~> 2.7.0" + - PUPPET_VERSION="~> 3.0.0" + - PUPPET_VERSION="~> 3.1.0" + - PUPPET_VERSION="~> 3.2.0" +matrix: + exclude: + - rvm: 1.9.3 + env: PUPPET_VERSION="~> 2.7.0" + - rvm: 2.0.0 + env: PUPPET_VERSION="~> 2.7.0" + - rvm: 2.0.0 + env: PUPPET_VERSION="~> 3.0.0" + - rvm: 2.0.0 + env: PUPPET_VERSION="~> 3.1.0" +notifications: + email: false + on_success: always + on_failure: always diff --git a/puppet/modules/openvpn/Gemfile b/puppet/modules/openvpn/Gemfile new file mode 100644 index 00000000..68e10e7d --- /dev/null +++ b/puppet/modules/openvpn/Gemfile @@ -0,0 +1,7 @@ +source :rubygems + +puppetversion = ENV['PUPPET_VERSION'] +gem 'puppet', puppetversion, :require => false +gem 'puppet-lint' +gem 'rspec-puppet' +gem 'puppetlabs_spec_helper' diff --git a/puppet/modules/openvpn/Gemfile.lock b/puppet/modules/openvpn/Gemfile.lock new file mode 100644 index 00000000..9fce3f98 --- /dev/null +++ b/puppet/modules/openvpn/Gemfile.lock @@ -0,0 +1,36 @@ +GEM + remote: http://rubygems.org/ + specs: + diff-lcs (1.1.3) + facter (1.6.17) + hiera (1.0.0) + metaclass (0.0.1) + mocha (0.13.1) + metaclass (~> 0.0.1) + puppet (3.0.2) + facter (~> 1.6.11) + hiera (~> 1.0.0) + puppetlabs_spec_helper (0.4.0) + mocha (>= 0.10.5) + rake + rspec (>= 2.9.0) + rspec-puppet (>= 0.1.1) + rake (10.0.3) + rspec (2.12.0) + rspec-core (~> 2.12.0) + rspec-expectations (~> 2.12.0) + rspec-mocks (~> 2.12.0) + rspec-core (2.12.2) + rspec-expectations (2.12.1) + diff-lcs (~> 1.1.3) + rspec-mocks (2.12.1) + rspec-puppet (0.1.5) + rspec + +PLATFORMS + ruby + +DEPENDENCIES + puppet + puppetlabs_spec_helper + rspec-puppet diff --git a/puppet/modules/openvpn/LICENSE b/puppet/modules/openvpn/LICENSE new file mode 100644 index 00000000..f433b1a5 --- /dev/null +++ b/puppet/modules/openvpn/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/puppet/modules/openvpn/Modulefile b/puppet/modules/openvpn/Modulefile new file mode 100644 index 00000000..679e7e64 --- /dev/null +++ b/puppet/modules/openvpn/Modulefile @@ -0,0 +1,11 @@ +name 'luxflux-openvpn' +version '2.1.0' +source 'https://github.com/luxflux/puppet-openvpn' +author 'luxflux' +license 'Apache 2.0' +summary 'OpenVPN server puppet module' +description 'Puppet module to manage OpenVPN servers' +project_page 'https://github.com/luxflux/puppet-openvpn' + +## Add dependencies, if any: +dependency 'ripienaar/concat', '0.2.0' diff --git a/puppet/modules/openvpn/Rakefile b/puppet/modules/openvpn/Rakefile new file mode 100644 index 00000000..14f1c246 --- /dev/null +++ b/puppet/modules/openvpn/Rakefile @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/rake_tasks' diff --git a/puppet/modules/openvpn/Readme.markdown b/puppet/modules/openvpn/Readme.markdown new file mode 100644 index 00000000..6bcf49ea --- /dev/null +++ b/puppet/modules/openvpn/Readme.markdown @@ -0,0 +1,54 @@ +# OpenVPN Puppet module + +Puppet module to manage OpenVPN servers + +## Features: + +* Client-specific rules and access policies +* Generated client configurations and SSL-Certificates +* Downloadable client configurations and SSL-Certificates for easy client configuration +* Support for multiple server instances + +Tested on Ubuntu Precise Pangolin, CentOS 6, RedHat 6. + + +## Dependencies + - [puppet-concat](https://github.com/ripienaar/puppet-concat) + + +## Example + +```puppet + # add a server instance + openvpn::server { 'winterthur': + country => 'CH', + province => 'ZH', + city => 'Winterthur', + organization => 'example.org', + email => 'root@example.org', + server => '10.200.200.0 255.255.255.0' + } + + # define clients + openvpn::client { 'client1': + server => 'winterthur' + } + openvpn::client { 'client2': + server => 'winterthur' + } + + openvpn::client_specific_config { 'client1': + server => 'winterthur', + ifconfig => '10.200.200.50 255.255.255.0' + } +``` + +Don't forget the [sysctl](https://github.com/luxflux/puppet-sysctl) directive ```net.ipv4.ip_forward```! + + +# Contributors + +These fine folks helped to get this far with this module: +* [@jlambert121](https://github.com/jlambert121) +* [@jlk](https://github.com/jlk) +* [@elisiano](https://github.com/elisiano) diff --git a/puppet/modules/openvpn/Vagrantfile b/puppet/modules/openvpn/Vagrantfile new file mode 100644 index 00000000..88875ff8 --- /dev/null +++ b/puppet/modules/openvpn/Vagrantfile @@ -0,0 +1,42 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +def server_config(config) + config.vm.provision :puppet, :module_path => '..' do |puppet| + puppet.manifests_path = "vagrant" + puppet.manifest_file = "server.pp" + end +end + +def client_config(config) + config.vm.provision :puppet, :module_path => '..' do |puppet| + puppet.manifests_path = "vagrant" + puppet.manifest_file = "client.pp" + end +end + +Vagrant::Config.run do |config| + + config.vm.define :server_ubuntu do |c| + c.vm.box = 'precise64' + server_config c + c.vm.network :hostonly, '10.255.255.10' + end + + config.vm.define :server_centos do |c| + c.vm.box = 'centos63' + + c.vm.provision :shell, :inline => 'if [ ! -f rpmforge-release-0.5.2-2.el6.rf.x86_64.rpm ]; then wget -q http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.2-2.el6.rf.x86_64.rpm; fi' + c.vm.provision :shell, :inline => 'yum install -y rpmforge-release-0.5.2-2.el6.rf.x86_64.rpm || exit 0' + + server_config c + c.vm.network :hostonly, '10.255.255.11' + end + + config.vm.define :client_ubuntu do |c| + c.vm.box = 'precise64' + client_config c + c.vm.network :hostonly, '10.255.255.20' + end + +end diff --git a/puppet/modules/openvpn/manifests/client.pp b/puppet/modules/openvpn/manifests/client.pp new file mode 100644 index 00000000..92c6aa4e --- /dev/null +++ b/puppet/modules/openvpn/manifests/client.pp @@ -0,0 +1,187 @@ +# == Define: openvpn::client +# +# This define creates the client certs for a specified openvpn server as well +# as creating a tarball that can be directly imported into openvpn clients +# +# +# === Parameters +# +# [*server*] +# String. Name of the corresponding openvpn endpoint +# Required +# +# [*compression*] +# String. Which compression algorithim to use +# Default: comp-lzo +# Options: comp-lzo or '' (disable compression) +# +# [*dev*] +# String. Device method +# Default: tun +# Options: tun (routed connections), tap (bridged connections) +# +# [*mute*] +# Integer. Set log mute level +# Default: 20 +# +# [*mute_replay_warnings*] +# Boolean. Silence duplicate packet warnings (common on wireless networks) +# Default: true +# +# [*nobind*] +# Boolean. Whether or not to bind to a specific port number +# Default: true +# +# [*persist_key*] +# Boolean. Try to retain access to resources that may be unavailable +# because of privilege downgrades +# Default: true +# +# [*persist_tun*] +# Boolean. Try to retain access to resources that may be unavailable +# because of privilege downgrades +# Default: true +# +# [*port*] +# Integer. The port the openvpn server service is running on +# Default: 1194 +# +# [*proto*] +# String. What IP protocol is being used. +# Default: tcp +# Options: tcp or udp +# +# [*remote_host*] +# String. The IP or hostname of the openvpn server service +# Default: FQDN +# +# [*resolv_retry*] +# Integer/String. How many seconds should the openvpn client try to resolve +# the server's hostname +# Default: infinite +# Options: Integer or infinite +# +# [*verb*] +# Integer. Level of logging verbosity +# Default: 3 +# +# +# === Examples +# +# openvpn::client { +# 'my_user': +# server => 'contractors', +# remote_host => 'vpn.mycompany.com' +# } +# +# * Removal: +# Manual process right now, todo for the future +# +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +define openvpn::client( + $server, + $compression = 'comp-lzo', + $dev = 'tun', + $mute = '20', + $mute_replay_warnings = true, + $nobind = true, + $persist_key = true, + $persist_tun = true, + $port = '1194', + $proto = 'tcp', + $remote_host = $::fqdn, + $resolv_retry = 'infinite', + $verb = '3', +) { + + Openvpn::Server[$server] -> + Openvpn::Client[$name] + + exec { + "generate certificate for ${name} in context of ${server}": + command => ". ./vars && ./pkitool ${name}", + cwd => "/etc/openvpn/${server}/easy-rsa", + creates => "/etc/openvpn/${server}/easy-rsa/keys/${name}.crt", + provider => 'shell'; + } + + file { + [ "/etc/openvpn/${server}/download-configs/${name}", + "/etc/openvpn/${server}/download-configs/${name}/keys"]: + ensure => directory; + + "/etc/openvpn/${server}/download-configs/${name}/keys/${name}.crt": + ensure => link, + target => "/etc/openvpn/${server}/easy-rsa/keys/${name}.crt", + require => Exec["generate certificate for ${name} in context of ${server}"]; + + "/etc/openvpn/${server}/download-configs/${name}/keys/${name}.key": + ensure => link, + target => "/etc/openvpn/${server}/easy-rsa/keys/${name}.key", + require => Exec["generate certificate for ${name} in context of ${server}"]; + + "/etc/openvpn/${server}/download-configs/${name}/keys/ca.crt": + ensure => link, + target => "/etc/openvpn/${server}/easy-rsa/keys/ca.crt", + require => Exec["generate certificate for ${name} in context of ${server}"]; + + "/etc/openvpn/${server}/download-configs/${name}/${name}.conf": + owner => root, + group => root, + mode => '0444', + content => template('openvpn/client.erb'), + notify => Exec["tar the thing ${server} with ${name}"]; + } + + exec { + "tar the thing ${server} with ${name}": + cwd => "/etc/openvpn/${server}/download-configs/", + command => "/bin/rm ${name}.tar.gz; tar --exclude=\\*.conf.d -chzvf ${name}.tar.gz ${name}", + refreshonly => true, + require => [ File["/etc/openvpn/${server}/download-configs/${name}/${name}.conf"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/ca.crt"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/${name}.key"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/${name}.crt"] + ], + notify => Exec["generate ${name}.ovpn in ${server}"]; + } + + exec { + "generate ${name}.ovpn in ${server}": + cwd => "/etc/openvpn/${server}/download-configs/", + command => "/bin/rm ${name}.ovpn; cat ${name}/${name}.conf|perl -lne 'if(m|^ca keys/ca.crt|){ chomp(\$ca=`cat ${name}/keys/ca.crt`); print \"<ca>\n\$ca\n</ca>\"} elsif(m|^cert keys/${name}.crt|) { chomp(\$crt=`cat ${name}/keys/${name}.crt`); print \"<cert>\n\$crt\n</cert>\"} elsif(m|^key keys/${name}.key|){ chomp(\$key=`cat ${name}/keys/${name}.key`); print \"<key>\n\$key\n</key>\"} else { print} ' > ${name}.ovpn", + refreshonly => true, + require => [ File["/etc/openvpn/${server}/download-configs/${name}/${name}.conf"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/ca.crt"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/${name}.key"], + File["/etc/openvpn/${server}/download-configs/${name}/keys/${name}.crt"], + ], + } + + file { "/etc/openvpn/${server}/download-configs/${name}.ovpn": + mode => '0400', + require => Exec["generate ${name}.ovpn in ${server}"], + } +} diff --git a/puppet/modules/openvpn/manifests/client_specific_config.pp b/puppet/modules/openvpn/manifests/client_specific_config.pp new file mode 100644 index 00000000..4287421a --- /dev/null +++ b/puppet/modules/openvpn/manifests/client_specific_config.pp @@ -0,0 +1,79 @@ +# == Define: openvpn::client_specific_config +# +# This define configures options which will be pushed by the server to a +# specific client only. This feature is explained here: +# http://openvpn.net/index.php/open-source/documentation/howto.html#policy +# +# === Parameters +# +# All the parameters are explained in the openvpn documentation: +# http://openvpn.net/index.php/open-source/documentation/howto.html#policy +# +# [*server*] +# String. Name of the corresponding openvpn endpoint +# Required +# +# [*iroute*] +# Array. Array of iroute combinations. +# Default: [] +# +# [*ifconfig*] +# String. IP configuration to push to the client. +# Default: false +# +# [*dhcp_options] +# Array. DHCP options to push to the client. +# Default: [] +# +# +# === Examples +# +# openvpn::client_specific_config { +# 'vpn_client': +# server => 'contractors', +# iroute => ['10.0.1.0 255.255.255.0'], +# ifconfig => '10.10.10.1 10.10.10.2', +# dhcp_options => ['DNS 8.8.8.8'] +# } +# +# * Removal: +# Manual process right now, todo for the future +# +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +define openvpn::client_specific_config( + $server, + $iroute = [], + $ifconfig = false, + $dhcp_options = [] +) { + + Openvpn::Server[$server] -> + Openvpn::Client[$name] -> + Openvpn::Client_specific_config[$name] + + file { "/etc/openvpn/${server}/client-configs/${name}": + ensure => present, + content => template('openvpn/client_specific_config.erb') + } + +} diff --git a/puppet/modules/openvpn/manifests/config.pp b/puppet/modules/openvpn/manifests/config.pp new file mode 100644 index 00000000..32b32094 --- /dev/null +++ b/puppet/modules/openvpn/manifests/config.pp @@ -0,0 +1,52 @@ +# == Class: openvpn::config +# +# This class sets up the openvpn enviornment as well as the default config file +# +# +# === Examples +# +# This class should not be directly invoked +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class openvpn::config { + + if $::osfamily == 'Debian' { + include concat::setup + + concat { + '/etc/default/openvpn': + owner => root, + group => root, + mode => 644, + warn => true; + } + + concat::fragment { + 'openvpn.default.header': + content => template('openvpn/etc-default-openvpn.erb'), + target => '/etc/default/openvpn', + order => 01; + } + } +} diff --git a/puppet/modules/openvpn/manifests/init.pp b/puppet/modules/openvpn/manifests/init.pp new file mode 100644 index 00000000..7e07f025 --- /dev/null +++ b/puppet/modules/openvpn/manifests/init.pp @@ -0,0 +1,43 @@ +# == Class: openvpn +# +# This module installs the openvpn service, configures vpn endpoints, generates +# client certificates, and generates client config files +# +# +# === Examples +# +# * Installation: +# class { 'openvpn': } +# +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class openvpn { + + class {'openvpn::params': } -> + class {'openvpn::install': } -> + class {'openvpn::config': } ~> + class {'openvpn::service': } -> + Class['openvpn'] + +} diff --git a/puppet/modules/openvpn/manifests/install.pp b/puppet/modules/openvpn/manifests/install.pp new file mode 100644 index 00000000..a230373a --- /dev/null +++ b/puppet/modules/openvpn/manifests/install.pp @@ -0,0 +1,46 @@ +# == Class: openvpn +# +# This module installs the openvpn service, configures vpn endpoints, generates +# client certificates, and generates client config files +# +# +# === Examples +# +# This class should not be directly invoked +# +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class openvpn::install { + + package { + 'openvpn': + ensure => installed; + } + + file { + [ '/etc/openvpn', '/etc/openvpn/keys' ]: + ensure => directory, + require => Package['openvpn']; + } +} diff --git a/puppet/modules/openvpn/manifests/params.pp b/puppet/modules/openvpn/manifests/params.pp new file mode 100644 index 00000000..33495270 --- /dev/null +++ b/puppet/modules/openvpn/manifests/params.pp @@ -0,0 +1,37 @@ +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class openvpn::params { + + $group = $::osfamily ? { + 'RedHat' => 'nobody', + default => 'nogroup' + } + + $easyrsa_source = $::osfamily ? { + 'RedHat' => $::operatingsystemmajrelease ? { + 6 => '/usr/share/openvpn/easy-rsa/2.0', + default => '/usr/share/doc/openvpn-2.2.2/easy-rsa/2.0' + }, + default => '/usr/share/doc/openvpn/examples/easy-rsa/2.0' + } + + $link_openssl_cnf = $::osfamily ? { + /(Debian|RedHat)/ => true, + default => false + } + +} diff --git a/puppet/modules/openvpn/manifests/server.pp b/puppet/modules/openvpn/manifests/server.pp new file mode 100644 index 00000000..649048c4 --- /dev/null +++ b/puppet/modules/openvpn/manifests/server.pp @@ -0,0 +1,233 @@ +# == Define: openvpn::server +# +# This define creates the openvpn server instance and ssl certificates +# +# +# === Parameters +# +# [*country*] +# String. Country to be used for the SSL certificate +# +# [*province*] +# String. Province to be used for the SSL certificate +# +# [*city*] +# String. City to be used for the SSL certificate +# +# [*organization*] +# String. Organization to be used for the SSL certificate +# +# [*email*] +# String. Email address to be used for the SSL certificate +# +# [*compression*] +# String. Which compression algorithim to use +# Default: comp-lzo +# Options: comp-lzo or '' (disable compression) +# +# [*dev*] +# String. Device method +# Default: tun +# Options: tun (routed connections), tap (bridged connections) +# +# [*user*] +# String. Group to drop privileges to after startup +# Default: nobody +# +# [*group*] +# String. User to drop privileges to after startup +# Default: depends on your $::osfamily +# +# [*ipp*] +# Boolean. Persist ifconfig information to a file to retain client IP +# addresses between sessions +# Default: false +# +# [*local*] +# String. Interface for openvpn to bind to. +# Default: $::ipaddress_eth0 +# Options: An IP address or '' to bind to all ip addresses +# +# [*logfile*] +# String. Logfile for this openvpn server +# Default: false +# Options: false (syslog) or log file name +# +# [*port*] +# Integer. The port the openvpn server service is running on +# Default: 1194 +# +# [*proto*] +# String. What IP protocol is being used. +# Default: tcp +# Options: tcp or udp +# +# [*status_log*] +# String. Logfile for periodic dumps of the vpn service status +# Default: "${name}/openvpn-status.log" +# +# [*server*] +# String. Network to assign client addresses out of +# Default: None. Required in tun mode, not in tap mode +# +# [*push*] +# Array. Options to push out to the client. This can include routes, DNS +# servers, DNS search domains, and many other options. +# Default: [] +# +# +# === Examples +# +# openvpn::client { +# 'my_user': +# server => 'contractors', +# remote_host => 'vpn.mycompany.com' +# } +# +# * Removal: +# Manual process right now, todo for the future +# +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +define openvpn::server( + $country, + $province, + $city, + $organization, + $email, + $compression = 'comp-lzo', + $dev = 'tun0', + $user = 'nobody', + $group = false, + $ipp = false, + $ip_pool = [], + $local = $::ipaddress_eth0, + $logfile = false, + $port = '1194', + $proto = 'tcp', + $status_log = "${name}/openvpn-status.log", + $server = '', + $push = [] +) { + + include openvpn + Class['openvpn::install'] -> + Openvpn::Server[$name] ~> + Class['openvpn::service'] + + $tls_server = $proto ? { + /tcp/ => true, + default => false + } + + $group_to_set = $group ? { + false => $openvpn::params::group, + default => $group + } + + file { + ["/etc/openvpn/${name}", "/etc/openvpn/${name}/client-configs", "/etc/openvpn/${name}/download-configs" ]: + ensure => directory; + } + + exec { + "copy easy-rsa to openvpn config folder ${name}": + command => "/bin/cp -r ${openvpn::params::easyrsa_source} /etc/openvpn/${name}/easy-rsa", + creates => "/etc/openvpn/${name}/easy-rsa", + notify => Exec["fix_easyrsa_file_permissions_${name}"], + require => File["/etc/openvpn/${name}"]; + } + + exec { + "fix_easyrsa_file_permissions_${name}": + refreshonly => true, + command => "/bin/chmod 755 /etc/openvpn/${name}/easy-rsa/*"; + } + + file { + "/etc/openvpn/${name}/easy-rsa/vars": + ensure => present, + content => template('openvpn/vars.erb'), + require => Exec["copy easy-rsa to openvpn config folder ${name}"]; + } + + file { + "/etc/openvpn/${name}/easy-rsa/openssl.cnf": + require => Exec["copy easy-rsa to openvpn config folder ${name}"]; + } + + if $openvpn::params::link_openssl_cnf == true { + File["/etc/openvpn/${name}/easy-rsa/openssl.cnf"] { + ensure => link, + target => "/etc/openvpn/${name}/easy-rsa/openssl-1.0.0.cnf" + } + } + + exec { + "generate dh param ${name}": + command => '. ./vars && ./clean-all && ./build-dh', + cwd => "/etc/openvpn/${name}/easy-rsa", + creates => "/etc/openvpn/${name}/easy-rsa/keys/dh1024.pem", + provider => 'shell', + require => File["/etc/openvpn/${name}/easy-rsa/vars"]; + + "initca ${name}": + command => '. ./vars && ./pkitool --initca', + cwd => "/etc/openvpn/${name}/easy-rsa", + creates => "/etc/openvpn/${name}/easy-rsa/keys/ca.key", + provider => 'shell', + require => [ Exec["generate dh param ${name}"], File["/etc/openvpn/${name}/easy-rsa/openssl.cnf"] ]; + + "generate server cert ${name}": + command => '. ./vars && ./pkitool --server server', + cwd => "/etc/openvpn/${name}/easy-rsa", + creates => "/etc/openvpn/${name}/easy-rsa/keys/server.key", + provider => 'shell', + require => Exec["initca ${name}"]; + } + + file { + "/etc/openvpn/${name}/keys": + ensure => link, + target => "/etc/openvpn/${name}/easy-rsa/keys", + require => Exec["copy easy-rsa to openvpn config folder ${name}"]; + } + + if $::osfamily == 'Debian' { + concat::fragment { + "openvpn.default.autostart.${name}": + content => "AUTOSTART=\"\$AUTOSTART ${name}\"\n", + target => '/etc/default/openvpn', + order => 10; + } + } + + file { + "/etc/openvpn/${name}.conf": + owner => root, + group => root, + mode => '0444', + content => template('openvpn/server.erb'); + } +} diff --git a/puppet/modules/openvpn/manifests/service.pp b/puppet/modules/openvpn/manifests/service.pp new file mode 100644 index 00000000..54e8db7d --- /dev/null +++ b/puppet/modules/openvpn/manifests/service.pp @@ -0,0 +1,36 @@ +# == Class: openvpn::config +# +# This class maintains the openvpn service +# +# +# === Examples +# +# This class should not be directly invoked +# +# === Authors +# +# * Raffael Schmid <mailto:raffael@yux.ch> +# * John Kinsella <mailto:jlkinsel@gmail.com> +# * Justin Lambert <mailto:jlambert@letsevenup.com> +# +# === License +# +# Copyright 2013 Raffael Schmid, <raffael@yux.ch> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# lied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class openvpn::service { + service { + 'openvpn': + ensure => running, + enable => true, + hasrestart => true, + hasstatus => true; + } +} diff --git a/puppet/modules/openvpn/spec/classes/openvpn_config_spec.rb b/puppet/modules/openvpn/spec/classes/openvpn_config_spec.rb new file mode 100644 index 00000000..bbb63a77 --- /dev/null +++ b/puppet/modules/openvpn/spec/classes/openvpn_config_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' + +describe 'openvpn::config', :type => :class do + + it { should create_class('openvpn::config') } + + context "on Debian based machines" do + let (:facts) { { :osfamily => 'Debian', :concat_basedir => '/var/lib/puppet/concat' } } + + it { should contain_class('concat::setup') } + it { should contain_concat('/etc/default/openvpn') } + it { should contain_concat__fragment('openvpn.default.header') } + end + +end diff --git a/puppet/modules/openvpn/spec/classes/openvpn_init_spec.rb b/puppet/modules/openvpn/spec/classes/openvpn_init_spec.rb new file mode 100644 index 00000000..45dcc9bf --- /dev/null +++ b/puppet/modules/openvpn/spec/classes/openvpn_init_spec.rb @@ -0,0 +1,9 @@ +require 'spec_helper' + +describe 'openvpn', :type => :class do + + let (:facts) { { :concat_basedir => '/var/lib/puppet/concat' } } + + it { should create_class('openvpn') } + +end diff --git a/puppet/modules/openvpn/spec/classes/openvpn_install_spec.rb b/puppet/modules/openvpn/spec/classes/openvpn_install_spec.rb new file mode 100644 index 00000000..cdb31358 --- /dev/null +++ b/puppet/modules/openvpn/spec/classes/openvpn_install_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe 'openvpn::install', :type => :class do + + it { should create_class('openvpn::install') } + it { should contain_package('openvpn') } + + it { should contain_file('/etc/openvpn').with('ensure' => 'directory') } + it { should contain_file('/etc/openvpn/keys').with('ensure' => 'directory') } + +end diff --git a/puppet/modules/openvpn/spec/classes/openvpn_service_spec.rb b/puppet/modules/openvpn/spec/classes/openvpn_service_spec.rb new file mode 100644 index 00000000..f427e7f1 --- /dev/null +++ b/puppet/modules/openvpn/spec/classes/openvpn_service_spec.rb @@ -0,0 +1,13 @@ +require 'spec_helper' + +describe 'openvpn::service', :type => :class do + + let (:facts) { { :concat_basedir => '/var/lib/puppet/concat' } } + + it { should create_class('openvpn::service') } + it { should contain_service('openvpn').with( + 'ensure' => 'running', + 'enable' => true + ) } + +end diff --git a/puppet/modules/openvpn/spec/defines/openvpn_client_spec.rb b/puppet/modules/openvpn/spec/defines/openvpn_client_spec.rb new file mode 100644 index 00000000..a4b580e8 --- /dev/null +++ b/puppet/modules/openvpn/spec/defines/openvpn_client_spec.rb @@ -0,0 +1,88 @@ +require 'spec_helper' + +describe 'openvpn::client', :type => :define do + let(:title) { 'test_client' } + let(:params) { { 'server' => 'test_server' } } + let(:facts) { { :fqdn => 'somehost', :concat_basedir => '/var/lib/puppet/concat' } } + let(:pre_condition) do + 'openvpn::server { "test_server": + country => "CO", + province => "ST", + city => "Some City", + organization => "example.org", + email => "testemail@example.org" + }' + end + + it { should contain_exec('generate certificate for test_client in context of test_server') } + + [ 'test_client', 'test_client/keys'].each do |directory| + it { should contain_file("/etc/openvpn/test_server/download-configs/#{directory}") } + end + + [ 'test_client.crt', 'test_client.key', 'ca.crt' ].each do |file| + it { should contain_file("/etc/openvpn/test_server/download-configs/test_client/keys/#{file}").with( + 'ensure' => 'link', + 'target' => "/etc/openvpn/test_server/easy-rsa/keys/#{file}" + )} + end + + it { should contain_exec('tar the thing test_server with test_client').with( + 'cwd' => '/etc/openvpn/test_server/download-configs/', + 'command' => '/bin/rm test_client.tar.gz; tar --exclude=\*.conf.d -chzvf test_client.tar.gz test_client' + ) } + + context "setting the minimum parameters" do + let(:params) { { 'server' => 'test_server' } } + let(:facts) { { :fqdn => 'somehost', :concat_basedir => '/var/lib/puppet/concat' } } + + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^client$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^ca\s+keys\/ca\.crt$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^cert\s+keys\/test_client.crt$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^key\s+keys\/test_client\.key$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^dev\s+tun$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^proto\s+tcp$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^remote\s+somehost\s+1194$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^comp-lzo$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^resolv-retry\s+infinite$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^nobind$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^persist-key$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^persist-tun$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^mute-replay-warnings$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^ns\-cert\-type\s+server$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^verb\s+3$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^mute\s+20$/)} + end + + context "setting all of the parameters" do + let(:params) { { + 'server' => 'test_server', + 'compression' => 'comp-something', + 'dev' => 'tap', + 'mute' => 10, + 'mute_replay_warnings' => false, + 'nobind' => false, + 'persist_key' => false, + 'persist_tun' => false, + 'port' => '123', + 'proto' => 'udp', + 'remote_host' => 'somewhere', + 'resolv_retry' => '2m', + 'verb' => '1' + } } + let(:facts) { { :fqdn => 'somehost', :concat_basedir => '/var/lib/puppet/concat' } } + + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^client$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^ca\s+keys\/ca\.crt$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^cert\s+keys\/test_client.crt$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^key\s+keys\/test_client\.key$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^dev\s+tap$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^proto\s+udp$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^remote\s+somewhere\s+123$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^comp-something$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^resolv-retry\s+2m$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^verb\s+1$/)} + it { should contain_file('/etc/openvpn/test_server/download-configs/test_client/test_client.conf').with_content(/^mute\s+10$/)} + end + +end diff --git a/puppet/modules/openvpn/spec/defines/openvpn_client_specific_config_spec.rb b/puppet/modules/openvpn/spec/defines/openvpn_client_specific_config_spec.rb new file mode 100644 index 00000000..cfdab389 --- /dev/null +++ b/puppet/modules/openvpn/spec/defines/openvpn_client_specific_config_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe 'openvpn::client_specific_config', :type => :define do + let(:title) { 'test_client' } + let(:params) { { 'server' => 'test_server' } } + let(:facts) { { :fqdn => 'somehost', :concat_basedir => '/var/lib/puppet/concat' } } + let(:pre_condition) do + [ + 'openvpn::server { "test_server": + country => "CO", + province => "ST", + city => "Some City", + organization => "example.org", + email => "testemail@example.org" + }', + 'openvpn::client { "test_client": + server => "test_server" + }' + ].join + end + + it { should contain_file('/etc/openvpn/test_server/client-configs/test_client') } + + describe "setting no paramter at all" do + it { should contain_file('/etc/openvpn/test_server/client-configs/test_client').with_content(/\A\n\z/) } + end + + describe "setting all parameters" do + let(:params) do + {:server => 'test_server', + :iroute => ['10.0.1.0 255.255.255.0'], + :ifconfig => '10.10.10.2 255.255.255.0', + :dhcp_options => ['DNS 8.8.8.8']} + end + + it { should contain_file('/etc/openvpn/test_server/client-configs/test_client').with_content(/^iroute 10.0.1.0 255.255.255.0$/) } + it { should contain_file('/etc/openvpn/test_server/client-configs/test_client').with_content(/^ifconfig-push 10.10.10.2 255.255.255.0$/) } + it { should contain_file('/etc/openvpn/test_server/client-configs/test_client').with_content(/^push dhcp-option DNS 8.8.8.8$/) } + end +end diff --git a/puppet/modules/openvpn/spec/defines/openvpn_server_spec.rb b/puppet/modules/openvpn/spec/defines/openvpn_server_spec.rb new file mode 100644 index 00000000..467be6aa --- /dev/null +++ b/puppet/modules/openvpn/spec/defines/openvpn_server_spec.rb @@ -0,0 +1,165 @@ +require 'spec_helper' + +describe 'openvpn::server', :type => :define do + + let(:title) { 'test_server' } + + context "creating a server with the minimum parameters" do + let(:params) { { + 'country' => 'CO', + 'province' => 'ST', + 'city' => 'Some City', + 'organization' => 'example.org', + 'email' => 'testemail@example.org' + } } + + let (:facts) { { + :ipaddress_eth0 => '1.2.3.4', + :network_eth0 => '1.2.3.0', + :netmask_eth0 => '255.255.255.0', + :concat_basedir => '/var/lib/puppet/concat', + :osfamily => 'anything_else' + } } + + # Files associated with a server config + it { should contain_file('/etc/openvpn/test_server').with('ensure' => 'directory')} + it { should contain_file('/etc/openvpn/test_server/client-configs').with('ensure' => 'directory')} + it { should contain_file('/etc/openvpn/test_server/download-configs').with('ensure' => 'directory')} + it { should contain_file('/etc/openvpn/test_server/easy-rsa/vars')} + it { should contain_file('/etc/openvpn/test_server/easy-rsa/openssl.cnf')} + it { should contain_file('/etc/openvpn/test_server/keys').with( + 'ensure' => 'link', + 'target' => '/etc/openvpn/test_server/easy-rsa/keys' + )} + + # Execs to working with certificates + it { should contain_exec('copy easy-rsa to openvpn config folder test_server').with( + 'command' => '/bin/cp -r /usr/share/doc/openvpn/examples/easy-rsa/2.0 /etc/openvpn/test_server/easy-rsa' + )} + it { should contain_exec('generate dh param test_server') } + it { should contain_exec('initca test_server') } + it { should contain_exec('generate server cert test_server') } + + # VPN server config file itself + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^mode\s+server$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^client\-config\-dir\s+\/etc\/openvpn\/test_server\/client\-configs$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^ca\s+\/etc\/openvpn\/test_server\/keys\/ca.crt$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^cert\s+\/etc\/openvpn\/test_server\/keys\/server.crt$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^key\s+\/etc\/openvpn\/test_server\/keys\/server.key$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^dh\s+\/etc\/openvpn\/test_server\/keys\/dh1024.pem$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^proto\s+tcp-server$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^tls-server$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^port\s+1194$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^comp-lzo$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^group\s+nogroup$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^user\s+nobody$/) } + it { should_not contain_file('/etc/openvpn/test_server.conf').with_content(/^log\-append\s+test_server\/openvpn\.log$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^status\s+test_server\/openvpn\-status\.log$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^dev\s+tun0$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^local\s+1\.2\.3\.4$/) } + it { should_not contain_file('/etc/openvpn/test_server.conf').with_content(/^ifconfig-pool-persist/) } + end + + context "creating a server setting all parameters" do + let(:params) { { + 'country' => 'CO', + 'province' => 'ST', + 'city' => 'Some City', + 'organization' => 'example.org', + 'email' => 'testemail@example.org', + 'compression' => 'fake_compression', + 'port' => '123', + 'proto' => 'udp', + 'group' => 'someone', + 'user' => 'someone', + 'logfile' => '/var/log/openvpn/test_server.log', + 'status_log' => '/var/log/openvpn/test_server_status.log', + 'dev' => 'tun1', + 'local' => '2.3.4.5', + 'ipp' => true, + 'server' => '2.3.4.0 255.255.0.0', + 'push' => [ 'dhcp-option DNS 172.31.0.30', 'route 172.31.0.0 255.255.0.0' ] + } } + + let (:facts) { { + :ipaddress_eth0 => '1.2.3.4', + :network_eth0 => '1.2.3.0', + :netmask_eth0 => '255.255.255.0', + :concat_basedir => '/var/lib/puppet/concat' + } } + + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^mode\s+server$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^client\-config\-dir\s+\/etc\/openvpn\/test_server\/client\-configs$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^ca\s+\/etc\/openvpn\/test_server\/keys\/ca.crt$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^cert\s+\/etc\/openvpn\/test_server\/keys\/server.crt$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^key\s+\/etc\/openvpn\/test_server\/keys\/server.key$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^dh\s+\/etc\/openvpn\/test_server\/keys\/dh1024.pem$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^proto\s+udp$/) } + it { should_not contain_file('/etc/openvpn/test_server.conf').with_content(/^proto\s+tls-server$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^port\s+123$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^fake_compression$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^group\s+someone$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^user\s+someone$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^log\-append\s+\/var\/log\/openvpn\/test_server\.log$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^status\s+\/var\/log\/openvpn\/test_server_status\.log$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^dev\s+tun1$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^local\s+2\.3\.4\.5$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^server\s+2\.3\.4\.0\s+255\.255\.0\.0$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^push\s+dhcp-option\s+DNS\s+172\.31\.0\.30$/) } + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^push\s+route\s+172\.31\.0\.0\s+255\.255\.0\.0$/) } + end + + context "when RedHat based machine" do + let(:params) { { + 'country' => 'CO', + 'province' => 'ST', + 'city' => 'Some City', + 'organization' => 'example.org', + 'email' => 'testemail@example.org' + } } + + let(:facts) { { :osfamily => 'RedHat', :concat_basedir => '/var/lib/puppet/concat' } } + + it { should contain_file('/etc/openvpn/test_server/easy-rsa/openssl.cnf').with( + 'ensure' => 'link', + 'target' => '/etc/openvpn/test_server/easy-rsa/openssl-1.0.0.cnf' + )} + + it { should contain_exec('copy easy-rsa to openvpn config folder test_server').with( + 'command' => '/bin/cp -r /usr/share/doc/openvpn-2.2.2/easy-rsa/2.0 /etc/openvpn/test_server/easy-rsa' + )} + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^group\s+nobody$/) } + + end + + context "when Debian based machine" do + let(:params) { { + 'country' => 'CO', + 'province' => 'ST', + 'city' => 'Some City', + 'organization' => 'example.org', + 'email' => 'testemail@example.org' + } } + + let(:facts) { { :osfamily => 'Debian', :concat_basedir => '/var/lib/puppet/concat' } } + + it { should contain_file('/etc/openvpn/test_server/easy-rsa/openssl.cnf').with( + 'ensure' => 'link', + 'target' => '/etc/openvpn/test_server/easy-rsa/openssl-1.0.0.cnf' + )} + + it { should contain_exec('copy easy-rsa to openvpn config folder test_server').with( + 'command' => '/bin/cp -r /usr/share/doc/openvpn/examples/easy-rsa/2.0 /etc/openvpn/test_server/easy-rsa' + )} + + # Configure to start vpn session + it { should contain_concat__fragment('openvpn.default.autostart.test_server').with( + 'content' => "AUTOSTART=\"$AUTOSTART test_server\"\n", + 'target' => '/etc/default/openvpn' + )} + + it { should contain_file('/etc/openvpn/test_server.conf').with_content(/^group\s+nogroup$/) } + + end + +end diff --git a/puppet/modules/openvpn/spec/spec_helper.rb b/puppet/modules/openvpn/spec/spec_helper.rb new file mode 100644 index 00000000..dc7e9f4a --- /dev/null +++ b/puppet/modules/openvpn/spec/spec_helper.rb @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/module_spec_helper' diff --git a/puppet/modules/openvpn/templates/client.erb b/puppet/modules/openvpn/templates/client.erb new file mode 100644 index 00000000..021ed617 --- /dev/null +++ b/puppet/modules/openvpn/templates/client.erb @@ -0,0 +1,26 @@ +client +ca keys/ca.crt +cert keys/<%= scope.lookupvar('name') %>.crt +key keys/<%= scope.lookupvar('name') %>.key +dev <%= scope.lookupvar('dev') %> +proto <%= scope.lookupvar('proto') %> +remote <%= scope.lookupvar('remote_host') %> <%= scope.lookupvar('port') %> +<% if scope.lookupvar('compression') != '' -%> +<%= scope.lookupvar('compression') %> +<% end -%> +resolv-retry <%= scope.lookupvar('resolv_retry') %> +<% if scope.lookupvar('nobind') -%> +nobind +<% end -%> +<% if scope.lookupvar('persist_key') -%> +persist-key +<% end -%> +<% if scope.lookupvar('persist_tun') -%> +persist-tun +<% end -%> +<% if scope.lookupvar('mute_replay_warnings') -%> +mute-replay-warnings +<% end -%> +ns-cert-type server +verb <%= scope.lookupvar('verb') %> +mute <%= scope.lookupvar('mute') %> diff --git a/puppet/modules/openvpn/templates/client_specific_config.erb b/puppet/modules/openvpn/templates/client_specific_config.erb new file mode 100644 index 00000000..62cc0e7a --- /dev/null +++ b/puppet/modules/openvpn/templates/client_specific_config.erb @@ -0,0 +1,10 @@ +<% scope.lookupvar('iroute').each do |route| -%> +iroute <%= route %> +<% end -%> +<% if ifconfig = scope.lookupvar('ifconfig') -%> +ifconfig-push <%= ifconfig %> +<% end -%> +<% scope.lookupvar('dhcp_options').each do |option| -%> +push dhcp-option <%= option %> +<% end -%> + diff --git a/puppet/modules/openvpn/templates/etc-default-openvpn.erb b/puppet/modules/openvpn/templates/etc-default-openvpn.erb new file mode 100644 index 00000000..310e462e --- /dev/null +++ b/puppet/modules/openvpn/templates/etc-default-openvpn.erb @@ -0,0 +1,20 @@ +# This is the configuration file for /etc/init.d/openvpn + +# +# Start only these VPNs automatically via init script. +# Allowed values are "all", "none" or space separated list of +# names of the VPNs. If empty, "all" is assumed. +# +#AUTOSTART="all" +#AUTOSTART="none" +#AUTOSTART="home office" +# +# Refresh interval (in seconds) of default status files +# located in /var/run/openvpn.$NAME.status +# Defaults to 10, 0 disables status file generation +# +#STATUSREFRESH=10 +#STATUSREFRESH=0 +# Optional arguments to openvpn's command line +OPTARGS="" +AUTOSTART="" diff --git a/puppet/modules/openvpn/templates/server.erb b/puppet/modules/openvpn/templates/server.erb new file mode 100644 index 00000000..6ef13263 --- /dev/null +++ b/puppet/modules/openvpn/templates/server.erb @@ -0,0 +1,37 @@ +mode server +client-config-dir /etc/openvpn/<%= scope.lookupvar('name') %>/client-configs +ca /etc/openvpn/<%= scope.lookupvar('name') %>/keys/ca.crt +cert /etc/openvpn/<%= scope.lookupvar('name') %>/keys/server.crt +key /etc/openvpn/<%= scope.lookupvar('name') %>/keys/server.key +dh /etc/openvpn/<%= scope.lookupvar('name') %>/keys/dh1024.pem +<% if scope.lookupvar('proto') == 'tcp' -%> +proto <%= scope.lookupvar('proto') %>-server +<% else -%> +proto <%= scope.lookupvar('proto') %> +<% end -%> +port <%= scope.lookupvar('port') %> +<% if scope.lookupvar('tls_server') -%> +tls-server +<% end -%> +<% if scope.lookupvar('compression') != '' -%> +<%= scope.lookupvar('compression') %> +<% end -%> +group <%= scope.lookupvar('group_to_set') %> +user <%= scope.lookupvar('user') %> +<% if scope.lookupvar('logfile') -%> +log-append <%= scope.lookupvar('logfile') %> +<% end -%> +status <%= scope.lookupvar('status_log') %> +dev <%= scope.lookupvar('dev') %> +<% if scope.lookupvar('local') != '' -%> +local <%= scope.lookupvar('local') %> +<% end -%> +<% if scope.lookupvar('ipp') -%> +ifconfig-pool-persist <%= scope.lookupvar('name') %>/vpn-ipp.txt +<% end -%> +<% if scope.lookupvar('server') != '' -%> +server <%= scope.lookupvar('server') %> +<% end -%> +<% scope.lookupvar('push').each do |item| -%> +push <%= item %> +<% end -%> diff --git a/puppet/modules/openvpn/templates/vars.erb b/puppet/modules/openvpn/templates/vars.erb new file mode 100644 index 00000000..20448b8b --- /dev/null +++ b/puppet/modules/openvpn/templates/vars.erb @@ -0,0 +1,68 @@ +# easy-rsa parameter settings + +# NOTE: If you installed from an RPM, +# don't edit this file in place in +# /usr/share/openvpn/easy-rsa -- +# instead, you should copy the whole +# easy-rsa directory to another location +# (such as /etc/openvpn) so that your +# edits will not be wiped out by a future +# OpenVPN package upgrade. + +# This variable should point to +# the top level of the easy-rsa +# tree. +export EASY_RSA="/etc/openvpn/<%= @name %>/easy-rsa" + +# +# This variable should point to +# the requested executables +# +export OPENSSL="openssl" +export PKCS11TOOL="pkcs11-tool" +export GREP="grep" + + +# This variable should point to +# the openssl.cnf file included +# with easy-rsa. +export KEY_CONFIG=`$EASY_RSA/whichopensslcnf $EASY_RSA` + +# Edit this variable to point to +# your soon-to-be-created key +# directory. +# +# WARNING: clean-all will do +# a rm -rf on this directory +# so make sure you define +# it correctly! +export KEY_DIR="$EASY_RSA/keys" + +# Issue rm -rf warning +echo NOTE: If you run ./clean-all, I will be doing a rm -rf on $KEY_DIR + +# PKCS11 fixes +export PKCS11_MODULE_PATH="dummy" +export PKCS11_PIN="dummy" + +# Increase this to 2048 if you +# are paranoid. This will slow +# down TLS negotiation performance +# as well as the one-time DH parms +# generation process. +export KEY_SIZE=1024 + +# In how many days should the root CA key expire? +export CA_EXPIRE=3650 + +# In how many days should certificates expire? +export KEY_EXPIRE=3650 + +# These are the default values for fields +# which will be placed in the certificate. +# Don't leave any of these fields blank. +export KEY_COUNTRY="<%= @country %>" +export KEY_PROVINCE="<%= @province %>" +export KEY_CITY="<%= @city %>" +export KEY_ORG="<%= @organization %>" +export KEY_EMAIL="<%= @email %>" diff --git a/puppet/modules/openvpn/vagrant/client.pp b/puppet/modules/openvpn/vagrant/client.pp new file mode 100644 index 00000000..7ebeb1d7 --- /dev/null +++ b/puppet/modules/openvpn/vagrant/client.pp @@ -0,0 +1,5 @@ +node default { + + package { 'openvpn': ensure => installed; } + +} diff --git a/puppet/modules/openvpn/vagrant/server.pp b/puppet/modules/openvpn/vagrant/server.pp new file mode 100644 index 00000000..a95def06 --- /dev/null +++ b/puppet/modules/openvpn/vagrant/server.pp @@ -0,0 +1,23 @@ +node default { + openvpn::server { 'winterthur': + country => 'CH', + province => 'ZH', + city => 'Winterthur', + organization => 'example.org', + email => 'root@example.org', + server => '10.200.200.0 255.255.255.0' + } + + openvpn::client { 'client1': + server => 'winterthur'; + } + + openvpn::client_specific_config { 'client1': + server => 'winterthur', + ifconfig => '10.200.200.100 255.255.255.0' + } + + openvpn::client { 'client2': + server => 'winterthur'; + } +} diff --git a/puppet/modules/postfwd/files/postfwd_default b/puppet/modules/postfwd/files/postfwd_default new file mode 100644 index 00000000..83742e40 --- /dev/null +++ b/puppet/modules/postfwd/files/postfwd_default @@ -0,0 +1,19 @@ +### This file managed by Puppet +# Global options for postfwd(8). + +# Set to '1' to enable startup (daemon mode) +STARTUP=1 + +# Config file +CONF=/etc/postfix/postfwd.cf +# IP where listen to +INET=127.0.0.1 +# Port where listen to +PORT=10040 +# run as user postfwd +RUNAS="postfw" +# Arguments passed on start (--daemon implied) +# disable summary and cache-no-size +#ARGS="--summary=600 --cache=600 --cache-rdomain-only --cache-no-size" +ARGS="--cache=600 --cache-rdomain-only --no-rulestats" + diff --git a/puppet/modules/postfwd/manifests/init.pp b/puppet/modules/postfwd/manifests/init.pp new file mode 100644 index 00000000..6db3fa52 --- /dev/null +++ b/puppet/modules/postfwd/manifests/init.pp @@ -0,0 +1,43 @@ +# This class provides rate-limiting for outgoing SMTP, using postfwd +# it is configured with some limits that seem reasonable for a generic +# use-case. Each of the following applies to sasl_authenticated users: +# +# . 150 recipients at a time +# . no more than 50 messages in 60 minutes +# . no more than 250 recipients in 60 minutes. +# +# This class could be easily extended to add overrides to these rules, +# maximum sizes per client, or additional rules +class postfwd { + + ensure_packages(['libnet-server-perl', 'libnet-dns-perl', 'postfwd']) + + file { + '/etc/default/postfwd': + source => 'puppet:///modules/postfwd/postfwd_default', + mode => '0644', + owner => root, + group => root, + before => Package['postfwd']; + + '/etc/postfix/postfwd.cf': + content => template('postfwd/postfwd.cf.erb'), + mode => '0644', + owner => root, + group => root, + require => Package['postfix'], + before => Package['postfwd']; + } + + service { + 'postfwd': + ensure => running, + name => postfwd, + pattern => '/usr/sbin/postfwd', + enable => true, + hasrestart => true, + hasstatus => false, + require => [ File['/etc/default/postfwd'], + File['/etc/postfix/postfwd.cf']]; + } +} diff --git a/puppet/modules/postfwd/templates/postfwd.cf.erb b/puppet/modules/postfwd/templates/postfwd.cf.erb new file mode 100644 index 00000000..1c45dd03 --- /dev/null +++ b/puppet/modules/postfwd/templates/postfwd.cf.erb @@ -0,0 +1,28 @@ +### This file managed by Puppet +# Before deploying a rule +# 1. test with an additional "sender==test@domain.org;" in the rule so it +# only applies to your test account +# 2. then when ready to test for all users, use WARN and watch the logs +# for a few days and make sure it working the way you like +# 3. Then when ready to deploy for real set a proper error code + +## Overrides - make like the following example +# id=exampleuser; sasl_username==exampleuser; action=dunno + +## Rules that apply to all senders +# Recipient Per Message Limit +# We only receive mail via smtp from sasl authenticated users +# directly. We want to limit to a lower amount to prevent phished accounts +# spamming +id=RCPTSENDER; recipient_count=150; action=REJECT Too many recipients, please try again. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTSENDER + +# Message Rate Limit +# This limits sasl authenticated users to no more than 50/60mins +# NOTE: sasl_username needs to be set to something or this check will fail +id=MSGRATE ; sasl_username=!!(^$); action==rate($$sasl_username/100/3600/450 4.7.1 exceeded message rate. Contact Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:MSGRATE) + +# Total Recipient Rate Limit +# This adds up the recipients for all the sasl authenticated users messages +# and can't exceed more than 250/60min +# NOTE: sasl_username needs to be set to something or this check will fail +id=RCPTRATE ; sasl_username=!!(^$); action==rcpt($$sasl_username/500/3600/450 4.7.1 exceeded message rate. Contact http://<%= @domain %>/tickets/new if this is in error. ERROR:RCPTRATE) diff --git a/puppet/modules/site_apache/files/conf.d/security b/puppet/modules/site_apache/files/conf.d/security new file mode 100644 index 00000000..a5ae5bdc --- /dev/null +++ b/puppet/modules/site_apache/files/conf.d/security @@ -0,0 +1,55 @@ +# +# Disable access to the entire file system except for the directories that +# are explicitly allowed later. +# +# This currently breaks the configurations that come with some web application +# Debian packages. It will be made the default for the release after lenny. +# +#<Directory /> +# AllowOverride None +# Order Deny,Allow +# Deny from all +#</Directory> + + +# Changing the following options will not really affect the security of the +# server, but might make attacks slightly more difficult in some cases. + +# +# ServerTokens +# This directive configures what you return as the Server HTTP response +# Header. The default is 'Full' which sends information about the OS-Type +# and compiled in modules. +# Set to one of: Full | OS | Minimal | Minor | Major | Prod +# where Full conveys the most information, and Prod the least. +# +#ServerTokens Minimal +ServerTokens Prod + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (internal error documents, FTP directory +# listings, mod_status and mod_info output etc., but not CGI generated +# documents or custom error documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +# +#ServerSignature Off +ServerSignature Off + +# +# Allow TRACE method +# +# Set to "extended" to also reflect the request body (only for testing and +# diagnostic purposes). +# +# Set to one of: On | Off | extended +# +#TraceEnable Off +TraceEnable On + +# Setting this header will prevent other sites from embedding pages from this +# site as frames. This defends against clickjacking attacks. +# Requires mod_headers to be enabled. +# +Header set X-Frame-Options: "DENY" diff --git a/puppet/modules/site_apache/files/include.d/ssl_common.inc b/puppet/modules/site_apache/files/include.d/ssl_common.inc new file mode 100644 index 00000000..2d282c84 --- /dev/null +++ b/puppet/modules/site_apache/files/include.d/ssl_common.inc @@ -0,0 +1,7 @@ +SSLEngine on +SSLProtocol all -SSLv2 -SSLv3 +SSLHonorCipherOrder on +SSLCompression off +SSLCipherSuite "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!RC4:!MD5:!PSK!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + +RequestHeader set X_FORWARDED_PROTO 'https'
\ No newline at end of file diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp new file mode 100644 index 00000000..8a11759a --- /dev/null +++ b/puppet/modules/site_apache/manifests/common.pp @@ -0,0 +1,30 @@ +# install basic apache modules needed for all services (nagios, webapp) +class site_apache::common { + + include apache::module::rewrite + include apache::module::env + + class { '::apache': + no_default_site => true, + ssl => true, + ssl_cipher_suite => 'HIGH:MEDIUM:!aNULL:!MD5' + } + + # needed for the mod_ssl config + include apache::module::mime + + # load mods depending on apache version + if ( $::lsbdistcodename == 'jessie' ) { + # apache >= 2.4, debian jessie + # needed for mod_ssl config + include apache::module::socache_shmcb + # generally needed + include apache::module::mpm_prefork + } else { + # apache < 2.4, debian wheezy + # for "Order" directive, i.e. main apache2.conf + include apache::module::authz_host + } + + include site_apache::common::tls +} diff --git a/puppet/modules/site_apache/manifests/common/tls.pp b/puppet/modules/site_apache/manifests/common/tls.pp new file mode 100644 index 00000000..040868bf --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/tls.pp @@ -0,0 +1,6 @@ +class site_apache::common::tls { + # class to setup common SSL configurations + + apache::config::include{ 'ssl_common.inc': } + +} diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb new file mode 100644 index 00000000..bfa5d04d --- /dev/null +++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb @@ -0,0 +1,48 @@ +<VirtualHost *:80> + ServerName <%= @api_domain %> + RewriteEngine On + RewriteRule ^.*$ https://<%= @api_domain -%>:<%= @api_port -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common +</VirtualHost> + +Listen 0.0.0.0:<%= @api_port %> + +<VirtualHost *:<%= @api_port -%>> + ServerName <%= @api_domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common + + SSLCACertificatePath /etc/ssl/certs + SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.key + SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.crt + + Include include.d/ssl_common.inc + + <IfModule mod_headers.c> +<% if @webapp['secure'] -%> + Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" +<% end -%> + Header always unset X-Powered-By + Header always unset X-Runtime + </IfModule> + + DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> + + # Check for maintenance file and redirect all requests + RewriteEngine On + RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f + RewriteCond %{SCRIPT_FILENAME} !maintenance.html + RewriteCond %{REQUEST_URI} !/images/maintenance.jpg + RewriteRule ^.*$ %{DOCUMENT_ROOT}/system/maintenance.html [L] + + # http://www.modrails.com/documentation/Users%20guide%20Apache.html#_passengerallowencodedslashes_lt_on_off_gt + AllowEncodedSlashes on + PassengerAllowEncodedSlashes on + PassengerFriendlyErrorPages off + SetEnv TMPDIR /var/tmp +</VirtualHost> diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb new file mode 100644 index 00000000..bf60e794 --- /dev/null +++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb @@ -0,0 +1,76 @@ +<VirtualHost *:80> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> + RewriteEngine On + RewriteRule ^.*$ https://<%= @webapp_domain -%>%{REQUEST_URI} [R=permanent,L] + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common +</VirtualHost> + +<VirtualHost *:443> + ServerName <%= @webapp_domain %> + ServerAlias <%= @domain_name %> + ServerAlias <%= @domain %> + ServerAlias www.<%= @domain %> + CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common + + SSLCACertificatePath /etc/ssl/certs + SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::commercial_cert_name') %>.key + SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::commercial_cert_name') %>.crt + + Include include.d/ssl_common.inc + + <IfModule mod_headers.c> +<% if (defined? @services) and (@services.include? 'webapp') and (@webapp['secure']) -%> + Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" +<% end -%> + Header always unset X-Powered-By + Header always unset X-Runtime + </IfModule> + +<% if (defined? @services) and (@services.include? 'webapp') -%> + DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> + + RewriteEngine On + # Check for maintenance file and redirect all requests + RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f + RewriteCond %{SCRIPT_FILENAME} !maintenance.html + RewriteCond %{REQUEST_URI} !/images/maintenance.jpg + RewriteRule ^.*$ %{DOCUMENT_ROOT}/system/maintenance.html [L] + + # http://www.modrails.com/documentation/Users%20guide%20Apache.html#_passengerallowencodedslashes_lt_on_off_gt + AllowEncodedSlashes on + PassengerAllowEncodedSlashes on + PassengerFriendlyErrorPages off + SetEnv TMPDIR /var/tmp + + # Allow rails assets to be cached for a very long time (since the URLs change whenever the content changes) + <Location /assets/> + Header unset ETag + FileETag None + ExpiresActive On + ExpiresDefault "access plus 1 year" + </Location> +<% end -%> + + +<% if (defined? @services) and (@services.include? 'monitor') -%> + <DirectoryMatch (/usr/share/nagios3/htdocs|/usr/lib/cgi-bin/nagios3|/etc/nagios3/stylesheets|/usr/share/pnp4nagios)> + <% if (defined? @services) and (@services.include? 'webapp') -%> + PassengerEnabled off + <% end -%> + AllowOverride all + # Nagios won't work with setting this option to "DENY", + # as set in conf.d/security (#4169). Therefor we allow + # it here, only for nagios. + Header set X-Frame-Options: "ALLOW" + </DirectoryMatch> +<% end -%> +</VirtualHost> diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb new file mode 100644 index 00000000..232b1577 --- /dev/null +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -0,0 +1,55 @@ +<VirtualHost 127.0.0.1:80> + ServerName <%= @tor_domain %> + + <IfModule mod_headers.c> + Header always unset X-Powered-By + Header always unset X-Runtime + </IfModule> + +<% if (defined? @services) and (@services.include? 'webapp') -%> + DocumentRoot /srv/leap/webapp/public + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/leap/webapp/public> + AllowOverride None + Require all granted + </Directory> + <% end %> + + RewriteEngine On + # Check for maintenance file and redirect all requests + RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f + RewriteCond %{SCRIPT_FILENAME} !maintenance.html + RewriteCond %{REQUEST_URI} !/images/maintenance.jpg + RewriteRule ^.*$ %{DOCUMENT_ROOT}/system/maintenance.html [L] + + # http://www.modrails.com/documentation/Users%20guide%20Apache.html#_passengerallowencodedslashes_lt_on_off_gt + AllowEncodedSlashes on + PassengerAllowEncodedSlashes on + PassengerFriendlyErrorPages off + SetEnv TMPDIR /var/tmp + + # Allow rails assets to be cached for a very long time (since the URLs change whenever the content changes) + <Location /assets/> + Header unset ETag + FileETag None + ExpiresActive On + ExpiresDefault "access plus 1 year" + </Location> +<% end -%> + +<% if (defined? @services) and (@services.include? 'static') -%> + DocumentRoot "/srv/static/root/public" + <% if scope.function_guess_apache_version([]) == '2.4' %> + <Directory /srv/static/root/public> + AllowOverride None + Require all granted + </Directory> + <% end %> + AccessFileName .htaccess + + Alias /provider.json /srv/leap/provider.json + <Location /provider.json> + Header set X-Minimum-Client-Version 0.5 + </Location> +<% end -%> +</VirtualHost> diff --git a/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap b/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap new file mode 100644 index 00000000..bbaac6a2 --- /dev/null +++ b/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap @@ -0,0 +1,6 @@ +// this file is managed by puppet ! + +Unattended-Upgrade::Allowed-Origins { + "leap.se:stable"; +} + diff --git a/puppet/modules/site_apt/files/keys/leap-archive.gpg b/puppet/modules/site_apt/files/keys/leap-archive.gpg Binary files differnew file mode 100644 index 00000000..dd7f3be6 --- /dev/null +++ b/puppet/modules/site_apt/files/keys/leap-archive.gpg diff --git a/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg b/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg Binary files differnew file mode 100644 index 00000000..5cc9064b --- /dev/null +++ b/puppet/modules/site_apt/files/keys/leap-experimental-archive.gpg diff --git a/puppet/modules/site_apt/manifests/dist_upgrade.pp b/puppet/modules/site_apt/manifests/dist_upgrade.pp new file mode 100644 index 00000000..0eb98cea --- /dev/null +++ b/puppet/modules/site_apt/manifests/dist_upgrade.pp @@ -0,0 +1,17 @@ +# upgrade all packages +class site_apt::dist_upgrade { + + # facter returns 'true' as string + # lint:ignore:quoted_booleans + if $::apt_running == 'true' { + # lint:endignore + fail ('apt-get is running in background - Please wait until it finishes. Exiting.') + } else { + exec{'initial_apt_dist_upgrade': + command => "/usr/bin/apt-get -q -y -o 'DPkg::Options::=--force-confold' dist-upgrade", + refreshonly => false, + timeout => 1200, + require => Exec['apt_updated'] + } + } +} diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp new file mode 100644 index 00000000..455425c1 --- /dev/null +++ b/puppet/modules/site_apt/manifests/init.pp @@ -0,0 +1,55 @@ +# setup apt on all nodes +class site_apt { + + $sources = hiera('sources') + $apt_config = $sources['apt'] + + # debian repo urls + $apt_url_basic = $apt_config['basic'] + $apt_url_security = $apt_config['security'] + $apt_url_backports = $apt_config['backports'] + + # leap repo url + $platform_sources = $sources['platform'] + $apt_url_platform_basic = $platform_sources['apt']['basic'] + + # needed on jessie hosts for getting pnp4nagios from testing + if ( $::operatingsystemmajrelease == '8' ) { + $use_next_release = true + } else { + $use_next_release = false + } + + class { 'apt': + custom_key_dir => 'puppet:///modules/site_apt/keys', + debian_url => $apt_url_basic, + security_url => $apt_url_security, + backports_url => $apt_url_backports, + use_next_release => $use_next_release + } + + # enable http://deb.leap.se debian package repository + include site_apt::leap_repo + + apt::apt_conf { '90disable-pdiffs': + content => 'Acquire::PDiffs "false";'; + } + + include ::site_apt::unattended_upgrades + + # not currently used + #apt::sources_list { 'secondary.list': + # content => template('site_apt/secondary.list'); + #} + + apt::preferences_snippet { 'leap': + priority => 999, + package => '*', + pin => 'origin "deb.leap.se"' + } + + # All packages should be installed after 'update_apt' is called, + # which does an 'apt-get update'. + Exec['update_apt'] -> Package <||> + +} diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp new file mode 100644 index 00000000..5eedce45 --- /dev/null +++ b/puppet/modules/site_apt/manifests/leap_repo.pp @@ -0,0 +1,16 @@ +# install leap deb repo together with leap-keyring package +# containing the apt signing key +class site_apt::leap_repo { + $platform = hiera_hash('platform') + $major_version = $platform['major_version'] + + apt::sources_list { 'leap.list': + content => "deb ${::site_apt::apt_url_platform_basic} ${::lsbdistcodename} main\n", + before => Exec[refresh_apt] + } + + package { 'leap-archive-keyring': + ensure => latest + } + +} diff --git a/puppet/modules/site_apt/manifests/preferences/check_mk.pp b/puppet/modules/site_apt/manifests/preferences/check_mk.pp new file mode 100644 index 00000000..580e0d3f --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/check_mk.pp @@ -0,0 +1,9 @@ +class site_apt::preferences::check_mk { + + apt::preferences_snippet { 'check-mk': + package => 'check-mk-*', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_apt/manifests/preferences/passenger.pp b/puppet/modules/site_apt/manifests/preferences/passenger.pp new file mode 100644 index 00000000..8cd41f91 --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/passenger.pp @@ -0,0 +1,14 @@ +# +# currently, this is only used by static_site to get passenger v4. +# +# UPGRADE: this is not needed for jessie. +# +class site_apt::preferences::passenger { + + apt::preferences_snippet { 'passenger': + package => 'libapache2-mod-passenger', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp new file mode 100644 index 00000000..bfeaa7da --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp @@ -0,0 +1,13 @@ +class site_apt::preferences::rsyslog { + + apt::preferences_snippet { + 'rsyslog_anon_depends': + package => 'libestr0 librelp0 rsyslog*', + priority => '999', + pin => 'release a=wheezy-backports', + before => Class['rsyslog::install']; + + 'fixed_rsyslog_anon_package': + ensure => absent; + } +} diff --git a/puppet/modules/site_apt/manifests/unattended_upgrades.pp b/puppet/modules/site_apt/manifests/unattended_upgrades.pp new file mode 100644 index 00000000..42f1f4c6 --- /dev/null +++ b/puppet/modules/site_apt/manifests/unattended_upgrades.pp @@ -0,0 +1,20 @@ +# configute unattended upgrades so packages from both Debian and LEAP +# repos get upgraded unattended +class site_apt::unattended_upgrades { + # override unattended-upgrades package resource to make sure + # that it is upgraded on every deploy (#6245) + + # configure upgrades for Debian + class { 'apt::unattended_upgrades': + ensure_version => latest + } + + # configure LEAP upgrades + apt::apt_conf { '51unattended-upgrades-leap': + source => [ + "puppet:///modules/site_apt/${::lsbdistid}/51unattended-upgrades-leap"], + require => Package['unattended-upgrades'], + refresh_apt => false, + } + +} diff --git a/puppet/modules/site_apt/templates/jessie/postfix.seeds b/puppet/modules/site_apt/templates/jessie/postfix.seeds new file mode 100644 index 00000000..1a878ccc --- /dev/null +++ b/puppet/modules/site_apt/templates/jessie/postfix.seeds @@ -0,0 +1 @@ +postfix postfix/main_mailer_type select No configuration diff --git a/puppet/modules/site_apt/templates/preferences.include_squeeze b/puppet/modules/site_apt/templates/preferences.include_squeeze new file mode 100644 index 00000000..d6d36b60 --- /dev/null +++ b/puppet/modules/site_apt/templates/preferences.include_squeeze @@ -0,0 +1,25 @@ +Explanation: Debian wheezy +Package: * +Pin: release o=Debian,n=wheezy +Pin-Priority: 990 + +Explanation: Debian wheezy-updates +Package: * +Pin: release o=Debian,n=wheezy-updates +Pin-Priority: 990 + +Explanation: Debian sid +Package: * +Pin: release o=Debian,n=sid +Pin-Priority: 1 + +Explanation: Debian squeeze +Package: * +Pin: release o=Debian,n=squeeze +Pin-Priority: 980 + +Explanation: Debian fallback +Package: * +Pin: release o=Debian +Pin-Priority: -10 + diff --git a/puppet/modules/site_apt/templates/secondary.list b/puppet/modules/site_apt/templates/secondary.list new file mode 100644 index 00000000..0c024549 --- /dev/null +++ b/puppet/modules/site_apt/templates/secondary.list @@ -0,0 +1,3 @@ +# basic +deb http://ftp.debian.org/debian/ <%= @lsbdistcodename %> main contrib non-free + diff --git a/puppet/modules/site_apt/templates/wheezy/postfix.seeds b/puppet/modules/site_apt/templates/wheezy/postfix.seeds new file mode 100644 index 00000000..1a878ccc --- /dev/null +++ b/puppet/modules/site_apt/templates/wheezy/postfix.seeds @@ -0,0 +1 @@ +postfix postfix/main_mailer_type select No configuration diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh b/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh new file mode 100644 index 00000000..1dd0afc9 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/all_hosts/run_node_tests.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# +# runs node tests + +/srv/leap/bin/run_tests --checkmk diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh new file mode 100755 index 00000000..c7477b18 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/couchdb/leap_couch_stats.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# todo: +# - thresholds +# - couch response time +# - make CURL/URL/DBLIST_EXCLUDE vars configurable +# - move load_nagios_utils() to helper library so we can use it from multiple scripts + +start_time=$(date +%s.%N) + +CURL='curl -s --netrc-file /etc/couchdb/couchdb.netrc' +URL='http://127.0.0.1:5984' +TMPFILE=$(mktemp) +DBLIST_EXCLUDE='(user-|sessions_|tokens_|_replicator|_users)' +PREFIX='Couchdb_' + + +load_nagios_utils () { + # load the nagios utils + # in debian, the package nagios-plugins-common installs utils.sh to /usr/lib/nagios/plugins/utils.sh + utilsfn= + for d in $PROGPATH /usr/lib/nagios/plugins /usr/lib64/nagios/plugins /usr/local/nagios/libexec /opt/nagios-plugins/libexec . ; do + if [ -f "$d/utils.sh" ]; then + utilsfn=$d/utils.sh; + fi + done + if [ "$utilsfn" = "" ]; then + echo "UNKNOWN - cannot find utils.sh (part of nagios plugins)"; + exit 3; + fi + . "$utilsfn"; + STATE[$STATE_OK]='OK' + STATE[$STATE_WARNING]='Warning' + STATE[$STATE_CRITICAL]='Critical' + STATE[$STATE_UNKNOWN]='Unknown' + STATE[$STATE_DEPENDENT]='Dependend' +} + +get_global_stats_perf () { + trap "localexit=3" ERR + local localexit db_count + localexit=0 + + # get a list of all dbs + $CURL -X GET $URL/_all_dbs | json_pp | egrep -v '(\[|\])' > $TMPFILE + + db_count=$( wc -l < $TMPFILE) + excluded_db_count=$( egrep -c "$DBLIST_EXCLUDE" $TMPFILE ) + + echo "db_count=$db_count|excluded_db_count=$excluded_db_count" + return ${localexit} +} + +db_stats () { + trap "localexit=3" ERR + local db db_stats doc_count del_doc_count localexit + localexit=0 + + db="$1" + name="$2" + + if [ -z "$name" ] + then + name="$db" + fi + + perf="$perf|${db}_docs=$( $CURL -s -X GET ${URL}/$db | json_pp |grep 'doc_count' | sed 's/[^0-9]//g' )" + db_stats=$( $CURL -s -X GET ${URL}/$db | json_pp ) + + doc_count=$( echo "$db_stats" | grep 'doc_count' | grep -v 'deleted_doc_count' | sed 's/[^0-9]//g' ) + del_doc_count=$( echo "$db_stats" | grep 'doc_del_count' | sed 's/[^0-9]//g' ) + + # don't divide by zero + if [ $del_doc_count -eq 0 ] + then + del_doc_perc=0 + else + del_doc_perc=$(( del_doc_count * 100 / doc_count )) + fi + + bytes=$( echo "$db_stats" | grep disk_size | sed 's/[^0-9]//g' ) + disk_size=$( echo "scale = 2; $bytes / 1024 / 1024" | bc -l ) + + echo -n "${localexit} ${PREFIX}${name}_database ${name}_docs=$doc_count|${name}_deleted_docs=$del_doc_count|${name}_deleted_docs_percentage=${del_doc_perc}%" + printf "|${name}_disksize_mb=%02.2fmb ${STATE[localexit]}: database $name\n" "$disk_size" + + return ${localexit} +} + +# main + +load_nagios_utils + +# per-db stats +# get a list of all dbs +$CURL -X GET $URL/_all_dbs | json_pp | egrep -v '(\[|\])' > $TMPFILE + +# get list of dbs to check +dbs=$( egrep -v "${DBLIST_EXCLUDE}" $TMPFILE | tr -d '\n"' | sed 's/,/ /g' ) + +for db in $dbs +do + db_stats "$db" +done + +# special handling for rotated dbs +suffix=$(($(date +'%s') / (60*60*24*30))) +db_stats "sessions_${suffix}" "sessions" +db_stats "tokens_${suffix}" "tokens" + + +# show global couchdb stats +global_stats_perf=$(get_global_stats_perf) +exitcode=$? + +end_time=$(date +%s.%N) +duration=$( echo "scale = 2; $end_time - $start_time" | bc -l ) + +printf "${exitcode} ${PREFIX}global_stats ${global_stats_perf}|script_duration=%02.2fs ${STATE[exitcode]}: global couchdb status\n" "$duration" + +rm "$TMPFILE" + diff --git a/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh new file mode 100755 index 00000000..4711e247 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/local_checks/mx/check_leap_mx.sh @@ -0,0 +1,33 @@ +#!/bin/bash + + +WARN=1 +CRIT=5 + +# in minutes +MAXAGE=10 + +STATUS[0]='OK' +STATUS[1]='Warning' +STATUS[2]='Critical' +CHECKNAME='Leap_MX_Queue' + +WATCHDIR='/var/mail/leap-mx/Maildir/new/' + + +total=`find $WATCHDIR -type f -mmin +$MAXAGE | wc -l` + +if [ $total -lt $WARN ] +then + exitcode=0 +else + if [ $total -le $CRIT ] + then + exitcode=1 + else + exitcode=2 + fi +fi + +echo "${exitcode} ${CHECKNAME} stale_files=${total} ${STATUS[exitcode]}: ${total} stale files (>=${MAXAGE} min) in ${WATCHDIR}." + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg new file mode 100644 index 00000000..0f378a5a --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg @@ -0,0 +1,28 @@ +/opt/bigcouch/var/log/bigcouch.log nocontext=1 +# ignore requests that are fine + I undefined - -.*200$ + I undefined - -.*201$ + I 127.0.0.1 undefined.* ok + I 127.0.0.1 localhost:5984 .* ok + # https://leap.se/code/issues/5246 + I Shutting down group server + # ignore bigcouch conflict errors + I Error in process.*{{nocatch,conflict} + # ignore "Uncaught error in HTTP request: {exit, normal}" error + # it's suppressed in later versions of bigcouch anhow + # see https://leap.se/code/issues/5226 + I Uncaught error in HTTP request: {exit,normal} + I Uncaught error in HTTP request: {exit, + # Ignore rexi_EXIT bigcouch error (Bug #6512) + I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{(killed|noproc|shutdown),\[{couch_db,collect_results + # Ignore "Generic server terminating" bigcouch message (Feature #6544) + I Generic server <.*> terminating + I {error_report,<.*>, + I {error_info, + C Uncaught error in HTTP request: {error, + C Response abnormally terminated: {nodedown, + C rexi_DOWN,noproc + C rexi_DOWN,noconnection + C error + C Connection attempt from disallowed node + W Apache CouchDB has started diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg new file mode 100644 index 00000000..166d0230 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/leap_mx.cfg @@ -0,0 +1,4 @@ +/var/log/leap/mx.log + W Don't know how to deliver mail + W No public key, stopping the processing chain + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg new file mode 100644 index 00000000..4f16d1bd --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/logwatch.cfg @@ -0,0 +1,31 @@ +# This file is managed by Puppet. DO NOT EDIT. + +# logwatch.cfg +# This file configures mk_logwatch. Define your logfiles +# and patterns to be looked for here. + +# Name one or more logfiles +/var/log/messages +# Patterns are indented with one space are prefixed with: +# C: Critical messages +# W: Warning messages +# I: ignore these lines (OK) +# The first match decided. Lines that do not match any pattern +# are ignored + C Fail event detected on md device + I mdadm.*: Rebuild.*event detected + W mdadm\[ + W ata.*hard resetting link + W ata.*soft reset failed (.*FIS failed) + W device-mapper: thin:.*reached low water mark + C device-mapper: thin:.*no free space + +/var/log/auth.log + W sshd.*Corrupted MAC on input + +/var/log/kern.log + C panic + C Oops + W generic protection rip + W .*Unrecovered read error - auto reallocate failed + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/openvpn.cfg new file mode 100644 index 00000000..d99dcde9 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/openvpn.cfg @@ -0,0 +1,19 @@ +/var/log/leap/openvpn.log +# ignore openvpn TLS initialization errors when clients +# suddenly hangup before properly establishing +# a tls connection + I ovpn-.*TLS Error: Unroutable control packet received from + I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds \(check your network connectivity\) + I ovpn-.*TLS Error: TLS handshake failed + I ovpn-.*TLS Error: TLS object -> incoming plaintext read error + I ovpn-.*Fatal TLS error \(check_tls_errors_co\), restarting + I ovpn-.*TLS_ERROR: BIO read tls_read_plaintext error: error:140890B2:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:no certificate + I ovpn-.*TLS_ERROR: BIO read tls_read_plaintext error: error:140890C7:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:peer did not return a certificate + I ovpn-.*TLS Error: unknown opcode received from + I ovpn-.*Authenticate/Decrypt packet error: packet HMAC authentication failed + I ovpn-.*TLS Error: reading acknowledgement record from packet + I ovpn-.*TLS Error: session-id not found in packet from + + I ovpn-.*SIGUSR1\[soft,tls-error\] received, client-instance restarting + I ovpn-.*VERIFY ERROR: depth=0, error=certificate has expired + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg new file mode 100644 index 00000000..3af5045b --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg @@ -0,0 +1,6 @@ +/var/log/soledad.log + C WSGI application error + C Error + C error +# Removed this line because we determined it was better to ignore it (#6566) +# W Timing out client: diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/stunnel.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/stunnel.cfg new file mode 100644 index 00000000..b1e6cf2f --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/stunnel.cfg @@ -0,0 +1,10 @@ +/var/log/leap/stunnel.log +# check for stunnel failures +# +# these are temporary failures and happen very often, so we +# ignore them until we tuned stunnel timeouts/logging, +# see https://leap.se/code/issues/5218 + I stunnel:.*Connection reset by peer + I stunnel:.*Peer suddenly disconnected + I stunnel:.*Connection refused + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg new file mode 100644 index 00000000..f53f0780 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/bigcouch.cfg @@ -0,0 +1,5 @@ +# on one-node bigcouch setups, we'll get this msg +# a lot, so we ignore it here until we fix +# https://leap.se/code/issues/5244 + I epmd: got partial packet only on file descriptor + diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg new file mode 100644 index 00000000..5f8d5b95 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/couchdb.cfg @@ -0,0 +1,2 @@ + C /usr/local/bin/couch-doc-update.*failed + C /usr/local/bin/couch-doc-update.*ERROR diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg new file mode 100644 index 00000000..f60d752b --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_header.cfg @@ -0,0 +1 @@ +/var/log/syslog diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg new file mode 100644 index 00000000..7daf0cac --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg @@ -0,0 +1,21 @@ +# some general patterns + I Error: Driver 'pcspkr' is already registered, aborting... +# ignore postfix errors on lost connection (Bug #6476) + I postfix/smtpd.*SSL_accept error from.*lost connection +# ignore postfix too many errors after DATA (#6545) + I postfix/smtpd.*too many errors after DATA from + C panic + C Oops + C Error +# ignore ipv6 icmp errors for now (Bug #6540) + I kernel: .*icmpv6_send: no reply to icmp error + C error + W generic protection rip + W .*Unrecovered read error - auto reallocate failed +# 401 Unauthorized error logged by webapp and possible other +# applications + C Unauthorized +# catch abnormal termination of processes (due to segfault/fpe +# signals etc). +# see https://github.com/pixelated/pixelated-user-agent/issues/683 + C systemd.*: main process exited, code=killed, status= diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg new file mode 100644 index 00000000..337d9ec6 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg @@ -0,0 +1,8 @@ +/var/log/leap/webapp.log +# check for webapp errors + C Completed 500 +# couch connection issues + C webapp.*Could not connect to couch database messages due to 401 Unauthorized: {"error":"unauthorized","reason":"You are not a server admin."} +# ignore RoutingErrors that rails throw when it can't handle a url +# see https://leap.se/code/issues/5173 + I webapp.*ActionController::RoutingError diff --git a/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl new file mode 100755 index 00000000..06163d49 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/nagios_plugins/check_unix_open_fds.pl @@ -0,0 +1,322 @@ +#!/usr/bin/perl -w + +# check_unix_open_fds Nagios Plugin +# +# TComm - Carlos Peris Pla +# +# This nagios plugin is free software, and comes with ABSOLUTELY +# NO WARRANTY. It may be used, redistributed and/or modified under +# the terms of the GNU General Public Licence (see +# http://www.fsf.org/licensing/licenses/gpl.txt). + + +# MODULE DECLARATION + +use strict; +use Nagios::Plugin; + + +# FUNCTION DECLARATION + +sub CreateNagiosManager (); +sub CheckArguments (); +sub PerformCheck (); + + +# CONSTANT DEFINITION + +use constant NAME => 'check_unix_open_fds'; +use constant VERSION => '0.1b'; +use constant USAGE => "Usage:\ncheck_unix_open_fds -w <process_threshold,application_threshold> -c <process_threshold,application_threshold>\n". + "\t\t[-V <version>]\n"; +use constant BLURB => "This plugin checks, in UNIX systems with the command lsof installed and with its SUID bit activated, the number\n". + "of file descriptors opened by an application and its processes.\n"; +use constant LICENSE => "This nagios plugin is free software, and comes with ABSOLUTELY\n". + "no WARRANTY. It may be used, redistributed and/or modified under\n". + "the terms of the GNU General Public Licence\n". + "(see http://www.fsf.org/licensing/licenses/gpl.txt).\n"; +use constant EXAMPLE => "\n\n". + "Example:\n". + "\n". + "check_unix_open_fds -a /usr/local/nagios/bin/ndo2db -w 20,75 -c 25,85\n". + "\n". + "It returns CRITICAL if number of file descriptors opened by ndo2db is higher than 85,\n". + "if not it returns WARNING if number of file descriptors opened by ndo2db is higher \n". + "than 75, if not it returns CRITICAL if number of file descriptors opened by any process\n". + "of ndo2db is higher than 25, if not it returns WARNING if number of file descriptors \n". + "opened by any process of ndo2db is higher than 20.\n". + "In other cases it returns OK if check has been performed succesfully.\n\n"; + + +# VARIABLE DEFINITION + +my $Nagios; +my $Error; +my $PluginResult; +my $PluginOutput; +my @WVRange; +my @CVRange; + + +# MAIN FUNCTION + +# Get command line arguments +$Nagios = &CreateNagiosManager(USAGE, VERSION, BLURB, LICENSE, NAME, EXAMPLE); +eval {$Nagios->getopts}; + +if (!$@) { + # Command line parsed + if (&CheckArguments($Nagios, \$Error, \@WVRange, \@CVRange)) { + # Argument checking passed + $PluginResult = &PerformCheck($Nagios, \$PluginOutput, \@WVRange, \@CVRange) + } + else { + # Error checking arguments + $PluginOutput = $Error; + $PluginResult = UNKNOWN; + } + $Nagios->nagios_exit($PluginResult,$PluginOutput); +} +else { + # Error parsing command line + $Nagios->nagios_exit(UNKNOWN,$@); +} + + + +# FUNCTION DEFINITIONS + +# Creates and configures a Nagios plugin object +# Input: strings (usage, version, blurb, license, name and example) to configure argument parsing functionality +# Return value: reference to a Nagios plugin object + +sub CreateNagiosManager() { + # Create GetOpt object + my $Nagios = Nagios::Plugin->new(usage => $_[0], version => $_[1], blurb => $_[2], license => $_[3], plugin => $_[4], extra => $_[5]); + + # Add argument units + $Nagios->add_arg(spec => 'application|a=s', + help => 'Application path for which you want to check the number of open file descriptors', + required => 1); + + # Add argument warning + $Nagios->add_arg(spec => 'warning|w=s', + help => "Warning thresholds. Format: <process_threshold,application_threshold>", + required => 1); + # Add argument critical + $Nagios->add_arg(spec => 'critical|c=s', + help => "Critical thresholds. Format: <process_threshold,application_threshold>", + required => 1); + + # Return value + return $Nagios; +} + + +# Checks argument values and sets some default values +# Input: Nagios Plugin object +# Output: reference to Error description string, Memory Unit, Swap Unit, reference to WVRange ($_[4]), reference to CVRange ($_[5]) +# Return value: True if arguments ok, false if not + +sub CheckArguments() { + my ($Nagios, $Error, $WVRange, $CVRange) = @_; + my $commas; + my $units; + my $i; + my $firstpos; + my $secondpos; + + # Check Warning thresholds list + $commas = $Nagios->opts->warning =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Warning list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $warning=$Nagios->opts->warning; + while ($warning =~ /[,]/g) { + $secondpos=pos $warning; + if ($secondpos - $firstpos==1){ + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->warning) - $firstpos==0){#La coma es el ultimo elemento del string + @{$WVRange}[$i] = "~:"; + } + else{ + @{$WVRange}[$i] = substr $Nagios->opts->warning, $firstpos, (length($Nagios->opts->warning)-$firstpos); + } + + if (@{$WVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Process Warning threshold in ${$WVRange[0]}"; + return 0; + }if (@{$WVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/){ + ${$Error} = "Invalid Application Warning threshold in ${$WVRange[1]}"; + return 0; + } + } + + # Check Critical thresholds list + $commas = $Nagios->opts->critical =~ tr/,//; + if ($commas !=1){ + ${$Error} = "Invalid Critical list format. One comma is expected."; + return 0; + } + else{ + $i=0; + $firstpos=0; + my $critical=$Nagios->opts->critical; + while ($critical =~ /[,]/g) { + $secondpos=pos $critical ; + if ($secondpos - $firstpos==1){ + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] =substr $Nagios->opts->critical, $firstpos, ($secondpos-$firstpos-1); + } + $firstpos=$secondpos; + $i++ + } + if (length($Nagios->opts->critical) - $firstpos==0){#La coma es el ultimo elemento del string + @{$CVRange}[$i] = "~:"; + } + else{ + @{$CVRange}[$i] = substr $Nagios->opts->critical, $firstpos, (length($Nagios->opts->critical)-$firstpos); + } + + if (@{$CVRange}[0] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Process Critical threshold in @{$CVRange}[0]"; + return 0; + } + if (@{$CVRange}[1] !~/^(@?(\d+|(\d+|~):(\d+)?))?$/) { + ${$Error} = "Invalid Application Critical threshold in @{$CVRange}[1]"; + return 0; + } + } + + return 1; +} + + +# Performs whole check: +# Input: Nagios Plugin object, reference to Plugin output string, Application, referece to WVRange, reference to CVRange +# Output: Plugin output string +# Return value: Plugin return value + +sub PerformCheck() { + my ($Nagios, $PluginOutput, $WVRange, $CVRange) = @_; + my $Application; + my @AppNameSplitted; + my $ApplicationName; + my $PsCommand; + my $PsResult; + my @PsResultLines; + my $ProcLine; + my $ProcPid; + my $LsofCommand; + my $LsofResult; + my $ProcCount = 0; + my $FDCount = 0; + my $ProcFDAvg = 0; + my $PerProcMaxFD = 0; + my $ProcOKFlag = 0; + my $ProcWarningFlag = 0; + my $ProcCriticalFlag = 0; + my $OKFlag = 0; + my $WarningFlag = 0; + my $CriticalFlag = 0; + my $LastWarningProcFDs = 0; + my $LastWarningProc = -1; + my $LastCriticalProcFDs = 0; + my $LastCriticalProc = -1; + my $ProcPluginReturnValue = UNKNOWN; + my $AppPluginReturnValue = UNKNOWN; + my $PluginReturnValue = UNKNOWN; + my $PerformanceData = ""; + my $PerfdataUnit = "FDs"; + + $Application = $Nagios->opts->application; + $PsCommand = "ps -eaf | grep $Application"; + $PsResult = `$PsCommand`; + @AppNameSplitted = split(/\//, $Application); + $ApplicationName = $AppNameSplitted[$#AppNameSplitted]; + @PsResultLines = split(/\n/, $PsResult); + if ( $#PsResultLines > 1 ) { + foreach my $Proc (split(/\n/, $PsResult)) { + if ($Proc !~ /check_unix_open_fds/ && $Proc !~ / grep /) { + $ProcCount += 1; + $ProcPid = (split(/\s+/, $Proc))[1]; + $LsofCommand = "lsof -p $ProcPid | wc -l"; + $LsofResult = `$LsofCommand`; + $LsofResult = ($LsofResult > 0 ) ? ($LsofResult - 1) : 0; + $FDCount += $LsofResult; + if ($LsofResult >= $PerProcMaxFD) { $PerProcMaxFD = $LsofResult; } + $ProcPluginReturnValue = $Nagios->check_threshold(check => $LsofResult,warning => @{$WVRange}[0],critical => @{$CVRange}[0]); + if ($ProcPluginReturnValue eq OK) { + $ProcOKFlag = 1; + } + elsif ($ProcPluginReturnValue eq WARNING) { + $ProcWarningFlag = 1; + if ($LsofResult >= $LastWarningProcFDs) { + $LastWarningProcFDs = $LsofResult; + $LastWarningProc = $ProcPid; + } + } + #if ($LsofResult >= $PCT) { + elsif ($ProcPluginReturnValue eq CRITICAL) { + $ProcCriticalFlag = 1; + if ($LsofResult >= $LastCriticalProcFDs) { + $LastCriticalProcFDs = $LsofResult; + $LastCriticalProc = $ProcPid; + } + } + } + } + if ($ProcCount) { $ProcFDAvg = int($FDCount / $ProcCount); } + $AppPluginReturnValue = $Nagios->check_threshold(check => $FDCount,warning => @{$WVRange}[1],critical => @{$CVRange}[1]); + #if ($FDCount >= $TWT) { + if ($AppPluginReturnValue eq OK) { $OKFlag = 1; } + elsif ($AppPluginReturnValue eq WARNING) { $WarningFlag = 1; } + elsif ($AppPluginReturnValue eq CRITICAL) { $CriticalFlag = 1; } + + # PluginReturnValue and PluginOutput + if ($CriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (critical threshold set to @{$CVRange}[1])"; + } + elsif ($WarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files (warning threshold set to @{$WVRange}[1])"; + } + elsif ($ProcCriticalFlag) { + $PluginReturnValue = CRITICAL; + ${$PluginOutput} .= "Process ID $LastCriticalProc handling $LastCriticalProcFDs files (critical threshold set to @{$CVRange}[0])"; + } + elsif ($ProcWarningFlag) { + $PluginReturnValue = WARNING; + ${$PluginOutput} .= "Process ID $LastWarningProc handling $LastWarningProcFDs files (warning threshold set to @{$WVRange}[0])"; + } + elsif ($OKFlag && $ProcOKFlag) { + $PluginReturnValue = OK; + ${$PluginOutput} .= "$ApplicationName handling $FDCount files"; + } + } + else { + ${$PluginOutput} .= "No existe la aplicacion $ApplicationName"; + } + + + $PerformanceData .= "ProcCount=$ProcCount$PerfdataUnit FDCount=$FDCount$PerfdataUnit ProcFDAvg=$ProcFDAvg$PerfdataUnit PerProcMaxFD=$PerProcMaxFD$PerfdataUnit"; + + # Output with performance data: + ${$PluginOutput} .= " | $PerformanceData"; + + return $PluginReturnValue; +} diff --git a/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 new file mode 100755 index 00000000..3dbca322 --- /dev/null +++ b/puppet/modules/site_check_mk/files/agent/plugins/mk_logwatch.1.2.4 @@ -0,0 +1,374 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2010 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# Call with -d for debug mode: colored output, no saving of status + +import sys, os, re, time +import glob + +if '-d' in sys.argv[1:] or '--debug' in sys.argv[1:]: + tty_red = '\033[1;31m' + tty_green = '\033[1;32m' + tty_yellow = '\033[1;33m' + tty_blue = '\033[1;34m' + tty_normal = '\033[0m' + debug = True +else: + tty_red = '' + tty_green = '' + tty_yellow = '' + tty_blue = '' + tty_normal = '' + debug = False + +# The configuration file and status file are searched +# in the directory named by the environment variable +# LOGWATCH_DIR. If that is not set, MK_CONFDIR is used. +# If that is not set either, the current directory ist +# used. +logwatch_dir = os.getenv("LOGWATCH_DIR") +if not logwatch_dir: + logwatch_dir = os.getenv("MK_CONFDIR") + if not logwatch_dir: + logwatch_dir = "." + +print "<<<logwatch>>>" + +config_filename = logwatch_dir + "/logwatch.cfg" +status_filename = logwatch_dir + "/logwatch.state" +config_dir = logwatch_dir + "/logwatch.d/*.cfg" + +def is_not_comment(line): + if line.lstrip().startswith('#') or \ + line.strip() == '': + return False + return True + +def parse_filenames(line): + return line.split() + +def parse_pattern(level, pattern): + if level not in [ 'C', 'W', 'I', 'O' ]: + raise(Exception("Invalid pattern line '%s'" % line)) + try: + compiled = re.compile(pattern) + except: + raise(Exception("Invalid regular expression in line '%s'" % line)) + return (level, compiled) + +def read_config(): + config_lines = [ line.rstrip() for line in filter(is_not_comment, file(config_filename).readlines()) ] + # Add config from a logwatch.d folder + for config_file in glob.glob(config_dir): + config_lines += [ line.rstrip() for line in filter(is_not_comment, file(config_file).readlines()) ] + + have_filenames = False + config = [] + + for line in config_lines: + rewrite = False + if line[0].isspace(): # pattern line + if not have_filenames: + raise Exception("Missing logfile names") + level, pattern = line.split(None, 1) + if level == 'A': + cont_list.append(parse_cont_pattern(pattern)) + elif level == 'R': + rewrite_list.append(pattern) + else: + level, compiled = parse_pattern(level, pattern) + cont_list = [] # List of continuation patterns + rewrite_list = [] # List of rewrite patterns + patterns.append((level, compiled, cont_list, rewrite_list)) + else: # filename line + patterns = [] + config.append((parse_filenames(line), patterns)) + have_filenames = True + return config + +def parse_cont_pattern(pattern): + try: + return int(pattern) + except: + try: + return re.compile(pattern) + except: + if debug: + raise + raise Exception("Invalid regular expression in line '%s'" % pattern) + +# structure of statusfile +# # LOGFILE OFFSET INODE +# /var/log/messages|7767698|32455445 +# /var/test/x12134.log|12345|32444355 +def read_status(): + if debug: + return {} + + status = {} + for line in file(status_filename): + # TODO: Remove variants with spaces. rsplit is + # not portable. split fails if logfilename contains + # spaces + inode = -1 + try: + parts = line.split('|') + filename = parts[0] + offset = parts[1] + if len(parts) >= 3: + inode = parts[2] + + except: + try: + filename, offset = line.rsplit(None, 1) + except: + filename, offset = line.split(None, 1) + status[filename] = int(offset), int(inode) + return status + +def save_status(status): + f = file(status_filename, "w") + for filename, (offset, inode) in status.items(): + f.write("%s|%d|%d\n" % (filename, offset, inode)) + +pushed_back_line = None +def next_line(f): + global pushed_back_line + if pushed_back_line != None: + line = pushed_back_line + pushed_back_line = None + return line + else: + try: + line = f.next() + return line + except: + return None + + +def process_logfile(logfile, patterns): + global pushed_back_line + + # Look at which file offset we have finished scanning + # the logfile last time. If we have never seen this file + # before, we set the offset to -1 + offset, prev_inode = status.get(logfile, (-1, -1)) + try: + fl = os.open(logfile, os.O_RDONLY) + inode = os.fstat(fl)[1] # 1 = st_ino + except: + if debug: + raise + print "[[[%s:cannotopen]]]" % logfile + return + + print "[[[%s]]]" % logfile + + # Seek to the current end in order to determine file size + current_end = os.lseek(fl, 0, 2) # os.SEEK_END not available in Python 2.4 + status[logfile] = current_end, inode + + # If we have never seen this file before, we just set the + # current pointer to the file end. We do not want to make + # a fuss about ancient log messages... + if offset == -1: + if not debug: + return + else: + offset = 0 + + + # If the inode of the logfile has changed it has appearently + # been started from new (logfile rotation). At least we must + # assume that. In some rare cases (restore of a backup, etc) + # we are wrong and resend old log messages + if prev_inode >= 0 and inode != prev_inode: + offset = 0 + + # Our previously stored offset is the current end -> + # no new lines in this file + if offset == current_end: + return # nothing new + + # If our offset is beyond the current end, the logfile has been + # truncated or wrapped while keeping the same inode. We assume + # that it contains all new data in that case and restart from + # offset 0. + if offset > current_end: + offset = 0 + + # now seek to offset where interesting data begins + os.lseek(fl, offset, 0) # os.SEEK_SET not available in Python 2.4 + f = os.fdopen(fl) + worst = -1 + outputtxt = "" + lines_parsed = 0 + start_time = time.time() + + while True: + line = next_line(f) + if line == None: + break # End of file + + lines_parsed += 1 + # Check if maximum number of new log messages is exceeded + if opt_maxlines != None and lines_parsed > opt_maxlines: + outputtxt += "%s Maximum number (%d) of new log messages exceeded.\n" % ( + opt_overflow, opt_maxlines) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + # Check if maximum processing time (per file) is exceeded. Check only + # every 100'th line in order to save system calls + if opt_maxtime != None and lines_parsed % 100 == 10 \ + and time.time() - start_time > opt_maxtime: + outputtxt += "%s Maximum parsing time (%.1f sec) of this log file exceeded.\n" % ( + opt_overflow, opt_maxtime) + worst = max(worst, opt_overflow_level) + os.lseek(fl, 0, 2) # Seek to end of file, skip all other messages + break + + level = "." + for lev, pattern, cont_patterns, replacements in patterns: + matches = pattern.search(line[:-1]) + if matches: + level = lev + levelint = {'C': 2, 'W': 1, 'O': 0, 'I': -1, '.': -1}[lev] + worst = max(levelint, worst) + + # Check for continuation lines + for cont_pattern in cont_patterns: + if type(cont_pattern) == int: # add that many lines + for x in range(cont_pattern): + cont_line = next_line(f) + if cont_line == None: # end of file + break + line = line[:-1] + "\1" + cont_line + + else: # pattern is regex + while True: + cont_line = next_line(f) + if cont_line == None: # end of file + break + elif cont_pattern.search(cont_line[:-1]): + line = line[:-1] + "\1" + cont_line + else: + pushed_back_line = cont_line # sorry for stealing this line + break + + # Replacement + for replace in replacements: + line = replace.replace('\\0', line) + "\n" + for nr, group in enumerate(matches.groups()): + line = line.replace('\\%d' % (nr+1), group) + + break # matching rule found and executed + + color = {'C': tty_red, 'W': tty_yellow, 'O': tty_green, 'I': tty_blue, '.': ''}[level] + if debug: + line = line.replace("\1", "\nCONT:") + if level == "I": + level = "." + if opt_nocontext and level == '.': + continue + outputtxt += "%s%s %s%s\n" % (color, level, line[:-1], tty_normal) + + new_offset = os.lseek(fl, 0, 1) # os.SEEK_CUR not available in Python 2.4 + status[logfile] = new_offset, inode + + # output all lines if at least one warning, error or ok has been found + if worst > -1: + sys.stdout.write(outputtxt) + sys.stdout.flush() + +try: + config = read_config() +except Exception, e: + if debug: + raise + print "CANNOT READ CONFIG FILE: %s" % e + sys.exit(1) + +# Simply ignore errors in the status file. In case of a corrupted status file we simply begin +# with an empty status. That keeps the monitoring up and running - even if we might loose a +# message in the extreme case of a corrupted status file. +try: + status = read_status() +except Exception, e: + status = {} + + +# The filename line may contain options like 'maxlines=100' or 'maxtime=10' +for filenames, patterns in config: + # Initialize options with default values + opt_maxlines = None + opt_maxtime = None + opt_regex = None + opt_overflow = 'C' + opt_overflow_level = 2 + opt_nocontext = False + try: + options = [ o.split('=', 1) for o in filenames if '=' in o ] + for key, value in options: + if key == 'maxlines': + opt_maxlines = int(value) + elif key == 'maxtime': + opt_maxtime = float(value) + elif key == 'overflow': + if value not in [ 'C', 'I', 'W', 'O' ]: + raise Exception("Invalid value %s for overflow. Allowed are C, I, O and W" % value) + opt_overflow = value + opt_overflow_level = {'C':2, 'W':1, 'O':0, 'I':0}[value] + elif key == 'regex': + opt_regex = re.compile(value) + elif key == 'iregex': + opt_regex = re.compile(value, re.I) + elif key == 'nocontext': + opt_nocontext = True + else: + raise Exception("Invalid option %s" % key) + except Exception, e: + if debug: + raise + print "INVALID CONFIGURATION: %s" % e + sys.exit(1) + + + for glob in filenames: + if '=' in glob: + continue + logfiles = [ l.strip() for l in os.popen("ls %s 2>/dev/null" % glob).readlines() ] + if opt_regex: + logfiles = [ f for f in logfiles if opt_regex.search(f) ] + if len(logfiles) == 0: + print '[[[%s:missing]]]' % glob + else: + for logfile in logfiles: + process_logfile(logfile, patterns) + +if not debug: + save_status(status) diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk new file mode 100644 index 00000000..c7120a96 --- /dev/null +++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk @@ -0,0 +1,14 @@ +# retry 3 times before setting a service into a hard state +# and send out notification +extra_service_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS , ALL_SERVICES ) +] + +# +# run check_mk_agent every 4 minutes if it terminates successfully. +# see https://leap.se/code/issues/6539 for the rationale +# +extra_service_conf["normal_check_interval"] = [ + ("4", ALL_HOSTS , "Check_MK" ) +] + diff --git a/puppet/modules/site_check_mk/files/ignored_services.mk b/puppet/modules/site_check_mk/files/ignored_services.mk new file mode 100644 index 00000000..35dc4433 --- /dev/null +++ b/puppet/modules/site_check_mk/files/ignored_services.mk @@ -0,0 +1,3 @@ +ignored_services = [ + ( ALL_HOSTS, [ "NTP Time" ] ) +] diff --git a/puppet/modules/site_check_mk/manifests/agent.pp b/puppet/modules/site_check_mk/manifests/agent.pp new file mode 100644 index 00000000..b95d5d64 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent.pp @@ -0,0 +1,35 @@ +# installs check-mk agent +class site_check_mk::agent { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + + + # /usr/bin/mk-job depends on /usr/bin/time + ensure_packages('time') + + class { 'site_apt::preferences::check_mk': } -> + + class { 'check_mk::agent': + agent_package_name => 'check-mk-agent', + agent_logwatch_package_name => 'check-mk-agent-logwatch', + method => 'ssh', + authdir => '/root/.ssh', + authfile => 'authorized_keys', + register_agent => false, + require => Package['time'] + } -> + + class { 'site_check_mk::agent::mrpe': } -> + class { 'site_check_mk::agent::logwatch': } -> + + file { + [ '/srv/leap/nagios', '/srv/leap/nagios/plugins' ]: + ensure => directory; + '/usr/lib/check_mk_agent/local/run_node_tests.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/all_hosts/run_node_tests.sh', + mode => '0755'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp new file mode 100644 index 00000000..1554fd3c --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp @@ -0,0 +1,34 @@ +# configure logwatch and nagios checks for couchdb (both bigcouch and plain +# couchdb installations) +class site_check_mk::agent::couchdb { + + concat::fragment { 'syslog_couchdb': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/couchdb.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # check different couchdb stats + file { '/usr/lib/check_mk_agent/local/leap_couch_stats.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/couchdb/leap_couch_stats.sh', + mode => '0755', + require => Package['check_mk-agent'] + } + + # check open files for bigcouch proc + include site_check_mk::agent::package::perl_plugin + file { '/srv/leap/nagios/plugins/check_unix_open_fds.pl': + source => 'puppet:///modules/site_check_mk/agent/nagios_plugins/check_unix_open_fds.pl', + mode => '0755' + } + augeas { + 'Couchdb_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Couchdb_open_files', + 'set Couchdb_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp new file mode 100644 index 00000000..82c3ac72 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/bigcouch.pp @@ -0,0 +1,49 @@ +# configure logwatch and nagios checks for bigcouch +class site_check_mk::agent::couchdb::bigcouch { + + # watch bigcouch logs + # currently disabled because bigcouch is too noisy + # see https://leap.se/code/issues/7375 for more details + # and site_config::remove_files for removing leftovers + #file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + # source => 'puppet:///modules/site_check_mk/agent/logwatch/bigcouch.cfg', + #} + + # check syslog msg from: + # - empd + # - /usr/local/bin/couch-doc-update + concat::fragment { 'syslog_bigcouch': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/bigcouch.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + + # check bigcouch processes + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + 'set Bigcouch_epmd_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/epmd\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + 'set Bigcouch_beam_procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /opt/bigcouch/erts-5.9.1/bin/beam\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + + augeas { + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + 'set Bigcouch_open_files \'/srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp new file mode 100644 index 00000000..3ec2267b --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/couchdb/plain.pp @@ -0,0 +1,23 @@ +# configure logwatch and nagios checks for plain single couchdb master +class site_check_mk::agent::couchdb::plain { + + # remove bigcouch leftovers + augeas { + 'Bigcouch_epmd_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_epmd_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_beam_procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_beam_procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Bigcouch_open_files': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Bigcouch_open_files', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/haproxy.pp b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp new file mode 100644 index 00000000..6d52efba --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp @@ -0,0 +1,15 @@ +class site_check_mk::agent::haproxy { + + include site_check_mk::agent::package::nagios_plugins_contrib + + # local nagios plugin checks via mrpe + augeas { 'haproxy': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Haproxy', + 'set Haproxy \'/usr/lib/nagios/plugins/check_haproxy -u "http://localhost:8000/haproxy;csv"\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/haveged.pp b/puppet/modules/site_check_mk/manifests/agent/haveged.pp new file mode 100644 index 00000000..cacbea8c --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/haveged.pp @@ -0,0 +1,15 @@ +class site_check_mk::agent::haveged { + +# check haveged process + augeas { + 'haveged_proc': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/haveged_proc', + 'set haveged_proc \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a /usr/sbin/haveged\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp new file mode 100644 index 00000000..423cace2 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch.pp @@ -0,0 +1,36 @@ +class site_check_mk::agent::logwatch { + # Deploy mk_logwatch 1.2.4 so we can split the config + # into multiple config files in /etc/check_mk/logwatch.d + # see https://leap.se/code/issues/5135 + + file { '/usr/lib/check_mk_agent/plugins/mk_logwatch': + source => 'puppet:///modules/site_check_mk/agent/plugins/mk_logwatch.1.2.4', + mode => '0755', + require => Package['check-mk-agent-logwatch'] + } + + # only config files that watch a distinct logfile should go in logwatch.d/ + file { '/etc/check_mk/logwatch.d': + ensure => directory, + recurse => true, + purge => true, + require => Package['check-mk-agent-logwatch'] + } + + # service that share a common logfile (i.e. /var/log/syslog) need to get + # concanated in one file, otherwise the last file sourced will override + # the config before + # see mk_logwatch: "logwatch.cfg overwrites config files in logwatch.d", + # https://leap.se/code/issues/5155 + + # first, we need to deploy a custom logwatch.cfg that doesn't include + # a section about /var/log/syslog + + file { '/etc/check_mk/logwatch.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/logwatch.cfg', + require => Package['check_mk-agent-logwatch'] + } + + include concat::setup + include site_check_mk::agent::logwatch::syslog +} diff --git a/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp new file mode 100644 index 00000000..c927780d --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/logwatch/syslog.pp @@ -0,0 +1,18 @@ +class site_check_mk::agent::logwatch::syslog { + + concat { '/etc/check_mk/logwatch.d/syslog.cfg': + warn => true + } + + concat::fragment { 'syslog_header': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog_header.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '01'; + } + concat::fragment { 'syslog_tail': + source => 'puppet:///modules/site_check_mk/agent/logwatch/syslog_tail.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '99'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mrpe.pp b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp new file mode 100644 index 00000000..5e1f087a --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/mrpe.pp @@ -0,0 +1,24 @@ +class site_check_mk::agent::mrpe { + # check_mk can use standard nagios plugins using + # a wrapper called mrpe + # see http://mathias-kettner.de/checkmk_mrpe.html + + package { 'nagios-plugins-basic': + ensure => latest, + } + + file { '/etc/check_mk/mrpe.cfg': + ensure => present, + require => Package['check-mk-agent'] + } -> + + augeas { + 'Apt': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/APT', + 'set APT \'/usr/lib/nagios/plugins/check_apt\'' ]; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/mx.pp b/puppet/modules/site_check_mk/manifests/agent/mx.pp new file mode 100644 index 00000000..20cbcade --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/mx.pp @@ -0,0 +1,27 @@ +# check check_mk agent checks for mx service +class site_check_mk::agent::mx { + + # watch logs + file { '/etc/check_mk/logwatch.d/leap_mx.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/leap_mx.cfg', + } + + # local nagios plugin checks via mrpe + # removed because leap_cli integrates a check for running mx procs already, + # which is also integrated into nagios (called "Mx/Are_MX_daemons_running") + augeas { + 'Leap_MX_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Leap_MX_Procs', + require => File['/etc/check_mk/mrpe.cfg']; + } + + # check stale files in queue dir + file { '/usr/lib/check_mk_agent/local/check_leap_mx.sh': + source => 'puppet:///modules/site_check_mk/agent/local_checks/mx/check_leap_mx.sh', + mode => '0755', + require => Package['check_mk-agent'] + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/openvpn.pp b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp new file mode 100644 index 00000000..0596a497 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/openvpn.pp @@ -0,0 +1,10 @@ +class site_check_mk::agent::openvpn { + + # check syslog + concat::fragment { 'syslog_openpvn': + source => 'puppet:///modules/site_check_mk/agent/logwatch/openvpn.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp new file mode 100644 index 00000000..95a60d17 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/nagios_plugins_contrib.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::nagios_plugins_contrib { + package { 'nagios-plugins-contrib': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp new file mode 100644 index 00000000..4feda375 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/package/perl_plugin.pp @@ -0,0 +1,5 @@ +class site_check_mk::agent::package::perl_plugin { + package { 'libnagios-plugin-perl': + ensure => installed, + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/soledad.pp b/puppet/modules/site_check_mk/manifests/agent/soledad.pp new file mode 100644 index 00000000..f4a3f3a6 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/soledad.pp @@ -0,0 +1,17 @@ +class site_check_mk::agent::soledad { + + file { '/etc/check_mk/logwatch.d/soledad.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/soledad.cfg', + } + + # local nagios plugin checks via mrpe + + augeas { 'Soledad_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ + 'rm /files/etc/check_mk/mrpe.cfg/Soledad_Procs', + 'set Soledad_Procs \'/usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a "/usr/bin/python /usr/bin/twistd --uid=soledad --gid=soledad --pidfile=/var/run/soledad.pid --logfile=/var/log/soledad.log web --wsgi=leap.soledad.server.application --port=ssl:2323:privateKey=/etc/x509/keys/leap.key:certKey=/etc/x509/certs/leap.crt:sslmethod=SSLv23_METHOD"\'' ], + require => File['/etc/check_mk/mrpe.cfg']; + } +} diff --git a/puppet/modules/site_check_mk/manifests/agent/stunnel.pp b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp new file mode 100644 index 00000000..7f765771 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/stunnel.pp @@ -0,0 +1,9 @@ +class site_check_mk::agent::stunnel { + + concat::fragment { 'syslog_stunnel': + source => 'puppet:///modules/site_check_mk/agent/logwatch/stunnel.cfg', + target => '/etc/check_mk/logwatch.d/syslog.cfg', + order => '02'; + } + +} diff --git a/puppet/modules/site_check_mk/manifests/agent/webapp.pp b/puppet/modules/site_check_mk/manifests/agent/webapp.pp new file mode 100644 index 00000000..9bf3b197 --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/agent/webapp.pp @@ -0,0 +1,15 @@ +class site_check_mk::agent::webapp { + + # remove leftovers of webapp python checks + file { + [ '/usr/lib/check_mk_agent/local/nagios-webapp_login.py', + '/usr/lib/check_mk_agent/local/soledad_sync.py' ]: + ensure => absent + } + + # watch logs + file { '/etc/check_mk/logwatch.d/webapp.cfg': + source => 'puppet:///modules/site_check_mk/agent/logwatch/webapp.cfg', + } + +} diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp new file mode 100644 index 00000000..7ff9eb4a --- /dev/null +++ b/puppet/modules/site_check_mk/manifests/server.pp @@ -0,0 +1,103 @@ +# setup check_mk on the monitoring server +class site_check_mk::server { + + $ssh_hash = hiera('ssh') + $pubkey = $ssh_hash['authorized_keys']['monitor']['key'] + $type = $ssh_hash['authorized_keys']['monitor']['type'] + $seckey = $ssh_hash['monitor']['private_key'] + + $nagios_hiera = hiera_hash('nagios') + $hosts = $nagios_hiera['hosts'] + + $all_hosts = inline_template ('<% @hosts.keys.sort.each do |key| -%><% if @hosts[key]["environment"] != "disabled" %>"<%= @hosts[key]["domain_internal"] %>", <% end -%><% end -%>') + $domains_internal = $nagios_hiera['domains_internal'] + $environments = $nagios_hiera['environments'] + + package { 'check-mk-server': + ensure => installed, + } + + # we don't use check-mk-multisite, and the jessie version + # of this config file breaks with apache 2.4 + # until https://gitlab.com/shared-puppet-modules-group/apache/issues/11 + # is not fixed, we need to use a generic file type here + #apache::config::global { 'check-mk-multisite.conf': + # ensure => absent + #} + + file { '/etc/apache2/conf-enabled/check-mk-multisite.conf': + ensure => absent, + require => Package['check-mk-server']; + } + + # override paths to use the system check_mk rather than OMD + class { 'check_mk::config': + site => '', + etc_dir => '/etc', + nagios_subdir => 'nagios3', + bin_dir => '/usr/bin', + host_groups => undef, + use_storedconfigs => false, + inventory_only_on_changes => false, + require => Package['check-mk-server'] + } + + Exec['check_mk-refresh'] -> + Exec['check_mk-refresh-inventory-daily'] -> + Exec['check_mk-reload'] -> + Service['nagios'] + + file { + '/etc/check_mk/conf.d/use_ssh.mk': + content => template('site_check_mk/use_ssh.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/hostgroups.mk': + content => template('site_check_mk/hostgroups.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/host_contactgroups.mk': + content => template('site_check_mk/host_contactgroups.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/ignored_services.mk': + source => 'puppet:///modules/site_check_mk/ignored_services.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/extra_service_conf.mk': + source => 'puppet:///modules/site_check_mk/extra_service_conf.mk', + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + '/etc/check_mk/conf.d/extra_host_conf.mk': + content => template('site_check_mk/extra_host_conf.mk'), + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + + '/etc/check_mk/all_hosts_static': + content => $all_hosts, + notify => Exec['check_mk-refresh'], + require => Package['check-mk-server']; + + '/etc/check_mk/.ssh': + ensure => directory, + require => Package['check-mk-server']; + '/etc/check_mk/.ssh/id_rsa': + content => $seckey, + owner => 'nagios', + mode => '0600', + require => Package['check-mk-server']; + '/etc/check_mk/.ssh/id_rsa.pub': + content => "${type} ${pubkey} monitor", + owner => 'nagios', + mode => '0644', + require => Package['check-mk-server']; + + # check_icmp must be suid root or called by sudo + # see https://leap.se/code/issues/5171 + '/usr/lib/nagios/plugins/check_icmp': + mode => '4755', + require => Package['nagios-plugins-basic']; + } + + include check_mk::agent::local_checks +} diff --git a/puppet/modules/site_check_mk/templates/extra_host_conf.mk b/puppet/modules/site_check_mk/templates/extra_host_conf.mk new file mode 100644 index 00000000..bc27b514 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/extra_host_conf.mk @@ -0,0 +1,13 @@ +# retry 3 times before setting a host into a hard state +# and send out notification +extra_host_conf["max_check_attempts"] = [ + ("4", ALL_HOSTS ) +] + +# Use hostnames as alias so notification mail subjects +# are more readable and not so long. Alias defaults to +# the fqdn of a host is not changed. +extra_host_conf["alias"] = [ +<% @hosts.keys.sort.each do |key| -%> ( "<%= key.strip %>", ["<%= @hosts[key]['domain_internal']%>"]), +<% end -%> +] diff --git a/puppet/modules/site_check_mk/templates/host_contactgroups.mk b/puppet/modules/site_check_mk/templates/host_contactgroups.mk new file mode 100644 index 00000000..6a534967 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/host_contactgroups.mk @@ -0,0 +1,17 @@ +<% + contact_groups = [] + @environments.keys.sort.each do |env_name| + hosts = "" + @nagios_hosts.keys.sort.each do |hostname| + hostdata = @nagios_hosts[hostname] + domain_internal = hostdata['domain_internal'] + if hostdata['environment'] == env_name + hosts << '"' + domain_internal + '", ' + end + end + contact_groups << ' ( "%s", [%s] )' % [env_name, hosts] + end +%> +host_contactgroups = [ +<%= contact_groups.join(",\n") %> +] diff --git a/puppet/modules/site_check_mk/templates/hostgroups.mk b/puppet/modules/site_check_mk/templates/hostgroups.mk new file mode 100644 index 00000000..7158dcd1 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/hostgroups.mk @@ -0,0 +1,17 @@ +<% + host_groups = [] + @environments.keys.sort.each do |env_name| + hosts = "" + @nagios_hosts.keys.sort.each do |hostname| + hostdata = @nagios_hosts[hostname] + domain_internal = hostdata['domain_internal'] + if hostdata['environment'] == env_name + hosts << '"' + domain_internal + '", ' + end + end + host_groups << ' ( "%s", [%s] )' % [env_name, hosts] + end +%> +host_groups = [ +<%= host_groups.join(",\n") %> +] diff --git a/puppet/modules/site_check_mk/templates/use_ssh.mk b/puppet/modules/site_check_mk/templates/use_ssh.mk new file mode 100644 index 00000000..55269536 --- /dev/null +++ b/puppet/modules/site_check_mk/templates/use_ssh.mk @@ -0,0 +1,6 @@ +# http://mathias-kettner.de/checkmk_datasource_programs.html +datasource_programs = [ +<% @nagios_hosts.sort.each do |name,config| %> + ( "ssh -l root -i /etc/check_mk/.ssh/id_rsa -p <%=config['ssh_port']%> <%=config['domain_internal']%> check_mk_agent", [ "<%=config['domain_internal']%>" ], ),<%- end -%> + +] diff --git a/puppet/modules/site_config/files/xterm-title.sh b/puppet/modules/site_config/files/xterm-title.sh new file mode 100644 index 00000000..3cff0e3a --- /dev/null +++ b/puppet/modules/site_config/files/xterm-title.sh @@ -0,0 +1,8 @@ +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*) + PROMPT_COMMAND='echo -ne "\033]0;${USER}@${HOSTNAME}: ${PWD}\007"' + ;; +*) + ;; +esac diff --git a/puppet/modules/site_config/lib/facter/dhcp_enabled.rb b/puppet/modules/site_config/lib/facter/dhcp_enabled.rb new file mode 100644 index 00000000..33220da3 --- /dev/null +++ b/puppet/modules/site_config/lib/facter/dhcp_enabled.rb @@ -0,0 +1,22 @@ +require 'facter' +def dhcp_enabled?(ifs, recurse=true) + dhcp = false + included_ifs = [] + if FileTest.exists?(ifs) + File.open(ifs) do |file| + dhcp = file.enum_for(:each_line).any? do |line| + if recurse && line =~ /^\s*source\s+([^\s]+)/ + included_ifs += Dir.glob($1) + end + line =~ /inet\s+dhcp/ + end + end + end + dhcp || included_ifs.any? { |ifs| dhcp_enabled?(ifs, false) } +end +Facter.add(:dhcp_enabled) do + confine :osfamily => 'Debian' + setcode do + dhcp_enabled?('/etc/network/interfaces') + end +end diff --git a/puppet/modules/site_config/lib/facter/ip_interface.rb b/puppet/modules/site_config/lib/facter/ip_interface.rb new file mode 100644 index 00000000..45764bfc --- /dev/null +++ b/puppet/modules/site_config/lib/facter/ip_interface.rb @@ -0,0 +1,13 @@ +require 'facter/util/ip' + +Facter::Util::IP.get_interfaces.each do |interface| + ip = Facter.value("ipaddress_#{interface}") + if ip != nil + Facter.add("interface_" + ip ) do + setcode do + interface + end + end + end +end + diff --git a/puppet/modules/site_config/manifests/caching_resolver.pp b/puppet/modules/site_config/manifests/caching_resolver.pp new file mode 100644 index 00000000..8bf465c1 --- /dev/null +++ b/puppet/modules/site_config/manifests/caching_resolver.pp @@ -0,0 +1,27 @@ +# deploy local caching resolver +class site_config::caching_resolver { + tag 'leap_base' + + class { 'unbound': + root_hints => false, + anchor => false, + ssl => false, + settings => { + server => { + verbosity => '1', + interface => [ '127.0.0.1', '::1' ], + port => '53', + hide-identity => 'yes', + hide-version => 'yes', + harden-glue => 'yes', + access-control => [ '127.0.0.0/8 allow', '::1 allow' ] + } + } + } + + concat::fragment { 'unbound glob include': + target => $unbound::params::config, + content => "include: /etc/unbound/unbound.conf.d/*.conf\n\n", + order => 10 + } +} diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp new file mode 100644 index 00000000..256de1a1 --- /dev/null +++ b/puppet/modules/site_config/manifests/default.pp @@ -0,0 +1,71 @@ +# common things to set up on every node +class site_config::default { + tag 'leap_base' + + $services = hiera('services', []) + $domain_hash = hiera('domain') + include site_config::params + include site_config::setup + + # default class, used by all hosts + + include lsb, git + + # configure sysctl parameters + include site_config::sysctl + + # configure ssh and include ssh-keys + include site_sshd + + # include classes for special environments + # i.e. openstack/aws nodes, vagrant nodes + + # fix dhclient from changing resolver information + # facter returns 'true' as string + # lint:ignore:quoted_booleans + if $::dhcp_enabled == 'true' { + # lint:endignore + include site_config::dhclient + } + + # configure /etc/resolv.conf + include site_config::resolvconf + + # configure caching, local resolver + include site_config::caching_resolver + + # install/configure syslog and core log rotations + include site_config::syslog + + # provide a basic level of quality entropy + include haveged + + # install/remove base packages + include site_config::packages + + # include basic shorewall config + include site_shorewall::defaults + + Package['git'] -> Vcsrepo<||> + + # include basic shell config + include site_config::shell + + # set up core leap files and directories + include site_config::files + + # remove leftovers from previous deploys + include site_config::remove + + if ! member($services, 'mx') { + include site_postfix::satellite + } + + # if class custom exists, include it. + # possibility for users to define custom puppet recipes + if defined( '::custom') { + include ::custom + } + + include site_check_mk::agent +} diff --git a/puppet/modules/site_config/manifests/dhclient.pp b/puppet/modules/site_config/manifests/dhclient.pp new file mode 100644 index 00000000..a1f87d41 --- /dev/null +++ b/puppet/modules/site_config/manifests/dhclient.pp @@ -0,0 +1,40 @@ +# Unfortunately, there does not seem to be a way to reload the dhclient.conf +# config file, or a convenient way to disable the modifications to +# /etc/resolv.conf. So the following makes the functions involved noops and +# ships a script to kill and restart dhclient. See the debian bugs: +# #681698, #712796 +class site_config::dhclient { + + + include site_config::params + + file { '/usr/local/sbin/reload_dhclient': + owner => 0, + group => 0, + mode => '0755', + content => template('site_config/reload_dhclient.erb'); + } + + exec { 'reload_dhclient': + refreshonly => true, + command => '/usr/local/sbin/reload_dhclient', + before => Class['site_config::resolvconf'], + require => File['/usr/local/sbin/reload_dhclient'], + } + + file { '/etc/dhcp/dhclient-enter-hooks.d': + ensure => directory, + mode => '0755', + owner => 'root', + group => 'root', + } + + file { '/etc/dhcp/dhclient-enter-hooks.d/disable_resolvconf': + content => 'make_resolv_conf() { : ; } ; set_hostname() { : ; }', + mode => '0644', + owner => 'root', + group => 'root', + require => File['/etc/dhcp/dhclient-enter-hooks.d'], + notify => Exec['reload_dhclient']; + } +} diff --git a/puppet/modules/site_config/manifests/files.pp b/puppet/modules/site_config/manifests/files.pp new file mode 100644 index 00000000..d2ef8a98 --- /dev/null +++ b/puppet/modules/site_config/manifests/files.pp @@ -0,0 +1,24 @@ +# set up core leap files and directories +class site_config::files { + + file { + '/srv/leap': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0711'; + + [ '/etc/leap', '/var/lib/leap']: + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + + '/var/log/leap': + ensure => directory, + owner => 'root', + group => 'adm', + mode => '0750'; + } + +} diff --git a/puppet/modules/site_config/manifests/hosts.pp b/puppet/modules/site_config/manifests/hosts.pp new file mode 100644 index 00000000..878b6af0 --- /dev/null +++ b/puppet/modules/site_config/manifests/hosts.pp @@ -0,0 +1,44 @@ +class site_config::hosts() { + $hosts = hiera('hosts', false) + + # calculate all the hostname aliases that might be used + $hostname = hiera('name') + $domain_hash = hiera('domain', {}) + $dns = hiera('dns', {}) + if $dns['aliases'] == undef { + $dns_aliases = [] + } else { + $dns_aliases = $dns['aliases'] + } + $my_hostnames = unique(concat( + [$domain_hash['full'], $hostname, $domain_hash['internal']], $dns_aliases + )) + + file { '/etc/hostname': + ensure => present, + content => $hostname + } + + exec { "/bin/hostname ${hostname}": + subscribe => [ File['/etc/hostname'], File['/etc/hosts'] ], + refreshonly => true; + } + + # we depend on reliable hostnames from /etc/hosts for the stunnel services + # so restart stunnel service when /etc/hosts is modified + # because this is done in an early stage, the stunnel module may not + # have been deployed and will not be available for overriding, so + # this is handled in an unorthodox manner + exec { '/etc/init.d/stunnel4 restart': + subscribe => File['/etc/hosts'], + refreshonly => true, + onlyif => 'test -f /etc/init.d/stunnel4'; + } + + file { '/etc/hosts': + content => template('site_config/hosts'), + mode => '0644', + owner => root, + group => root; + } +} diff --git a/puppet/modules/site_config/manifests/initial_firewall.pp b/puppet/modules/site_config/manifests/initial_firewall.pp new file mode 100644 index 00000000..93cfb847 --- /dev/null +++ b/puppet/modules/site_config/manifests/initial_firewall.pp @@ -0,0 +1,64 @@ +class site_config::initial_firewall { + + # This class is intended to setup an initial firewall, before shorewall is + # configured. The purpose of this is for the rare case where shorewall fails + # to start, we should not expose services to the public. + + $ssh_config = hiera('ssh') + $ssh_port = $ssh_config['port'] + + package { 'iptables': + ensure => present + } + + file { + # This firewall enables ssh access, dns lookups and web lookups (for + # package installation) but otherwise restricts all outgoing and incoming + # ports + '/etc/network/ipv4firewall_up.rules': + content => template('site_config/ipv4firewall_up.rules.erb'), + owner => root, + group => 0, + mode => '0644'; + + # This firewall denys all ipv6 traffic - we will need to change this + # when we begin to support ipv6 + '/etc/network/ipv6firewall_up.rules': + content => template('site_config/ipv6firewall_up.rules.erb'), + owner => root, + group => 0, + mode => '0644'; + + # Run the iptables-restore in if-pre-up so that the network is locked down + # until the correct interfaces and ips are connected + '/etc/network/if-pre-up.d/ipv4tables': + content => "#!/bin/sh\n/sbin/iptables-restore < /etc/network/ipv4firewall_up.rules\n", + owner => root, + group => 0, + mode => '0744'; + + # Same as above for IPv6 + '/etc/network/if-pre-up.d/ipv6tables': + content => "#!/bin/sh\n/sbin/ip6tables-restore < /etc/network/ipv6firewall_up.rules\n", + owner => root, + group => 0, + mode => '0744'; + } + + # Immediately setup these firewall rules, but only if shorewall is not running + exec { + 'default_ipv4_firewall': + command => '/sbin/iptables-restore < /etc/network/ipv4firewall_up.rules', + logoutput => true, + unless => 'test -x /etc/init.d/shorewall && /etc/init.d/shorewall status', + subscribe => File['/etc/network/ipv4firewall_up.rules'], + require => File['/etc/network/ipv4firewall_up.rules']; + + 'default_ipv6_firewall': + command => '/sbin/ip6tables-restore < /etc/network/ipv6firewall_up.rules', + logoutput => true, + unless => 'test -x /etc/init.d/shorewall6 && /etc/init.d/shorewall6 status', + subscribe => File['/etc/network/ipv6firewall_up.rules'], + require => File['/etc/network/ipv6firewall_up.rules']; + } +} diff --git a/puppet/modules/site_config/manifests/packages.pp b/puppet/modules/site_config/manifests/packages.pp new file mode 100644 index 00000000..140189a4 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages.pp @@ -0,0 +1,32 @@ +# install default packages and remove unwanted packages +class site_config::packages { + + + # base set of packages that we want to have installed everywhere + package { [ 'etckeeper', 'screen', 'less', 'ntp' ]: + ensure => installed, + } + + # base set of packages that we want to remove everywhere + package { [ + 'acpi', 'build-essential', + 'cpp', 'cpp-4.6', 'cpp-4.7', 'cpp-4.8', 'cpp-4.9', + 'eject', 'ftp', + 'g++', 'g++-4.6', 'g++-4.7', 'g++-4.8', 'g++-4.9', + 'gcc', 'gcc-4.6', 'gcc-4.7', 'gcc-4.8', 'gcc-4.9', + 'laptop-detect', 'libc6-dev', 'libssl-dev', 'lpr', 'make', + 'pppconfig', 'pppoe', 'pump', 'qstat', + 'samba-common', 'samba-common-bin', 'smbclient', + 'tcl8.5', 'tk8.5', 'os-prober', 'unzip', 'xauth', 'x11-common', + 'x11-utils', 'xterm' ]: + ensure => purged; + } + + # leave a few packages installed on local environments + # vagrant i.e. needs them for mounting shared folders + if $::site_config::params::environment != 'local' { + package { [ 'nfs-common', 'nfs-kernel-server', 'rpcbind', 'portmap' ]: + ensure => purged; + } + } +} diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp new file mode 100644 index 00000000..2b3e13b9 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -0,0 +1,28 @@ +# +# include this whenever you want to ensure build-essential package and related compilers are installed. +# +class site_config::packages::build_essential inherits ::site_config::packages { + + # NICKSERVER CODE NOTE: in order to support TLS, libssl-dev must be installed + # before EventMachine gem is built/installed. + Package[ 'gcc', 'make', 'g++', 'cpp', 'libssl-dev', 'libc6-dev' ] { + ensure => present + } + + case $::operatingsystemrelease { + /^8.*/: { + Package[ 'gcc-4.9','g++-4.9', 'cpp-4.9' ] { + ensure => present + } + } + + /^7.*/: { + Package[ 'gcc-4.7','g++-4.7', 'cpp-4.7' ] { + ensure => present + } + } + + default: { } + } + +} diff --git a/puppet/modules/site_config/manifests/packages/gnutls.pp b/puppet/modules/site_config/manifests/packages/gnutls.pp new file mode 100644 index 00000000..b1f17480 --- /dev/null +++ b/puppet/modules/site_config/manifests/packages/gnutls.pp @@ -0,0 +1,5 @@ +class site_config::packages::gnutls { + + package { 'gnutls-bin': ensure => installed } + +} diff --git a/puppet/modules/site_config/manifests/params.pp b/puppet/modules/site_config/manifests/params.pp new file mode 100644 index 00000000..012b3ce0 --- /dev/null +++ b/puppet/modules/site_config/manifests/params.pp @@ -0,0 +1,35 @@ +class site_config::params { + + $ip_address = hiera('ip_address') + $ip_address_interface = getvar("interface_${ip_address}") + $ec2_local_ipv4_interface = getvar("interface_${::ec2_local_ipv4}") + $environment = hiera('environment', undef) + + + if $environment == 'local' { + $interface = 'eth1' + include site_config::packages::build_essential + } + elsif hiera('interface','') != '' { + $interface = hiera('interface') + } + elsif $ip_address_interface != '' { + $interface = $ip_address_interface + } + elsif $ec2_local_ipv4_interface != '' { + $interface = $ec2_local_ipv4_interface + } + elsif $::interfaces =~ /eth0/ { + $interface = 'eth0' + } + else { + fail("unable to determine a valid interface, please set a valid interface for this node in nodes/${::hostname}.json") + } + + $ca_name = 'leap_ca' + $client_ca_name = 'leap_client_ca' + $ca_bundle_name = 'leap_ca_bundle' + $cert_name = 'leap' + $commercial_ca_name = 'leap_commercial_ca' + $commercial_cert_name = 'leap_commercial' +} diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp new file mode 100644 index 00000000..443df9c2 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove.pp @@ -0,0 +1,11 @@ +# remove leftovers from previous deploys +class site_config::remove { + include site_config::remove::files + + case $::operatingsystemrelease { + /^8.*/: { + include site_config::remove::jessie + } + default: { } + } +} diff --git a/puppet/modules/site_config/manifests/remove/bigcouch.pp b/puppet/modules/site_config/manifests/remove/bigcouch.pp new file mode 100644 index 00000000..3535c3c1 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/bigcouch.pp @@ -0,0 +1,42 @@ +# remove bigcouch leftovers from previous installations +class site_config::remove::bigcouch { + + # Don't use check_mk logwatch to watch bigcouch logs anymore + # see https://leap.se/code/issues/7375 for more details + file { '/etc/check_mk/logwatch.d/bigcouch.cfg': + ensure => absent, + notify => [ + Exec['remove_bigcouch_logwatch_stateline'] + ] + } + + exec { 'remove_bigcouch_logwatch_stateline': + command => "sed -i '/bigcouch.log/d' /etc/check_mk/logwatch.state", + refreshonly => true, + } + + cron { 'compact_all_shards': + ensure => absent + } + + + exec { 'kill_bigcouch_stunnel_procs': + refreshonly => true, + command => '/usr/bin/pkill -f "/usr/bin/stunnel4 /etc/stunnel/(ednp|epmd)_server.conf"' + } + + # 'tidy' doesn't notify other resources, so we need to use file here instead + # see https://tickets.puppetlabs.com/browse/PUP-6021 + file { + [ '/etc/stunnel/ednp_server.conf', '/etc/stunnel/epmd_server.conf']: + ensure => absent, + # notifying Service[stunnel] doesn't work here because the config + # files contain the pid of the procs to stop/start. + # If we remove the config, and restart stunnel then it will only + # stop/start the procs for which config files are found and the stale + # service will continue to run. + # So we simply kill them. + notify => Exec['kill_bigcouch_stunnel_procs'] + } + +} diff --git a/puppet/modules/site_config/manifests/remove/files.pp b/puppet/modules/site_config/manifests/remove/files.pp new file mode 100644 index 00000000..41d6462e --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/files.pp @@ -0,0 +1,56 @@ +# +# Sometimes when we upgrade the platform, we need to ensure that files that +# the platform previously created will get removed. +# +# These file removals don't need to be kept forever: we only need to remove +# files that are present in the prior platform release. +# +# We can assume that the every node is upgraded from the previous platform +# release. +# + +class site_config::remove::files { + + # Platform 0.8 removals + tidy { + '/etc/default/leap_mx':; + '/etc/logrotate.d/mx':; + '/etc/rsyslog.d/50-mx.conf':; + '/etc/apt/preferences.d/openvpn':; + '/etc/apt/sources.list.d/secondary.list.disabled.list':; + } + + # + # Platform 0.7 removals + # + + tidy { + '/etc/rsyslog.d/99-tapicero.conf':; + '/etc/rsyslog.d/01-webapp.conf':; + '/etc/rsyslog.d/50-stunnel.conf':; + '/etc/logrotate.d/stunnel':; + '/var/log/stunnel4/stunnel.log':; + 'leap_mx': + path => '/var/log/', + recurse => true, + matches => ['leap_mx*', 'mx.log.[1-5]', 'mx.log.[6-9](.gz)?', + 'mx.log.[0-9][0-9](.gz)?']; + '/srv/leap/webapp/public/provider.json':; + '/srv/leap/couchdb/designs/tmp_users': + recurse => true, + rmdirs => true; + '/etc/leap/soledad-server.conf':; + '/var/log/leap/openvpn.log':; + '/etc/rsyslog.d/50-openvpn.conf':; + } + + # leax-mx logged to /var/log/leap_mx.log in the past + # we need to use a dumb exec here because file_line doesn't + # allow removing lines that match a regex in the current version + # of stdlib, see https://tickets.puppetlabs.com/browse/MODULES-1903 + exec { 'rm_old_leap_mx_log_destination': + command => "/bin/sed -i '/leap_mx.log/d' /etc/check_mk/logwatch.state", + onlyif => "/bin/grep -qe 'leap_mx.log' /etc/check_mk/logwatch.state" + } + +} diff --git a/puppet/modules/site_config/manifests/remove/jessie.pp b/puppet/modules/site_config/manifests/remove/jessie.pp new file mode 100644 index 00000000..e9497baf --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/jessie.pp @@ -0,0 +1,14 @@ +# remove possible leftovers after upgrading from wheezy to jessie +class site_config::remove::jessie { + + tidy { + '/etc/apt/preferences.d/rsyslog_anon_depends': + notify => Exec['apt_updated']; + } + + apt::preferences_snippet { + [ 'facter', 'obfsproxy', 'python-twisted', 'unbound' ]: + ensure => absent; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/monitoring.pp b/puppet/modules/site_config/manifests/remove/monitoring.pp new file mode 100644 index 00000000..18e2949b --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/monitoring.pp @@ -0,0 +1,13 @@ +# remove leftovers on monitoring nodes +class site_config::remove::monitoring { + + # Remove check_mk loggwatch spoolfiles for + # tapicero and bigcouch + tidy { + 'remove_logwatch_spoolfiles': + path => '/var/lib/check_mk/logwatch', + recurse => true, + matches => [ '*tapicero.log', '*bigcouch.log']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/tapicero.pp b/puppet/modules/site_config/manifests/remove/tapicero.pp new file mode 100644 index 00000000..07c3c6c6 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/tapicero.pp @@ -0,0 +1,72 @@ +# remove tapicero leftovers from previous deploys on couchdb nodes +class site_config::remove::tapicero { + + ensure_packages('curl') + + # remove tapicero couchdb user + $couchdb_config = hiera('couch') + $couchdb_mode = $couchdb_config['mode'] + + if $couchdb_mode == 'multimaster' + { + $port = 5986 + } else { + $port = 5984 + } + + exec { 'remove_couchdb_user': + onlyif => "/usr/bin/curl -s 127.0.0.1:${port}/_users/org.couchdb.user:tapicero | grep -qv 'not_found'", + command => "/usr/local/bin/couch-doc-update --host 127.0.0.1:${port} --db _users --id org.couchdb.user:tapicero --delete", + require => Package['curl'] + } + + + exec { 'kill_tapicero': + onlyif => '/usr/bin/test -s /var/run/tapicero.pid', + command => '/usr/bin/pkill --pidfile /var/run/tapicero.pid' + } + + user { 'tapicero': + ensure => absent; + } + + group { 'tapicero': + ensure => absent, + require => User['tapicero']; + } + + tidy { + '/srv/leap/tapicero': + recurse => true, + require => [ Exec['kill_tapicero'] ]; + '/var/lib/leap/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/var/run/tapicero': + require => [ Exec['kill_tapicero'] ]; + '/etc/leap/tapicero.yaml': + require => [ Exec['kill_tapicero'] ]; + '/etc/init.d/tapicero': + require => [ Exec['kill_tapicero'] ]; + 'tapicero_logs': + path => '/var/log/leap', + recurse => true, + matches => 'tapicero*', + require => [ Exec['kill_tapicero'] ]; + '/etc/check_mk/logwatch.d/tapicero.cfg':; + } + + # remove local nagios plugin checks via mrpe + augeas { + 'Tapicero_Procs': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm /files/etc/check_mk/mrpe.cfg/Tapicero_Procs', + require => File['/etc/check_mk/mrpe.cfg']; + 'Tapicero_Heartbeat': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => 'rm Tapicero_Heartbeat', + require => File['/etc/check_mk/mrpe.cfg']; + } + +} diff --git a/puppet/modules/site_config/manifests/remove/webapp.pp b/puppet/modules/site_config/manifests/remove/webapp.pp new file mode 100644 index 00000000..58f59815 --- /dev/null +++ b/puppet/modules/site_config/manifests/remove/webapp.pp @@ -0,0 +1,7 @@ +# remove leftovers on webapp nodes +class site_config::remove::webapp { + tidy { + '/etc/apache/sites-enabled/leap_webapp.conf': + notify => Service['apache']; + } +} diff --git a/puppet/modules/site_config/manifests/resolvconf.pp b/puppet/modules/site_config/manifests/resolvconf.pp new file mode 100644 index 00000000..09f0b405 --- /dev/null +++ b/puppet/modules/site_config/manifests/resolvconf.pp @@ -0,0 +1,14 @@ +class site_config::resolvconf { + + $domain_public = $site_config::default::domain_hash['full_suffix'] + + class { '::resolvconf': + domain => $domain_public, + search => $domain_public, + nameservers => [ + '127.0.0.1 # local caching-only, unbound', + '85.214.20.141 # Digitalcourage, a german privacy organisation: (https://en.wikipedia.org/wiki/Digitalcourage)', + '172.81.176.146 # OpenNIC (https://servers.opennicproject.org/edit.php?srv=ns1.tor.ca.dns.opennic.glue)' + ] + } +} diff --git a/puppet/modules/site_config/manifests/ruby.pp b/puppet/modules/site_config/manifests/ruby.pp new file mode 100644 index 00000000..5c13233d --- /dev/null +++ b/puppet/modules/site_config/manifests/ruby.pp @@ -0,0 +1,8 @@ +# install ruby, rubygems and bundler +# configure ruby settings common to all servers +class site_config::ruby { + Class[Ruby] -> Class[rubygems] -> Class[bundler::install] + class { '::ruby': } + class { 'bundler::install': install_method => 'package' } + include rubygems +} diff --git a/puppet/modules/site_config/manifests/ruby/dev.pp b/puppet/modules/site_config/manifests/ruby/dev.pp new file mode 100644 index 00000000..2b0b106d --- /dev/null +++ b/puppet/modules/site_config/manifests/ruby/dev.pp @@ -0,0 +1,8 @@ +# install ruby dev packages needed for building some gems +class site_config::ruby::dev { + include site_config::ruby + include ::ruby::devel + + # building gems locally probably requires build-essential and gcc: + include site_config::packages::build_essential +} diff --git a/puppet/modules/site_config/manifests/setup.pp b/puppet/modules/site_config/manifests/setup.pp new file mode 100644 index 00000000..82dfe76d --- /dev/null +++ b/puppet/modules/site_config/manifests/setup.pp @@ -0,0 +1,50 @@ +# common things to set up on every node +# leftover from the past, where we did two puppetruns +# after another. We should consolidate this into site_config::default +# in the future. +class site_config::setup { + tag 'leap_base' + + # + # this is applied before each run of site.pp + # + + Exec { path => '/usr/bin:/usr/sbin/:/bin:/sbin:/usr/local/bin:/usr/local/sbin' } + + include site_config::params + + include concat::setup + include stdlib + + # configure /etc/hosts + class { 'site_config::hosts': } + + include site_config::initial_firewall + + include site_apt + + package { 'facter': + ensure => latest + } + + # if squid_deb_proxy_client is set to true, install and configure + # squid_deb_proxy_client for apt caching + if hiera('squid_deb_proxy_client', false) { + include site_squid_deb_proxy::client + } + + # shorewall is installed/half-configured during setup.pp (Bug #3871) + # we need to include shorewall::interface{eth0} in setup.pp so + # packages can be installed during main puppetrun, even before shorewall + # is configured completly + if ( $::site_config::params::environment == 'local' ) { + include site_config::vagrant + } + + # if class site_custom::setup exists, include it. + # possibility for users to define custom puppet recipes + if defined( '::site_custom::setup') { + include ::site_custom::setup + } + +} diff --git a/puppet/modules/site_config/manifests/shell.pp b/puppet/modules/site_config/manifests/shell.pp new file mode 100644 index 00000000..5b8c025d --- /dev/null +++ b/puppet/modules/site_config/manifests/shell.pp @@ -0,0 +1,22 @@ +class site_config::shell { + + file { + '/etc/profile.d/leap_path.sh': + content => 'PATH=$PATH:/srv/leap/bin', + mode => '0644', + owner => root, + group => root; + } + + ## + ## XTERM TITLE + ## + + file { '/etc/profile.d/xterm-title.sh': + source => 'puppet:///modules/site_config/xterm-title.sh', + owner => root, + group => 0, + mode => '0644'; + } + +} diff --git a/puppet/modules/site_config/manifests/slow.pp b/puppet/modules/site_config/manifests/slow.pp new file mode 100644 index 00000000..8e9b7035 --- /dev/null +++ b/puppet/modules/site_config/manifests/slow.pp @@ -0,0 +1,10 @@ +# this class is run by default, but can be excluded +# for testing purposes by calling "leap deploy" with +# the "--fast" parameter +class site_config::slow { + tag 'leap_slow' + + include site_config::default + include apt::update + class { 'site_apt::dist_upgrade': } +} diff --git a/puppet/modules/site_config/manifests/sysctl.pp b/puppet/modules/site_config/manifests/sysctl.pp new file mode 100644 index 00000000..99f75123 --- /dev/null +++ b/puppet/modules/site_config/manifests/sysctl.pp @@ -0,0 +1,8 @@ +class site_config::sysctl { + + sysctl::config { + 'net.ipv4.ip_nonlocal_bind': + value => 1, + comment => 'Allow applications to bind to an address when link is down (see https://leap.se/code/issues/4506)' + } +} diff --git a/puppet/modules/site_config/manifests/syslog.pp b/puppet/modules/site_config/manifests/syslog.pp new file mode 100644 index 00000000..591e0601 --- /dev/null +++ b/puppet/modules/site_config/manifests/syslog.pp @@ -0,0 +1,62 @@ +# configure rsyslog on all nodes +class site_config::syslog { + + # only pin rsyslog packages to backports on wheezy + case $::operatingsystemrelease { + /^7.*/: { + include ::site_apt::preferences::rsyslog + } + # on jessie+ systems, systemd and journald are enabled, + # and journald logs IP addresses, so we need to disable + # it until a solution is found, (#7863): + # https://github.com/systemd/systemd/issues/2447 + default: { + include ::journald + augeas { + 'disable_journald': + incl => '/etc/systemd/journald.conf', + lens => 'Puppet.lns', + changes => 'set /files/etc/systemd/journald.conf/Journal/Storage \'none\'', + notify => Service['systemd-journald']; + } + } + } + + class { '::rsyslog::client': + log_remote => false, + log_local => true, + custom_config => 'site_rsyslog/client.conf.erb' + } + + rsyslog::snippet { '00-anonymize_logs': + content => '$ModLoad mmanon +action(type="mmanon" ipv4.bits="32" mode="rewrite")' + } + + augeas { + 'logrotate_leap_deploy': + context => '/files/etc/logrotate.d/leap_deploy/rule', + changes => [ + 'set file /var/log/leap/deploy.log', + 'set rotate 5', + 'set size 1M', + 'set compress compress', + 'set missingok missingok', + 'set copytruncate copytruncate' ]; + + # NOTE: + # the puppet_command script requires the option delaycompress + # be set on the summary log file. + + 'logrotate_leap_deploy_summary': + context => '/files/etc/logrotate.d/leap_deploy_summary/rule', + changes => [ + 'set file /var/log/leap/deploy-summary.log', + 'set rotate 5', + 'set size 100k', + 'set delaycompress delaycompress', + 'set compress compress', + 'set missingok missingok', + 'set copytruncate copytruncate' ] + } +} diff --git a/puppet/modules/site_config/manifests/vagrant.pp b/puppet/modules/site_config/manifests/vagrant.pp new file mode 100644 index 00000000..8f50b305 --- /dev/null +++ b/puppet/modules/site_config/manifests/vagrant.pp @@ -0,0 +1,11 @@ +class site_config::vagrant { + # class for vagrant nodes + + include site_shorewall::defaults + # eth0 on vagrant nodes is the uplink if + shorewall::interface { 'eth0': + zone => 'net', + options => 'tcpflags,blacklist,nosmurfs'; + } + +} diff --git a/puppet/modules/site_config/manifests/x509/ca.pp b/puppet/modules/site_config/manifests/x509/ca.pp new file mode 100644 index 00000000..2880ecaf --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/ca.pp @@ -0,0 +1,11 @@ +class site_config::x509::ca { + + include ::site_config::params + + $x509 = hiera('x509') + $ca = $x509['ca_cert'] + + x509::ca { $site_config::params::ca_name: + content => $ca + } +} diff --git a/puppet/modules/site_config/manifests/x509/ca_bundle.pp b/puppet/modules/site_config/manifests/x509/ca_bundle.pp new file mode 100644 index 00000000..5808e29e --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/ca_bundle.pp @@ -0,0 +1,17 @@ +class site_config::x509::ca_bundle { + + # CA bundle -- we want to have the possibility of allowing multiple CAs. + # For now, the reason is to transition to using client CA. In the future, + # we will want to be able to smoothly phase out one CA and phase in another. + # I tried "--capath" for this, but it did not work. + + include ::site_config::params + + $x509 = hiera('x509') + $ca = $x509['ca_cert'] + $client_ca = $x509['client_ca_cert'] + + x509::ca { $site_config::params::ca_bundle_name: + content => "${ca}${client_ca}" + } +} diff --git a/puppet/modules/site_config/manifests/x509/cert.pp b/puppet/modules/site_config/manifests/x509/cert.pp new file mode 100644 index 00000000..7e5a36b9 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/cert.pp @@ -0,0 +1,12 @@ +class site_config::x509::cert { + + include ::site_config::params + + $x509 = hiera('x509') + $cert = $x509['cert'] + + x509::cert { $site_config::params::cert_name: + content => $cert + } + +} diff --git a/puppet/modules/site_config/manifests/x509/client_ca/ca.pp b/puppet/modules/site_config/manifests/x509/client_ca/ca.pp new file mode 100644 index 00000000..3fbafa98 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/client_ca/ca.pp @@ -0,0 +1,16 @@ +class site_config::x509::client_ca::ca { + + ## + ## This is for the special CA that is used exclusively for generating + ## client certificates by the webapp. + ## + + include ::site_config::params + + $x509 = hiera('x509') + $cert = $x509['client_ca_cert'] + + x509::ca { $site_config::params::client_ca_name: + content => $cert + } +} diff --git a/puppet/modules/site_config/manifests/x509/client_ca/key.pp b/puppet/modules/site_config/manifests/x509/client_ca/key.pp new file mode 100644 index 00000000..0b537e76 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/client_ca/key.pp @@ -0,0 +1,16 @@ +class site_config::x509::client_ca::key { + + ## + ## This is for the special CA that is used exclusively for generating + ## client certificates by the webapp. + ## + + include ::site_config::params + + $x509 = hiera('x509') + $key = $x509['client_ca_key'] + + x509::key { $site_config::params::client_ca_name: + content => $key + } +} diff --git a/puppet/modules/site_config/manifests/x509/commercial/ca.pp b/puppet/modules/site_config/manifests/x509/commercial/ca.pp new file mode 100644 index 00000000..c76a9dbb --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/commercial/ca.pp @@ -0,0 +1,11 @@ +class site_config::x509::commercial::ca { + + include ::site_config::params + + $x509 = hiera('x509') + $ca = $x509['commercial_ca_cert'] + + x509::ca { $site_config::params::commercial_ca_name: + content => $ca + } +} diff --git a/puppet/modules/site_config/manifests/x509/commercial/cert.pp b/puppet/modules/site_config/manifests/x509/commercial/cert.pp new file mode 100644 index 00000000..9dd6ffcd --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/commercial/cert.pp @@ -0,0 +1,15 @@ +class site_config::x509::commercial::cert { + + include ::site_config::params + + $x509 = hiera('x509') + $cert = $x509['commercial_cert'] + $ca = $x509['commercial_ca_cert'] + + $cafile = "${cert}\n${ca}" + + x509::cert { $site_config::params::commercial_cert_name: + content => $cafile + } + +} diff --git a/puppet/modules/site_config/manifests/x509/commercial/key.pp b/puppet/modules/site_config/manifests/x509/commercial/key.pp new file mode 100644 index 00000000..2be439fd --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/commercial/key.pp @@ -0,0 +1,11 @@ +class site_config::x509::commercial::key { + + include ::site_config::params + + $x509 = hiera('x509') + $key = $x509['commercial_key'] + + x509::key { $site_config::params::commercial_cert_name: + content => $key + } +} diff --git a/puppet/modules/site_config/manifests/x509/key.pp b/puppet/modules/site_config/manifests/x509/key.pp new file mode 100644 index 00000000..448dc6a6 --- /dev/null +++ b/puppet/modules/site_config/manifests/x509/key.pp @@ -0,0 +1,11 @@ +class site_config::x509::key { + + include ::site_config::params + + $x509 = hiera('x509') + $key = $x509['key'] + + x509::key { $site_config::params::cert_name: + content => $key + } +} diff --git a/puppet/modules/site_config/templates/hosts b/puppet/modules/site_config/templates/hosts new file mode 100644 index 00000000..d62cbc3f --- /dev/null +++ b/puppet/modules/site_config/templates/hosts @@ -0,0 +1,19 @@ +# This file is managed by puppet, any changes will be overwritten! + +127.0.0.1 localhost +127.0.1.1 <%= @my_hostnames.join(' ') %> + +<%- if @hosts then -%> +<% @hosts.keys.sort.each do |name| -%> +<%- props = @hosts[name] -%> +<%- aliases = props["aliases"] ? props["aliases"].join(' ') : nil -%> +<%= [props["ip_address"], props["domain_full"], props["domain_internal"], aliases, name].compact.uniq.join(' ') %> +<% end -%> +<% end -%> + +# The following lines are desirable for IPv6 capable hosts +::1 ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters diff --git a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb new file mode 100644 index 00000000..b0c2b7ad --- /dev/null +++ b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb @@ -0,0 +1,14 @@ +# Generated by iptables-save v1.4.14 on Tue Aug 20 14:40:40 2013 +*filter +:INPUT DROP [0:0] +:FORWARD DROP [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -i lo -j ACCEPT +-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT +-A INPUT -p tcp -m state --state NEW,ESTABLISHED --dport 22 -j ACCEPT +-A INPUT -p tcp -m state --state NEW,ESTABLISHED --dport <%= @ssh_port %> -j ACCEPT +-A INPUT -p udp -m udp --sport 53 -j ACCEPT +-A INPUT -p icmp -m icmp --icmp-type 8 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT +-A INPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT +-A INPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7 +COMMIT diff --git a/puppet/modules/site_config/templates/ipv6firewall_up.rules.erb b/puppet/modules/site_config/templates/ipv6firewall_up.rules.erb new file mode 100644 index 00000000..e2c92524 --- /dev/null +++ b/puppet/modules/site_config/templates/ipv6firewall_up.rules.erb @@ -0,0 +1,8 @@ +# Generated by ip6tables-save v1.4.20 on Tue Aug 20 12:19:43 2013 +*filter +:INPUT DROP [24:1980] +:FORWARD DROP [0:0] +:OUTPUT DROP [14:8030] +-A OUTPUT -j REJECT --reject-with icmp6-port-unreachable +COMMIT +# Completed on Tue Aug 20 12:19:43 2013 diff --git a/puppet/modules/site_config/templates/reload_dhclient.erb b/puppet/modules/site_config/templates/reload_dhclient.erb new file mode 100644 index 00000000..075828b7 --- /dev/null +++ b/puppet/modules/site_config/templates/reload_dhclient.erb @@ -0,0 +1,13 @@ +#!/bin/sh + +# Get the PID +PIDFILE='/var/run/dhclient.<%= scope.lookupvar('site_config::params::interface') %>.pid' + +# Capture how dhclient is currently running so we can relaunch it +dhclient=`/bin/ps --no-headers --pid $(cat $PIDFILE) -f | /usr/bin/awk '{for(i=8;i<=NF;++i) printf("%s ", $i) }'` + +# Kill the current dhclient +/usr/bin/pkill -F $PIDFILE + +# Restart dhclient with the arguments it had previously +$dhclient diff --git a/puppet/modules/site_couchdb/files/couchdb_scripts_defaults.conf b/puppet/modules/site_couchdb/files/couchdb_scripts_defaults.conf new file mode 100644 index 00000000..1565e1a1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/couchdb_scripts_defaults.conf @@ -0,0 +1,4 @@ +# space separated list of excluded DBs for dumping +# sourced by couchdb_dumpall.sh +EXCLUDE_DBS='sessions tokens' + diff --git a/puppet/modules/site_couchdb/files/designs/Readme.md b/puppet/modules/site_couchdb/files/designs/Readme.md new file mode 100644 index 00000000..983f629f --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/Readme.md @@ -0,0 +1,14 @@ +This directory contains design documents for the leap platform. + +They need to be uploaded to the couch database in order to query the +database in certain ways. + +Each subdirectory corresponds to a couch database and contains the design +documents that need to be added to that particular database. + +Here's an example of how to upload the users design document: +```bash +HOST="http://localhost:5984" +curl -X PUT $HOST/users/_design/User --data @users/User.json + +``` diff --git a/puppet/modules/site_couchdb/files/designs/customers/Customer.json b/puppet/modules/site_couchdb/files/designs/customers/Customer.json new file mode 100644 index 00000000..1b4bbddd --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/customers/Customer.json @@ -0,0 +1,18 @@ +{ + "_id": "_design/Customer", + "language": "javascript", + "views": { + "by_user_id": { + "map": " function(doc) {\n if ((doc['type'] == 'Customer') && (doc['user_id'] != null)) {\n emit(doc['user_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_braintree_customer_id": { + "map": " function(doc) {\n if ((doc['type'] == 'Customer') && (doc['braintree_customer_id'] != null)) {\n emit(doc['braintree_customer_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Customer') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "688c401ec0230b75625c176a88fc4a02" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/identities/Identity.json b/puppet/modules/site_couchdb/files/designs/identities/Identity.json new file mode 100644 index 00000000..b1c567c1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/identities/Identity.json @@ -0,0 +1,34 @@ +{ + "_id": "_design/Identity", + "language": "javascript", + "views": { + "by_address_and_destination": { + "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['address'] != null) && (doc['destination'] != null)) {\n emit([doc['address'], doc['destination']], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Identity') {\n emit(doc._id, null);\n }\n }\n" + }, + "cert_fingerprints_by_expiry": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(doc.cert_fingerprints[fp], fp);\n }\n }\n }\n}\n" + }, + "cert_expiry_by_fingerprint": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(fp, doc.cert_fingerprints[fp]);\n }\n }\n }\n}\n" + }, + "disabled": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.user_id === \"undefined\") {\n emit(doc._id, 1);\n }\n}\n" + }, + "pgp_key_by_email": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.keys === \"object\") {\n emit(doc.address, doc.keys[\"pgp\"]);\n }\n}\n" + }, + "by_user_id": { + "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['user_id'] != null)) {\n emit(doc['user_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_address": { + "map": " function(doc) {\n if ((doc['type'] == 'Identity') && (doc['address'] != null)) {\n emit(doc['address'], 1);\n }\n }\n", + "reduce": "_sum" + } + }, + "couchrest-hash": "4a774c3f56122b655a314670403b27e2" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json new file mode 100644 index 00000000..006c1ea1 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json @@ -0,0 +1,22 @@ +{ + "_id": "_design/InviteCode", + "language": "javascript", + "views": { + "by__id": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['_id'] != null)) {\n emit(doc['_id'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_code": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_count": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "83fb8f504520b4a9c7ddbb7928cd0ce3" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/messages/Message.json b/puppet/modules/site_couchdb/files/designs/messages/Message.json new file mode 100644 index 00000000..6a48fc4d --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/messages/Message.json @@ -0,0 +1,18 @@ +{ + "_id": "_design/Message", + "language": "javascript", + "views": { + "by_user_ids_to_show": { + "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_user_ids_to_show_and_created_at": { + "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Message') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "ba80168e51015d2678cad88fc6c5b986" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/sessions/Session.json b/puppet/modules/site_couchdb/files/designs/sessions/Session.json new file mode 100644 index 00000000..70202780 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/sessions/Session.json @@ -0,0 +1,8 @@ +{ + "views": { + "by_expires": { + "reduce": "_sum", + "map": "function(doc) {\n if(typeof doc.expires !== \"undefined\") {\n emit(doc.expires, 1);\n }\n}\n" + } + } +} diff --git a/puppet/modules/site_couchdb/files/designs/shared/docs.json b/puppet/modules/site_couchdb/files/designs/shared/docs.json new file mode 100644 index 00000000..004180cd --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/shared/docs.json @@ -0,0 +1,8 @@ +{ + "_id": "_design/docs", + "views": { + "get": { + "map": "function(doc) {\n if (doc.u1db_rev) {\n var is_tombstone = true;\n var has_conflicts = false;\n if (doc._attachments) {\n if (doc._attachments.u1db_content)\n is_tombstone = false;\n if (doc._attachments.u1db_conflicts)\n has_conflicts = true;\n }\n emit(doc._id,\n {\n \"couch_rev\": doc._rev,\n \"u1db_rev\": doc.u1db_rev,\n \"is_tombstone\": is_tombstone,\n \"has_conflicts\": has_conflicts,\n }\n );\n }\n}\n" + } + } +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/shared/syncs.json b/puppet/modules/site_couchdb/files/designs/shared/syncs.json new file mode 100644 index 00000000..bab5622f --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/shared/syncs.json @@ -0,0 +1,11 @@ +{ + "_id": "_design/syncs", + "updates": { + "put": "function(doc, req){\n if (!doc) {\n doc = {}\n doc['_id'] = 'u1db_sync_log';\n doc['syncs'] = [];\n }\n body = JSON.parse(req.body);\n // remove outdated info\n doc['syncs'] = doc['syncs'].filter(\n function (entry) {\n return entry[0] != body['other_replica_uid'];\n }\n );\n // store u1db rev\n doc['syncs'].push([\n body['other_replica_uid'],\n body['other_generation'],\n body['other_transaction_id']\n ]);\n return [doc, 'ok'];\n}\n\n" + }, + "views": { + "log": { + "map": "function(doc) {\n if (doc._id == 'u1db_sync_log') {\n if (doc.syncs)\n doc.syncs.forEach(function (entry) {\n emit(entry[0],\n {\n 'known_generation': entry[1],\n 'known_transaction_id': entry[2]\n });\n });\n }\n}\n" + } + } +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/shared/transactions.json b/puppet/modules/site_couchdb/files/designs/shared/transactions.json new file mode 100644 index 00000000..106ad46c --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/shared/transactions.json @@ -0,0 +1,13 @@ +{ + "_id": "_design/transactions", + "lists": { + "generation": "function(head, req) {\n var row;\n var rows=[];\n // fetch all rows\n while(row = getRow()) {\n rows.push(row);\n }\n if (rows.length > 0)\n send(JSON.stringify({\n \"generation\": rows.length,\n \"doc_id\": rows[rows.length-1]['id'],\n \"transaction_id\": rows[rows.length-1]['value']\n }));\n else\n send(JSON.stringify({\n \"generation\": 0,\n \"doc_id\": \"\",\n \"transaction_id\": \"\",\n }));\n}\n", + "trans_id_for_gen": "function(head, req) {\n var row;\n var rows=[];\n var i = 1;\n var gen = 1;\n if (req.query.gen)\n gen = parseInt(req.query['gen']);\n // fetch all rows\n while(row = getRow())\n rows.push(row);\n if (gen <= rows.length)\n send(JSON.stringify({\n \"generation\": gen,\n \"doc_id\": rows[gen-1]['id'],\n \"transaction_id\": rows[gen-1]['value'],\n }));\n else\n send('{}');\n}\n", + "whats_changed": "function(head, req) {\n var row;\n var gen = 1;\n var old_gen = 0;\n if (req.query.old_gen)\n old_gen = parseInt(req.query['old_gen']);\n send('{\"transactions\":[\\n');\n // fetch all rows\n while(row = getRow()) {\n if (gen > old_gen) {\n if (gen > old_gen+1)\n send(',\\n');\n send(JSON.stringify({\n \"generation\": gen,\n \"doc_id\": row[\"id\"],\n \"transaction_id\": row[\"value\"]\n }));\n }\n gen++;\n }\n send('\\n]}');\n}\n" + }, + "views": { + "log": { + "map": "function(doc) {\n if (doc.u1db_transactions)\n doc.u1db_transactions.forEach(function(t) {\n emit(t[0], // use timestamp as key so the results are ordered\n t[1]); // value is the transaction_id\n });\n}\n" + } + } +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json new file mode 100644 index 00000000..578f632b --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json @@ -0,0 +1,50 @@ +{ + "_id": "_design/Ticket", + "language": "javascript", + "views": { + "by_updated_at": { + "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['updated_at'] != null)) {\n emit(doc['updated_at'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_created_at": { + "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_created_by": { + "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['created_by'] != null)) {\n emit(doc['created_by'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_is_open_and_created_at": { + "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['is_open'] != null) && (doc['created_at'] != null)) {\n emit([doc['is_open'], doc['created_at']], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_is_open_and_updated_at": { + "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['is_open'] != null) && (doc['updated_at'] != null)) {\n emit([doc['is_open'], doc['updated_at']], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_includes_post_by_and_is_open_and_created_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.created_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by": { + "map": "// TODO: This view is only used in tests--should we keep it?\nfunction(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit(comment.posted_by, 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_is_open_and_updated_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.updated_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_created_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.created_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_updated_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.updated_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Ticket') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "b21eaeea8ea66bfda65581b1b7ce06af" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/tokens/Token.json b/puppet/modules/site_couchdb/files/designs/tokens/Token.json new file mode 100644 index 00000000..b9025f15 --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/tokens/Token.json @@ -0,0 +1,14 @@ +{ + "_id": "_design/Token", + "language": "javascript", + "views": { + "by_last_seen_at": { + "map": " function(doc) {\n if ((doc['type'] == 'Token') && (doc['last_seen_at'] != null)) {\n emit(doc['last_seen_at'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'Token') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "541dd924551c42a2317b345effbe65cc" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/users/User.json b/puppet/modules/site_couchdb/files/designs/users/User.json new file mode 100644 index 00000000..8a82cf4a --- /dev/null +++ b/puppet/modules/site_couchdb/files/designs/users/User.json @@ -0,0 +1,22 @@ +{ + "_id": "_design/User", + "language": "javascript", + "views": { + "by_login": { + "map": " function(doc) {\n if ((doc['type'] == 'User') && (doc['login'] != null)) {\n emit(doc['login'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'User') {\n emit(doc._id, null);\n }\n }\n" + }, + "by_created_at_and_one_month_warning_not_sent": { + "map": "function (doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null) && (doc['one_month_warning_sent'] == null)) {\n emit(doc['created_at'], 1);\n } \n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_created_at": { + "map": " function(doc) {\n if ((doc['type'] == 'User') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", + "reduce": "_sum" + } + }, + "couchrest-hash": "d854607d299887a347e554176cb79e20" +}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/leap_ca_daemon b/puppet/modules/site_couchdb/files/leap_ca_daemon new file mode 100755 index 00000000..9a1a0bc7 --- /dev/null +++ b/puppet/modules/site_couchdb/files/leap_ca_daemon @@ -0,0 +1,157 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: leap_ca_daemon +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: leap_ca_daemon initscript +# Description: Controls leap_ca_daemon (see https://github.com/leapcode/leap_ca +# for more information. +### END INIT INFO + +# Author: varac <varac@leap.se> +# + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="leap_ca_daemon initscript" +NAME=leap_ca_daemon +DAEMON=/usr/local/bin/$NAME +DAEMON_ARGS="run " +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/puppet/modules/site_couchdb/files/local.ini b/puppet/modules/site_couchdb/files/local.ini new file mode 100644 index 00000000..b921a927 --- /dev/null +++ b/puppet/modules/site_couchdb/files/local.ini @@ -0,0 +1,8 @@ +; Puppet modified file !! + +; Custom settings should be made in this file. They will override settings +; in default.ini, but unlike changes made to default.ini, this file won't be +; overwritten on server upgrade. + +[compactions] +_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "03:00"}, {to, "05:00"}] diff --git a/puppet/modules/site_couchdb/files/runit_config b/puppet/modules/site_couchdb/files/runit_config new file mode 100644 index 00000000..169b4832 --- /dev/null +++ b/puppet/modules/site_couchdb/files/runit_config @@ -0,0 +1,6 @@ +#!/bin/bash +exec 2>&1 +export HOME=/home/bigcouch +ulimit -H -n 32768 +ulimit -S -n 32768 +exec chpst -u bigcouch /opt/bigcouch/bin/bigcouch diff --git a/puppet/modules/site_couchdb/lib/puppet/parser/functions/rotated_db_name.rb b/puppet/modules/site_couchdb/lib/puppet/parser/functions/rotated_db_name.rb new file mode 100644 index 00000000..6458ae81 --- /dev/null +++ b/puppet/modules/site_couchdb/lib/puppet/parser/functions/rotated_db_name.rb @@ -0,0 +1,24 @@ +module Puppet::Parser::Functions + newfunction(:rotated_db_name, :type => :rvalue, :doc => <<-EOS +This function takes a database name string and returns a database name with the current rotation stamp appended. +The first argument is the base name of the database. Subsequent arguments may contain these options: + * 'next' -- return the db name for the next rotation, not the current one. + * 'monthly' -- rotate monthly (default) + * 'weekly' -- rotate weekly +*Examples:* + rotated_db_name('tokens') => 'tokens_551' + EOS + ) do |arguments| + if arguments.include?('weekly') + rotation_period = 604800 # 1 week + else + rotation_period = 2592000 # 1 month + end + suffix = Time.now.utc.to_i / rotation_period + if arguments.include?('next') + suffix += 1 + end + "#{arguments.first}_#{suffix}" + end +end + diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp new file mode 100644 index 00000000..c905316b --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/add_users.pp @@ -0,0 +1,57 @@ +# add couchdb users for all services +class site_couchdb::add_users { + + Class['site_couchdb::create_dbs'] + -> Class['site_couchdb::add_users'] + + # Couchdb users + + ## leap_mx couchdb user + ## read: identities + ## write access to user-<uuid> + couchdb::add_user { $site_couchdb::couchdb_leap_mx_user: + roles => '["identities"]', + pw => $site_couchdb::couchdb_leap_mx_pw, + salt => $site_couchdb::couchdb_leap_mx_salt, + require => Couchdb::Query::Setup['localhost'] + } + + ## nickserver couchdb user + ## r: identities + ## r/w: keycache + couchdb::add_user { $site_couchdb::couchdb_nickserver_user: + roles => '["identities","keycache"]', + pw => $site_couchdb::couchdb_nickserver_pw, + salt => $site_couchdb::couchdb_nickserver_salt, + require => Couchdb::Query::Setup['localhost'] + } + + ## soledad couchdb user + ## r/w: user-<uuid>, shared + ## read: tokens + couchdb::add_user { $site_couchdb::couchdb_soledad_user: + roles => '["tokens"]', + pw => $site_couchdb::couchdb_soledad_pw, + salt => $site_couchdb::couchdb_soledad_salt, + require => Couchdb::Query::Setup['localhost'] + } + + ## webapp couchdb user + ## read/write: users, tokens, sessions, tickets, identities, customer + couchdb::add_user { $site_couchdb::couchdb_webapp_user: + roles => '["tokens","identities","users"]', + pw => $site_couchdb::couchdb_webapp_pw, + salt => $site_couchdb::couchdb_webapp_salt, + require => Couchdb::Query::Setup['localhost'] + } + + ## replication couchdb user + ## read/write: all databases for replication + couchdb::add_user { $site_couchdb::couchdb_replication_user: + roles => '["replication"]', + pw => $site_couchdb::couchdb_replication_pw, + salt => $site_couchdb::couchdb_replication_salt, + require => Couchdb::Query::Setup['localhost'] + } + +} diff --git a/puppet/modules/site_couchdb/manifests/backup.pp b/puppet/modules/site_couchdb/manifests/backup.pp new file mode 100644 index 00000000..8b5aa6ea --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/backup.pp @@ -0,0 +1,23 @@ +class site_couchdb::backup { + + # general backupninja config + backupninja::config { 'backupninja_config': + usecolors => false, + } + + # dump all DBs locally to /var/backups/couchdb once a day + backupninja::sh { 'couchdb_backup': + command_string => "cd /srv/leap/couchdb/scripts \n./couchdb_dumpall.sh" + } + + # Deploy /etc/leap/couchdb_scripts_defaults.conf so we can exclude + # some databases + + file { '/etc/leap/couchdb_scripts_defaults.conf': + source => 'puppet:///modules/site_couchdb/couchdb_scripts_defaults.conf', + mode => '0644', + owner => 'root', + group => 'root', + } + +} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp new file mode 100644 index 00000000..2de3d4d0 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp @@ -0,0 +1,50 @@ +# sets up bigcouch on couchdb node +class site_couchdb::bigcouch { + + $config = $::site_couchdb::couchdb_config['bigcouch'] + $cookie = $config['cookie'] + $ednp_port = $config['ednp_port'] + + class { 'couchdb': + admin_pw => $::site_couchdb::couchdb_admin_pw, + admin_salt => $::site_couchdb::couchdb_admin_salt, + bigcouch => true, + bigcouch_cookie => $cookie, + ednp_port => $ednp_port, + chttpd_bind_address => '127.0.0.1' + } + + # + # stunnel must running correctly before bigcouch dbs can be set up. + # + Class['site_config::default'] + -> Class['site_config::resolvconf'] + -> Class['couchdb::bigcouch::package::cloudant'] + -> Service['shorewall'] + -> Exec['refresh_stunnel'] + -> Class['site_couchdb::setup'] + -> Class['site_couchdb::bigcouch::add_nodes'] + -> Class['site_couchdb::bigcouch::settle_cluster'] + -> Class['site_couchdb::create_dbs'] + + include site_couchdb::bigcouch::add_nodes + include site_couchdb::bigcouch::settle_cluster + include site_couchdb::bigcouch::compaction + + file { '/var/log/bigcouch': + ensure => directory + } + + file { '/etc/sv/bigcouch/run': + ensure => present, + source => 'puppet:///modules/site_couchdb/runit_config', + owner => root, + group => root, + mode => '0755', + require => Package['couchdb'], + notify => Service['couchdb'] + } + + include site_check_mk::agent::couchdb::bigcouch + +} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp new file mode 100644 index 00000000..c8c43275 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp @@ -0,0 +1,8 @@ +class site_couchdb::bigcouch::add_nodes { + # loop through neighbors array and add nodes + $nodes = $::site_couchdb::bigcouch::config['neighbors'] + + couchdb::bigcouch::add_node { $nodes: + require => Couchdb::Query::Setup['localhost'] + } +} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp new file mode 100644 index 00000000..84aab4ef --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/compaction.pp @@ -0,0 +1,8 @@ +class site_couchdb::bigcouch::compaction { + cron { + 'compact_all_shards': + command => '/srv/leap/couchdb/scripts/bigcouch_compact_all_shards.sh >> /var/log/bigcouch/compaction.log', + hour => 3, + minute => 17; + } +} diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp new file mode 100644 index 00000000..820b5be2 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp @@ -0,0 +1,11 @@ +class site_couchdb::bigcouch::settle_cluster { + + exec { 'wait_for_couch_nodes': + command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 12 --wait 10' + } + + exec { 'settle_cluster_membership': + command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 12 --wait 10', + require => Exec['wait_for_couch_nodes'] + } +} diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp new file mode 100644 index 00000000..a2d1c655 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp @@ -0,0 +1,102 @@ +# creates neccesary databases +class site_couchdb::create_dbs { + + Class['site_couchdb::setup'] + -> Class['site_couchdb::create_dbs'] + + ### customer database + ### r/w: webapp, + couchdb::create_db { 'customers': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## identities database + ## r: nickserver, leap_mx - needs to be restrict with design document + ## r/w: webapp + couchdb::create_db { 'identities': + members => "{ \"names\": [], \"roles\": [\"replication\", \"identities\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## keycache database + ## r/w: nickserver + couchdb::create_db { 'keycache': + members => "{ \"names\": [], \"roles\": [\"replication\", \"keycache\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## sessions database + ## r/w: webapp + $sessions_db = rotated_db_name('sessions', 'monthly') + couchdb::create_db { $sessions_db: + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + $sessions_next_db = rotated_db_name('sessions', 'monthly', 'next') + couchdb::create_db { $sessions_next_db: + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## shared database + ## r/w: soledad + couchdb::create_db { 'shared': + members => "{ \"names\": [\"${site_couchdb::couchdb_soledad_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## tickets database + ## r/w: webapp + couchdb::create_db { 'tickets': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## tokens database + ## r: soledad - needs to be restricted with a design document + ## r/w: webapp + $tokens_db = rotated_db_name('tokens', 'monthly') + couchdb::create_db { $tokens_db: + members => "{ \"names\": [], \"roles\": [\"replication\", \"tokens\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + $tokens_next_db = rotated_db_name('tokens', 'monthly', 'next') + couchdb::create_db { $tokens_next_db: + members => "{ \"names\": [], \"roles\": [\"replication\", \"tokens\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## users database + ## r/w: webapp + couchdb::create_db { 'users': + members => "{ \"names\": [], \"roles\": [\"replication\", \"users\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## tmp_users database + ## r/w: webapp + couchdb::create_db { 'tmp_users': + members => "{ \"names\": [], \"roles\": [\"replication\", \"users\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## messages db + ## store messages to the clients such as payment reminders + ## r/w: webapp + couchdb::create_db { 'messages': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + + ## invite_codes db + ## store invite codes for new signups + ## r/w: webapp + couchdb::create_db { 'invite_codes': + members => "{ \"names\": [\"${site_couchdb::couchdb_webapp_user}\"], \"roles\": [\"replication\"] }", + require => Couchdb::Query::Setup['localhost'] + } + +} diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp new file mode 100644 index 00000000..e5fd94c6 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -0,0 +1,46 @@ +class site_couchdb::designs { + + Class['site_couchdb::create_dbs'] + -> Class['site_couchdb::designs'] + + file { '/srv/leap/couchdb/designs': + ensure => directory, + source => 'puppet:///modules/site_couchdb/designs', + recurse => true, + purge => true, + mode => '0755' + } + + site_couchdb::upload_design { + 'customers': design => 'customers/Customer.json'; + 'identities': design => 'identities/Identity.json'; + 'tickets': design => 'tickets/Ticket.json'; + 'messages': design => 'messages/Message.json'; + 'users': design => 'users/User.json'; + 'tmp_users': design => 'users/User.json'; + 'invite_codes': design => 'invite_codes/InviteCode.json'; + 'shared_docs': + db => 'shared', + design => 'shared/docs.json'; + 'shared_syncs': + db => 'shared', + design => 'shared/syncs.json'; + 'shared_transactions': + db => 'shared', + design => 'shared/transactions.json'; + } + + $sessions_db = rotated_db_name('sessions', 'monthly') + $sessions_next_db = rotated_db_name('sessions', 'monthly', 'next') + site_couchdb::upload_design { + $sessions_db: design => 'sessions/Session.json'; + $sessions_next_db: design => 'sessions/Session.json'; + } + + $tokens_db = rotated_db_name('tokens', 'monthly') + $tokens_next_db = rotated_db_name('tokens', 'monthly', 'next') + site_couchdb::upload_design { + $tokens_db: design => 'tokens/Token.json'; + $tokens_next_db: design => 'tokens/Token.json'; + } +} diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp new file mode 100644 index 00000000..c4fe6277 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/init.pp @@ -0,0 +1,81 @@ +# entry class for configuring couchdb/bigcouch node +# couchdb node +class site_couchdb { + tag 'leap_service' + + $couchdb_config = hiera('couch') + $couchdb_users = $couchdb_config['users'] + + $couchdb_admin = $couchdb_users['admin'] + $couchdb_admin_user = $couchdb_admin['username'] + $couchdb_admin_pw = $couchdb_admin['password'] + $couchdb_admin_salt = $couchdb_admin['salt'] + + $couchdb_leap_mx = $couchdb_users['leap_mx'] + $couchdb_leap_mx_user = $couchdb_leap_mx['username'] + $couchdb_leap_mx_pw = $couchdb_leap_mx['password'] + $couchdb_leap_mx_salt = $couchdb_leap_mx['salt'] + + $couchdb_nickserver = $couchdb_users['nickserver'] + $couchdb_nickserver_user = $couchdb_nickserver['username'] + $couchdb_nickserver_pw = $couchdb_nickserver['password'] + $couchdb_nickserver_salt = $couchdb_nickserver['salt'] + + $couchdb_soledad = $couchdb_users['soledad'] + $couchdb_soledad_user = $couchdb_soledad['username'] + $couchdb_soledad_pw = $couchdb_soledad['password'] + $couchdb_soledad_salt = $couchdb_soledad['salt'] + + $couchdb_webapp = $couchdb_users['webapp'] + $couchdb_webapp_user = $couchdb_webapp['username'] + $couchdb_webapp_pw = $couchdb_webapp['password'] + $couchdb_webapp_salt = $couchdb_webapp['salt'] + + $couchdb_replication = $couchdb_users['replication'] + $couchdb_replication_user = $couchdb_replication['username'] + $couchdb_replication_pw = $couchdb_replication['password'] + $couchdb_replication_salt = $couchdb_replication['salt'] + + $couchdb_backup = $couchdb_config['backup'] + $couchdb_mode = $couchdb_config['mode'] + + # ensure bigcouch has been purged from the system: + # TODO: remove this check in 0.9 release + if file('/opt/bigcouch/bin/bigcouch', '/dev/null') != '' { + fail 'ERROR: BigCouch appears to be installed. Make sure you have migrated to CouchDB before proceeding. See https://leap.se/upgrade-0-8' + } + + include site_couchdb::plain + + Class['site_config::default'] + -> Service['shorewall'] + -> Exec['refresh_stunnel'] + -> Class['couchdb'] + -> Class['site_couchdb::setup'] + + include ::site_config::default + include site_stunnel + + include site_couchdb::setup + include site_couchdb::create_dbs + include site_couchdb::add_users + include site_couchdb::designs + include site_couchdb::logrotate + + if $couchdb_backup { include site_couchdb::backup } + + include site_check_mk::agent::couchdb + + # remove tapicero leftovers on couchdb nodes + include site_config::remove::tapicero + + # Destroy every per-user storage database + # where the corresponding user record does not exist. + cron { 'cleanup_stale_userdbs': + command => '(/bin/date; /srv/leap/couchdb/scripts/cleanup-user-dbs) >> /var/log/leap/couchdb-cleanup.log', + user => 'root', + hour => 4, + minute => 7; + } + +} diff --git a/puppet/modules/site_couchdb/manifests/logrotate.pp b/puppet/modules/site_couchdb/manifests/logrotate.pp new file mode 100644 index 00000000..bb8843bb --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/logrotate.pp @@ -0,0 +1,14 @@ +# configure couchdb logrotation +class site_couchdb::logrotate { + + augeas { + 'logrotate_bigcouch': + context => '/files/etc/logrotate.d/bigcouch/rule', + changes => [ + 'set file /opt/bigcouch/var/log/*.log', 'set rotate 7', + 'set schedule daily', 'set compress compress', + 'set missingok missingok', 'set ifempty notifempty', + 'set copytruncate copytruncate' ] + } + +} diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp new file mode 100644 index 00000000..fb82b897 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/mirror.pp @@ -0,0 +1,78 @@ +# configure mirroring of couch nodes +class site_couchdb::mirror { + + Class['site_couchdb::add_users'] + -> Class['site_couchdb::mirror'] + + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + + $masters = $site_couchdb::couchdb_config['replication']['masters'] + $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters']) + $master_node = $masters[$master_node_names[0]] + $user = $site_couchdb::couchdb_replication_user + $password = $site_couchdb::couchdb_replication_pw + $from_host = $master_node['domain_internal'] + $from_port = $master_node['couch_port'] + $from = "http://${user}:${password}@${from_host}:${from_port}" + + notice("mirror from: ${from}") + + ### customer database + couchdb::mirror_db { 'customers': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## identities database + couchdb::mirror_db { 'identities': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## keycache database + couchdb::mirror_db { 'keycache': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## sessions database + couchdb::mirror_db { 'sessions': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## shared database + couchdb::mirror_db { 'shared': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tickets database + couchdb::mirror_db { 'tickets': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## tokens database + couchdb::mirror_db { 'tokens': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## users database + couchdb::mirror_db { 'users': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + + ## messages db + couchdb::mirror_db { 'messages': + from => $from, + require => Couchdb::Query::Setup['localhost'] + } + +} diff --git a/puppet/modules/site_couchdb/manifests/plain.pp b/puppet/modules/site_couchdb/manifests/plain.pp new file mode 100644 index 00000000..b40fc100 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/plain.pp @@ -0,0 +1,14 @@ +# this class sets up a single, plain couchdb node +class site_couchdb::plain { + class { 'couchdb': + admin_pw => $site_couchdb::couchdb_admin_pw, + admin_salt => $site_couchdb::couchdb_admin_salt, + chttpd_bind_address => '127.0.0.1' + } + + include site_check_mk::agent::couchdb::plain + + # remove bigcouch leftovers from previous installations + include ::site_config::remove::bigcouch + +} diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp new file mode 100644 index 00000000..710d3c1c --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/setup.pp @@ -0,0 +1,61 @@ +# +# An initial setup class. All the other classes depend on this +# +class site_couchdb::setup { + + # ensure that we don't have leftovers from previous installations + # where we installed the cloudant bigcouch package + # https://leap.se/code/issues/4971 + class { 'couchdb::bigcouch::package::cloudant': + ensure => absent + } + + $user = $site_couchdb::couchdb_admin_user + + # setup /etc/couchdb/couchdb-admin.netrc for couchdb admin access + couchdb::query::setup { 'localhost': + user => $user, + pw => $site_couchdb::couchdb_admin_pw + } + + # We symlink /etc/couchdb/couchdb-admin.netrc to /etc/couchdb/couchdb.netrc + # for puppet commands, and to to /root/.netrc for couchdb_scripts + # (eg. backup) and to makes life easier for the admin on the command line + # (i.e. using curl/wget without passing credentials) + file { + '/etc/couchdb/couchdb.netrc': + ensure => link, + target => "/etc/couchdb/couchdb-${user}.netrc"; + '/root/.netrc': + ensure => link, + target => '/etc/couchdb/couchdb.netrc'; + } + + # setup /etc/couchdb/couchdb-soledad-admin.netrc file for couchdb admin + # access, accessible only for the soledad-admin user to create soledad + # userdbs + if member(hiera('services', []), 'soledad') { + file { '/etc/couchdb/couchdb-soledad-admin.netrc': + content => "machine localhost login ${user} password ${site_couchdb::couchdb_admin_pw}", + mode => '0400', + owner => 'soledad-admin', + group => 'root', + require => [ Package['couchdb'], User['soledad-admin'] ]; + } + } + + # Checkout couchdb_scripts repo + file { + '/srv/leap/couchdb': + ensure => directory + } + + vcsrepo { '/srv/leap/couchdb/scripts': + ensure => present, + provider => git, + source => 'https://leap.se/git/couchdb_scripts', + revision => 'origin/master', + require => File['/srv/leap/couchdb'] + } + +} diff --git a/puppet/modules/site_couchdb/manifests/upload_design.pp b/puppet/modules/site_couchdb/manifests/upload_design.pp new file mode 100644 index 00000000..bd73ebf2 --- /dev/null +++ b/puppet/modules/site_couchdb/manifests/upload_design.pp @@ -0,0 +1,14 @@ +# upload a design doc to a db +define site_couchdb::upload_design($design, $db = $title) { + $design_name = regsubst($design, '^.*\/(.*)\.json$', '\1') + $id = "_design/${design_name}" + $file = "/srv/leap/couchdb/designs/${design}" + exec { + "upload_design_${name}": + command => "/usr/local/bin/couch-doc-update --host 127.0.0.1:5984 --db '${db}' --id '${id}' --data '{}' --file '${file}'", + refreshonly => false, + loglevel => debug, + logoutput => on_failure, + require => File['/srv/leap/couchdb/designs']; + } +} diff --git a/puppet/modules/site_haproxy/files/haproxy-stats.cfg b/puppet/modules/site_haproxy/files/haproxy-stats.cfg new file mode 100644 index 00000000..e6335ba2 --- /dev/null +++ b/puppet/modules/site_haproxy/files/haproxy-stats.cfg @@ -0,0 +1,6 @@ +# provide access to stats for the nagios plugin +listen stats 127.0.0.1:8000 + mode http + stats enable + stats uri /haproxy + diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp new file mode 100644 index 00000000..b28ce80e --- /dev/null +++ b/puppet/modules/site_haproxy/manifests/init.pp @@ -0,0 +1,41 @@ +class site_haproxy { + $haproxy = hiera('haproxy') + + class { 'haproxy': + enable => true, + manage_service => true, + global_options => { + 'log' => '127.0.0.1 local0', + 'maxconn' => '4096', + 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', + 'chroot' => '/usr/share/haproxy', + 'user' => 'haproxy', + 'group' => 'haproxy', + 'daemon' => '' + }, + defaults_options => { + 'log' => 'global', + 'retries' => '3', + 'option' => 'redispatch', + 'timeout connect' => '4000', + 'timeout client' => '20000', + 'timeout server' => '20000' + } + } + + # monitor haproxy + concat::fragment { 'stats': + target => '/etc/haproxy/haproxy.cfg', + order => '90', + source => 'puppet:///modules/site_haproxy/haproxy-stats.cfg'; + } + + # Template uses $haproxy + concat::fragment { 'leap_haproxy_webapp_couchdb': + target => '/etc/haproxy/haproxy.cfg', + order => '20', + content => template('site_haproxy/haproxy.cfg.erb'), + } + + include site_check_mk::agent::haproxy +} diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb new file mode 100644 index 00000000..f42e8368 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/couch.erb @@ -0,0 +1,32 @@ +frontend couch + bind localhost:<%= @listen_port %> + mode http + option httplog + option dontlognull + option http-server-close # use client keep-alive, but close server connection. + use_backend couch_read if METH_GET + default_backend couch_write + +backend couch_write + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> +<%- next unless server['writable'] -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + +backend couch_read + mode http + balance roundrobin + option httpchk GET / # health check using simple get to root + option allbackups # balance among all backups, not just one. + default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 +<%- @servers.sort.each do |name,server| -%> + # <%=name%> + server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check +<%- end -%> + diff --git a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb new file mode 100644 index 00000000..8311b1a5 --- /dev/null +++ b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb @@ -0,0 +1,11 @@ +<%- @haproxy.each do |frontend, options| -%> +<%- if options['servers'] -%> + +## +## <%= frontend %> +## + +<%= scope.function_templatewlv(["site_haproxy/#{frontend}.erb", options]) %> +<%- end -%> +<%- end -%> + diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp new file mode 100644 index 00000000..a9b0198b --- /dev/null +++ b/puppet/modules/site_mx/manifests/init.pp @@ -0,0 +1,20 @@ +class site_mx { + tag 'leap_service' + Class['site_config::default'] -> Class['site_mx'] + + include site_config::default + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include site_config::x509::client_ca::ca + include site_config::x509::client_ca::key + + include site_stunnel + + include site_postfix::mx + include site_haproxy + include site_shorewall::mx + include site_shorewall::service::smtp + include leap_mx + include site_check_mk::agent::mx +} diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg new file mode 100644 index 00000000..62f26f2c --- /dev/null +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -0,0 +1,1302 @@ +############################################################################## +# +# NAGIOS.CFG - Sample Main Config File for Nagios +# +# +############################################################################## + + +# LOG FILE +# This is the main log file where service and host events are logged +# for historical purposes. This should be the first option specified +# in the config file!!! + +log_file=/var/log/nagios3/nagios.log + + + +# OBJECT CONFIGURATION FILE(S) +# These are the object configuration files in which you define hosts, +# host groups, contacts, contact groups, services, etc. +# You can split your object definitions across several config files +# if you wish (as shown below), or keep them all in a single config file. +#cfg_file=/etc/nagios3/commands.cfg + +# Check_mk configuration files +cfg_dir=/etc/nagios3/conf.d/check_mk +cfg_dir=/etc/nagios3/local + +# Puppet-managed configuration files +cfg_file=/etc/nagios3/nagios_templates.cfg +cfg_file=/etc/nagios3/nagios_command.cfg +cfg_file=/etc/nagios3/nagios_contact.cfg +cfg_file=/etc/nagios3/nagios_contactgroup.cfg +cfg_file=/etc/nagios3/nagios_host.cfg +cfg_file=/etc/nagios3/nagios_hostdependency.cfg +cfg_file=/etc/nagios3/nagios_hostescalation.cfg +cfg_file=/etc/nagios3/nagios_hostextinfo.cfg +cfg_file=/etc/nagios3/nagios_hostgroup.cfg +cfg_file=/etc/nagios3/nagios_hostgroupescalation.cfg +cfg_file=/etc/nagios3/nagios_service.cfg +cfg_file=/etc/nagios3/nagios_servicedependency.cfg +cfg_file=/etc/nagios3/nagios_serviceescalation.cfg +cfg_file=/etc/nagios3/nagios_serviceextinfo.cfg +cfg_file=/etc/nagios3/nagios_servicegroup.cfg +cfg_file=/etc/nagios3/nagios_timeperiod.cfg + +# Debian also defaults to using the check commands defined by the debian +# nagios-plugins package +cfg_dir=/etc/nagios-plugins/config + + +# OBJECT CACHE FILE +# This option determines where object definitions are cached when +# Nagios starts/restarts. The CGIs read object definitions from +# this cache file (rather than looking at the object config files +# directly) in order to prevent inconsistencies that can occur +# when the config files are modified after Nagios starts. + +object_cache_file=/var/cache/nagios3/objects.cache + + + +# PRE-CACHED OBJECT FILE +# This options determines the location of the precached object file. +# If you run Nagios with the -p command line option, it will preprocess +# your object configuration file(s) and write the cached config to this +# file. You can then start Nagios with the -u option to have it read +# object definitions from this precached file, rather than the standard +# object configuration files (see the cfg_file and cfg_dir options above). +# Using a precached object file can speed up the time needed to (re)start +# the Nagios process if you've got a large and/or complex configuration. +# Read the documentation section on optimizing Nagios to find our more +# about how this feature works. + +precached_object_file=/var/lib/nagios3/objects.precache + + + +# RESOURCE FILE +# This is an optional resource file that contains $USERx$ macro +# definitions. Multiple resource files can be specified by using +# multiple resource_file definitions. The CGIs will not attempt to +# read the contents of resource files, so information that is +# considered to be sensitive (usernames, passwords, etc) can be +# defined as macros in this file and restrictive permissions (600) +# can be placed on this file. + +resource_file=/etc/nagios3/resource.cfg + + + +# STATUS FILE +# This is where the current status of all monitored services and +# hosts is stored. Its contents are read and processed by the CGIs. +# The contents of the status file are deleted every time Nagios +# restarts. + +status_file=/var/cache/nagios3/status.dat + + + +# STATUS FILE UPDATE INTERVAL +# This option determines the frequency (in seconds) that +# Nagios will periodically dump program, host, and +# service status data. + +status_update_interval=10 + + + +# NAGIOS USER +# This determines the effective user that Nagios should run as. +# You can either supply a username or a UID. + +nagios_user=nagios + + + +# NAGIOS GROUP +# This determines the effective group that Nagios should run as. +# You can either supply a group name or a GID. + +nagios_group=nagios + + + +# EXTERNAL COMMAND OPTION +# This option allows you to specify whether or not Nagios should check +# for external commands (in the command file defined below). By default +# Nagios will *not* check for external commands, just to be on the +# cautious side. If you want to be able to use the CGI command interface +# you will have to enable this. +# Values: 0 = disable commands, 1 = enable commands + +check_external_commands=1 + + + +# EXTERNAL COMMAND CHECK INTERVAL +# This is the interval at which Nagios should check for external commands. +# This value works of the interval_length you specify later. If you leave +# that at its default value of 60 (seconds), a value of 1 here will cause +# Nagios to check for external commands every minute. If you specify a +# number followed by an "s" (i.e. 15s), this will be interpreted to mean +# actual seconds rather than a multiple of the interval_length variable. +# Note: In addition to reading the external command file at regularly +# scheduled intervals, Nagios will also check for external commands after +# event handlers are executed. +# NOTE: Setting this value to -1 causes Nagios to check the external +# command file as often as possible. + +#command_check_interval=15s +command_check_interval=-1 + + + +# EXTERNAL COMMAND FILE +# This is the file that Nagios checks for external command requests. +# It is also where the command CGI will write commands that are submitted +# by users, so it must be writeable by the user that the web server +# is running as (usually 'nobody'). Permissions should be set at the +# directory level instead of on the file, as the file is deleted every +# time its contents are processed. +# Debian Users: In case you didn't read README.Debian yet, _NOW_ is the +# time to do it. + +command_file=/var/lib/nagios3/rw/nagios.cmd + + + +# EXTERNAL COMMAND BUFFER SLOTS +# This settings is used to tweak the number of items or "slots" that +# the Nagios daemon should allocate to the buffer that holds incoming +# external commands before they are processed. As external commands +# are processed by the daemon, they are removed from the buffer. + +external_command_buffer_slots=4096 + + + +# LOCK FILE +# This is the lockfile that Nagios will use to store its PID number +# in when it is running in daemon mode. + +lock_file=/var/run/nagios3/nagios3.pid + + + +# TEMP FILE +# This is a temporary file that is used as scratch space when Nagios +# updates the status log, cleans the comment file, etc. This file +# is created, used, and deleted throughout the time that Nagios is +# running. + +temp_file=/var/cache/nagios3/nagios.tmp + + + +# TEMP PATH +# This is path where Nagios can create temp files for service and +# host check results, etc. + +temp_path=/tmp + + + +# EVENT BROKER OPTIONS +# Controls what (if any) data gets sent to the event broker. +# Values: 0 = Broker nothing +# -1 = Broker everything +# <other> = See documentation + +event_broker_options=-1 + + + +# EVENT BROKER MODULE(S) +# This directive is used to specify an event broker module that should +# by loaded by Nagios at startup. Use multiple directives if you want +# to load more than one module. Arguments that should be passed to +# the module at startup are seperated from the module path by a space. +# +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# WARNING !!! WARNING !!! WARNING !!! WARNING !!! WARNING !!! WARNING +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# Do NOT overwrite modules while they are being used by Nagios or Nagios +# will crash in a fiery display of SEGFAULT glory. This is a bug/limitation +# either in dlopen(), the kernel, and/or the filesystem. And maybe Nagios... +# +# The correct/safe way of updating a module is by using one of these methods: +# 1. Shutdown Nagios, replace the module file, restart Nagios +# 2. Delete the original module file, move the new module file into place, restart Nagios +# +# Example: +# +# broker_module=<modulepath> [moduleargs] + +#broker_module=/somewhere/module1.o +#broker_module=/somewhere/module2.o arg1 arg2=3 debug=0 + + + +# LOG ROTATION METHOD +# This is the log rotation method that Nagios should use to rotate +# the main log file. Values are as follows.. +# n = None - don't rotate the log +# h = Hourly rotation (top of the hour) +# d = Daily rotation (midnight every day) +# w = Weekly rotation (midnight on Saturday evening) +# m = Monthly rotation (midnight last day of month) + +log_rotation_method=n + + + +# LOG ARCHIVE PATH +# This is the directory where archived (rotated) log files should be +# placed (assuming you've chosen to do log rotation). + +log_archive_path=/var/log/nagios3/archives + + + +# LOGGING OPTIONS +# If you want messages logged to the syslog facility, as well as the +# Nagios log file set this option to 1. If not, set it to 0. + +use_syslog=0 + + + +# NOTIFICATION LOGGING OPTION +# If you don't want notifications to be logged, set this value to 0. +# If notifications should be logged, set the value to 1. + +log_notifications=1 + + + +# SERVICE RETRY LOGGING OPTION +# If you don't want service check retries to be logged, set this value +# to 0. If retries should be logged, set the value to 1. + +log_service_retries=1 + + + +# HOST RETRY LOGGING OPTION +# If you don't want host check retries to be logged, set this value to +# 0. If retries should be logged, set the value to 1. + +log_host_retries=1 + + + +# EVENT HANDLER LOGGING OPTION +# If you don't want host and service event handlers to be logged, set +# this value to 0. If event handlers should be logged, set the value +# to 1. + +log_event_handlers=1 + + + +# INITIAL STATES LOGGING OPTION +# If you want Nagios to log all initial host and service states to +# the main log file (the first time the service or host is checked) +# you can enable this option by setting this value to 1. If you +# are not using an external application that does long term state +# statistics reporting, you do not need to enable this option. In +# this case, set the value to 0. + +log_initial_states=0 + + + +# EXTERNAL COMMANDS LOGGING OPTION +# If you don't want Nagios to log external commands, set this value +# to 0. If external commands should be logged, set this value to 1. +# Note: This option does not include logging of passive service +# checks - see the option below for controlling whether or not +# passive checks are logged. + +log_external_commands=1 + + + +# PASSIVE CHECKS LOGGING OPTION +# If you don't want Nagios to log passive host and service checks, set +# this value to 0. If passive checks should be logged, set +# this value to 1. + +log_passive_checks=1 + + + +# GLOBAL HOST AND SERVICE EVENT HANDLERS +# These options allow you to specify a host and service event handler +# command that is to be run for every host or service state change. +# The global event handler is executed immediately prior to the event +# handler that you have optionally specified in each host or +# service definition. The command argument is the short name of a +# command definition that you define in your host configuration file. +# Read the HTML docs for more information. + +#global_host_event_handler=somecommand +#global_service_event_handler=somecommand + + + +# SERVICE INTER-CHECK DELAY METHOD +# This is the method that Nagios should use when initially +# "spreading out" service checks when it starts monitoring. The +# default is to use smart delay calculation, which will try to +# space all service checks out evenly to minimize CPU load. +# Using the dumb setting will cause all checks to be scheduled +# at the same time (with no delay between them)! This is not a +# good thing for production, but is useful when testing the +# parallelization functionality. +# n = None - don't use any delay between checks +# d = Use a "dumb" delay of 1 second between checks +# s = Use "smart" inter-check delay calculation +# x.xx = Use an inter-check delay of x.xx seconds + +service_inter_check_delay_method=s + + + +# MAXIMUM SERVICE CHECK SPREAD +# This variable determines the timeframe (in minutes) from the +# program start time that an initial check of all services should +# be completed. Default is 30 minutes. + +max_service_check_spread=30 + + + +# SERVICE CHECK INTERLEAVE FACTOR +# This variable determines how service checks are interleaved. +# Interleaving the service checks allows for a more even +# distribution of service checks and reduced load on remote +# hosts. Setting this value to 1 is equivalent to how versions +# of Nagios previous to 0.0.5 did service checks. Set this +# value to s (smart) for automatic calculation of the interleave +# factor unless you have a specific reason to change it. +# s = Use "smart" interleave factor calculation +# x = Use an interleave factor of x, where x is a +# number greater than or equal to 1. + +service_interleave_factor=s + + + +# HOST INTER-CHECK DELAY METHOD +# This is the method that Nagios should use when initially +# "spreading out" host checks when it starts monitoring. The +# default is to use smart delay calculation, which will try to +# space all host checks out evenly to minimize CPU load. +# Using the dumb setting will cause all checks to be scheduled +# at the same time (with no delay between them)! +# n = None - don't use any delay between checks +# d = Use a "dumb" delay of 1 second between checks +# s = Use "smart" inter-check delay calculation +# x.xx = Use an inter-check delay of x.xx seconds + +host_inter_check_delay_method=s + + + +# MAXIMUM HOST CHECK SPREAD +# This variable determines the timeframe (in minutes) from the +# program start time that an initial check of all hosts should +# be completed. Default is 30 minutes. + +max_host_check_spread=30 + + + +# MAXIMUM CONCURRENT SERVICE CHECKS +# This option allows you to specify the maximum number of +# service checks that can be run in parallel at any given time. +# Specifying a value of 1 for this variable essentially prevents +# any service checks from being parallelized. A value of 0 +# will not restrict the number of concurrent checks that are +# being executed. + +max_concurrent_checks=0 + + + +# HOST AND SERVICE CHECK REAPER FREQUENCY +# This is the frequency (in seconds!) that Nagios will process +# the results of host and service checks. + +check_result_reaper_frequency=10 + + + + +# MAX CHECK RESULT REAPER TIME +# This is the max amount of time (in seconds) that a single +# check result reaper event will be allowed to run before +# returning control back to Nagios so it can perform other +# duties. + +max_check_result_reaper_time=30 + + + + +# CHECK RESULT PATH +# This is directory where Nagios stores the results of host and +# service checks that have not yet been processed. +# +# Note: Make sure that only one instance of Nagios has access +# to this directory! + +check_result_path=/var/lib/nagios3/spool/checkresults + + + + +# MAX CHECK RESULT FILE AGE +# This option determines the maximum age (in seconds) which check +# result files are considered to be valid. Files older than this +# threshold will be mercilessly deleted without further processing. + +max_check_result_file_age=3600 + + + + +# CACHED HOST CHECK HORIZON +# This option determines the maximum amount of time (in seconds) +# that the state of a previous host check is considered current. +# Cached host states (from host checks that were performed more +# recently that the timeframe specified by this value) can immensely +# improve performance in regards to the host check logic. +# Too high of a value for this option may result in inaccurate host +# states being used by Nagios, while a lower value may result in a +# performance hit for host checks. Use a value of 0 to disable host +# check caching. + +cached_host_check_horizon=15 + + + +# CACHED SERVICE CHECK HORIZON +# This option determines the maximum amount of time (in seconds) +# that the state of a previous service check is considered current. +# Cached service states (from service checks that were performed more +# recently that the timeframe specified by this value) can immensely +# improve performance in regards to predictive dependency checks. +# Use a value of 0 to disable service check caching. + +cached_service_check_horizon=15 + + + +# ENABLE PREDICTIVE HOST DEPENDENCY CHECKS +# This option determines whether or not Nagios will attempt to execute +# checks of hosts when it predicts that future dependency logic test +# may be needed. These predictive checks can help ensure that your +# host dependency logic works well. +# Values: +# 0 = Disable predictive checks +# 1 = Enable predictive checks (default) + +enable_predictive_host_dependency_checks=1 + + + +# ENABLE PREDICTIVE SERVICE DEPENDENCY CHECKS +# This option determines whether or not Nagios will attempt to execute +# checks of service when it predicts that future dependency logic test +# may be needed. These predictive checks can help ensure that your +# service dependency logic works well. +# Values: +# 0 = Disable predictive checks +# 1 = Enable predictive checks (default) + +enable_predictive_service_dependency_checks=1 + + + +# SOFT STATE DEPENDENCIES +# This option determines whether or not Nagios will use soft state +# information when checking host and service dependencies. Normally +# Nagios will only use the latest hard host or service state when +# checking dependencies. If you want it to use the latest state (regardless +# of whether its a soft or hard state type), enable this option. +# Values: +# 0 = Don't use soft state dependencies (default) +# 1 = Use soft state dependencies + +soft_state_dependencies=0 + + + +# TIME CHANGE ADJUSTMENT THRESHOLDS +# These options determine when Nagios will react to detected changes +# in system time (either forward or backwards). + +#time_change_threshold=900 + + + +# AUTO-RESCHEDULING OPTION +# This option determines whether or not Nagios will attempt to +# automatically reschedule active host and service checks to +# "smooth" them out over time. This can help balance the load on +# the monitoring server. +# WARNING: THIS IS AN EXPERIMENTAL FEATURE - IT CAN DEGRADE +# PERFORMANCE, RATHER THAN INCREASE IT, IF USED IMPROPERLY + +auto_reschedule_checks=0 + + + +# AUTO-RESCHEDULING INTERVAL +# This option determines how often (in seconds) Nagios will +# attempt to automatically reschedule checks. This option only +# has an effect if the auto_reschedule_checks option is enabled. +# Default is 30 seconds. +# WARNING: THIS IS AN EXPERIMENTAL FEATURE - IT CAN DEGRADE +# PERFORMANCE, RATHER THAN INCREASE IT, IF USED IMPROPERLY + +auto_rescheduling_interval=30 + + + +# AUTO-RESCHEDULING WINDOW +# This option determines the "window" of time (in seconds) that +# Nagios will look at when automatically rescheduling checks. +# Only host and service checks that occur in the next X seconds +# (determined by this variable) will be rescheduled. This option +# only has an effect if the auto_reschedule_checks option is +# enabled. Default is 180 seconds (3 minutes). +# WARNING: THIS IS AN EXPERIMENTAL FEATURE - IT CAN DEGRADE +# PERFORMANCE, RATHER THAN INCREASE IT, IF USED IMPROPERLY + +auto_rescheduling_window=180 + + + +# SLEEP TIME +# This is the number of seconds to sleep between checking for system +# events and service checks that need to be run. + +sleep_time=0.25 + + + +# TIMEOUT VALUES +# These options control how much time Nagios will allow various +# types of commands to execute before killing them off. Options +# are available for controlling maximum time allotted for +# service checks, host checks, event handlers, notifications, the +# ocsp command, and performance data commands. All values are in +# seconds. + +service_check_timeout=60 +host_check_timeout=30 +event_handler_timeout=30 +notification_timeout=30 +ocsp_timeout=5 +perfdata_timeout=5 + + + +# RETAIN STATE INFORMATION +# This setting determines whether or not Nagios will save state +# information for services and hosts before it shuts down. Upon +# startup Nagios will reload all saved service and host state +# information before starting to monitor. This is useful for +# maintaining long-term data on state statistics, etc, but will +# slow Nagios down a bit when it (re)starts. Since its only +# a one-time penalty, I think its well worth the additional +# startup delay. + +retain_state_information=1 + + + +# STATE RETENTION FILE +# This is the file that Nagios should use to store host and +# service state information before it shuts down. The state +# information in this file is also read immediately prior to +# starting to monitor the network when Nagios is restarted. +# This file is used only if the preserve_state_information +# variable is set to 1. + +state_retention_file=/var/lib/nagios3/retention.dat + + + +# RETENTION DATA UPDATE INTERVAL +# This setting determines how often (in minutes) that Nagios +# will automatically save retention data during normal operation. +# If you set this value to 0, Nagios will not save retention +# data at regular interval, but it will still save retention +# data before shutting down or restarting. If you have disabled +# state retention, this option has no effect. + +retention_update_interval=60 + + + +# USE RETAINED PROGRAM STATE +# This setting determines whether or not Nagios will set +# program status variables based on the values saved in the +# retention file. If you want to use retained program status +# information, set this value to 1. If not, set this value +# to 0. + +use_retained_program_state=1 + + + +# USE RETAINED SCHEDULING INFO +# This setting determines whether or not Nagios will retain +# the scheduling info (next check time) for hosts and services +# based on the values saved in the retention file. If you +# If you want to use retained scheduling info, set this +# value to 1. If not, set this value to 0. + +use_retained_scheduling_info=1 + + + +# RETAINED ATTRIBUTE MASKS (ADVANCED FEATURE) +# The following variables are used to specify specific host and +# service attributes that should *not* be retained by Nagios during +# program restarts. +# +# The values of the masks are bitwise ANDs of values specified +# by the "MODATTR_" definitions found in include/common.h. +# For example, if you do not want the current enabled/disabled state +# of flap detection and event handlers for hosts to be retained, you +# would use a value of 24 for the host attribute mask... +# MODATTR_EVENT_HANDLER_ENABLED (8) + MODATTR_FLAP_DETECTION_ENABLED (16) = 24 + +# This mask determines what host attributes are not retained +retained_host_attribute_mask=0 + +# This mask determines what service attributes are not retained +retained_service_attribute_mask=0 + +# These two masks determine what process attributes are not retained. +# There are two masks, because some process attributes have host and service +# options. For example, you can disable active host checks, but leave active +# service checks enabled. +retained_process_host_attribute_mask=0 +retained_process_service_attribute_mask=0 + +# These two masks determine what contact attributes are not retained. +# There are two masks, because some contact attributes have host and +# service options. For example, you can disable host notifications for +# a contact, but leave service notifications enabled for them. +retained_contact_host_attribute_mask=0 +retained_contact_service_attribute_mask=0 + + + +# INTERVAL LENGTH +# This is the seconds per unit interval as used in the +# host/contact/service configuration files. Setting this to 60 means +# that each interval is one minute long (60 seconds). Other settings +# have not been tested much, so your mileage is likely to vary... + +interval_length=60 + + + +# AGGRESSIVE HOST CHECKING OPTION +# If you don't want to turn on aggressive host checking features, set +# this value to 0 (the default). Otherwise set this value to 1 to +# enable the aggressive check option. Read the docs for more info +# on what aggressive host check is or check out the source code in +# base/checks.c + +use_aggressive_host_checking=0 + + + +# SERVICE CHECK EXECUTION OPTION +# This determines whether or not Nagios will actively execute +# service checks when it initially starts. If this option is +# disabled, checks are not actively made, but Nagios can still +# receive and process passive check results that come in. Unless +# you're implementing redundant hosts or have a special need for +# disabling the execution of service checks, leave this enabled! +# Values: 1 = enable checks, 0 = disable checks + +execute_service_checks=1 + + + +# PASSIVE SERVICE CHECK ACCEPTANCE OPTION +# This determines whether or not Nagios will accept passive +# service checks results when it initially (re)starts. +# Values: 1 = accept passive checks, 0 = reject passive checks + +accept_passive_service_checks=1 + + + +# HOST CHECK EXECUTION OPTION +# This determines whether or not Nagios will actively execute +# host checks when it initially starts. If this option is +# disabled, checks are not actively made, but Nagios can still +# receive and process passive check results that come in. Unless +# you're implementing redundant hosts or have a special need for +# disabling the execution of host checks, leave this enabled! +# Values: 1 = enable checks, 0 = disable checks + +execute_host_checks=1 + + + +# PASSIVE HOST CHECK ACCEPTANCE OPTION +# This determines whether or not Nagios will accept passive +# host checks results when it initially (re)starts. +# Values: 1 = accept passive checks, 0 = reject passive checks + +accept_passive_host_checks=1 + + + +# NOTIFICATIONS OPTION +# This determines whether or not Nagios will sent out any host or +# service notifications when it is initially (re)started. +# Values: 1 = enable notifications, 0 = disable notifications + +enable_notifications=1 + + + +# EVENT HANDLER USE OPTION +# This determines whether or not Nagios will run any host or +# service event handlers when it is initially (re)started. Unless +# you're implementing redundant hosts, leave this option enabled. +# Values: 1 = enable event handlers, 0 = disable event handlers + +enable_event_handlers=1 + + + +# PROCESS PERFORMANCE DATA OPTION +# This determines whether or not Nagios will process performance +# data returned from service and host checks. If this option is +# enabled, host performance data will be processed using the +# host_perfdata_command (defined below) and service performance +# data will be processed using the service_perfdata_command (also +# defined below). Read the HTML docs for more information on +# performance data. +# Values: 1 = process performance data, 0 = do not process performance data + +process_performance_data=0 + + + +# HOST AND SERVICE PERFORMANCE DATA PROCESSING COMMANDS +# These commands are run after every host and service check is +# performed. These commands are executed only if the +# enable_performance_data option (above) is set to 1. The command +# argument is the short name of a command definition that you +# define in your host configuration file. Read the HTML docs for +# more information on performance data. + +#host_perfdata_command=process-host-perfdata +#service_perfdata_command=process-service-perfdata + + + +# HOST AND SERVICE PERFORMANCE DATA FILES +# These files are used to store host and service performance data. +# Performance data is only written to these files if the +# enable_performance_data option (above) is set to 1. + +#host_perfdata_file=/tmp/host-perfdata +#service_perfdata_file=/tmp/service-perfdata + + + +# HOST AND SERVICE PERFORMANCE DATA FILE TEMPLATES +# These options determine what data is written (and how) to the +# performance data files. The templates may contain macros, special +# characters (\t for tab, \r for carriage return, \n for newline) +# and plain text. A newline is automatically added after each write +# to the performance data file. Some examples of what you can do are +# shown below. + +#host_perfdata_file_template=[HOSTPERFDATA]\t$TIMET$\t$HOSTNAME$\t$HOSTEXECUTIONTIME$\t$HOSTOUTPUT$\t$HOSTPERFDATA$ +#service_perfdata_file_template=[SERVICEPERFDATA]\t$TIMET$\t$HOSTNAME$\t$SERVICEDESC$\t$SERVICEEXECUTIONTIME$\t$SERVICELATENCY$\t$SERVICEOUTPUT$\t$SERVICEPERFDATA$ + + + +# HOST AND SERVICE PERFORMANCE DATA FILE MODES +# This option determines whether or not the host and service +# performance data files are opened in write ("w") or append ("a") +# mode. If you want to use named pipes, you should use the special +# pipe ("p") mode which avoid blocking at startup, otherwise you will +# likely want the defult append ("a") mode. + +#host_perfdata_file_mode=a +#service_perfdata_file_mode=a + + + +# HOST AND SERVICE PERFORMANCE DATA FILE PROCESSING INTERVAL +# These options determine how often (in seconds) the host and service +# performance data files are processed using the commands defined +# below. A value of 0 indicates the files should not be periodically +# processed. + +#host_perfdata_file_processing_interval=0 +#service_perfdata_file_processing_interval=0 + + + +# HOST AND SERVICE PERFORMANCE DATA FILE PROCESSING COMMANDS +# These commands are used to periodically process the host and +# service performance data files. The interval at which the +# processing occurs is determined by the options above. + +#host_perfdata_file_processing_command=process-host-perfdata-file +#service_perfdata_file_processing_command=process-service-perfdata-file + + + +# OBSESS OVER SERVICE CHECKS OPTION +# This determines whether or not Nagios will obsess over service +# checks and run the ocsp_command defined below. Unless you're +# planning on implementing distributed monitoring, do not enable +# this option. Read the HTML docs for more information on +# implementing distributed monitoring. +# Values: 1 = obsess over services, 0 = do not obsess (default) + +obsess_over_services=0 + + + +# OBSESSIVE COMPULSIVE SERVICE PROCESSOR COMMAND +# This is the command that is run for every service check that is +# processed by Nagios. This command is executed only if the +# obsess_over_services option (above) is set to 1. The command +# argument is the short name of a command definition that you +# define in your host configuration file. Read the HTML docs for +# more information on implementing distributed monitoring. + +#ocsp_command=somecommand + + + +# OBSESS OVER HOST CHECKS OPTION +# This determines whether or not Nagios will obsess over host +# checks and run the ochp_command defined below. Unless you're +# planning on implementing distributed monitoring, do not enable +# this option. Read the HTML docs for more information on +# implementing distributed monitoring. +# Values: 1 = obsess over hosts, 0 = do not obsess (default) + +obsess_over_hosts=0 + + + +# OBSESSIVE COMPULSIVE HOST PROCESSOR COMMAND +# This is the command that is run for every host check that is +# processed by Nagios. This command is executed only if the +# obsess_over_hosts option (above) is set to 1. The command +# argument is the short name of a command definition that you +# define in your host configuration file. Read the HTML docs for +# more information on implementing distributed monitoring. + +#ochp_command=somecommand + + + +# TRANSLATE PASSIVE HOST CHECKS OPTION +# This determines whether or not Nagios will translate +# DOWN/UNREACHABLE passive host check results into their proper +# state for this instance of Nagios. This option is useful +# if you have distributed or failover monitoring setup. In +# these cases your other Nagios servers probably have a different +# "view" of the network, with regards to the parent/child relationship +# of hosts. If a distributed monitoring server thinks a host +# is DOWN, it may actually be UNREACHABLE from the point of +# this Nagios instance. Enabling this option will tell Nagios +# to translate any DOWN or UNREACHABLE host states it receives +# passively into the correct state from the view of this server. +# Values: 1 = perform translation, 0 = do not translate (default) + +translate_passive_host_checks=0 + + + +# PASSIVE HOST CHECKS ARE SOFT OPTION +# This determines whether or not Nagios will treat passive host +# checks as being HARD or SOFT. By default, a passive host check +# result will put a host into a HARD state type. This can be changed +# by enabling this option. +# Values: 0 = passive checks are HARD, 1 = passive checks are SOFT + +passive_host_checks_are_soft=0 + + + +# ORPHANED HOST/SERVICE CHECK OPTIONS +# These options determine whether or not Nagios will periodically +# check for orphaned host service checks. Since service checks are +# not rescheduled until the results of their previous execution +# instance are processed, there exists a possibility that some +# checks may never get rescheduled. A similar situation exists for +# host checks, although the exact scheduling details differ a bit +# from service checks. Orphaned checks seem to be a rare +# problem and should not happen under normal circumstances. +# If you have problems with service checks never getting +# rescheduled, make sure you have orphaned service checks enabled. +# Values: 1 = enable checks, 0 = disable checks + +check_for_orphaned_services=1 +check_for_orphaned_hosts=1 + + + +# SERVICE FRESHNESS CHECK OPTION +# This option determines whether or not Nagios will periodically +# check the "freshness" of service results. Enabling this option +# is useful for ensuring passive checks are received in a timely +# manner. +# Values: 1 = enabled freshness checking, 0 = disable freshness checking + +check_service_freshness=1 + + + +# SERVICE FRESHNESS CHECK INTERVAL +# This setting determines how often (in seconds) Nagios will +# check the "freshness" of service check results. If you have +# disabled service freshness checking, this option has no effect. + +service_freshness_check_interval=60 + + + +# HOST FRESHNESS CHECK OPTION +# This option determines whether or not Nagios will periodically +# check the "freshness" of host results. Enabling this option +# is useful for ensuring passive checks are received in a timely +# manner. +# Values: 1 = enabled freshness checking, 0 = disable freshness checking + +check_host_freshness=0 + + + +# HOST FRESHNESS CHECK INTERVAL +# This setting determines how often (in seconds) Nagios will +# check the "freshness" of host check results. If you have +# disabled host freshness checking, this option has no effect. + +host_freshness_check_interval=60 + + + + +# ADDITIONAL FRESHNESS THRESHOLD LATENCY +# This setting determines the number of seconds that Nagios +# will add to any host and service freshness thresholds that +# it calculates (those not explicitly specified by the user). + +additional_freshness_latency=15 + + + + +# FLAP DETECTION OPTION +# This option determines whether or not Nagios will try +# and detect hosts and services that are "flapping". +# Flapping occurs when a host or service changes between +# states too frequently. When Nagios detects that a +# host or service is flapping, it will temporarily suppress +# notifications for that host/service until it stops +# flapping. Flap detection is very experimental, so read +# the HTML documentation before enabling this feature! +# Values: 1 = enable flap detection +# 0 = disable flap detection (default) + +enable_flap_detection=1 + + + +# FLAP DETECTION THRESHOLDS FOR HOSTS AND SERVICES +# Read the HTML documentation on flap detection for +# an explanation of what this option does. This option +# has no effect if flap detection is disabled. + +low_service_flap_threshold=5.0 +high_service_flap_threshold=20.0 +low_host_flap_threshold=5.0 +high_host_flap_threshold=20.0 + + + +# DATE FORMAT OPTION +# This option determines how short dates are displayed. Valid options +# include: +# us (MM-DD-YYYY HH:MM:SS) +# euro (DD-MM-YYYY HH:MM:SS) +# iso8601 (YYYY-MM-DD HH:MM:SS) +# strict-iso8601 (YYYY-MM-DDTHH:MM:SS) +# + +date_format=iso8601 + + + + +# TIMEZONE OFFSET +# This option is used to override the default timezone that this +# instance of Nagios runs in. If not specified, Nagios will use +# the system configured timezone. +# +# NOTE: In order to display the correct timezone in the CGIs, you +# will also need to alter the Apache directives for the CGI path +# to include your timezone. Example: +# +# <Directory "/usr/local/nagios/sbin/"> +# SetEnv TZ "Australia/Brisbane" +# ... +# </Directory> + +#use_timezone=US/Mountain +#use_timezone=Australia/Brisbane + + + + +# P1.PL FILE LOCATION +# This value determines where the p1.pl perl script (used by the +# embedded Perl interpreter) is located. If you didn't compile +# Nagios with embedded Perl support, this option has no effect. + +p1_file=/usr/lib/nagios3/p1.pl + + + +# EMBEDDED PERL INTERPRETER OPTION +# This option determines whether or not the embedded Perl interpreter +# will be enabled during runtime. This option has no effect if Nagios +# has not been compiled with support for embedded Perl. +# Values: 0 = disable interpreter, 1 = enable interpreter + +enable_embedded_perl=1 + + + +# EMBEDDED PERL USAGE OPTION +# This option determines whether or not Nagios will process Perl plugins +# and scripts with the embedded Perl interpreter if the plugins/scripts +# do not explicitly indicate whether or not it is okay to do so. Read +# the HTML documentation on the embedded Perl interpreter for more +# information on how this option works. + +use_embedded_perl_implicitly=1 + + + +# ILLEGAL OBJECT NAME CHARACTERS +# This option allows you to specify illegal characters that cannot +# be used in host names, service descriptions, or names of other +# object types. + +illegal_object_name_chars=`~!$%^&*|'"<>?,()= + + + +# ILLEGAL MACRO OUTPUT CHARACTERS +# This option allows you to specify illegal characters that are +# stripped from macros before being used in notifications, event +# handlers, etc. This DOES NOT affect macros used in service or +# host check commands. +# The following macros are stripped of the characters you specify: +# $HOSTOUTPUT$ +# $HOSTPERFDATA$ +# $HOSTACKAUTHOR$ +# $HOSTACKCOMMENT$ +# $SERVICEOUTPUT$ +# $SERVICEPERFDATA$ +# $SERVICEACKAUTHOR$ +# $SERVICEACKCOMMENT$ + +illegal_macro_output_chars=`~$&|'"<> + + + +# REGULAR EXPRESSION MATCHING +# This option controls whether or not regular expression matching +# takes place in the object config files. Regular expression +# matching is used to match host, hostgroup, service, and service +# group names/descriptions in some fields of various object types. +# Values: 1 = enable regexp matching, 0 = disable regexp matching + +use_regexp_matching=0 + + + +# "TRUE" REGULAR EXPRESSION MATCHING +# This option controls whether or not "true" regular expression +# matching takes place in the object config files. This option +# only has an effect if regular expression matching is enabled +# (see above). If this option is DISABLED, regular expression +# matching only occurs if a string contains wildcard characters +# (* and ?). If the option is ENABLED, regexp matching occurs +# all the time (which can be annoying). +# Values: 1 = enable true matching, 0 = disable true matching + +use_true_regexp_matching=0 + + + +# ADMINISTRATOR EMAIL/PAGER ADDRESSES +# The email and pager address of a global administrator (likely you). +# Nagios never uses these values itself, but you can access them by +# using the $ADMINEMAIL$ and $ADMINPAGER$ macros in your notification +# commands. + +admin_email=root@localhost +admin_pager=pageroot@localhost + + + +# DAEMON CORE DUMP OPTION +# This option determines whether or not Nagios is allowed to create +# a core dump when it runs as a daemon. Note that it is generally +# considered bad form to allow this, but it may be useful for +# debugging purposes. Enabling this option doesn't guarantee that +# a core file will be produced, but that's just life... +# Values: 1 - Allow core dumps +# 0 - Do not allow core dumps (default) + +daemon_dumps_core=0 + + + +# LARGE INSTALLATION TWEAKS OPTION +# This option determines whether or not Nagios will take some shortcuts +# which can save on memory and CPU usage in large Nagios installations. +# Read the documentation for more information on the benefits/tradeoffs +# of enabling this option. +# Values: 1 - Enabled tweaks +# 0 - Disable tweaks (default) + +use_large_installation_tweaks=0 + + + +# ENABLE ENVIRONMENT MACROS +# This option determines whether or not Nagios will make all standard +# macros available as environment variables when host/service checks +# and system commands (event handlers, notifications, etc.) are +# executed. Enabling this option can cause performance issues in +# large installations, as it will consume a bit more memory and (more +# importantly) consume more CPU. +# Values: 1 - Enable environment variable macros (default) +# 0 - Disable environment variable macros + +enable_environment_macros=1 + + + +# CHILD PROCESS MEMORY OPTION +# This option determines whether or not Nagios will free memory in +# child processes (processed used to execute system commands and host/ +# service checks). If you specify a value here, it will override +# program defaults. +# Value: 1 - Free memory in child processes +# 0 - Do not free memory in child processes + +#free_child_process_memory=1 + + + +# CHILD PROCESS FORKING BEHAVIOR +# This option determines how Nagios will fork child processes +# (used to execute system commands and host/service checks). Normally +# child processes are fork()ed twice, which provides a very high level +# of isolation from problems. Fork()ing once is probably enough and will +# save a great deal on CPU usage (in large installs), so you might +# want to consider using this. If you specify a value here, it will +# program defaults. +# Value: 1 - Child processes fork() twice +# 0 - Child processes fork() just once + +#child_processes_fork_twice=1 + + + +# DEBUG LEVEL +# This option determines how much (if any) debugging information will +# be written to the debug file. OR values together to log multiple +# types of information. +# Values: +# -1 = Everything +# 0 = Nothing +# 1 = Functions +# 2 = Configuration +# 4 = Process information +# 8 = Scheduled events +# 16 = Host/service checks +# 32 = Notifications +# 64 = Event broker +# 128 = External commands +# 256 = Commands +# 512 = Scheduled downtime +# 1024 = Comments +# 2048 = Macros + +debug_level=0 + + + +# DEBUG VERBOSITY +# This option determines how verbose the debug log out will be. +# Values: 0 = Brief output +# 1 = More detailed +# 2 = Very detailed + +debug_verbosity=1 + + + +# DEBUG FILE +# This option determines where Nagios should write debugging information. + +debug_file=/var/lib/nagios3/nagios.debug + + + +# MAX DEBUG FILE SIZE +# This option determines the maximum size (in bytes) of the debug file. If +# the file grows larger than this size, it will be renamed with a .old +# extension. If a file already exists with a .old extension it will +# automatically be deleted. This helps ensure your disk space usage doesn't +# get out of control when debugging Nagios. + +max_debug_file_size=1000000 + +process_performance_data=1 +service_perfdata_file=/var/lib/nagios3/service-perfdata +service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$ +service_perfdata_file_mode=a +service_perfdata_file_processing_interval=15 +service_perfdata_file_processing_command=process-service-perfdata-file-pnp4nagios-bulk-npcd +host_perfdata_file=/var/lib/nagios3/host-perfdata +host_perfdata_file_template=DATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$ +host_perfdata_file_mode=a +host_perfdata_file_processing_interval=15 +host_perfdata_file_processing_command=process-host-perfdata-file-pnp4nagios-bulk-npcd + diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log new file mode 100755 index 00000000..47569388 --- /dev/null +++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log @@ -0,0 +1,85 @@ +#!/bin/sh +# +# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh +# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe +# class + +set -e + +usage() +{ +cat << EOF +usage: $0 -w <sec> -c <sec> -r <regexp> -f <filename> + +OPTIONS: + -h Show this message + -r <regex> regex to grep for + -f <file> logfile to search in + -w <sec> warning state after X seconds + -c <sec> critical state after x seconds + +example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600 +EOF +} + + +. /usr/lib/nagios/plugins/utils.sh + + +warn=0 +crit=0 +log='' +regex='' + +set -- $(getopt hr:f:w:c: "$@") +while [ $# -gt 0 ] +do + case "$1" in + (-h) usage; exit 0 ;; + (-f) log="$2"; shift;; + (-r) regex="$2"; shift;; + (-w) warn="$2"; shift;; + (-c) crit="$2"; shift;; + (--) shift; break;; + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift +done + +[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN) +[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN) + +lastmsg=$(tac $log | grep -i $regex | head -1 | sed 's/ / /g' | cut -d' ' -f 1-3) + +if [ -z "$lastmsg" ] +then + summary="\"$regex\" in $log was not found" + state=$STATE_CRITICAL + state_text='CRITICAL' + diff_sec=0 +else + lastmsg_sec=$(date '+%s' -d "$lastmsg") + now_sec=$(date '+%s') + + diff_sec=$(($now_sec - $lastmsg_sec)) + + if [ $diff_sec -lt $warn ]; then + state=$STATE_OK + state_text='OK' + elif [ $diff_sec -lt $crit ]; then + state=$STATE_WARNING + state_text='WARNING' + else + state=$STATE_CRITICAL + state_text='CRITICAL' + fi + + summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago" +fi + +# check_mk_agent output +# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary" + +echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;" +exit $state diff --git a/puppet/modules/site_nagios/manifests/add_host_services.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp new file mode 100644 index 00000000..bd968e6f --- /dev/null +++ b/puppet/modules/site_nagios/manifests/add_host_services.pp @@ -0,0 +1,32 @@ +define site_nagios::add_host_services ( + $domain_full_suffix, + $domain_internal, + $domain_internal_suffix, + $ip_address, + $services, + $ssh_port, + $environment, + $openvpn_gateway_address='', + ) { + + $nagios_hostname = $domain_internal + + # Add Nagios service + + # First, we need to turn the serice array into hash, using a "hash template" + # see https://github.com/ashak/puppet-resource-looping + $nagios_service_hashpart = { + 'hostname' => $nagios_hostname, + 'ip_address' => $ip_address, + 'openvpn_gw' => $openvpn_gateway_address, + 'environment' => $environment + } + $dynamic_parameters = { + 'service' => '%s' + } + $nagios_servicename = "${nagios_hostname}_%s" + + $nagios_service_hash = create_resources_hash_from($nagios_servicename, $services, $nagios_service_hashpart, $dynamic_parameters) + + create_resources ( site_nagios::add_service, $nagios_service_hash ) +} diff --git a/puppet/modules/site_nagios/manifests/add_service.pp b/puppet/modules/site_nagios/manifests/add_service.pp new file mode 100644 index 00000000..72cd038a --- /dev/null +++ b/puppet/modules/site_nagios/manifests/add_service.pp @@ -0,0 +1,32 @@ +define site_nagios::add_service ( + $hostname, $ip_address, $service, $environment, $openvpn_gw = '') { + + $ssh = hiera_hash('ssh') + $ssh_port = $ssh['port'] + + case $service { + 'webapp': { + nagios_service { + "${name}_ssh": + use => 'generic-service', + check_command => "check_ssh_port!${ssh_port}", + service_description => 'SSH', + host_name => $hostname, + contact_groups => $environment; + "${name}_cert": + use => 'generic-service', + check_command => 'check_https_cert', + service_description => 'Website Certificate', + host_name => $hostname, + contact_groups => $environment; + "${name}_website": + use => 'generic-service', + check_command => 'check_https', + service_description => 'Website', + host_name => $hostname, + contact_groups => $environment; + } + } + default: {} + } +} diff --git a/puppet/modules/site_nagios/manifests/init.pp b/puppet/modules/site_nagios/manifests/init.pp new file mode 100644 index 00000000..f91bfc26 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/init.pp @@ -0,0 +1,13 @@ +# setup nagios on monitoring node +class site_nagios { + tag 'leap_service' + + include site_config::default + + Class['site_config::default'] -> Class['site_nagios'] + + include site_nagios::server + + # remove leftovers on monitoring nodes + include site_config::remove::monitoring +} diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp new file mode 100644 index 00000000..90a01cfb --- /dev/null +++ b/puppet/modules/site_nagios/manifests/plugins.pp @@ -0,0 +1,16 @@ +# Deploy generic plugins useful to all nodes +# nagios::plugin won't work to deploy a plugin +# because it complains with: +# Could not find dependency Package[nagios-plugins] … +# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18 +class site_nagios::plugins { + + file { [ + '/usr/local/lib', '/usr/local/lib/nagios', + '/usr/local/lib/nagios/plugins' ]: + ensure => directory; + '/usr/local/lib/nagios/plugins/check_last_regex_in_log': + source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log', + mode => '0755'; + } +} diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp new file mode 100644 index 00000000..6537124d --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server.pp @@ -0,0 +1,97 @@ +# configures nagios on monitoring node +# lint:ignore:inherits_across_namespaces +class site_nagios::server inherits nagios::base { +# lint:endignore + + $nagios_hiera = hiera('nagios') + $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw']) + $nagios_hosts = $nagios_hiera['hosts'] + $nagios_contacts = hiera('contacts') + $environment = $nagios_hiera['environments'] + + include nagios::base + include nagios::defaults::commands + include nagios::defaults::templates + include nagios::defaults::timeperiods + include nagios::pnp4nagios + include nagios::pnp4nagios::popup + + class { 'nagios': + # don't manage apache class from nagios, cause we already include + # it in site_apache::common + httpd => 'absent', + allow_external_cmd => true, + storeconfigs => false, + } + + # Delete nagios config files provided by packages + # These don't get parsed by nagios.conf, but are + # still irritating duplicates to the real config + # files deployed by puppet in /etc/nagios3/ + file { [ + '/etc/nagios3/conf.d/contacts_nagios2.cfg', + '/etc/nagios3/conf.d/extinfo_nagios2.cfg', + '/etc/nagios3/conf.d/generic-host_nagios2.cfg', + '/etc/nagios3/conf.d/generic-service_nagios2.cfg', + '/etc/nagios3/conf.d/hostgroups_nagios2.cfg', + '/etc/nagios3/conf.d/localhost_nagios2.cfg', + '/etc/nagios3/conf.d/pnp4nagios.cfg', + '/etc/nagios3/conf.d/services_nagios2.cfg', + '/etc/nagios3/conf.d/timeperiods_nagios2.cfg' ]: + ensure => absent; + } + + # deploy apache nagios3 config + # until https://gitlab.com/shared-puppet-modules-group/apache/issues/11 + # is not fixed, we need to manually deploy the config file + file { + '/etc/apache2/conf-available/nagios3.conf': + ensure => present, + source => 'puppet:///modules/nagios/configs/apache2.conf', + require => [ Package['nagios3'], Package['apache2'] ]; + '/etc/apache2/conf-enabled/nagios3.conf': + ensure => link, + target => '/etc/apache2/conf-available/nagios3.conf', + require => [ Package['nagios3'], Package['apache2'] ]; + } + + include site_apache::common + include site_webapp::common_vhost + include apache::module::headers + + File['nagios_htpasswd'] { + source => undef, + content => "nagiosadmin:${nagiosadmin_pw}", + mode => '0640', + } + + + # deploy serverside plugins + file { '/usr/lib/nagios/plugins/check_openvpn_server.pl': + source => 'puppet:///modules/nagios/plugins/check_openvpn_server.pl', + mode => '0755', + owner => 'nagios', + group => 'nagios', + require => Package['nagios-plugins']; + } + + create_resources ( site_nagios::add_host_services, $nagios_hosts ) + + include site_nagios::server::apache + include site_check_mk::server + include site_shorewall::monitor + include site_nagios::server::icli + + augeas { + 'logrotate_nagios': + context => '/files/etc/logrotate.d/nagios/rule', + changes => [ 'set file /var/log/nagios3/nagios.log', 'set rotate 7', + 'set schedule daily', 'set compress compress', + 'set missingok missingok', 'set ifempty notifempty', + 'set copytruncate copytruncate' ] + } + + create_resources ( site_nagios::server::hostgroup, $environment ) + create_resources ( site_nagios::server::contactgroup, $environment ) + create_resources ( site_nagios::server::add_contacts, $environment ) +} diff --git a/puppet/modules/site_nagios/manifests/server/add_contacts.pp b/puppet/modules/site_nagios/manifests/server/add_contacts.pp new file mode 100644 index 00000000..b5c6f0a5 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/add_contacts.pp @@ -0,0 +1,18 @@ +# configure a nagios_contact +define site_nagios::server::add_contacts ($contact_emails) { + + $environment = $name + + nagios_contact { + $environment: + alias => $environment, + service_notification_period => '24x7', + host_notification_period => '24x7', + service_notification_options => 'w,u,c,r', + host_notification_options => 'd,r', + service_notification_commands => 'notify-service-by-email', + host_notification_commands => 'notify-host-by-email', + email => join($contact_emails, ', '), + require => Package['nagios'] + } +} diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp new file mode 100644 index 00000000..82962e89 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -0,0 +1,25 @@ +# set up apache for nagios +class site_nagios::server::apache { + + include x509::variables + + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + include apache::module::authn_file + # "AuthUserFile" + include apache::module::authz_user + # "AuthType Basic" + include apache::module::auth_basic + # "DirectoryIndex" + include apache::module::dir + include apache::module::php5 + include apache::module::cgi + + # apache >= 2.4, debian jessie + if ( $::lsbdistcodename == 'jessie' ) { + include apache::module::authn_core + } + +} diff --git a/puppet/modules/site_nagios/manifests/server/contactgroup.pp b/puppet/modules/site_nagios/manifests/server/contactgroup.pp new file mode 100644 index 00000000..5e60dd06 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/contactgroup.pp @@ -0,0 +1,8 @@ +# configure a contactgroup +define site_nagios::server::contactgroup ($contact_emails) { + + nagios_contactgroup { $name: + members => $name, + require => Package['nagios'] + } +} diff --git a/puppet/modules/site_nagios/manifests/server/hostgroup.pp b/puppet/modules/site_nagios/manifests/server/hostgroup.pp new file mode 100644 index 00000000..0692fced --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/hostgroup.pp @@ -0,0 +1,7 @@ +# create a nagios hostsgroup +define site_nagios::server::hostgroup ($contact_emails) { + nagios_hostgroup { $name: + ensure => present, + require => Package['nagios'] + } +} diff --git a/puppet/modules/site_nagios/manifests/server/icli.pp b/puppet/modules/site_nagios/manifests/server/icli.pp new file mode 100644 index 00000000..26fba725 --- /dev/null +++ b/puppet/modules/site_nagios/manifests/server/icli.pp @@ -0,0 +1,26 @@ +# Install icli package and configure ncli aliases +class site_nagios::server::icli { + $nagios_hiera = hiera('nagios') + $environments = $nagios_hiera['environments'] + + package { 'icli': + ensure => installed; + } + + file { '/root/.bashrc': + ensure => present; + } + + file_line { 'icli aliases': + path => '/root/.bashrc', + line => 'source /root/.icli_aliases'; + } + + file { '/root/.icli_aliases': + content => template("${module_name}/icli_aliases.erb"), + mode => '0644', + owner => root, + group => 0, + require => Package['icli']; + } +}
\ No newline at end of file diff --git a/puppet/modules/site_nagios/templates/icli_aliases.erb b/puppet/modules/site_nagios/templates/icli_aliases.erb new file mode 100644 index 00000000..bcb2abb0 --- /dev/null +++ b/puppet/modules/site_nagios/templates/icli_aliases.erb @@ -0,0 +1,7 @@ +alias ncli='icli -c /var/cache/nagios3/objects.cache -f /var/cache/nagios3/status.dat -F /var/lib/nagios3/rw/nagios.cmd' +alias ncli_problems='ncli -z '!o,!A'' + +<% @environments.keys.sort.each do |env_name| %> +alias ncli_<%= env_name %>='ncli -z '!o,!A' -g <%= env_name %>' +alias ncli_<%= env_name %>_recheck='ncli -s Check_MK -g <%= env_name %> -a R' +<% end -%> diff --git a/puppet/modules/site_nickserver/manifests/init.pp b/puppet/modules/site_nickserver/manifests/init.pp new file mode 100644 index 00000000..eb4415e7 --- /dev/null +++ b/puppet/modules/site_nickserver/manifests/init.pp @@ -0,0 +1,178 @@ +# +# TODO: currently, this is dependent on some things that are set up in +# site_webapp +# +# (1) HAProxy -> couchdb +# (2) Apache +# +# It would be good in the future to make nickserver installable independently of +# site_webapp. +# + +class site_nickserver { + tag 'leap_service' + Class['site_config::default'] -> Class['site_nickserver'] + + include site_config::ruby::dev + + # + # VARIABLES + # + + $nickserver = hiera('nickserver') + $nickserver_domain = $nickserver['domain'] + $couchdb_user = $nickserver['couchdb_nickserver_user']['username'] + $couchdb_password = $nickserver['couchdb_nickserver_user']['password'] + + # the port that public connects to (should be 6425) + $nickserver_port = $nickserver['port'] + # the port that nickserver is actually running on + $nickserver_local_port = '64250' + + # couchdb is available on localhost via haproxy, which is bound to 4096. + $couchdb_host = 'localhost' + # See site_webapp/templates/haproxy_couchdb.cfg.erg + $couchdb_port = '4096' + + $sources = hiera('sources') + + # temporarily for now: + $domain = hiera('domain') + $address_domain = $domain['full_suffix'] + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + # + # USER AND GROUP + # + + group { 'nickserver': + ensure => present, + allowdupe => false; + } + + user { 'nickserver': + ensure => present, + allowdupe => false, + gid => 'nickserver', + home => '/srv/leap/nickserver', + require => Group['nickserver']; + } + + vcsrepo { '/srv/leap/nickserver': + ensure => present, + revision => $sources['nickserver']['revision'], + provider => $sources['nickserver']['type'], + source => $sources['nickserver']['source'], + owner => 'nickserver', + group => 'nickserver', + require => [ User['nickserver'], Group['nickserver'] ], + notify => Exec['nickserver_bundler_update']; + } + + exec { 'nickserver_bundler_update': + cwd => '/srv/leap/nickserver', + command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle"', + unless => '/usr/bin/bundle check', + user => 'nickserver', + timeout => 600, + require => [ + Class['bundler::install'], Vcsrepo['/srv/leap/nickserver'], + Package['libssl-dev'], Class['site_config::ruby::dev'] ], + + notify => Service['nickserver']; + } + + # + # NICKSERVER CONFIG + # + + file { '/etc/nickserver.yml': + content => template('site_nickserver/nickserver.yml.erb'), + owner => nickserver, + group => nickserver, + mode => '0600', + notify => Service['nickserver']; + } + + # + # NICKSERVER DAEMON + # + + file { + '/usr/bin/nickserver': + ensure => link, + target => '/srv/leap/nickserver/bin/nickserver', + require => Vcsrepo['/srv/leap/nickserver']; + + '/etc/init.d/nickserver': + owner => root, + group => 0, + mode => '0755', + source => '/srv/leap/nickserver/dist/debian-init-script', + require => Vcsrepo['/srv/leap/nickserver']; + } + + # register initscript at systemd on nodes newer than wheezy + # see https://leap.se/code/issues/7614 + case $::operatingsystemrelease { + /^7.*/: { } + default: { + exec { 'register_systemd_nickserver': + refreshonly => true, + command => '/bin/systemctl enable nickserver', + subscribe => File['/etc/init.d/nickserver'], + before => Service['nickserver']; + } + } + } + + service { 'nickserver': + ensure => running, + enable => true, + hasrestart => true, + hasstatus => true, + require => [ + File['/etc/init.d/nickserver'], + File['/usr/bin/nickserver'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + # + # FIREWALL + # poke a hole in the firewall to allow nickserver requests + # + + file { '/etc/shorewall/macro.nickserver': + content => "PARAM - - tcp ${nickserver_port}", + notify => Service['shorewall'], + require => Package['shorewall']; + } + + shorewall::rule { 'net2fw-nickserver': + source => 'net', + destination => '$FW', + action => 'nickserver(ACCEPT)', + order => 200; + } + + # + # APACHE REVERSE PROXY + # nickserver doesn't speak TLS natively, let Apache handle that. + # + + apache::module { + 'proxy': ensure => present; + 'proxy_http': ensure => present + } + + apache::vhost::file { + 'nickserver': + content => template('site_nickserver/nickserver-proxy.conf.erb') + } + +} diff --git a/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb new file mode 100644 index 00000000..8f59fe38 --- /dev/null +++ b/puppet/modules/site_nickserver/templates/nickserver-proxy.conf.erb @@ -0,0 +1,19 @@ +# +# Apache reverse proxy configuration for the Nickserver +# + +Listen 0.0.0.0:<%= @nickserver_port -%> + +<VirtualHost *:<%= @nickserver_port -%>> + ServerName <%= @nickserver_domain %> + ServerAlias <%= @address_domain %> + + SSLCACertificatePath /etc/ssl/certs + SSLCertificateKeyFile <%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.key + SSLCertificateFile <%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.crt + + Include include.d/ssl_common.inc + + ProxyPass / http://localhost:<%= @nickserver_local_port %>/ + ProxyPreserveHost On # preserve Host header in HTTP request +</VirtualHost> diff --git a/puppet/modules/site_nickserver/templates/nickserver.yml.erb b/puppet/modules/site_nickserver/templates/nickserver.yml.erb new file mode 100644 index 00000000..e717cbaa --- /dev/null +++ b/puppet/modules/site_nickserver/templates/nickserver.yml.erb @@ -0,0 +1,19 @@ +# +# configuration for nickserver. +# + +domain: "<%= @address_domain %>" + +couch_host: "<%= @couchdb_host %>" +couch_port: <%= @couchdb_port %> +couch_database: "identities" +couch_user: "<%= @couchdb_user %>" +couch_password: "<%= @couchdb_password %>" + +hkp_url: "https://hkps.pool.sks-keyservers.net:/pks/lookup" + +user: "nickserver" +port: <%= @nickserver_local_port %> +pid_file: "/var/run/nickserver" +log_file: "/var/log/nickserver.log" + diff --git a/puppet/modules/site_obfsproxy/README b/puppet/modules/site_obfsproxy/README new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/puppet/modules/site_obfsproxy/README diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp new file mode 100644 index 00000000..2ed5ec9e --- /dev/null +++ b/puppet/modules/site_obfsproxy/manifests/init.pp @@ -0,0 +1,38 @@ +class site_obfsproxy { + tag 'leap_service' + Class['site_config::default'] -> Class['site_obfsproxy'] + + $transport = 'scramblesuit' + + $obfsproxy = hiera('obfsproxy') + $scramblesuit = $obfsproxy['scramblesuit'] + $scram_pass = $scramblesuit['password'] + $scram_port = $scramblesuit['port'] + $dest_ip = $obfsproxy['gateway_address'] + $dest_port = '443' + + if member($::services, 'openvpn') { + $openvpn = hiera('openvpn') + $bind_address = $openvpn['gateway_address'] + } + elsif member($::services, 'obfsproxy') { + $bind_address = hiera('ip_address') + } + + include site_config::default + + class { 'obfsproxy': + transport => $transport, + bind_address => $bind_address, + port => $scram_port, + param => $scram_pass, + dest_ip => $dest_ip, + dest_port => $dest_port, + } + + include site_shorewall::obfsproxy + +} + + + diff --git a/puppet/modules/site_openvpn/README b/puppet/modules/site_openvpn/README new file mode 100644 index 00000000..cef5be23 --- /dev/null +++ b/puppet/modules/site_openvpn/README @@ -0,0 +1,20 @@ +Place to look when debugging problems +======================================== + +Log files: + + openvpn: /var/log/syslog + shorewall: /var/log/syslog + shorewall startup: /var/log/shorewall-init.log + +Check NAT masq: + + iptables -t nat --list-rules + +Check interfaces: + + ip addr ls + +Scripts: + + /usr/local/bin/add_gateway_ips.sh
\ No newline at end of file diff --git a/puppet/modules/site_openvpn/manifests/dh_key.pp b/puppet/modules/site_openvpn/manifests/dh_key.pp new file mode 100644 index 00000000..13cc0f5b --- /dev/null +++ b/puppet/modules/site_openvpn/manifests/dh_key.pp @@ -0,0 +1,10 @@ +class site_openvpn::dh_key { + + $x509_config = hiera('x509') + + file { '/etc/openvpn/keys/dh.pem': + content => $x509_config['dh'], + mode => '0644', + } + +} diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp new file mode 100644 index 00000000..f1ecefb9 --- /dev/null +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -0,0 +1,238 @@ +# +# An openvpn gateway can support three modes: +# +# (1) limited and unlimited +# (2) unlimited only +# (3) limited only +# +# The difference is that 'unlimited' gateways only allow client certs that match +# the 'unlimited_prefix', and 'limited' gateways only allow certs that match the +# 'limited_prefix'. +# +# We potentially create four openvpn config files (thus four daemons): +# +# (1) unlimited + tcp => tcp_config.conf +# (2) unlimited + udp => udp_config.conf +# (3) limited + tcp => limited_tcp_config.conf +# (4) limited + udp => limited_udp_config.conf +# + +class site_openvpn { + tag 'leap_service' + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca_bundle + + include site_config::default + Class['site_config::default'] -> Class['site_openvpn'] + + include ::site_obfsproxy + + $openvpn = hiera('openvpn') + $openvpn_ports = $openvpn['ports'] + $openvpn_config = $openvpn['configuration'] + + if $::ec2_instance_id { + $openvpn_gateway_address = $::ipaddress + } else { + $openvpn_gateway_address = $openvpn['gateway_address'] + if $openvpn['second_gateway_address'] { + $openvpn_second_gateway_address = $openvpn['second_gateway_address'] + } else { + $openvpn_second_gateway_address = undef + } + } + + $openvpn_allow_unlimited = $openvpn['allow_unlimited'] + $openvpn_unlimited_prefix = $openvpn['unlimited_prefix'] + $openvpn_unlimited_tcp_network_prefix = '10.41.0' + $openvpn_unlimited_tcp_netmask = '255.255.248.0' + $openvpn_unlimited_tcp_cidr = '21' + $openvpn_unlimited_udp_network_prefix = '10.42.0' + $openvpn_unlimited_udp_netmask = '255.255.248.0' + $openvpn_unlimited_udp_cidr = '21' + + if !$::ec2_instance_id { + $openvpn_allow_limited = $openvpn['allow_limited'] + $openvpn_limited_prefix = $openvpn['limited_prefix'] + $openvpn_rate_limit = $openvpn['rate_limit'] + $openvpn_limited_tcp_network_prefix = '10.43.0' + $openvpn_limited_tcp_netmask = '255.255.248.0' + $openvpn_limited_tcp_cidr = '21' + $openvpn_limited_udp_network_prefix = '10.44.0' + $openvpn_limited_udp_netmask = '255.255.248.0' + $openvpn_limited_udp_cidr = '21' + } + + # find out the netmask in cidr format of the primary IF + # thx to https://blog.kumina.nl/tag/puppet-tips-and-tricks/ + # we can do this using an inline_template: + $factname_primary_netmask = "netmask_cidr_${::site_config::params::interface}" + $primary_netmask = inline_template('<%= scope.lookupvar(@factname_primary_netmask) %>') + + # deploy dh keys + include site_openvpn::dh_key + + if $openvpn_allow_unlimited and $openvpn_allow_limited { + $unlimited_gateway_address = $openvpn_gateway_address + $limited_gateway_address = $openvpn_second_gateway_address + } elsif $openvpn_allow_unlimited { + $unlimited_gateway_address = $openvpn_gateway_address + $limited_gateway_address = undef + } elsif $openvpn_allow_limited { + $unlimited_gateway_address = undef + $limited_gateway_address = $openvpn_gateway_address + } + + if $openvpn_allow_unlimited { + site_openvpn::server_config { 'tcp_config': + port => '1194', + proto => 'tcp', + local => $unlimited_gateway_address, + tls_remote => "\"${openvpn_unlimited_prefix}\"", + server => "${openvpn_unlimited_tcp_network_prefix}.0 ${openvpn_unlimited_tcp_netmask}", + push => "\"dhcp-option DNS ${openvpn_unlimited_tcp_network_prefix}.1\"", + management => '127.0.0.1 1000', + config => $openvpn_config + } + site_openvpn::server_config { 'udp_config': + port => '1194', + proto => 'udp', + local => $unlimited_gateway_address, + tls_remote => "\"${openvpn_unlimited_prefix}\"", + server => "${openvpn_unlimited_udp_network_prefix}.0 ${openvpn_unlimited_udp_netmask}", + push => "\"dhcp-option DNS ${openvpn_unlimited_udp_network_prefix}.1\"", + management => '127.0.0.1 1001', + config => $openvpn_config + } + } else { + tidy { '/etc/openvpn/tcp_config.conf': } + tidy { '/etc/openvpn/udp_config.conf': } + } + + if $openvpn_allow_limited { + site_openvpn::server_config { 'limited_tcp_config': + port => '1194', + proto => 'tcp', + local => $limited_gateway_address, + tls_remote => "\"${openvpn_limited_prefix}\"", + server => "${openvpn_limited_tcp_network_prefix}.0 ${openvpn_limited_tcp_netmask}", + push => "\"dhcp-option DNS ${openvpn_limited_tcp_network_prefix}.1\"", + management => '127.0.0.1 1002', + config => $openvpn_config + } + site_openvpn::server_config { 'limited_udp_config': + port => '1194', + proto => 'udp', + local => $limited_gateway_address, + tls_remote => "\"${openvpn_limited_prefix}\"", + server => "${openvpn_limited_udp_network_prefix}.0 ${openvpn_limited_udp_netmask}", + push => "\"dhcp-option DNS ${openvpn_limited_udp_network_prefix}.1\"", + management => '127.0.0.1 1003', + config => $openvpn_config + } + } else { + tidy { '/etc/openvpn/limited_tcp_config.conf': } + tidy { '/etc/openvpn/limited_udp_config.conf': } + } + + file { + '/usr/local/bin/add_gateway_ips.sh': + content => template('site_openvpn/add_gateway_ips.sh.erb'), + mode => '0755'; + } + + exec { '/usr/local/bin/add_gateway_ips.sh': + subscribe => File['/usr/local/bin/add_gateway_ips.sh'], + } + + exec { 'restart_openvpn': + command => '/etc/init.d/openvpn restart', + refreshonly => true, + subscribe => [ + File['/etc/openvpn'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca_bundle'] ], + require => [ + Package['openvpn'], + File['/etc/openvpn'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca_bundle'] ]; + } + + cron { 'add_gateway_ips.sh': + command => '/usr/local/bin/add_gateway_ips.sh', + user => 'root', + special => 'reboot', + } + + # setup the resolver to listen on the vpn IP + include site_openvpn::resolver + + include site_shorewall::eip + + package { + 'openvpn': ensure => latest + } + + service { + 'openvpn': + ensure => running, + hasrestart => true, + hasstatus => true, + require => [ + Package['openvpn'], + Exec['concat_/etc/default/openvpn'] ]; + } + + file { + '/etc/openvpn': + ensure => directory, + notify => Exec['restart_openvpn'], + require => Package['openvpn']; + } + + file { + '/etc/openvpn/keys': + ensure => directory, + require => Package['openvpn']; + } + + concat { + '/etc/default/openvpn': + owner => root, + group => root, + mode => 644, + warn => true, + notify => Service['openvpn']; + } + + concat::fragment { + 'openvpn.default.header': + content => template('openvpn/etc-default-openvpn.erb'), + target => '/etc/default/openvpn', + order => 01; + } + + concat::fragment { + "openvpn.default.autostart.${name}": + content => 'AUTOSTART=all', + target => '/etc/default/openvpn', + order => 10; + } + + leap::logfile { 'openvpn_tcp': } + leap::logfile { 'openvpn_udp': } + + # Because we currently do not support ipv6 and instead block it (so no leaks + # happen), we get a large number of these messages, so we ignore them (#6540) + rsyslog::snippet { '01-ignore_icmpv6_send': + content => ':msg, contains, "icmpv6_send: no reply to icmp error" ~' + } + + include site_check_mk::agent::openvpn + +} diff --git a/puppet/modules/site_openvpn/manifests/resolver.pp b/puppet/modules/site_openvpn/manifests/resolver.pp new file mode 100644 index 00000000..cea0153a --- /dev/null +++ b/puppet/modules/site_openvpn/manifests/resolver.pp @@ -0,0 +1,50 @@ +class site_openvpn::resolver { + + if $site_openvpn::openvpn_allow_unlimited { + $ensure_unlimited = 'present' + file { + '/etc/unbound/unbound.conf.d/vpn_unlimited_udp_resolver.conf': + content => "server:\n\tinterface: ${site_openvpn::openvpn_unlimited_udp_network_prefix}.1\n\taccess-control: ${site_openvpn::openvpn_unlimited_udp_network_prefix}.0/${site_openvpn::openvpn_unlimited_udp_cidr} allow\n", + owner => root, + group => root, + mode => '0644', + require => [ Class['site_config::caching_resolver'], Service['openvpn'] ], + notify => Service['unbound']; + '/etc/unbound/unbound.conf.d/vpn_unlimited_tcp_resolver.conf': + content => "server:\n\tinterface: ${site_openvpn::openvpn_unlimited_tcp_network_prefix}.1\n\taccess-control: ${site_openvpn::openvpn_unlimited_tcp_network_prefix}.0/${site_openvpn::openvpn_unlimited_tcp_cidr} allow\n", + owner => root, + group => root, + mode => '0644', + require => [ Class['site_config::caching_resolver'], Service['openvpn'] ], + notify => Service['unbound']; + } + } else { + $ensure_unlimited = 'absent' + tidy { '/etc/unbound/unbound.conf.d/vpn_unlimited_udp_resolver.conf': } + tidy { '/etc/unbound/unbound.conf.d/vpn_unlimited_tcp_resolver.conf': } + } + + if $site_openvpn::openvpn_allow_limited { + $ensure_limited = 'present' + file { + '/etc/unbound/unbound.conf.d/vpn_limited_udp_resolver.conf': + content => "server:\n\tinterface: ${site_openvpn::openvpn_limited_udp_network_prefix}.1\n\taccess-control: ${site_openvpn::openvpn_limited_udp_network_prefix}.0/${site_openvpn::openvpn_limited_udp_cidr} allow\n", + owner => root, + group => root, + mode => '0644', + require => [ Class['site_config::caching_resolver'], Service['openvpn'] ], + notify => Service['unbound']; + '/etc/unbound/unbound.conf.d/vpn_limited_tcp_resolver.conf': + content => "server\n\tinterface: ${site_openvpn::openvpn_limited_tcp_network_prefix}.1\n\taccess-control: ${site_openvpn::openvpn_limited_tcp_network_prefix}.0/${site_openvpn::openvpn_limited_tcp_cidr} allow\n", + owner => root, + group => root, + mode => '0644', + require => [ Class['site_config::caching_resolver'], Service['openvpn'] ], + notify => Service['unbound']; + } + } else { + $ensure_limited = 'absent' + tidy { '/etc/unbound/unbound.conf.d/vpn_limited_udp_resolver.conf': } + tidy { '/etc/unbound/unbound.conf.d/vpn_limited_tcp_resolver.conf': } + } +} diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp new file mode 100644 index 00000000..15e6fb38 --- /dev/null +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -0,0 +1,228 @@ +# +# Cipher discussion +# ================================ +# +# We want to specify explicit values for the crypto options to prevent a MiTM from forcing +# a weaker cipher. These should be set in both the server and the client ('auth' and 'cipher' +# MUST be the same on both ends or no data will get transmitted). +# +# tls-cipher DHE-RSA-AES128-SHA +# +# dkg: For the TLS control channel, we want to make sure we choose a +# key exchange mechanism that has PFS (meaning probably some form of ephemeral +# Diffie-Hellman key exchange), and that uses a standard, well-tested cipher +# (I recommend AES, and 128 bits is probably fine, since there are some known +# weaknesses in the 192- and 256-bit key schedules). That leaves us with the +# choice of public key algorithms: /usr/sbin/openvpn --show-tls | grep DHE | +# grep AES128 | grep GCM. +# +# elijah: +# I could not get any of these working: +# * openvpn --show-tls | grep GCM +# * openvpn --show-tls | grep DHE | grep AES128 | grep SHA256 +# so, i went with this: +# * openvpn --show-tls | grep DHE | grep AES128 | grep -v SHA256 | grep -v GCM +# Also, i couldn't get any of the elliptical curve algorithms to work. Not sure how +# our cert generation interacts with the tls-cipher algorithms. +# +# note: in my tests, DHE-RSA-AES256-SHA is the one it negotiates if no value is set. +# +# auth SHA1 +# +# dkg: For HMAC digest to authenticate packets, we just want SHA256. OpenVPN lists +# a number of "digest" with names like "RSA-SHA256", but this are legacy and +# should be avoided. +# +# elijah: i am not so sure that the digest algo matters for 'auth' option, because +# i think an attacker would have to forge the digest in real time, which is still far from +# a possibility for SHA1. So, i am leaving the default for now (SHA1). +# +# cipher AES-128-CBC +# +# dkg: For the choice of cipher, we need to select an algorithm and a +# cipher mode. OpenVPN defaults to Blowfish, which is a fine algorithm - but +# our control channel is already relying on AES not being broken; if the +# control channel is cracked, then the key material for the tunnel is exposed, +# and the choice of algorithm is moot. So it makes more sense to me to rely on +# the same cipher here: AES128. As for the cipher mode, OFB seems cleaner to +# me, but CBC is more well-tested, and the OpenVPN man page (at least as of +# version 2.2.1) says "CBC is recommended and CFB and OFB should be considered +# advanced modes." +# +# note: the default is BF-CBC (blowfish) +# + +define site_openvpn::server_config( + $port, $proto, $local, $server, $push, + $management, $config, $tls_remote = undef) { + + $openvpn_configname = $name + $shortname = regsubst(regsubst($name, '_config', ''), '_', '-') + $openvpn_status_filename = "/var/run/openvpn-status-${shortname}" + + concat { + "/etc/openvpn/${openvpn_configname}.conf": + owner => root, + group => root, + mode => 644, + warn => true, + require => File['/etc/openvpn'], + before => Service['openvpn'], + notify => Exec['restart_openvpn']; + } + + if $tls_remote != undef { + openvpn::option { + "tls-remote ${openvpn_configname}": + key => 'tls-remote', + value => $tls_remote, + server => $openvpn_configname; + } + } + + # according to openvpn man page: tcp-nodelay is a "generally a good latency optimization". + if $proto == 'tcp' { + openvpn::option { + "tcp-nodelay ${openvpn_configname}": + key => 'tcp-nodelay', + server => $openvpn_configname; + } + } elsif $proto == 'udp' { + if $config['fragment'] != 1500 { + openvpn::option { + "fragment ${openvpn_configname}": + key => 'fragment', + value => $config['fragment'], + server => $openvpn_configname; + "mssfix ${openvpn_configname}": + key => 'mssfix', + server => $openvpn_configname; + } + } + } + + openvpn::option { + "ca ${openvpn_configname}": + key => 'ca', + value => "${x509::variables::local_CAs}/${site_config::params::ca_bundle_name}.crt", + server => $openvpn_configname; + "cert ${openvpn_configname}": + key => 'cert', + value => "${x509::variables::certs}/${site_config::params::cert_name}.crt", + server => $openvpn_configname; + "key ${openvpn_configname}": + key => 'key', + value => "${x509::variables::keys}/${site_config::params::cert_name}.key", + server => $openvpn_configname; + "dh ${openvpn_configname}": + key => 'dh', + value => '/etc/openvpn/keys/dh.pem', + server => $openvpn_configname; + "tls-cipher ${openvpn_configname}": + key => 'tls-cipher', + value => $config['tls-cipher'], + server => $openvpn_configname; + "auth ${openvpn_configname}": + key => 'auth', + value => $config['auth'], + server => $openvpn_configname; + "cipher ${openvpn_configname}": + key => 'cipher', + value => $config['cipher'], + server => $openvpn_configname; + "dev ${openvpn_configname}": + key => 'dev', + value => 'tun', + server => $openvpn_configname; + "tun-ipv6 ${openvpn_configname}": + key => 'tun-ipv6', + server => $openvpn_configname; + "duplicate-cn ${openvpn_configname}": + key => 'duplicate-cn', + server => $openvpn_configname; + "keepalive ${openvpn_configname}": + key => 'keepalive', + value => $config['keepalive'], + server => $openvpn_configname; + "local ${openvpn_configname}": + key => 'local', + value => $local, + server => $openvpn_configname; + "mute ${openvpn_configname}": + key => 'mute', + value => '5', + server => $openvpn_configname; + "mute-replay-warnings ${openvpn_configname}": + key => 'mute-replay-warnings', + server => $openvpn_configname; + "management ${openvpn_configname}": + key => 'management', + value => $management, + server => $openvpn_configname; + "proto ${openvpn_configname}": + key => 'proto', + value => $proto, + server => $openvpn_configname; + "push1 ${openvpn_configname}": + key => 'push', + value => $push, + server => $openvpn_configname; + "push2 ${openvpn_configname}": + key => 'push', + value => '"redirect-gateway def1"', + server => $openvpn_configname; + "push-ipv6 ${openvpn_configname}": + key => 'push', + value => '"route-ipv6 2000::/3"', + server => $openvpn_configname; + "script-security ${openvpn_configname}": + key => 'script-security', + value => '1', + server => $openvpn_configname; + "server ${openvpn_configname}": + key => 'server', + value => $server, + server => $openvpn_configname; + "server-ipv6 ${openvpn_configname}": + key => 'server-ipv6', + value => '2001:db8:123::/64', + server => $openvpn_configname; + "status ${openvpn_configname}": + key => 'status', + value => "${openvpn_status_filename} 10", + server => $openvpn_configname; + "status-version ${openvpn_configname}": + key => 'status-version', + value => '3', + server => $openvpn_configname; + "topology ${openvpn_configname}": + key => 'topology', + value => 'subnet', + server => $openvpn_configname; + "verb ${openvpn_configname}": + key => 'verb', + value => '3', + server => $openvpn_configname; + "log-append /var/log/leap/openvpn_${proto}.log": + key => 'log-append', + value => "/var/log/leap/openvpn_${proto}.log", + server => $openvpn_configname; + } + + # register openvpn services at systemd on nodes newer than wheezy + # see https://leap.se/code/issues/7798 + case $::operatingsystemrelease { + /^7.*/: { } + default: { + exec { "enable_systemd_${openvpn_configname}": + refreshonly => true, + command => "/bin/systemctl enable openvpn@${openvpn_configname}", + subscribe => File["/etc/openvpn/${openvpn_configname}.conf"], + notify => Service["openvpn@${openvpn_configname}"]; + } + service { "openvpn@${openvpn_configname}": + ensure => running + } + } + } +} diff --git a/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb b/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb new file mode 100644 index 00000000..e76b756b --- /dev/null +++ b/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb @@ -0,0 +1,11 @@ +#!/bin/sh + +ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q <%= @openvpn_gateway_address %>/<%= @primary_netmask %> || + ip addr add <%= @openvpn_gateway_address %>/<%= @primary_netmask %> dev <%= scope.lookupvar('site_config::params::interface') %> + +<% if @openvpn_second_gateway_address %> +ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q <%= @openvpn_second_gateway_address %>/<%= @primary_netmask %> || + ip addr add <%= @openvpn_second_gateway_address %>/<%= @primary_netmask %> dev <%= scope.lookupvar('site_config::params::interface') %> +<% end %> + +/bin/echo 1 > /proc/sys/net/ipv4/ip_forward diff --git a/puppet/modules/site_postfix/files/checks/received_anon b/puppet/modules/site_postfix/files/checks/received_anon new file mode 100644 index 00000000..9de25e63 --- /dev/null +++ b/puppet/modules/site_postfix/files/checks/received_anon @@ -0,0 +1,2 @@ +/^Received: from (.* \([-._[:alnum:]]+ \[[.[:digit:]]{7,15}\]\))([[:space:]]+).*(\(using [.[:alnum:]]+ with cipher [-A-Z0-9]+ \([0-9]+\/[0-9]+ bits\)\))[[:space:]]+\(Client CN "([-._@[:alnum:]]+)", Issuer "[[:print:]]+" \(verified OK\)\)[[:space:]]+by ([.[:alnum:]]+) \(([^)]+)\) with (E?SMTPS?A?) id ([A-F[:digit:]]+).*/ + REPLACE Received: from [127.0.0.1] (localhost [127.0.0.1])${2}${3}${2}(Authenticated sender: $4)${2}with $7 id $8 diff --git a/puppet/modules/site_postfix/manifests/debug.pp b/puppet/modules/site_postfix/manifests/debug.pp new file mode 100644 index 00000000..f370d166 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/debug.pp @@ -0,0 +1,9 @@ +class site_postfix::debug { + + postfix::config { + 'debug_peer_list': value => '127.0.0.1'; + 'debug_peer_level': value => '1'; + 'smtpd_tls_loglevel': value => '1'; + } + +} diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp new file mode 100644 index 00000000..c269946b --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -0,0 +1,152 @@ +# +# configure mx node +# +class site_postfix::mx { + + $domain_hash = hiera('domain') + $domain = $domain_hash['full_suffix'] + $host_domain = $domain_hash['full'] + $cert_name = hiera('name') + $mynetworks = join(hiera('mynetworks', ''), ' ') + $rbls = suffix(prefix(hiera('rbls', []), 'reject_rbl_client '), ',') + + $root_mail_recipient = hiera('contacts') + $postfix_smtp_listen = 'all' + $postfix_use_postscreen = 'yes' + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::client_ca::ca + include site_config::x509::client_ca::key + + postfix::config { + 'mynetworks': + value => "127.0.0.0/8 [::1]/128 [fe80::]/64 ${mynetworks}"; + # Note: mydestination should not include @domain, because this is + # used in virtual alias maps. + 'mydestination': + value => "\$myorigin, localhost, localhost.\$mydomain"; + 'myhostname': + value => $host_domain; + 'mailbox_size_limit': + value => '0'; + 'home_mailbox': + value => ''; + 'virtual_mailbox_domains': + value => 'deliver.local'; + 'virtual_mailbox_base': + value => '/var/mail/leap-mx'; + 'virtual_mailbox_maps': + value => 'static:Maildir/'; + # Note: virtual-aliases map will take precedence over leap-mx + # lookup (tcp:localhost) + 'virtual_alias_maps': + value => 'hash:/etc/postfix/virtual-aliases tcp:localhost:4242'; + 'luser_relay': + value => ''; + # uid and gid are set to an arbitrary hard-coded value here, this + # must match the 'leap-mx' user/group + 'virtual_uid_maps': + value => 'static:42424'; + 'virtual_gid_maps': + value => 'static:42424'; + # the two following configs are needed for matching user's client cert + # fingerprints to enable relaying (#3634). Satellites do not have + # these configured. + 'smtpd_tls_fingerprint_digest': + value => 'sha1'; + 'relay_clientcerts': + value => 'tcp:localhost:2424'; + # Note: we are setting this here, instead of in site_postfix::mx::smtp_tls + # because the satellites need to have a different value + 'smtp_tls_security_level': + value => 'may'; + # reject inbound mail to system users + # see https://leap.se/code/issues/6829 + # this blocks *only* mails to system users, that don't appear in the + # alias map + 'local_recipient_maps': + value => '$alias_maps'; + # setup clamav and opendkim on smtpd + 'smtpd_milters': + value => 'unix:/run/clamav/milter.ctl,inet:localhost:8891'; + # setup opendkim for smtp (non-smtpd) outgoing mail + 'non_smtpd_milters': + value => 'inet:localhost:8891'; + 'milter_default_action': + value => 'accept'; + # Make sure that the right values are set, these could be set to different + # things on install, depending on preseed or debconf options + # selected (see #7478) + 'relay_transport': + value => 'relay'; + 'default_transport': + value => 'smtp'; + 'mailbox_command': + value => ''; + 'header_checks': + value => ''; + 'postscreen_access_list': + value => 'permit_mynetworks'; + 'postscreen_greet_action': + value => 'enforce'; + } + + # Make sure that the cleanup serivce is not chrooted, otherwise it cannot + # access the opendkim milter socket (#8020) + exec { 'unset_cleanup_chroot': + command => '/usr/sbin/postconf -F "cleanup/unix/chroot=n"', + onlyif => '/usr/sbin/postconf -h -F "cleanup/unix/chroot" | egrep -q ^n', + notify => Service['postfix'], + require => File['/etc/postfix/master.cf'] + } + + include ::site_postfix::mx::smtpd_checks + include ::site_postfix::mx::checks + include ::site_postfix::mx::smtp_tls + include ::site_postfix::mx::smtpd_tls + include ::site_postfix::mx::static_aliases + include ::site_postfix::mx::rewrite_openpgp_header + include ::site_postfix::mx::received_anon + include ::clamav + include ::opendkim + include ::postfwd + + # greater verbosity for debugging, take out for production + #include site_postfix::debug + + case $::operatingsystemrelease { + /^7.*/: { + $smtpd_relay_restrictions='' + } + default: { + $smtpd_relay_restrictions=" -o smtpd_relay_restrictions=\$smtps_relay_restrictions\n" + } + } + + $mastercf_tail = " +smtps inet n - - - - smtpd + -o smtpd_tls_wrappermode=yes + -o smtpd_tls_security_level=encrypt + -o tls_preempt_cipherlist=yes +${smtpd_relay_restrictions} -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions + -o smtpd_helo_restrictions=\$smtps_helo_restrictions + -o smtpd_client_restrictions= + -o cleanup_service_name=clean_smtps +clean_smtps unix n - n - 0 cleanup + -o header_checks=pcre:/etc/postfix/checks/rewrite_openpgp_headers,pcre:/etc/postfix/checks/received_anon" + + class { 'postfix': + preseed => true, + root_mail_recipient => $root_mail_recipient, + smtp_listen => 'all', + mastercf_tail => $mastercf_tail, + use_postscreen => 'yes', + require => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Client_ca::Key'], + Class['Site_config::X509::Client_ca::Ca'], + User['leap-mx'] ] + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/checks.pp b/puppet/modules/site_postfix/manifests/mx/checks.pp new file mode 100644 index 00000000..f406ad34 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/checks.pp @@ -0,0 +1,23 @@ +class site_postfix::mx::checks { + + file { + '/etc/postfix/checks': + ensure => directory, + mode => '0755', + owner => root, + group => postfix, + require => Package['postfix']; + + '/etc/postfix/checks/helo_checks': + content => template('site_postfix/checks/helo_access.erb'), + mode => '0644', + owner => root, + group => root; + } + + exec { + '/usr/sbin/postmap /etc/postfix/checks/helo_checks': + refreshonly => true, + subscribe => File['/etc/postfix/checks/helo_checks']; + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/received_anon.pp b/puppet/modules/site_postfix/manifests/mx/received_anon.pp new file mode 100644 index 00000000..51ba3faa --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/received_anon.pp @@ -0,0 +1,13 @@ +# Anonymize the user's home IP from the email headers (Feature #3866) +class site_postfix::mx::received_anon { + + package { 'postfix-pcre': ensure => installed, require => Package['postfix'] } + + file { '/etc/postfix/checks/received_anon': + source => 'puppet:///modules/site_postfix/checks/received_anon', + mode => '0644', + owner => root, + group => root, + notify => Service['postfix'] + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp new file mode 100644 index 00000000..71f945b8 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/rewrite_openpgp_header.pp @@ -0,0 +1,11 @@ +class site_postfix::mx::rewrite_openpgp_header { + $mx = hiera('mx') + $correct_domain = $mx['key_lookup_domain'] + + file { '/etc/postfix/checks/rewrite_openpgp_headers': + content => template('site_postfix/checks/rewrite_openpgp_headers.erb'), + mode => '0644', + owner => root, + group => root; + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_auth.pp b/puppet/modules/site_postfix/manifests/mx/smtp_auth.pp new file mode 100644 index 00000000..afa70527 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/smtp_auth.pp @@ -0,0 +1,6 @@ +class site_postfix::mx::smtp_auth { + + postfix::config { + 'smtpd_tls_ask_ccert': value => 'yes'; + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp new file mode 100644 index 00000000..c93c3ba2 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp @@ -0,0 +1,43 @@ +# configure smtp tls +class site_postfix::mx::smtp_tls { + + include site_config::x509::ca + include x509::variables + $cert_name = hiera('name') + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + include site_config::x509::cert + include site_config::x509::key + + # smtp TLS + postfix::config { + 'smtp_use_tls': value => 'yes'; + 'smtp_tls_CApath': value => '/etc/ssl/certs/'; + 'smtp_tls_CAfile': value => $ca_path; + 'smtp_tls_cert_file': value => $cert_path; + 'smtp_tls_key_file': value => $key_path; + 'smtp_tls_loglevel': value => '1'; + 'smtp_tls_exclude_ciphers': + value => 'aNULL, MD5, DES'; + # upstream default is md5 (since 2.5 and older used it), we force sha1 + 'smtp_tls_fingerprint_digest': + value => 'sha1'; + 'smtp_tls_session_cache_database': + value => "btree:\${data_directory}/smtp_cache"; + # see issue #4011 + 'smtp_tls_protocols': + value => '!SSLv2, !SSLv3'; + 'smtp_tls_mandatory_protocols': + value => '!SSLv2, !SSLv3'; + 'tls_ssl_options': + value => 'NO_COMPRESSION'; + # We can switch between the different postfix internal list of ciphers by + # using smtpd_tls_ciphers. For server-to-server connections we leave this + # at its default because of opportunistic encryption combined with many mail + # servers only support outdated protocols and ciphers and if we are too + # strict with required ciphers, then connections *will* fall-back to + # plain-text. Bad ciphers are still better than plain text transmission. + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp new file mode 100644 index 00000000..291d7ee4 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_checks.pp @@ -0,0 +1,36 @@ +# smtpd checks for incoming mail on smtp port 25 and +# mail sent via the bitmask client using smtps port 465 +class site_postfix::mx::smtpd_checks { + + postfix::config { + 'smtpd_helo_required': + value => 'yes'; + 'checks_dir': + value => '$config_directory/checks'; + 'smtpd_client_restrictions': + value => "permit_mynetworks,${site_postfix::mx::rbls},permit"; + 'smtpd_data_restrictions': + value => 'permit_mynetworks, reject_unauth_pipelining, permit'; + 'smtpd_delay_reject': + value => 'yes'; + 'smtpd_helo_restrictions': + value => 'permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, check_helo_access hash:$checks_dir/helo_checks, permit'; + 'smtpd_recipient_restrictions': + value => 'reject_unknown_recipient_domain, permit_mynetworks, check_recipient_access tcp:localhost:2244, reject_unauth_destination, permit'; + + # permit_tls_clientcerts will lookup client cert fingerprints from the tcp + # lookup on port 2424 (based on what is configured in relay_clientcerts + # paramter, see site_postfix::mx postfix::config resource) to determine + # if a client is allowed to relay mail through us. This enables us to + # disable a user by removing their valid client cert (#3634) + 'smtps_recipient_restrictions': + value => 'permit_tls_clientcerts, check_recipient_access tcp:localhost:2244, reject_unauth_destination, permit'; + 'smtps_relay_restrictions': + value => 'permit_mynetworks, permit_tls_clientcerts, defer_unauth_destination'; + 'smtps_helo_restrictions': + value => 'permit_mynetworks, check_helo_access hash:$checks_dir/helo_checks, permit'; + 'smtpd_sender_restrictions': + value => 'permit_mynetworks, reject_non_fqdn_sender, reject_unknown_sender_domain, permit'; + } + +} diff --git a/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp new file mode 100644 index 00000000..66297f55 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/smtpd_tls.pp @@ -0,0 +1,69 @@ +# configure smtpd tls +class site_postfix::mx::smtpd_tls { + + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::client_ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + + postfix::config { + 'smtpd_use_tls': value => 'yes'; + 'smtpd_tls_CAfile': value => $ca_path; + 'smtpd_tls_cert_file': value => $cert_path; + 'smtpd_tls_key_file': value => $key_path; + 'smtpd_tls_ask_ccert': value => 'yes'; + 'smtpd_tls_received_header': + value => 'yes'; + 'smtpd_tls_security_level': + value => 'may'; + 'smtpd_tls_eecdh_grade': + value => 'ultra'; + 'smtpd_tls_session_cache_database': + value => "btree:\${data_directory}/smtpd_scache"; + # see issue #4011 + 'smtpd_tls_mandatory_protocols': + value => '!SSLv2, !SSLv3'; + 'smtpd_tls_protocols': + value => '!SSLv2, !SSLv3'; + # For connections to MUAs, TLS is mandatory and the ciphersuite is modified. + # MX and SMTP client configuration + 'smtpd_tls_mandatory_ciphers': + value => 'high'; + 'tls_high_cipherlist': + value => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!RC4:!MD5:!PSK!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; + } + + # Setup DH parameters + # Instead of using the dh parameters that are created by leap cli, it is more + # secure to generate new parameter files that will only be used for postfix, + # for each machine + + include site_config::packages::gnutls + + # Note, the file name is called dh_1024.pem, but we are generating 2048bit dh + # parameters Neither Postfix nor OpenSSL actually care about the size of the + # prime in "smtpd_tls_dh1024_param_file". You can make it 2048 bits + + exec { 'certtool-postfix-gendh': + command => 'certtool --generate-dh-params --bits 2048 --outfile /etc/postfix/smtpd_tls_dh_param.pem', + user => root, + group => root, + creates => '/etc/postfix/smtpd_tls_dh_param.pem', + require => [ Package['gnutls-bin'], Package['postfix'] ] + } + + # Make sure the dh params file has correct ownership and mode + file { + '/etc/postfix/smtpd_tls_dh_param.pem': + owner => root, + group => root, + mode => '0600', + require => Exec['certtool-postfix-gendh']; + } + + postfix::config { 'smtpd_tls_dh1024_param_file': + value => '/etc/postfix/smtpd_tls_dh_param.pem', + require => File['/etc/postfix/smtpd_tls_dh_param.pem'] + } +} diff --git a/puppet/modules/site_postfix/manifests/mx/static_aliases.pp b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp new file mode 100644 index 00000000..9cd7ca02 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/mx/static_aliases.pp @@ -0,0 +1,88 @@ +# +# Defines static, hard coded aliases that are not in the database. +# These aliases take precedence over the database aliases. +# +# There are three classes of reserved names: +# +# (1) forbidden_usernames: +# Some usernames are forbidden and cannot be registered. +# this is defined in node property webapp.forbidden_usernames +# This is enforced by the webapp. +# +# (2) public aliases: +# Some aliases for root, and are publicly exposed so that anyone +# can deliver mail to them. For example, postmaster. +# These are implemented in the virtual alias map, which takes +# precedence over the local alias map. +# +# (3) local aliases: +# Some aliases are only available locally: mail can be delivered +# to the alias if the mail originates from the local host, or is +# hostname qualified, but otherwise it will be rejected. +# These are implemented in the local alias map. +# +# The alias for local 'root' is defined elsewhere. In this file, we +# define the virtual 'root@domain' (which can be overwritten by +# defining an entry for root in node property mx.aliases). +# + +class site_postfix::mx::static_aliases { + + $mx = hiera('mx') + $root_recipients = hiera('contacts') + + # + # LOCAL ALIASES + # + + # NOTE: if you remove one of these, they will still appear in the + # /etc/aliases file + $local_aliases = [ + 'admin', 'administrator', 'bin', 'cron', 'games', 'ftp', 'lp', 'maildrop', + 'mysql', 'news', 'nobody', 'noc', 'postgresql', 'ssladmin', 'sys', + 'usenet', 'uucp', 'www', 'www-data', 'leap-mx' + ] + + postfix::mailalias { + $local_aliases: + ensure => present, + recipient => 'root' + } + + # + # PUBLIC ALIASES + # + + $public_aliases = $mx['aliases'] + + $default_public_aliases = { + 'root' => $root_recipients, + 'abuse' => 'postmaster', + 'arin-admin' => 'root', + 'certmaster' => 'hostmaster', + 'domainadmin' => 'hostmaster', + 'hostmaster' => 'root', + 'mailer-daemon' => 'postmaster', + 'postmaster' => 'root', + 'security' => 'root', + 'webmaster' => 'hostmaster', + } + + $aliases = merge($default_public_aliases, $public_aliases) + + exec { 'postmap_virtual_aliases': + command => '/usr/sbin/postmap /etc/postfix/virtual-aliases', + refreshonly => true, + user => root, + group => root, + require => Package['postfix'], + subscribe => File['/etc/postfix/virtual-aliases'] + } + file { '/etc/postfix/virtual-aliases': + content => template('site_postfix/virtual-aliases.erb'), + owner => root, + group => root, + mode => '0600', + require => Package['postfix'] + } +} diff --git a/puppet/modules/site_postfix/manifests/satellite.pp b/puppet/modules/site_postfix/manifests/satellite.pp new file mode 100644 index 00000000..5725e6b8 --- /dev/null +++ b/puppet/modules/site_postfix/manifests/satellite.pp @@ -0,0 +1,47 @@ +class site_postfix::satellite { + + $root_mail_recipient = hiera ('contacts') + $mail = hiera ('mail') + $relayhost = $mail['smarthost'] + $cert_name = hiera('name') + + class { '::postfix::satellite': + relayhost => $relayhost, + root_mail_recipient => $root_mail_recipient + } + + # There are special conditions for satellite hosts that will make them not be + # able to contact their relayhost: + # + # 1. they are on openstack/amazon/PC and are on the same cluster as the relay + # host, the MX lookup for the relay host will use the public IP, which cannot + # be contacted + # + # 2. When a domain is used that is not in DNS, because it is internal, + # a testing domain, etc. eg. a .local domain cannot be looked up in DNS + # + # to resolve this, so the satellite can contact the relayhost, we need to set + # the http://www.postfix.org/postconf.5.html#smtp_host_lookup to be 'native' + # which will cause the lookup to use the native naming service + # (nsswitch.conf), which typically defaults to 'files, dns' allowing the + # /etc/hosts to be consulted first, then DNS if the entry doesn't exist. + # + # NOTE: this will make it not possible to enable DANE support through DNSSEC + # with http://www.postfix.org/postconf.5.html#smtp_dns_support_level - but + # this parameter is not available until 2.11. If this ends up being important + # we could also make this an optional parameter for providers without + # dns / local domains + + postfix::config { + 'smtp_host_lookup': + value => 'native'; + + # Note: we are setting this here, instead of in site_postfix::mx::smtp_tls + # because the mx server has to have a different value + 'smtp_tls_security_level': + value => 'encrypt'; + } + + include site_postfix::mx::smtp_tls + +} diff --git a/puppet/modules/site_postfix/templates/checks/helo_access.erb b/puppet/modules/site_postfix/templates/checks/helo_access.erb new file mode 100644 index 00000000..bac2c45a --- /dev/null +++ b/puppet/modules/site_postfix/templates/checks/helo_access.erb @@ -0,0 +1,21 @@ +# THIS FILE IS MANAGED BY PUPPET +# To make changes to this file, please edit your platform directory under +# puppet/modules/site_postfix/templates/checks/helo_access.erb and then deploy + +# The format of this file is the HELO/EHLO domain followed by an action. +# The action could be OK to allow it, REJECT to reject it, or a custom +# status code and message. Any lines that are prefixed by an octothorpe (#) +# will be considered comments. + +# Some examples: +# +# Reject anyone that HELO's with foobar: +# foobar REJECT +# +# Allow the switches to skip this check: +# switch1 OK +# switch2 OK + +# Reject anybody that HELO's as being in our own domain(s) +# anyone who identifies themselves as us is a virus/spammer +<%= @domain %> 554 You are not in domain <%= @domain %> diff --git a/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb new file mode 100644 index 00000000..7af14f7d --- /dev/null +++ b/puppet/modules/site_postfix/templates/checks/rewrite_openpgp_headers.erb @@ -0,0 +1,13 @@ +# THIS FILE IS MANAGED BY PUPPET +# +# This will replace the OpenPGP header that the client adds, because it is +# sometimes incorrect (due to the client not always knowing what the proper URL +# is for the webapp). +# e.g. This will rewrite this header: +# OpenPGP: id=4C0E01CD50E2F653; url="https://leap.se/key/elijah"; preference="signencrypt +# with this replacement: +# OpenPGP: id=4C0E01CD50E2F653; url="https://user.leap.se/key/elijah"; preference="signencrypt +# +# Note: whitespace in the pattern is represented by [[:space:]] to avoid these warnings from postmap: +# "record is in "key: value" format; is this an alias file?" and "duplicate entry" +/^(OpenPGP:[[:space:]]id=[[:alnum:]]+;[[:space:]]url="https:\/\/)<%= @domain %>(\/key\/[[:alpha:]]+";.*)/i REPLACE ${1}<%= @correct_domain %>${2} diff --git a/puppet/modules/site_postfix/templates/virtual-aliases.erb b/puppet/modules/site_postfix/templates/virtual-aliases.erb new file mode 100644 index 00000000..8373de97 --- /dev/null +++ b/puppet/modules/site_postfix/templates/virtual-aliases.erb @@ -0,0 +1,21 @@ +# +# This file is managed by puppet. +# +# These virtual aliases take precedence over all other aliases. +# + +# +# enable these virtual domains: +# +<%= @domain %> enabled +<%- @aliases.keys.map {|addr| addr.split('@')[1] }.compact.sort.uniq.each do |virt_domain| -%> +<%= virt_domain %> enabled +<%- end %> + +# +# virtual aliases: +# +<%- @aliases.keys.sort.each do |from| -%> +<%- full_address = from =~ /@/ ? from : from + "@" + @domain -%> +<%= full_address %> <%= [@aliases[from]].flatten.map{|a| a =~ /@/ ? a : a + "@" + @domain}.join(', ') %> +<%- end -%> diff --git a/puppet/modules/site_rsyslog/templates/client.conf.erb b/puppet/modules/site_rsyslog/templates/client.conf.erb new file mode 100644 index 00000000..7f94759d --- /dev/null +++ b/puppet/modules/site_rsyslog/templates/client.conf.erb @@ -0,0 +1,134 @@ + +# An "In-Memory Queue" is created for remote logging. +$WorkDirectory <%= scope.lookupvar('rsyslog::spool_dir') -%> # where to place spool files +$ActionQueueFileName queue # unique name prefix for spool files +$ActionQueueMaxDiskSpace <%= scope.lookupvar('rsyslog::client::spool_size') -%> # spool space limit (use as much as possible) +$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +$ActionQueueType LinkedList # run asynchronously +$ActionResumeRetryCount -1 # infinety retries if host is down +<% if scope.lookupvar('rsyslog::client::log_templates') and ! scope.lookupvar('rsyslog::client::log_templates').empty?-%> + +# Define custom logging templates +<% scope.lookupvar('rsyslog::client::log_templates').flatten.compact.each do |log_template| -%> +$template <%= log_template['name'] %>,"<%= log_template['template'] %>" +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::actionfiletemplate') -%> + +# Using specified format for default logging format: +$ActionFileDefaultTemplate <%= scope.lookupvar('rsyslog::client::actionfiletemplate') %> +<% else -%> + +#Using default format for default logging format: +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat +<% end -%> +<% if scope.lookupvar('rsyslog::client::ssl') -%> + +# Setup SSL connection. +# CA/Cert +$DefaultNetStreamDriverCAFile <%= scope.lookupvar('rsyslog::client::ssl_ca') %> + +# Connection settings. +$DefaultNetstreamDriver gtls +$ActionSendStreamDriverMode 1 +$ActionSendStreamDriverAuthMode anon +<% end -%> +<% if scope.lookupvar('rsyslog::client::remote_servers') -%> + +<% scope.lookupvar('rsyslog::client::remote_servers').flatten.compact.each do |server| -%> +<% if server['pattern'] and server['pattern'] != ''-%> +<% pattern = server['pattern'] -%> +<% else -%> +<% pattern = '*.*' -%> +<% end -%> +<% if server['protocol'] == 'TCP' or server['protocol'] == 'tcp'-%> +<% protocol = '@@' -%> +<% protocol_type = 'TCP' -%> +<% else -%> +<% protocol = '@' -%> +<% protocol_type = 'UDP' -%> +<% end -%> +<% if server['host'] and server['host'] != ''-%> +<% host = server['host'] -%> +<% else -%> +<% host = 'localhost' -%> +<% end -%> +<% if server['port'] and server['port'] != ''-%> +<% port = server['port'] -%> +<% else -%> +<% port = '514' -%> +<% end -%> +<% if server['format'] -%> +<% format = ";#{server['format']}" -%> +<% format_type = server['format'] -%> +<% else -%> +<% format = '' -%> +<% format_type = 'the default' -%> +<% end -%> +# Sending logs that match <%= pattern %> to <%= host %> via <%= protocol_type %> on <%= port %> using <%=format_type %> format. +<%= pattern %> <%= protocol %><%= host %>:<%= port %><%= format %> +<% end -%> +<% elsif scope.lookupvar('rsyslog::client::log_remote') -%> + +# Log to remote syslog server using <%= scope.lookupvar('rsyslog::client::remote_type') %> +<% if scope.lookupvar('rsyslog::client::remote_type') == 'tcp' -%> +*.* @@<%= scope.lookupvar('rsyslog::client::server') -%>:<%= scope.lookupvar('rsyslog::client::port') -%>;<%= scope.lookupvar('remote_forward_format') -%> +<% else -%> +*.* @<%= scope.lookupvar('rsyslog::client::server') -%>:<%= scope.lookupvar('rsyslog::client::port') -%>;<%= scope.lookupvar('remote_forward_format') -%> +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::log_auth_local') or scope.lookupvar('rsyslog::client::log_local') -%> + +# Logging locally. + +<% if scope.lookupvar('rsyslog::log_style') == 'debian' -%> +# Log auth messages locally +.*;auth,authpriv.none;mail.none -/var/log/syslog +<% elsif scope.lookupvar('rsyslog::log_style') == 'redhat' -%> +# Log auth messages locally +auth,authpriv.* /var/log/secure +<% end -%> +<% end -%> +<% if scope.lookupvar('rsyslog::client::log_local') -%> +<% if scope.lookupvar('rsyslog::log_style') == 'debian' -%> +# First some standard log files. Log by facility. +# +*.*;auth,authpriv.none -/var/log/syslog +cron.* /var/log/cron.log +daemon.* -/var/log/daemon.log +kern.* -/var/log/kern.log +mail.* -/var/log/mail.log +user.* -/var/log/user.log + +# +# Some "catch-all" log files. +# +*.=debug;\ + auth,authpriv.none;\ + news.none;mail.none -/var/log/debug +*.=info;*.=notice;*.=warn;\ + auth,authpriv.none;\ + cron,daemon.none;\ + mail,news.none -/var/log/messages + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +<% if @rsyslog_version and @rsyslog_version.split('.')[0].to_i >= 8 -%> +*.emerg :omusrmsg:* +<% else -%> +*.emerg * +<% end -%> + +# Save boot messages also to boot.log +local7.* -/var/log/boot.log +<% end -%> +<% end -%> + + + diff --git a/puppet/modules/site_shorewall/files/Debian/shorewall.service b/puppet/modules/site_shorewall/files/Debian/shorewall.service new file mode 100644 index 00000000..ec250ef1 --- /dev/null +++ b/puppet/modules/site_shorewall/files/Debian/shorewall.service @@ -0,0 +1,23 @@ +# +# The Shoreline Firewall (Shorewall) Packet Filtering Firewall +# +# Copyright 2011 Jonathan Underwood <jonathan.underwood@gmail.com> +# Copyright 2015 Tom Eastep <teastep@shorewall.net> +# +[Unit] +Description=Shorewall IPv4 firewall +Wants=network-online.target +After=network-online.target +Conflicts=iptables.service firewalld.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=-/etc/default/shorewall +StandardOutput=syslog +ExecStart=/sbin/shorewall $OPTIONS start $STARTOPTIONS +ExecStop=/sbin/shorewall $OPTIONS stop +ExecReload=/sbin/shorewall $OPTIONS reload $RELOADOPTIONS + +[Install] +WantedBy=basic.target diff --git a/puppet/modules/site_shorewall/manifests/defaults.pp b/puppet/modules/site_shorewall/manifests/defaults.pp new file mode 100644 index 00000000..ceb17868 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/defaults.pp @@ -0,0 +1,86 @@ +class site_shorewall::defaults { + + include shorewall + include site_config::params + + # be safe for development + # if ( $::site_config::params::environment == 'local' ) { + # $shorewall_startup='0' + # } + + # If you want logging: + shorewall::params { + 'LOG': value => 'debug'; + } + + shorewall::zone {'net': type => 'ipv4'; } + + # define interfaces + shorewall::interface { $site_config::params::interface: + zone => 'net', + options => 'tcpflags,blacklist,nosmurfs'; + } + + shorewall::policy { + 'fw-to-all': + sourcezone => 'fw', + destinationzone => 'all', + policy => 'ACCEPT', + order => 100; + 'all-to-all': + sourcezone => 'all', + destinationzone => 'all', + policy => 'DROP', + order => 200; + } + + shorewall::rule { + # ping party + 'all2all-ping': + source => 'all', + destination => 'all', + action => 'Ping(ACCEPT)', + order => 200; + } + + package { 'shorewall-init': + ensure => installed + } + + include ::systemd + file { '/etc/systemd/system/shorewall.service': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/site_shorewall/Debian/shorewall.service', + require => Package['shorewall'], + notify => Service['shorewall'], + } ~> + Exec['systemctl-daemon-reload'] + + augeas { + # stop instead of clear firewall on shutdown + 'shorewall_SAFESTOP': + changes => 'set /files/etc/shorewall/shorewall.conf/SAFESTOP Yes', + lens => 'Shellvars.lns', + incl => '/etc/shorewall/shorewall.conf', + require => Package['shorewall'], + notify => Service['shorewall']; + # require that the interface exist + 'shorewall_REQUIRE_INTERFACE': + changes => 'set /files/etc/shorewall/shorewall.conf/REQUIRE_INTERFACE Yes', + lens => 'Shellvars.lns', + incl => '/etc/shorewall/shorewall.conf', + require => Package['shorewall'], + notify => Service['shorewall']; + # configure shorewall-init + 'shorewall-init': + changes => 'set /files/etc/default/shorewall-init/PRODUCTS shorewall', + lens => 'Shellvars.lns', + incl => '/etc/default/shorewall-init', + require => [ Package['shorewall-init'], Service['shorewall'] ] + } + + include site_shorewall::sshd +} diff --git a/puppet/modules/site_shorewall/manifests/dnat.pp b/puppet/modules/site_shorewall/manifests/dnat.pp new file mode 100644 index 00000000..a73294cc --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/dnat.pp @@ -0,0 +1,19 @@ +define site_shorewall::dnat ( + $source, + $destination, + $proto, + $destinationport, + $originaldest ) { + + + shorewall::rule { + "dnat_${name}_${destinationport}": + action => 'DNAT', + source => $source, + destination => $destination, + proto => $proto, + destinationport => $destinationport, + originaldest => $originaldest, + order => 200 + } +} diff --git a/puppet/modules/site_shorewall/manifests/dnat_rule.pp b/puppet/modules/site_shorewall/manifests/dnat_rule.pp new file mode 100644 index 00000000..f9fbe950 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/dnat_rule.pp @@ -0,0 +1,50 @@ +define site_shorewall::dnat_rule { + + $port = $name + if $port != 1194 { + if $site_openvpn::openvpn_allow_unlimited { + shorewall::rule { + "dnat_tcp_port_${port}": + action => 'DNAT', + source => 'net', + destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", + proto => 'tcp', + destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, + order => 100; + } + shorewall::rule { + "dnat_udp_port_${port}": + action => 'DNAT', + source => 'net', + destination => "\$FW:${site_openvpn::unlimited_gateway_address}:1194", + proto => 'udp', + destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, + order => 100; + } + } + if $site_openvpn::openvpn_allow_limited { + shorewall::rule { + "dnat_free_tcp_port_${port}": + action => 'DNAT', + source => 'net', + destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", + proto => 'tcp', + destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, + order => 100; + } + shorewall::rule { + "dnat_free_udp_port_${port}": + action => 'DNAT', + source => 'net', + destination => "\$FW:${site_openvpn::limited_gateway_address}:1194", + proto => 'udp', + destinationport => $port, + originaldest => $site_openvpn::unlimited_gateway_address, + order => 100; + } + } + } +} diff --git a/puppet/modules/site_shorewall/manifests/eip.pp b/puppet/modules/site_shorewall/manifests/eip.pp new file mode 100644 index 00000000..8fbba658 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/eip.pp @@ -0,0 +1,92 @@ +class site_shorewall::eip { + + include site_shorewall::defaults + include site_config::params + include site_shorewall::ip_forward + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_eip': + content => "PARAM - - tcp 1194 + PARAM - - udp 1194 + ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + shorewall::interface { + 'tun0': + zone => 'eip', + options => 'tcpflags,blacklist,nosmurfs'; + 'tun1': + zone => 'eip', + options => 'tcpflags,blacklist,nosmurfs'; + 'tun2': + zone => 'eip', + options => 'tcpflags,blacklist,nosmurfs'; + 'tun3': + zone => 'eip', + options => 'tcpflags,blacklist,nosmurfs'; + } + + shorewall::zone { + 'eip': + type => 'ipv4'; + } + + $interface = $site_config::params::interface + + shorewall::masq { + "${interface}_unlimited_tcp": + interface => $interface, + source => "${site_openvpn::openvpn_unlimited_tcp_network_prefix}.0/${site_openvpn::openvpn_unlimited_tcp_cidr}"; + "${interface}_unlimited_udp": + interface => $interface, + source => "${site_openvpn::openvpn_unlimited_udp_network_prefix}.0/${site_openvpn::openvpn_unlimited_udp_cidr}"; + } + if ! $::ec2_instance_id { + shorewall::masq { + "${interface}_limited_tcp": + interface => $interface, + source => "${site_openvpn::openvpn_limited_tcp_network_prefix}.0/${site_openvpn::openvpn_limited_tcp_cidr}"; + "${interface}_limited_udp": + interface => $interface, + source => "${site_openvpn::openvpn_limited_udp_network_prefix}.0/${site_openvpn::openvpn_limited_udp_cidr}"; + } + } + + shorewall::policy { + 'eip-to-all': + sourcezone => 'eip', + destinationzone => 'all', + policy => 'ACCEPT', + order => 100; + } + + shorewall::rule { + 'net2fw-openvpn': + source => 'net', + destination => '$FW', + action => 'leap_eip(ACCEPT)', + order => 200; + + 'block_eip_dns_udp': + action => 'REJECT', + source => 'eip', + destination => 'net', + proto => 'udp', + destinationport => 'domain', + order => 300; + + 'block_eip_dns_tcp': + action => 'REJECT', + source => 'eip', + destination => 'net', + proto => 'tcp', + destinationport => 'domain', + order => 301; + } + + # create dnat rule for each port + site_shorewall::dnat_rule { $site_openvpn::openvpn_ports: } + +} diff --git a/puppet/modules/site_shorewall/manifests/ip_forward.pp b/puppet/modules/site_shorewall/manifests/ip_forward.pp new file mode 100644 index 00000000..d53ee8a5 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/ip_forward.pp @@ -0,0 +1,10 @@ +class site_shorewall::ip_forward { + include augeas + augeas { 'enable_ip_forwarding': + changes => 'set /files/etc/shorewall/shorewall.conf/IP_FORWARDING Yes', + lens => 'Shellvars.lns', + incl => '/etc/shorewall/shorewall.conf', + notify => Service[shorewall], + require => [ Class[augeas], Package[shorewall] ]; + } +} diff --git a/puppet/modules/site_shorewall/manifests/monitor.pp b/puppet/modules/site_shorewall/manifests/monitor.pp new file mode 100644 index 00000000..f4ed4f7c --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/monitor.pp @@ -0,0 +1,8 @@ +class site_shorewall::monitor { + + include site_shorewall::defaults + include site_shorewall::service::http + include site_shorewall::service::https + + +} diff --git a/puppet/modules/site_shorewall/manifests/mx.pp b/puppet/modules/site_shorewall/manifests/mx.pp new file mode 100644 index 00000000..332f164e --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/mx.pp @@ -0,0 +1,24 @@ +class site_shorewall::mx { + + include site_shorewall::defaults + + $smtpd_ports = '25,465,587' + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_mx': + content => "PARAM - - tcp ${smtpd_ports} ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + + shorewall::rule { + 'net2fw-mx': + source => 'net', + destination => '$FW', + action => 'leap_mx(ACCEPT)', + order => 200; + } + + include site_shorewall::service::smtp +} diff --git a/puppet/modules/site_shorewall/manifests/obfsproxy.pp b/puppet/modules/site_shorewall/manifests/obfsproxy.pp new file mode 100644 index 00000000..75846705 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/obfsproxy.pp @@ -0,0 +1,25 @@ +# configure shorewell for obfsproxy +class site_shorewall::obfsproxy { + + include site_shorewall::defaults + + $obfsproxy = hiera('obfsproxy') + $scramblesuit = $obfsproxy['scramblesuit'] + $scram_port = $scramblesuit['port'] + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_obfsproxy': + content => "PARAM - - tcp ${scram_port} ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + shorewall::rule { + 'net2fw-obfs': + source => 'net', + destination => '$FW', + action => 'leap_obfsproxy(ACCEPT)', + order => 200; + } + +} diff --git a/puppet/modules/site_shorewall/manifests/service/http.pp b/puppet/modules/site_shorewall/manifests/service/http.pp new file mode 100644 index 00000000..74b874d5 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/service/http.pp @@ -0,0 +1,13 @@ +class site_shorewall::service::http { + + include site_shorewall::defaults + + shorewall::rule { + 'net2fw-http': + source => 'net', + destination => '$FW', + action => 'HTTP(ACCEPT)', + order => 200; + } + +} diff --git a/puppet/modules/site_shorewall/manifests/service/https.pp b/puppet/modules/site_shorewall/manifests/service/https.pp new file mode 100644 index 00000000..4a8b119c --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/service/https.pp @@ -0,0 +1,12 @@ +class site_shorewall::service::https { + + include site_shorewall::defaults + + shorewall::rule { + 'net2fw-https': + source => 'net', + destination => '$FW', + action => 'HTTPS(ACCEPT)', + order => 200; + } +} diff --git a/puppet/modules/site_shorewall/manifests/service/smtp.pp b/puppet/modules/site_shorewall/manifests/service/smtp.pp new file mode 100644 index 00000000..7fbdf14e --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/service/smtp.pp @@ -0,0 +1,13 @@ +class site_shorewall::service::smtp { + + include site_shorewall::defaults + + shorewall::rule { + 'fw2net-http': + source => '$FW', + destination => 'net', + action => 'SMTP(ACCEPT)', + order => 200; + } + +} diff --git a/puppet/modules/site_shorewall/manifests/service/webapp_api.pp b/puppet/modules/site_shorewall/manifests/service/webapp_api.pp new file mode 100644 index 00000000..d3a1aeed --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/service/webapp_api.pp @@ -0,0 +1,23 @@ +# configure shorewall for webapp api +class site_shorewall::service::webapp_api { + + $api = hiera('api') + $api_port = $api['port'] + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_webapp_api': + content => "PARAM - - tcp ${api_port} ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + + shorewall::rule { + 'net2fw-webapp_api': + source => 'net', + destination => '$FW', + action => 'leap_webapp_api(ACCEPT)', + order => 200; + } + +} diff --git a/puppet/modules/site_shorewall/manifests/soledad.pp b/puppet/modules/site_shorewall/manifests/soledad.pp new file mode 100644 index 00000000..518d8689 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/soledad.pp @@ -0,0 +1,23 @@ +class site_shorewall::soledad { + + $soledad = hiera('soledad') + $soledad_port = $soledad['port'] + + include site_shorewall::defaults + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_soledad': + content => "PARAM - - tcp ${soledad_port}", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + shorewall::rule { + 'net2fw-soledad': + source => 'net', + destination => '$FW', + action => 'leap_soledad(ACCEPT)', + order => 200; + } +} + diff --git a/puppet/modules/site_shorewall/manifests/sshd.pp b/puppet/modules/site_shorewall/manifests/sshd.pp new file mode 100644 index 00000000..e2332592 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/sshd.pp @@ -0,0 +1,31 @@ +# configure shorewall for sshd +class site_shorewall::sshd { + + $ssh_config = hiera('ssh') + $ssh_port = $ssh_config['port'] + + include shorewall + + # define macro for incoming sshd + file { '/etc/shorewall/macro.leap_sshd': + content => "PARAM - - tcp ${ssh_port}", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + + shorewall::rule { + # outside to server + 'net2fw-ssh': + source => 'net', + destination => '$FW', + action => 'leap_sshd(ACCEPT)', + order => 200; + } + + # setup a routestopped rule to allow ssh when shorewall is stopped + shorewall::routestopped { $site_config::params::interface: + options => "- tcp ${ssh_port}" + } + +} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/client.pp b/puppet/modules/site_shorewall/manifests/stunnel/client.pp new file mode 100644 index 00000000..9a89a244 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/client.pp @@ -0,0 +1,40 @@ +# +# Adds some firewall magic to the stunnel. +# +# Using DNAT, this firewall rule allow a locally running program +# to try to connect to the normal remote IP and remote port of the +# service on another machine, but have this connection magically +# routed through the locally running stunnel client. +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> localhost:original_port +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +define site_shorewall::stunnel::client( + $accept_port, + $connect, + $connect_port, + $original_port) { + + include site_shorewall::defaults + + shorewall::rule { + "stunnel_dnat_${name}": + action => 'DNAT', + source => '$FW', + destination => "\$FW:127.0.0.1:${accept_port}", + proto => 'tcp', + destinationport => $original_port, + originaldest => $connect, + order => 200 + } +} diff --git a/puppet/modules/site_shorewall/manifests/stunnel/server.pp b/puppet/modules/site_shorewall/manifests/stunnel/server.pp new file mode 100644 index 00000000..798cd631 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/stunnel/server.pp @@ -0,0 +1,22 @@ +# +# Allow all incoming connections to stunnel server port +# + +define site_shorewall::stunnel::server($port) { + + include site_shorewall::defaults + + file { "/etc/shorewall/macro.stunnel_server_${name}": + content => "PARAM - - tcp ${port}", + notify => Service['shorewall'], + require => Package['shorewall'] + } + shorewall::rule { + "net2fw-stunnel-server-${name}": + source => 'net', + destination => '$FW', + action => "stunnel_server_${name}(ACCEPT)", + order => 200; + } + +}
\ No newline at end of file diff --git a/puppet/modules/site_shorewall/manifests/tor.pp b/puppet/modules/site_shorewall/manifests/tor.pp new file mode 100644 index 00000000..324b4844 --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/tor.pp @@ -0,0 +1,26 @@ +# configure shorewall for tor +class site_shorewall::tor { + + include site_shorewall::defaults + include site_shorewall::ip_forward + + $tor_port = '9001' + + # define macro for incoming services + file { '/etc/shorewall/macro.leap_tor': + content => "PARAM - - tcp ${tor_port} ", + notify => Service['shorewall'], + require => Package['shorewall'] + } + + + shorewall::rule { + 'net2fw-tor': + source => 'net', + destination => '$FW', + action => 'leap_tor(ACCEPT)', + order => 200; + } + + include site_shorewall::service::http +} diff --git a/puppet/modules/site_shorewall/manifests/webapp.pp b/puppet/modules/site_shorewall/manifests/webapp.pp new file mode 100644 index 00000000..a8d2aa5b --- /dev/null +++ b/puppet/modules/site_shorewall/manifests/webapp.pp @@ -0,0 +1,7 @@ +class site_shorewall::webapp { + + include site_shorewall::defaults + include site_shorewall::service::https + include site_shorewall::service::http + include site_shorewall::service::webapp_api +} diff --git a/puppet/modules/site_squid_deb_proxy/manifests/client.pp b/puppet/modules/site_squid_deb_proxy/manifests/client.pp new file mode 100644 index 00000000..27844270 --- /dev/null +++ b/puppet/modules/site_squid_deb_proxy/manifests/client.pp @@ -0,0 +1,5 @@ +class site_squid_deb_proxy::client { + include squid_deb_proxy::client + include site_shorewall::defaults + include shorewall::rules::mdns +} diff --git a/puppet/modules/site_sshd/manifests/authorized_keys.pp b/puppet/modules/site_sshd/manifests/authorized_keys.pp new file mode 100644 index 00000000..a1fde3f6 --- /dev/null +++ b/puppet/modules/site_sshd/manifests/authorized_keys.pp @@ -0,0 +1,34 @@ +# We want to purge unmanaged keys from the authorized_keys file so that only +# keys added in the provider are valid. Any manually added keys will be +# overridden. +# +# In order to do this, we have to use a custom define to deploy the +# authorized_keys file because puppet's internal resource doesn't allow +# purging before populating this file. +# +# See the following for more information: +# https://tickets.puppetlabs.com/browse/PUP-1174 +# https://leap.se/code/issues/2990 +# https://leap.se/code/issues/3010 +# +define site_sshd::authorized_keys ($keys, $ensure = 'present', $home = '') { + # This line allows default homedir based on $title variable. + # If $home is empty, the default is used. + $homedir = $home ? {'' => "/home/${title}", default => $home} + $owner = $ensure ? {'present' => $title, default => undef } + $group = $ensure ? {'present' => $title, default => undef } + file { + "${homedir}/.ssh": + ensure => 'directory', + owner => $title, + group => $title, + mode => '0700'; + "${homedir}/.ssh/authorized_keys": + ensure => $ensure, + owner => $owner, + group => $group, + mode => '0600', + require => File["${homedir}/.ssh"], + content => template('site_sshd/authorized_keys.erb'); + } +} diff --git a/puppet/modules/site_sshd/manifests/deploy_authorized_keys.pp b/puppet/modules/site_sshd/manifests/deploy_authorized_keys.pp new file mode 100644 index 00000000..97ca058f --- /dev/null +++ b/puppet/modules/site_sshd/manifests/deploy_authorized_keys.pp @@ -0,0 +1,9 @@ +class site_sshd::deploy_authorized_keys ( $keys ) { + tag 'leap_authorized_keys' + + site_sshd::authorized_keys {'root': + keys => $keys, + home => '/root' + } + +} diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp new file mode 100644 index 00000000..a9202da4 --- /dev/null +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -0,0 +1,82 @@ +# configures sshd, mosh, authorized keys and known hosts +class site_sshd { + $ssh = hiera_hash('ssh') + $ssh_config = $ssh['config'] + $hosts = hiera('hosts', '') + + ## + ## SETUP AUTHORIZED KEYS + ## + + $authorized_keys = $ssh['authorized_keys'] + + class { 'site_sshd::deploy_authorized_keys': + keys => $authorized_keys + } + + ## + ## SETUP KNOWN HOSTS and SSH_CONFIG + ## + + file { + '/etc/ssh/ssh_known_hosts': + owner => root, + group => root, + mode => '0644', + content => template('site_sshd/ssh_known_hosts.erb'); + + '/etc/ssh/ssh_config': + owner => root, + group => root, + mode => '0644', + content => template('site_sshd/ssh_config.erb'); + } + + ## + ## OPTIONAL MOSH SUPPORT + ## + + $mosh = $ssh['mosh'] + + if $mosh['enabled'] { + class { 'site_sshd::mosh': + ensure => present, + ports => $mosh['ports'] + } + } + else { + class { 'site_sshd::mosh': + ensure => absent + } + } + + # we cannot use the 'hardened' parameter because leap_cli uses an + # old net-ssh gem that is incompatible with the included + # "KexAlgorithms curve25519-sha256@libssh.org", + # see https://leap.se/code/issues/7591 + # therefore we don't use it here, but include all other options + # that would be applied by the 'hardened' parameter + # not all options are available on wheezy + if ( $::lsbdistcodename == 'wheezy' ) { + $tail_additional_options = 'Ciphers aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } else { + $tail_additional_options = 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + } + + ## + ## SSHD SERVER CONFIGURATION + ## + class { '::sshd': + manage_nagios => false, + ports => [ $ssh['port'] ], + use_pam => 'yes', + print_motd => 'no', + tcp_forwarding => $ssh_config['AllowTcpForwarding'], + manage_client => false, + use_storedconfigs => false, + tail_additional_options => $tail_additional_options, + hostkey_type => [ 'rsa', 'dsa', 'ecdsa' ] + } +} diff --git a/puppet/modules/site_sshd/manifests/mosh.pp b/puppet/modules/site_sshd/manifests/mosh.pp new file mode 100644 index 00000000..49f56ca0 --- /dev/null +++ b/puppet/modules/site_sshd/manifests/mosh.pp @@ -0,0 +1,21 @@ +class site_sshd::mosh ( $ensure = present, $ports = '60000-61000' ) { + + package { 'mosh': + ensure => $ensure + } + + file { '/etc/shorewall/macro.mosh': + ensure => $ensure, + content => "PARAM - - udp ${ports}", + notify => Service['shorewall'], + require => Package['shorewall']; + } + + shorewall::rule { 'net2fw-mosh': + ensure => $ensure, + source => 'net', + destination => '$FW', + action => 'mosh(ACCEPT)', + order => 200; + } +} diff --git a/puppet/modules/site_sshd/templates/authorized_keys.erb b/puppet/modules/site_sshd/templates/authorized_keys.erb new file mode 100644 index 00000000..51bdc5b3 --- /dev/null +++ b/puppet/modules/site_sshd/templates/authorized_keys.erb @@ -0,0 +1,10 @@ +# NOTICE: This file is autogenerated by Puppet +# all manually added keys will be overridden + +<% @keys.sort.each do |user, hash| -%> +<% if user == 'monitor' -%> +command="/usr/bin/check_mk_agent",no-port-forwarding,no-x11-forwarding,no-agent-forwarding,no-pty,no-user-rc, <%=hash['type']-%> <%=hash['key']%> <%=user%> +<% else -%> +<%=hash['type']-%> <%=hash['key']%> <%=user%> +<% end -%> +<% end -%> diff --git a/puppet/modules/site_sshd/templates/ssh_config.erb b/puppet/modules/site_sshd/templates/ssh_config.erb new file mode 100644 index 00000000..36c0b6d5 --- /dev/null +++ b/puppet/modules/site_sshd/templates/ssh_config.erb @@ -0,0 +1,40 @@ +# This file is generated by Puppet +# This is the ssh client system-wide configuration file. See +# ssh_config(5) for more information. This file provides defaults for +# users, and the values can be changed in per-user configuration files +# or on the command line. + +Host * + SendEnv LANG LC_* + HashKnownHosts yes + GSSAPIAuthentication yes + GSSAPIDelegateCredentials no +<% if scope.lookupvar('::site_config::params::environment') == 'local' -%> + # + # Vagrant nodes should have strict host key checking + # turned off. The problem is that the host key for a vagrant + # node is specific to the particular instance of the vagrant + # node you have running locally. For this reason, we can't + # track the host keys, or your host key for vpn1 would conflict + # with my host key for vpn1. + # + StrictHostKeyChecking no +<% end -%> + +# +# Tell SSH what host key algorithm we should use. I don't understand why this +# is needed, since the man page says that "if hostkeys are known for the +# destination host then [HostKeyAlgorithms default] is modified to prefer +# their algorithms." +# + +<% @hosts.sort.each do |name, host| -%> +Host <%= name %> <%= host['domain_full'] %> <%= host['domain_internal'] %> <%= host['ip_address'] %> +<% if host['host_pub_key'] -%> +HostKeyAlgorithms <%= host['host_pub_key'].split(" ").first %> +<% end -%> +<% if host['port'] -%> +Port <%= host['port'] %> +<% end -%> + +<% end -%> diff --git a/puppet/modules/site_sshd/templates/ssh_known_hosts.erb b/puppet/modules/site_sshd/templates/ssh_known_hosts.erb new file mode 100644 index 00000000..002ab732 --- /dev/null +++ b/puppet/modules/site_sshd/templates/ssh_known_hosts.erb @@ -0,0 +1,7 @@ +# This file is generated by Puppet + +<% @hosts.sort.each do |name, hash| -%> +<% if hash['host_pub_key'] -%> +<%= name%>,<%=hash['domain_full']%>,<%=hash['domain_internal']%>,<%=hash['ip_address']%> <%=hash['host_pub_key']%> +<% end -%> +<% end -%> diff --git a/puppet/modules/site_static/README b/puppet/modules/site_static/README new file mode 100644 index 00000000..bc719782 --- /dev/null +++ b/puppet/modules/site_static/README @@ -0,0 +1,3 @@ +Deploy one or more static websites to a node. + +For now, it only supports `amber` based static sites. Should support plain html and jekyll in the future. diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp new file mode 100644 index 00000000..b26cc9e3 --- /dev/null +++ b/puppet/modules/site_static/manifests/domain.pp @@ -0,0 +1,33 @@ +# configure static service for domain +define site_static::domain ( + $ca_cert, + $key, + $cert, + $tls_only=true, + $locations=undef, + $aliases=undef, + $apache_config=undef) { + + $domain = $name + $base_dir = '/srv/static' + + $cafile = "${cert}\n${ca_cert}" + + if is_hash($locations) { + create_resources(site_static::location, $locations) + } + + x509::cert { $domain: + content => $cafile, + notify => Service[apache] + } + x509::key { $domain: + content => $key, + notify => Service[apache] + } + + apache::vhost::file { $domain: + content => template('site_static/apache.conf.erb') + } + +} diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp new file mode 100644 index 00000000..4a722d62 --- /dev/null +++ b/puppet/modules/site_static/manifests/init.pp @@ -0,0 +1,72 @@ +# deploy static service +class site_static { + tag 'leap_service' + + include site_config::default + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca_bundle + + $static = hiera('static') + $domains = $static['domains'] + $formats = $static['formats'] + $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) + + if $bootstrap['enabled'] { + $bootstrap_domain = $bootstrap['domain'] + $bootstrap_client = $bootstrap['client_version'] + file { '/srv/leap/provider.json': + content => $bootstrap['provider_json'], + owner => 'www-data', + group => 'www-data', + mode => '0444'; + } + # It is important to always touch provider.json: the client needs to check x-min-client-version header, + # but this is only sent when the file has been modified (otherwise 304 is sent by apache). The problem + # is that changing min client version won't alter the content of provider.json, so we must touch it. + exec { '/bin/touch /srv/leap/provider.json': + require => File['/srv/leap/provider.json']; + } + } + + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include apache::module::dir + include apache::module::negotiation + include site_apache::common + include site_config::ruby::dev + + if (member($formats, 'rack')) { + include site_apt::preferences::passenger + class { 'passenger': + use_munin => false, + require => Class['site_apt::preferences::passenger'] + } + } + + if (member($formats, 'amber')) { + rubygems::gem{'amber-0.3.8': + require => Package['zlib1g-dev'] + } + + package { 'zlib1g-dev': + ensure => installed + } + } + + create_resources(site_static::domain, $domains) + + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + + include site_shorewall::defaults + include site_shorewall::service::http + include site_shorewall::service::https +} diff --git a/puppet/modules/site_static/manifests/location.pp b/puppet/modules/site_static/manifests/location.pp new file mode 100644 index 00000000..d116de2f --- /dev/null +++ b/puppet/modules/site_static/manifests/location.pp @@ -0,0 +1,36 @@ +# configure static service for location +define site_static::location($path, $format, $source) { + + $file_path = "/srv/static/${name}" + $allowed_formats = ['amber','rack'] + + if $format == undef { + fail("static_site location `${path}` is missing `format` field.") + } + + if ! member($allowed_formats, $format) { + $formats_str = join($allowed_formats, ', ') + fail("Unsupported static_site location format `${format}`. Supported formats include ${formats_str}.") + } + + if ($format == 'amber') { + exec {"amber_build_${name}": + cwd => $file_path, + command => 'amber rebuild', + user => 'www-data', + timeout => 600, + subscribe => Vcsrepo[$file_path] + } + } + + vcsrepo { $file_path: + ensure => present, + force => true, + revision => $source['revision'], + provider => $source['type'], + source => $source['repo'], + owner => 'www-data', + group => 'www-data' + } + +} diff --git a/puppet/modules/site_static/templates/amber.erb b/puppet/modules/site_static/templates/amber.erb new file mode 100644 index 00000000..694f1136 --- /dev/null +++ b/puppet/modules/site_static/templates/amber.erb @@ -0,0 +1,13 @@ +<%- if @location_path != '' -%> + AliasMatch ^/[a-z]{2}/<%=@location_path%>(/.+|/|)$ "<%=@directory%>/$1" + Alias /<%=@location_path%> "<%=@directory%>/" +<%- end -%> + <Directory "<%=@directory%>/"> + AllowOverride FileInfo Indexes Options=All,MultiViews +<% if scope.function_guess_apache_version([]) == '2.4' %> + Require all granted +<% else %> + Order deny,allow + Allow from all +<% end %> + </Directory> diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb new file mode 100644 index 00000000..6b969d1c --- /dev/null +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -0,0 +1,88 @@ +<%- + ## + ## An apache config for static websites. + ## + + def location_directory(name, location) + if ['amber', 'rack'].include?(location['format']) + File.join(@base_dir, name, 'public') + else + File.join(@base_dir, name) + end + end + + @document_root = begin + root = '/var/www' + @locations && @locations.each do |name, location| + root = location_directory(name, location) if location['path'] == '/' + end + root.gsub(%r{^/|/$}, '') + end + + bootstrap_domain = scope.lookupvar('site_static::bootstrap_domain') + bootstrap_client = scope.lookupvar('site_static::bootstrap_client') +-%> + +<VirtualHost *:80> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> +<%- @aliases && @aliases.each do |domain_alias| -%> + ServerAlias <%= domain_alias %> +<%- end -%> +<%- if @tls_only -%> + RewriteEngine On + RewriteRule ^.*$ https://<%= @domain -%>%{REQUEST_URI} [R=permanent,L] +<%- end -%> +</VirtualHost> + +<VirtualHost *:443> + ServerName <%= @domain %> + ServerAlias www.<%= @domain %> +<%- @aliases && @aliases.each do |domain_alias| -%> + ServerAlias <%= domain_alias %> +<%- end -%> + + #RewriteLog "/var/log/apache2/rewrite.log" + #RewriteLogLevel 3 + + Include include.d/ssl_common.inc + +<%- if @tls_only -%> + Header always set Strict-Transport-Security: "max-age=15768000;includeSubdomains" +<%- end -%> + Header set X-Frame-Options "deny" + Header always unset X-Powered-By + Header always unset X-Runtime + + SSLCertificateKeyFile /etc/x509/keys/<%= @domain %>.key + SSLCertificateFile /etc/x509/certs/<%= @domain %>.crt + + RequestHeader set X_FORWARDED_PROTO 'https' + + DocumentRoot "/<%= @document_root %>/" + AccessFileName .htaccess + +<%- if ([@aliases]+[@domain]).flatten.include?(bootstrap_domain) -%> + Alias /provider.json /srv/leap/provider.json + <Location /provider.json> + Header set X-Minimum-Client-Version <%= bootstrap_client['min'] %> + </Location> +<%- end -%> + +<%- if @apache_config -%> +<%= @apache_config.gsub(':percent:','%') %> +<%- end -%> + +<%- @locations && @locations.each do |name, location| -%> +<%- location_path = location['path'].gsub(%r{^/|/$}, '') -%> +<%- directory = location_directory(name, location) -%> +<%- local_vars = {'location_path'=>location_path, 'directory'=>directory, 'location'=>location, 'name'=>name} -%> +<%- template_path = File.join(File.dirname(__FILE__), location['format']) + '.erb' -%> +<%- break unless File.exists?(template_path) -%> + ## + ## <%= name %> (<%= location['format'] %>) + ## +<%= scope.function_templatewlv([template_path, local_vars]) %> +<%- end -%> + +</VirtualHost> diff --git a/puppet/modules/site_static/templates/rack.erb b/puppet/modules/site_static/templates/rack.erb new file mode 100644 index 00000000..431778bb --- /dev/null +++ b/puppet/modules/site_static/templates/rack.erb @@ -0,0 +1,19 @@ + #PassengerLogLevel 1 + #PassengerAppEnv production + #PassengerFriendlyErrorPages on +<%- if @location_path != '' -%> + Alias /<%=@location_path%> "<%=@directory%>" + <Location /<%=@location_path%>> + PassengerBaseURI /<%=@location_path%> + PassengerAppRoot "<%=File.dirname(@directory)%>" + </Location> +<%- end -%> + <Directory "<%=@directory%>"> + Options -MultiViews +<% if scope.function_guess_apache_version([]) == '2.4' %> + Require all granted +<% else %> + Order deny,allow + Allow from all +<% end %> + </Directory> diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp new file mode 100644 index 00000000..c9e034f1 --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/client.pp @@ -0,0 +1,64 @@ +# +# Sets up stunnel and firewall configuration for +# a single stunnel client +# +# As a client, we accept connections on localhost, +# and connect to a remote $connect:$connect_port +# + +define site_stunnel::client ( + $accept_port, + $connect_port, + $connect, + $original_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = 'warning' ) { + + $logfile = "/var/log/stunnel4/${name}.log" + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => "127.0.0.1:${accept_port}", + connect => "${connect}:${connect_port}", + client => true, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => $rndfile, + debuglevel => $debuglevel, + sslversion => 'TLSv1', + syslog => 'no', + output => $logfile; + } + + # define the log files so that we can purge the + # files from /var/log/stunnel4 that are not defined. + file { + $logfile:; + "${logfile}.1.gz":; + "${logfile}.2.gz":; + "${logfile}.3.gz":; + "${logfile}.4.gz":; + "${logfile}.5.gz":; + } + + site_shorewall::stunnel::client { $name: + accept_port => $accept_port, + connect => $connect, + connect_port => $connect_port, + original_port => $original_port + } + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp new file mode 100644 index 00000000..c0958b5f --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/clients.pp @@ -0,0 +1,23 @@ +# +# example hiera yaml: +# +# stunnel: +# clients: +# ednp_clients: +# thrips_9002: +# accept_port: 4001 +# connect: thrips.demo.bitmask.i +# connect_port: 19002 +# epmd_clients: +# thrips_4369: +# accept_port: 4000 +# connect: thrips.demo.bitmask.i +# connect_port: 14369 +# +# In the above example, this resource definition is called twice, with $name +# 'ednp_clients' and 'epmd_clients' +# + +define site_stunnel::clients { + create_resources(site_stunnel::client, $site_stunnel::clients[$name]) +} diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp new file mode 100644 index 00000000..a874721f --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -0,0 +1,48 @@ +# +# If you need something to happen after stunnel is started, +# you can depend on Service['stunnel'] or Class['site_stunnel'] +# + +class site_stunnel { + + # include the generic stunnel module + # increase the number of open files to allow for 800 connections + class { 'stunnel': default_extra => 'ulimit -n 4096' } + + # The stunnel.conf provided by the Debian package is broken by default + # so we get rid of it and just define our own. See #549384 + if !defined(File['/etc/stunnel/stunnel.conf']) { + file { + # this file is a broken config installed by the package + '/etc/stunnel/stunnel.conf': + ensure => absent; + } + } + + $stunnel = hiera('stunnel') + + # add server stunnels + create_resources(site_stunnel::servers, $stunnel['servers']) + + # add client stunnels + $clients = $stunnel['clients'] + $client_sections = keys($clients) + site_stunnel::clients { $client_sections: } + + # remove any old stunnel logs that are not + # defined by this puppet run + file {'/var/log/stunnel4': purge => true;} + + # the default is to keep 356 log files for each stunnel. + # here we set a more reasonable number. + augeas { + 'logrotate_stunnel': + context => '/files/etc/logrotate.d/stunnel4/rule', + changes => [ + 'set rotate 5', + ] + } + + include site_stunnel::override_service +} + diff --git a/puppet/modules/site_stunnel/manifests/override_service.pp b/puppet/modules/site_stunnel/manifests/override_service.pp new file mode 100644 index 00000000..435b9aa0 --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/override_service.pp @@ -0,0 +1,18 @@ +# override stunnel::debian defaults +# +# ignore puppet lint error about inheriting from different namespace +# lint:ignore:inherits_across_namespaces +class site_stunnel::override_service inherits stunnel::debian { +# lint:endignore + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + Service[stunnel] { + subscribe => [ + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ] + } +} diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp new file mode 100644 index 00000000..e76d1e9d --- /dev/null +++ b/puppet/modules/site_stunnel/manifests/servers.pp @@ -0,0 +1,51 @@ +# +# example hiera yaml: +# +# stunnel: +# servers: +# couch_server: +# accept_port: 15984 +# connect_port: 5984 +# + +define site_stunnel::servers ( + $accept_port, + $connect_port, + $verify = '2', + $pid = $name, + $rndfile = '/var/lib/stunnel4/.rnd', + $debuglevel = '4' ) { + + $logfile = "/var/log/stunnel4/${name}.log" + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include x509::variables + $ca_path = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt" + $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt" + $key_path = "${x509::variables::keys}/${site_config::params::cert_name}.key" + + stunnel::service { $name: + accept => $accept_port, + connect => "127.0.0.1:${connect_port}", + client => false, + cafile => $ca_path, + key => $key_path, + cert => $cert_path, + verify => $verify, + pid => "/var/run/stunnel4/${pid}.pid", + rndfile => '/var/lib/stunnel4/.rnd', + debuglevel => $debuglevel, + sslversion => 'TLSv1', + syslog => 'no', + output => $logfile; + } + + # allow incoming connections on $accept_port + site_shorewall::stunnel::server { $name: + port => $accept_port + } + + include site_check_mk::agent::stunnel +} diff --git a/puppet/modules/site_tor/manifests/disable_exit.pp b/puppet/modules/site_tor/manifests/disable_exit.pp new file mode 100644 index 00000000..078f80ae --- /dev/null +++ b/puppet/modules/site_tor/manifests/disable_exit.pp @@ -0,0 +1,7 @@ +class site_tor::disable_exit { + tor::daemon::exit_policy { + 'no_exit_at_all': + reject => [ '*:*' ]; + } +} + diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp new file mode 100644 index 00000000..2207a5a9 --- /dev/null +++ b/puppet/modules/site_tor/manifests/init.pp @@ -0,0 +1,45 @@ +class site_tor { + tag 'leap_service' + Class['site_config::default'] -> Class['site_tor'] + + $tor = hiera('tor') + $bandwidth_rate = $tor['bandwidth_rate'] + $tor_type = $tor['type'] + $nickname = $tor['nickname'] + $contact_emails = join($tor['contacts'],', ') + $family = $tor['family'] + + $address = hiera('ip_address') + + $openvpn = hiera('openvpn', undef) + if $openvpn { + $openvpn_ports = $openvpn['ports'] + } + else { + $openvpn_ports = [] + } + + include site_config::default + include tor::daemon + tor::daemon::relay { $nickname: + port => 9001, + address => $address, + contact_info => obfuscate_email($contact_emails), + bandwidth_rate => $bandwidth_rate, + my_family => $family + } + + if ( $tor_type == 'exit'){ + # Only enable the daemon directory if the node isn't also a webapp node + # or running openvpn on port 80 + if ! member($::services, 'webapp') and ! member($openvpn_ports, '80') { + tor::daemon::directory { $::hostname: port => 80 } + } + } + else { + include site_tor::disable_exit + } + + include site_shorewall::tor + +} diff --git a/puppet/modules/site_webapp/files/server-status.conf b/puppet/modules/site_webapp/files/server-status.conf new file mode 100644 index 00000000..10b2d4ed --- /dev/null +++ b/puppet/modules/site_webapp/files/server-status.conf @@ -0,0 +1,26 @@ +# Keep track of extended status information for each request +ExtendedStatus On + +# Determine if mod_status displays the first 63 characters of a request or +# the last 63, assuming the request itself is greater than 63 chars. +# Default: Off +#SeeRequestTail On + +Listen 127.0.0.1:8162 + +<VirtualHost 127.0.0.1:8162> + +<Location /server-status> + SetHandler server-status + Require all granted + Allow from 127.0.0.1 +</Location> + +</VirtualHost> + + +<IfModule mod_proxy.c> + # Show Proxy LoadBalancer status in mod_status + ProxyStatus On +</IfModule> + diff --git a/puppet/modules/site_webapp/manifests/apache.pp b/puppet/modules/site_webapp/manifests/apache.pp new file mode 100644 index 00000000..80c7b29b --- /dev/null +++ b/puppet/modules/site_webapp/manifests/apache.pp @@ -0,0 +1,28 @@ +# configure apache and passenger to serve the webapp +class site_webapp::apache { + + $web_api = hiera('api') + $api_domain = $web_api['domain'] + $api_port = $web_api['port'] + + $web_domain = hiera('domain') + $domain_name = $web_domain['name'] + + $webapp = hiera('webapp') + $webapp_domain = $webapp['domain'] + + include site_apache::common + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + include site_webapp::common_vhost + + class { 'passenger': use_munin => false } + + apache::vhost::file { + 'api': + content => template('site_apache/vhosts.d/api.conf.erb'); + } + +} diff --git a/puppet/modules/site_webapp/manifests/common_vhost.pp b/puppet/modules/site_webapp/manifests/common_vhost.pp new file mode 100644 index 00000000..c57aad57 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/common_vhost.pp @@ -0,0 +1,18 @@ +class site_webapp::common_vhost { + # installs x509 cert + key and common config + # that both nagios + leap webapp use + + include x509::variables + include site_config::x509::commercial::cert + include site_config::x509::commercial::key + include site_config::x509::commercial::ca + + Class['Site_config::X509::Commercial::Key'] ~> Service[apache] + Class['Site_config::X509::Commercial::Cert'] ~> Service[apache] + Class['Site_config::X509::Commercial::Ca'] ~> Service[apache] + + apache::vhost::file { + 'common': + content => template('site_apache/vhosts.d/common.conf.erb') + } +} diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp new file mode 100644 index 00000000..71450370 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -0,0 +1,52 @@ +class site_webapp::couchdb { + + $webapp = hiera('webapp') + # haproxy listener on port localhost:4096, see site_webapp::haproxy + $couchdb_host = 'localhost' + $couchdb_port = '4096' + $couchdb_webapp_user = $webapp['couchdb_webapp_user']['username'] + $couchdb_webapp_password = $webapp['couchdb_webapp_user']['password'] + $couchdb_admin_user = $webapp['couchdb_admin_user']['username'] + $couchdb_admin_password = $webapp['couchdb_admin_user']['password'] + + include x509::variables + + file { + '/srv/leap/webapp/config/couchdb.yml': + content => template('site_webapp/couchdb.yml.erb'), + owner => 'leap-webapp', + group => 'leap-webapp', + mode => '0600', + require => Vcsrepo['/srv/leap/webapp']; + + # couchdb.admin.yml is a symlink to prevent the vcsrepo resource + # from changing its user permissions every time. + '/srv/leap/webapp/config/couchdb.admin.yml': + ensure => 'link', + target => '/etc/leap/couchdb.admin.yml', + require => Vcsrepo['/srv/leap/webapp']; + + '/etc/leap/couchdb.admin.yml': + content => template('site_webapp/couchdb.admin.yml.erb'), + owner => 'root', + group => 'root', + mode => '0600', + require => File['/etc/leap']; + + '/srv/leap/webapp/log': + ensure => directory, + owner => 'leap-webapp', + group => 'leap-webapp', + mode => '0755', + require => Vcsrepo['/srv/leap/webapp']; + + '/srv/leap/webapp/log/production.log': + ensure => present, + owner => 'leap-webapp', + group => 'leap-webapp', + mode => '0666', + require => Vcsrepo['/srv/leap/webapp']; + } + + include site_stunnel +} diff --git a/puppet/modules/site_webapp/manifests/cron.pp b/puppet/modules/site_webapp/manifests/cron.pp new file mode 100644 index 00000000..70b9da04 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/cron.pp @@ -0,0 +1,37 @@ +# setup webapp cronjobs +class site_webapp::cron { + + # cron tasks that need to be performed to cleanup the database + cron { + 'rotate_databases': + command => 'cd /srv/leap/webapp && bundle exec rake db:rotate', + environment => 'RAILS_ENV=production', + user => 'root', + hour => [0,6,12,18], + minute => 0; + + 'delete_tmp_databases': + command => 'cd /srv/leap/webapp && bundle exec rake db:deletetmp', + environment => 'RAILS_ENV=production', + user => 'root', + hour => 1, + minute => 1; + + # there is no longer a need to remove expired sessions, since the database + # will get destroyed. + 'remove_expired_sessions': + ensure => absent, + command => 'cd /srv/leap/webapp && bundle exec rake cleanup:sessions', + environment => 'RAILS_ENV=production', + user => 'leap-webapp', + hour => 2, + minute => 30; + + 'remove_expired_tokens': + command => 'cd /srv/leap/webapp && bundle exec rake cleanup:tokens', + environment => 'RAILS_ENV=production', + user => 'leap-webapp', + hour => 3, + minute => 0; + } +} diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp new file mode 100644 index 00000000..72a2ce95 --- /dev/null +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -0,0 +1,52 @@ +class site_webapp::hidden_service { + $tor = hiera('tor') + $hidden_service = $tor['hidden_service'] + $tor_domain = "${hidden_service['address']}.onion" + + include site_apache::common + include apache::module::headers + include apache::module::alias + include apache::module::expires + include apache::module::removeip + + include tor::daemon + tor::daemon::hidden_service { 'webapp': ports => [ '80 127.0.0.1:80'] } + + file { + '/var/lib/tor/webapp/': + ensure => directory, + owner => 'debian-tor', + group => 'debian-tor', + mode => '2700'; + + '/var/lib/tor/webapp/private_key': + ensure => present, + source => "/srv/leap/files/nodes/${::hostname}/tor.key", + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600'; + + '/var/lib/tor/webapp/hostname': + ensure => present, + content => $tor_domain, + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600'; + } + + # it is necessary to zero out the config of the status module + # because we are configuring our own version that is unavailable + # over the hidden service (see: #7456 and #7776) + apache::module { 'status': ensure => present, conf_content => ' ' } + # the access_compat module is required to enable Allow directives + apache::module { 'access_compat': ensure => present } + + apache::vhost::file { + 'hidden_service': + content => template('site_apache/vhosts.d/hidden_service.conf.erb'); + 'server_status': + vhost_source => 'modules/site_webapp/server-status.conf'; + } + + include site_shorewall::tor +} diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp new file mode 100644 index 00000000..15925aba --- /dev/null +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -0,0 +1,179 @@ +# configure webapp service +class site_webapp { + tag 'leap_service' + $definition_files = hiera('definition_files') + $provider = $definition_files['provider'] + $eip_service = $definition_files['eip_service'] + $soledad_service = $definition_files['soledad_service'] + $smtp_service = $definition_files['smtp_service'] + $node_domain = hiera('domain') + $provider_domain = $node_domain['full_suffix'] + $webapp = hiera('webapp') + $api_version = $webapp['api_version'] + $secret_token = $webapp['secret_token'] + $tor = hiera('tor', false) + $sources = hiera('sources') + + Class['site_config::default'] -> Class['site_webapp'] + + include site_config::ruby::dev + include site_webapp::apache + include site_webapp::couchdb + include site_haproxy + include site_webapp::cron + include site_config::default + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + include site_config::x509::client_ca::ca + include site_config::x509::client_ca::key + include site_nickserver + + # remove leftovers from previous installations on webapp nodes + include site_config::remove::webapp + + group { 'leap-webapp': + ensure => present, + allowdupe => false; + } + + user { 'leap-webapp': + ensure => present, + allowdupe => false, + gid => 'leap-webapp', + groups => 'ssl-cert', + home => '/srv/leap/webapp', + require => [ Group['leap-webapp'] ]; + } + + vcsrepo { '/srv/leap/webapp': + ensure => present, + force => true, + revision => $sources['webapp']['revision'], + provider => $sources['webapp']['type'], + source => $sources['webapp']['source'], + owner => 'leap-webapp', + group => 'leap-webapp', + require => [ User['leap-webapp'], Group['leap-webapp'] ], + notify => Exec['bundler_update'] + } + + exec { 'bundler_update': + cwd => '/srv/leap/webapp', + command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development debug"', + unless => '/usr/bin/bundle check --path vendor/bundle', + user => 'leap-webapp', + timeout => 600, + require => [ + Class['bundler::install'], + Vcsrepo['/srv/leap/webapp'], + Class['site_config::ruby::dev'], + Service['shorewall'] ], + notify => Service['apache']; + } + + # + # NOTE: in order to support a webapp that is running on a subpath and not the + # root of the domain assets:precompile needs to be run with + # RAILS_RELATIVE_URL_ROOT=/application-root + # + + exec { 'compile_assets': + cwd => '/srv/leap/webapp', + command => '/bin/bash -c "RAILS_ENV=production /usr/bin/bundle exec rake assets:precompile"', + user => 'leap-webapp', + logoutput => on_failure, + require => Exec['bundler_update'], + notify => Service['apache']; + } + + file { + '/srv/leap/webapp/config/provider': + ensure => directory, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0755'; + + '/srv/leap/webapp/config/provider/provider.json': + content => $provider, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0644'; + + '/srv/leap/webapp/public/ca.crt': + ensure => link, + require => Vcsrepo['/srv/leap/webapp'], + target => "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"; + + "/srv/leap/webapp/public/${api_version}": + ensure => directory, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0755'; + + "/srv/leap/webapp/public/${api_version}/config/": + ensure => directory, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0755'; + + "/srv/leap/webapp/public/${api_version}/config/eip-service.json": + content => $eip_service, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0644'; + + "/srv/leap/webapp/public/${api_version}/config/soledad-service.json": + content => $soledad_service, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0644'; + + "/srv/leap/webapp/public/${api_version}/config/smtp-service.json": + content => $smtp_service, + require => Vcsrepo['/srv/leap/webapp'], + owner => leap-webapp, group => leap-webapp, mode => '0644'; + } + + try::file { + '/srv/leap/webapp/config/customization': + ensure => directory, + recurse => true, + purge => true, + force => true, + owner => leap-webapp, + group => leap-webapp, + mode => 'u=rwX,go=rX', + require => Vcsrepo['/srv/leap/webapp'], + notify => Exec['compile_assets'], + source => $webapp['customization_dir']; + } + + git::changes { + 'public/favicon.ico': + cwd => '/srv/leap/webapp', + require => Vcsrepo['/srv/leap/webapp'], + user => 'leap-webapp'; + } + + file { + '/srv/leap/webapp/config/config.yml': + content => template('site_webapp/config.yml.erb'), + owner => leap-webapp, + group => leap-webapp, + mode => '0600', + require => Vcsrepo['/srv/leap/webapp'], + notify => Service['apache']; + } + + if $tor { + $hidden_service = $tor['hidden_service'] + if $hidden_service['active'] { + include site_webapp::hidden_service + } + } + + + # needed for the soledad-sync check which is run on the + # webapp node + include soledad::client + + leap::logfile { 'webapp': } + + include site_shorewall::webapp + include site_check_mk::agent::webapp +} diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb new file mode 100644 index 00000000..dd55d3e9 --- /dev/null +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -0,0 +1,36 @@ +<% +cert_options = @webapp['client_certificates'] +production = { + "admins" => @webapp['admins'], + "default_locale" => @webapp['default_locale'], + "available_locales" => @webapp['locales'], + "domain" => @provider_domain, + "force_ssl" => @webapp['secure'], + "client_ca_key" => "%s/%s.key" % [scope.lookupvar('x509::variables::keys'), scope.lookupvar('site_config::params::client_ca_name')], + "client_ca_cert" => "%s/%s.crt" % [scope.lookupvar('x509::variables::local_CAs'), scope.lookupvar('site_config::params::client_ca_name')], + "secret_token" => @secret_token, + "client_cert_lifespan" => cert_options['life_span'], + "client_cert_bit_size" => cert_options['bit_size'].to_i, + "client_cert_hash" => cert_options['digest'], + "allow_limited_certs" => @webapp['allow_limited_certs'], + "allow_unlimited_certs" => @webapp['allow_unlimited_certs'], + "allow_anonymous_certs" => @webapp['allow_anonymous_certs'], + "limited_cert_prefix" => cert_options['limited_prefix'], + "unlimited_cert_prefix" => cert_options['unlimited_prefix'], + "minimum_client_version" => @webapp['client_version']['min'], + "default_service_level" => @webapp['default_service_level'], + "service_levels" => @webapp['service_levels'], + "allow_registration" => @webapp['allow_registration'], + "handle_blacklist" => @webapp['forbidden_usernames'], + "invite_required" => @webapp['invite_required'], + "api_tokens" => @webapp['api_tokens'] +} + +if @webapp['engines'] && @webapp['engines'].any? + production["engines"] = @webapp['engines'] +end +-%> +# +# This file is generated by puppet. This file inherits from defaults.yml. +# +<%= scope.function_sorted_yaml([{"production" => production}]) %> diff --git a/puppet/modules/site_webapp/templates/couchdb.admin.yml.erb b/puppet/modules/site_webapp/templates/couchdb.admin.yml.erb new file mode 100644 index 00000000..a0921add --- /dev/null +++ b/puppet/modules/site_webapp/templates/couchdb.admin.yml.erb @@ -0,0 +1,9 @@ +production: + prefix: "" + protocol: 'http' + host: <%= @couchdb_host %> + port: <%= @couchdb_port %> + auto_update_design_doc: false + username: <%= @couchdb_admin_user %> + password: <%= @couchdb_admin_password %> + diff --git a/puppet/modules/site_webapp/templates/couchdb.yml.erb b/puppet/modules/site_webapp/templates/couchdb.yml.erb new file mode 100644 index 00000000..2bef0af5 --- /dev/null +++ b/puppet/modules/site_webapp/templates/couchdb.yml.erb @@ -0,0 +1,9 @@ +production: + prefix: "" + protocol: 'http' + host: <%= @couchdb_host %> + port: <%= @couchdb_port %> + auto_update_design_doc: false + username: <%= @couchdb_webapp_user %> + password: <%= @couchdb_webapp_password %> + diff --git a/puppet/modules/soledad/manifests/client.pp b/puppet/modules/soledad/manifests/client.pp new file mode 100644 index 00000000..e470adeb --- /dev/null +++ b/puppet/modules/soledad/manifests/client.pp @@ -0,0 +1,16 @@ +# setup soledad-client +# currently needed on webapp node to run the soledad-sync test +class soledad::client { + + tag 'leap_service' + include soledad::common + + package { + 'soledad-client': + ensure => latest, + require => Class['site_apt::leap_repo']; + 'python-u1db': + ensure => latest; + } + +} diff --git a/puppet/modules/soledad/manifests/common.pp b/puppet/modules/soledad/manifests/common.pp new file mode 100644 index 00000000..8d8339d4 --- /dev/null +++ b/puppet/modules/soledad/manifests/common.pp @@ -0,0 +1,8 @@ +# install soledad-common, both needed both soledad-client and soledad-server +class soledad::common { + + package { 'soledad-common': + ensure => latest; + } + +} diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp new file mode 100644 index 00000000..8674f421 --- /dev/null +++ b/puppet/modules/soledad/manifests/server.pp @@ -0,0 +1,104 @@ +# setup soledad-server +class soledad::server { + tag 'leap_service' + + include site_config::default + include soledad::common + + $soledad = hiera('soledad') + $couchdb_user = $soledad['couchdb_soledad_user']['username'] + $couchdb_password = $soledad['couchdb_soledad_user']['password'] + $couchdb_leap_mx_user = $soledad['couchdb_leap_mx_user']['username'] + + $couchdb_host = 'localhost' + $couchdb_port = '5984' + + $soledad_port = $soledad['port'] + + $sources = hiera('sources') + + include site_config::x509::cert + include site_config::x509::key + include site_config::x509::ca + + # + # SOLEDAD CONFIG + # + + file { + '/etc/soledad': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + '/etc/soledad/soledad-server.conf': + content => template('soledad/soledad-server.conf.erb'), + owner => 'soledad', + group => 'soledad', + mode => '0640', + notify => Service['soledad-server'], + require => [ User['soledad'], Group['soledad'] ]; + '/srv/leap/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; + '/var/lib/soledad': + ensure => directory, + owner => 'soledad', + group => 'soledad', + require => [ User['soledad'], Group['soledad'] ]; + } + + package { $sources['soledad']['package']: + ensure => $sources['soledad']['revision'], + require => Class['site_apt::leap_repo']; + } + + file { '/etc/default/soledad': + content => template('soledad/default-soledad.erb'), + owner => 'soledad', + group => 'soledad', + mode => '0600', + notify => Service['soledad-server'], + require => [ User['soledad'], Group['soledad'] ]; + } + + service { 'soledad-server': + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + require => [ User['soledad'], Group['soledad'] ], + subscribe => [ + Package['soledad-server'], + Class['Site_config::X509::Key'], + Class['Site_config::X509::Cert'], + Class['Site_config::X509::Ca'] ]; + } + + include site_shorewall::soledad + include site_check_mk::agent::soledad + + # set up users, group and directories for soledad-server + # although the soledad users are already created by the + # soledad-server package + group { 'soledad': + ensure => present, + system => true, + } + user { + 'soledad': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + 'soledad-admin': + ensure => present, + system => true, + gid => 'soledad', + home => '/srv/leap/soledad', + require => Group['soledad']; + } +} diff --git a/puppet/modules/soledad/templates/default-soledad.erb b/puppet/modules/soledad/templates/default-soledad.erb new file mode 100644 index 00000000..32504e38 --- /dev/null +++ b/puppet/modules/soledad/templates/default-soledad.erb @@ -0,0 +1,5 @@ +# this file is managed by puppet +START=yes +CERT_PATH=<%= scope.lookupvar('x509::variables::certs') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.crt +PRIVKEY_PATH=<%= scope.lookupvar('x509::variables::keys') %>/<%= scope.lookupvar('site_config::params::cert_name') %>.key +HTTPS_PORT=<%=@soledad_port%> diff --git a/puppet/modules/soledad/templates/soledad-server.conf.erb b/puppet/modules/soledad/templates/soledad-server.conf.erb new file mode 100644 index 00000000..1c6a0d19 --- /dev/null +++ b/puppet/modules/soledad/templates/soledad-server.conf.erb @@ -0,0 +1,12 @@ +[soledad-server] +couch_url = http://<%= @couchdb_user %>:<%= @couchdb_password %>@<%= @couchdb_host %>:<%= @couchdb_port %> +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc + +[database-security] +members = <%= @couchdb_user %>, <%= @couchdb_leap_mx_user %> +# not needed, but for documentation: +# members_roles = replication +# admins = admin +# admins_roles = replication + diff --git a/puppet/modules/templatewlv/Modulefile b/puppet/modules/templatewlv/Modulefile new file mode 100644 index 00000000..8007a070 --- /dev/null +++ b/puppet/modules/templatewlv/Modulefile @@ -0,0 +1,11 @@ +name 'duritong-templatewlv' +version '0.0.1' +source 'https://github.com/duritong/puppet-templatewlv.git' +author 'duritong' +license 'Apache License, Version 2.0' +summary 'Template With Local Variables' +description 'Pass local variables to templates' +project_page 'https://github.com/duritong/puppet-templatewlv' + +## Add dependencies, if any: +# dependency 'username/name', '>= 1.2.0' diff --git a/puppet/modules/templatewlv/README.md b/puppet/modules/templatewlv/README.md new file mode 100644 index 00000000..5ab01e45 --- /dev/null +++ b/puppet/modules/templatewlv/README.md @@ -0,0 +1,21 @@ +# templatewlv + +## Template With Local Variables + +A wrapper around puppet's template function. See +[the templating docs](http://docs.puppetlabs.com/guides/templating.html) for +the basic functionality. + +Additionally, you can pass a hash, as the last argument, which will be turned into +local variables and available to the template itself. This will allow you to define +variables in a template and pass them down to a template you include in the current +template. An example: + + scope.function_templatewlv(['sub_template', { 'local_var' => 'value' }]) + +Note that if multiple templates are specified, their output is all +concatenated and returned as the output of the function. + +# Who - License + +duritong - Apache License, Version 2.0 diff --git a/puppet/modules/templatewlv/lib/puppet/parser/functions/templatewlv.rb b/puppet/modules/templatewlv/lib/puppet/parser/functions/templatewlv.rb new file mode 100644 index 00000000..c9579e2c --- /dev/null +++ b/puppet/modules/templatewlv/lib/puppet/parser/functions/templatewlv.rb @@ -0,0 +1,41 @@ +require File.join(File.dirname(__FILE__),'../templatewrapperwlv') +Puppet::Parser::Functions::newfunction(:templatewlv, :type => :rvalue, :arity => -2, :doc => + "A wrapper around puppet's template function. See + [the templating docs](http://docs.puppetlabs.com/guides/templating.html) for + the basic functionality. + + Additionally, you can pass a hash, as the last argument, which will be turned into + local variables and available to the template itself. This will allow you to define + variables in a template and pass them down to a template you include in the current + template. An example: + + scope.function_templatewlv(['sub_template', { 'local_var' => 'value' }]) + + Note that if multiple templates are specified, their output is all + concatenated and returned as the output of the function.") do |vals| + + if vals.last.is_a?(Hash) + local_vars = vals.last + local_vals = vals[0..-2] + else + local_vars = {} + local_vals = vals + end + + result = nil + local_vals.collect do |file| + # Use a wrapper, so the template can't get access to the full + # Scope object. + debug "Retrieving template #{file}" + + wrapper = Puppet::Parser::TemplateWrapperWlv.new(self,local_vars) + wrapper.file = file + begin + wrapper.result + rescue => detail + info = detail.backtrace.first.split(':') + raise Puppet::ParseError, + "Failed to parse template #{file}:\n Filepath: #{info[0]}\n Line: #{info[1]}\n Detail: #{detail}\n" + end + end.join("") +end diff --git a/puppet/modules/templatewlv/lib/puppet/parser/templatewrapperwlv.rb b/puppet/modules/templatewlv/lib/puppet/parser/templatewrapperwlv.rb new file mode 100644 index 00000000..f1753e18 --- /dev/null +++ b/puppet/modules/templatewlv/lib/puppet/parser/templatewrapperwlv.rb @@ -0,0 +1,39 @@ +# A wrapper for templates, that allows you to additionally define +# local variables +class Puppet::Parser::TemplateWrapperWlv < Puppet::Parser::TemplateWrapper + attr_reader :local_vars + def initialize(scope, local_vars) + super(scope) + @local_vars = local_vars + end + + # Should return true if a variable is defined, false if it is not + def has_variable?(name) + super(name) || local_vars.keys.include?(name.to_s) + end + + def method_missing(name, *args) + if local_vars.keys.include?(n=name.to_s) + local_vars[n] + else + super(name, *args) + end + end + + def result(string = nil) + # Expose all the variables in our scope as instance variables of the + # current object, making it possible to access them without conflict + # to the regular methods. + benchmark(:debug, "Bound local template variables for #{@__file__}") do + local_vars.each do |name, value| + if name.kind_of?(String) + realname = name.gsub(/[^\w]/, "_") + else + realname = name + end + instance_variable_set("@#{realname}", value) + end + end + super(string) + end +end diff --git a/puppet/modules/try/README.md b/puppet/modules/try/README.md new file mode 100644 index 00000000..3888661e --- /dev/null +++ b/puppet/modules/try/README.md @@ -0,0 +1,13 @@ +This module provides a "try" wrapper around common resource types. + +For example: + + try::file { + '/path/to/file': + ensure => 'link', + target => $target; + } + +This will work just like `file`, but will silently fail if `$target` is undefined or the file does not exist. + +So far, only `file` type with symlinks works. diff --git a/puppet/modules/try/manifests/file.pp b/puppet/modules/try/manifests/file.pp new file mode 100644 index 00000000..2493d343 --- /dev/null +++ b/puppet/modules/try/manifests/file.pp @@ -0,0 +1,114 @@ +# +# Works like the built-in type "file", but gets gracefully ignored if the target/source does not exist or is undefined. +# +# Also, if the source or target doesn't exist, and the destination is a git repo, then the file is restored from git. +# +# All executable paths are hardcoded to their paths in debian. +# +# known limitations: +# * this is far too noisy +# * $restore does not work for directories +# * only file:// $source is supported +# * $content is not supported, only $target or $source. +# * does not auto-require all the parent directories like 'file' does +# +define try::file ( + $ensure = undef, + $target = undef, + $source = undef, + $owner = undef, + $group = undef, + $recurse = undef, + $purge = undef, + $force = undef, + $mode = undef, + $restore = true) { + + # dummy exec to propagate requires: + # metaparameter 'require' will get triggered by this dummy exec + # so then we just need to depend on this to capture all requires. + # exec { $name: command => "/bin/true" } + + exec { + "chmod_${name}": + command => "/bin/chmod -R ${mode} '${name}'", + onlyif => "/usr/bin/test ${mode}", + refreshonly => true, + loglevel => debug; + "chown_${name}": + command => "/bin/chown -R ${owner} '${name}'", + onlyif => "/usr/bin/test ${owner}", + refreshonly => true, + loglevel => debug; + "chgrp_${name}": + command => "/bin/chgrp -R ${group} '${name}'", + onlyif => "/usr/bin/test ${group}", + refreshonly => true, + loglevel => debug; + } + + if $target { + exec { "symlink_${name}": + command => "/bin/ln -s ${target} ${name}", + onlyif => "/usr/bin/test -d '${target}'", + } + } elsif $source { + if $ensure == 'directory' { + if $purge { + exec { "rsync_${name}": + command => "/usr/bin/rsync -r --delete '${source}/' '${name}'", + onlyif => "/usr/bin/test -d '${source}'", + unless => "/usr/bin/diff -rq '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + } + } else { + exec { "cp_r_${name}": + command => "/bin/cp -r '${source}' '${name}'", + onlyif => "/usr/bin/test -d '${source}'", + unless => "/usr/bin/diff -rq '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + } + } + } else { + exec { "cp_${name}": + command => "/bin/cp --remove-destination '${source}' '${name}'", + onlyif => "/usr/bin/test -e '${source}'", + unless => "/usr/bin/test ! -h '${name}' && /usr/bin/diff -q '${source}' '${name}'", + notify => [Exec["chmod_${name}"], Exec["chown_${name}"], Exec["chgrp_${name}"]] + } + } + } + + # + # if the target/source does not exist (or is undef), and the file happens to be in a git repo, + # then restore the file to its original state. + # + + if $target { + $target_or_source = $target + } else { + $target_or_source = $source + } + + if ($target_or_source == undef) or $restore { + $file_basename = basename($name) + $file_dirname = dirname($name) + $command = "git rev-parse && unlink '${name}'; git checkout -- '${file_basename}' && chown --reference='${file_dirname}' '${name}'; true" + debug($command) + + if $target_or_source == undef { + exec { "restore_${name}": + command => $command, + cwd => $file_dirname, + loglevel => info; + } + } else { + exec { "restore_${name}": + unless => "/usr/bin/test -e '${target_or_source}'", + command => $command, + cwd => $file_dirname, + loglevel => info; + } + } + } +} diff --git a/puppet/modules/try/manifests/init.pp b/puppet/modules/try/manifests/init.pp new file mode 100644 index 00000000..1d2108c9 --- /dev/null +++ b/puppet/modules/try/manifests/init.pp @@ -0,0 +1,3 @@ +class try { + +} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..814c25b1 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,25 @@ +Tests +--------------------------------- + +tests/white-box/ + + These tests are run on the server as superuser. They are for + troubleshooting any problems with the internal setup of the server. + +tests/black-box/ + + These test are run the user's local machine. They are for troubleshooting + any external problems with the service exposed by the server. + +Additional Files +--------------------------------- + +tests/helpers/ + + Utility functions made available to all tests. + +tests/order.rb + + Configuration file to specify which nodes should be tested in which order. + + diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb new file mode 100644 index 00000000..5b886228 --- /dev/null +++ b/tests/helpers/bonafide_helper.rb @@ -0,0 +1,235 @@ +# +# helper for the communication with the provider API for creating, authenticating, and deleting accounts. +# + +class LeapTest + + def assert_tmp_user + user = assert_create_user + assert_authenticate_user(user) + yield user if block_given? + assert_delete_user(user) + rescue StandardError, MiniTest::Assertion => exc + begin + assert_delete_user(user) + rescue + end + raise exc + end + + # + # attempts to create a user account via the API, + # returning the user object if successful. + # + def assert_create_user(username=nil, auth=nil) + user = SRP::User.new(username) + url = api_url("/users.json") + params = user.to_params + if auth + options = api_options(:auth => auth) + else + options = api_options + if property('webapp.invite_required') + @invite_code = generate_invite_code + params['user[invite_code]'] = @invite_code + end + end + + assert_post(url, params, options) do |body| + assert response = JSON.parse(body), 'response should be JSON' + assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead." + user.ok = true + user.id = response['id'] + end + return user + end + + # TODO: use the api for this instead. + def generate_invite_code + `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "") + end + + # + # attempts to authenticate user. if successful, + # user object is updated with id and session token. + # + def assert_authenticate_user(user) + url = api_url("/sessions.json") + session = SRP::Session.new(user) + params = {'login' => user.username, 'A' => session.aa} + assert_post(url, params, api_options) do |body, response| + cookie = response['Set-Cookie'].split(';').first + assert(response = JSON.parse(body), 'response should be JSON') + assert(session.bb = response["B"], 'response should include "B"') + url = api_url("/sessions/login.json") + params = {'client_auth' => session.m, 'A' => session.aa} + assert_put(url, params, api_options('Cookie' => cookie)) do |body| + assert(response = JSON.parse(body), 'response should be JSON') + assert(response['M2'], 'response should include M2') + user.session_token = response['token'] + user.id = response['id'] + assert(user.session_token, 'response should include token') + assert(user.id, 'response should include user id') + end + end + end + + # + # attempts to destroy a user account via the API. + # + def assert_delete_user(user) + if user.is_a? String + assert_delete_user_by_login(user) + elsif user.is_a? SRP::User + assert_delete_srp_user(user) + end + end + + # + # returns true if the identity exists, uses monitor token auth + # + def identity_exists?(address) + url = api_url("/identities/#{URI.encode(address)}.json") + options = {:ok_codes => [200, 404]}.merge( + api_options(:auth => :monitor) + ) + assert_get(url, nil, options) do |body, response| + return response.code == "200" + end + end + + def upload_public_key(user_id, public_key) + url = api_url("/users/#{user_id}.json") + params = {"user[public_key]" => public_key} + assert_put(url, params, api_options(:auth => :monitor)) + end + + # + # return user document as a Hash. uses monitor token auth + # + def find_user_by_id(user_id) + url = api_url("/users/#{user_id}.json") + assert_get(url, nil, api_options(:auth => :monitor)) do |body| + return JSON.parse(body) + end + end + + # + # return user document as a Hash. uses monitor token auth + # NOTE: this relies on deprecated behavior of the API + # and will not work when multi-domain support is added. + # + def find_user_by_login(login) + url = api_url("/users/0.json?login=#{login}") + options = {:ok_codes => [200, 404]}.merge( + api_options(:auth => :monitor) + ) + assert_get(url, nil, options) do |body, response| + if response.code == "200" + return JSON.parse(body) + else + return nil + end + end + end + + private + + def api_url(path) + unless path =~ /^\// + path = '/' + path + end + if property('testing.api_uri') + return property('testing.api_uri') + path + elsif property('api') + api = property('api') + return "https://%{domain}:%{port}/%{version}#{path}" % { + :domain => api['domain'], + :port => api['port'], + :version => api['version'] || 1 + } + else + fail 'This node needs to have either testing.api_url or api.{domain,port} configured.' + end + end + + # + # produces an options hash used for api http requests. + # + # argument options hash gets added to "headers" + # of the http request. + # + # special :auth key in argument will expand to + # add api_token_auth header. + # + # if you want to try manually: + # + # export API_URI=`grep api_uri /etc/leap/hiera.yaml | cut -d\" -f2` + # export TOKEN=`grep monitor_auth_token /etc/leap/hiera.yaml | awk '{print $2}'` + # curl -H "Accept: application/json" -H "Token: $TOKEN" $API_URI + # + def api_options(options={}) + # note: must be :headers, not "headers" + hsh = { + :headers => { + "Accept" => "application/json" + } + } + if options[:auth] + hsh[:headers].merge!(api_token_auth(options.delete(:auth))) + end + hsh[:headers].merge!(options) + return hsh + end + + # + # add token authentication to a http request. + # + # returns a hash suitable for adding to the 'headers' option + # of an http function. + # + def api_token_auth(token) + if token.is_a?(Symbol) && property('testing') + if token == :monitor + token_str = property('testing.monitor_auth_token') + else + raise ArgumentError.new 'no such token' + end + else + token_str = token + end + {"Authorization" => "Token token=\"#{token_str}\""} + end + + # + # not actually used in any test, but useful when + # writing new tests. + # + def assert_delete_user_by_login(login_name) + user = find_user_by_login(login_name) + url = api_url("/users/#{user['id']}.json") + params = {:identities => 'destroy'} + delete(url, params, api_options(:auth => :monitor)) do |body, response, error| + assert error.nil?, "Error deleting user: #{error}" + assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" + assert(response = JSON.parse(body), 'Delete response should be JSON') + assert(response["success"], 'Deleting user should be a success') + end + end + + def assert_delete_srp_user(user) + if user && user.ok && user.id && user.session_token && !user.deleted + url = api_url("users/#{user.id}.json") + params = {:identities => 'destroy'} + user.deleted = true + delete(url, params, api_options(:auth => user.session_token)) do |body, response, error| + assert error.nil?, "Error deleting user: #{error}" + assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}" + assert(response = JSON.parse(body), 'Delete response should be JSON') + assert(response["success"], 'Deleting user should be a success') + end + end + end + + +end diff --git a/tests/helpers/client_side_db.py b/tests/helpers/client_side_db.py new file mode 100644 index 00000000..2f8c220f --- /dev/null +++ b/tests/helpers/client_side_db.py @@ -0,0 +1,167 @@ +import logging +import os +import tempfile +import getpass +import binascii +import json + +try: + import requests + import srp._pysrp as srp +except ImportError: + pass + +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad + + +""" +Helper functions to give access to client-side Soledad database. +Copied over from soledad/scripts folder. +""" + +# create a logger +logger = logging.getLogger(__name__) + +# DEBUG: enable debug logs +# LOG_FORMAT = '%(asctime)s %(message)s' +# logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_api_info(provider): + info = requests.get( + 'https://'+provider+'/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb new file mode 100644 index 00000000..b9085c1e --- /dev/null +++ b/tests/helpers/couchdb_helper.rb @@ -0,0 +1,142 @@ +class LeapTest + + # + # generates a couchdb url for when couchdb is running + # remotely and is available via stunnel. + # + # example properties: + # + # stunnel: + # clients: + # couch_client: + # couch1_5984: + # accept_port: 4000 + # connect: couch1.bitmask.i + # connect_port: 15984 + # + def couchdb_urls_via_stunnel(path="", options=nil) + path = path.gsub('"', '%22') + if options && options[:username] && options[:password] + userpart = "%{username}:%{password}@" % options + else + userpart = "" + end + assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf| + assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' + URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url| + remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last + url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address] + } + end + end + + # + # generates a couchdb url for accessing couchdb via haproxy + # + # example properties: + # + # haproxy: + # couch: + # listen_port: 4096 + # servers: + # panda: + # backup: false + # host: localhost + # port: 4000 + # weight: 100 + # writable: true + # + def couchdb_url_via_haproxy(path="", options=nil) + path = path.gsub('"', '%22') + if options && options[:username] && options[:password] + userpart = "%{username}:%{password}@" % options + else + userpart = "" + end + port = assert_property('haproxy.couch.listen_port') + return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| + url.memo = '(via haproxy)' + } + end + + # + # generates a couchdb url for when couchdb is running locally. + # + # example properties: + # + # couch: + # port: 5984 + # + def couchdb_url_via_localhost(path="", options=nil) + path = path.gsub('"', '%22') + port = (options && options[:port]) || assert_property('couch.port') + if options && options[:username] + password = property("couch.users.%{username}.password" % options) + userpart = "%s:%s@" % [options[:username], password] + else + userpart = "" + end + return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| + url.memo = '(via direct localhost connection)' + } + end + + # + # returns a single url for accessing couchdb + # + def couchdb_url(path="", options=nil) + if property('couch.port') + couchdb_url_via_localhost(path, options) + elsif property('stunnel.clients.couch_client') + couchdb_urls_via_stunnel(path, options).first + end + end + + # + # returns an array of urls for accessing couchdb + # + def couchdb_urls(path="", options=nil) + if property('couch.port') + [couchdb_url_via_localhost(path, options)] + elsif property('stunnel.clients.couch_client') + couchdb_urls_via_stunnel(path, options) + end + end + + def assert_destroy_user_db(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + http_options = {:ok_codes => [200, 404]} # ignore missing dbs + assert_delete(url, nil, http_options) + end + + def assert_create_user_db(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + http_options = {:ok_codes => [200, 404]} # ignore missing dbs + assert_put(url, nil, :format => :json) do |body| + assert response = JSON.parse(body), "PUT response should be JSON" + assert response["ok"], "PUT response should be OK" + end + end + + # + # returns true if the per-user db created by soledad-server exists. + # + def user_db_exists?(user_id, options=nil) + db_name = "user-#{user_id}" + url = couchdb_url("/#{db_name}", options) + get(url) do |body, response, error| + if response.nil? + fail "could not query couchdb #{url}: #{error}\n#{body}" + elsif response.code.to_i == 200 + return true + elsif response.code.to_i == 404 + return false + else + fail ["could not query couchdb #{url}: expected response code 200 or 404, but got #{response.code}.", error, body].compact.join("\n") + end + end + end + +end
\ No newline at end of file diff --git a/tests/helpers/files_helper.rb b/tests/helpers/files_helper.rb new file mode 100644 index 00000000..d6795889 --- /dev/null +++ b/tests/helpers/files_helper.rb @@ -0,0 +1,54 @@ +class LeapTest + + # + # Matches the regexp in the file, and returns the first matched string (or fails if no match). + # + def file_match(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures.first + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # Matches the regexp in the file, and returns array of matched strings (or fails if no match). + # + def file_matches(filename, regexp) + if match = File.read(filename).match(regexp) + match.captures + else + fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}." + end + end + + # + # checks to make sure the given property path exists in $node (e.g. hiera.yaml) + # and returns the value + # + def assert_property(property) + latest = $node + property.split('.').each do |segment| + latest = latest[segment] + fail "Required node property `#{property}` is missing." if latest.nil? + end + return latest + end + + # + # a handy function to get the value of a long property path + # without needing to test the existance individually of each part + # in the tree. + # + # e.g. property("stunnel.clients.couch_client") + # + def property(property) + latest = $node + property.split('.').each do |segment| + latest = latest[segment] + return nil if latest.nil? + end + return latest + end + +end
\ No newline at end of file diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb new file mode 100644 index 00000000..0d0bb7d5 --- /dev/null +++ b/tests/helpers/http_helper.rb @@ -0,0 +1,157 @@ +require 'net/http' + +class LeapTest + + # + # In order to easily provide detailed error messages, it is useful + # to append a memo to a url string that details what this url is for + # (e.g. stunnel, haproxy, etc). + # + # So, the url happens to be a UrlString, the memo field is used + # if there is an error in assert_get. + # + class URLString < String + attr_accessor :memo + end + + # + # aliases for http_send() + # + def get(url, params=nil, options=nil, &block) + http_send("GET", url, params, options, &block) + end + def delete(url, params=nil, options=nil, &block) + http_send("DELETE", url, params, options, &block) + end + def post(url, params=nil, options=nil, &block) + http_send("POST", url, params, options, &block) + end + def put(url, params=nil, options=nil, &block) + http_send("PUT", url, params, options, &block) + end + + # + # send a GET, DELETE, POST, or PUT + # yields |body, response, error| + # + def http_send(method, url, params=nil, options=nil) + options ||= {} + response = nil + + # build uri + uri = URI(url) + if params && (method == 'GET' || method == 'DELETE') + uri.query = URI.encode_www_form(params) + end + + # build http + http = Net::HTTP.new uri.host, uri.port + if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.use_ssl = true + end + + # build request + request = build_request(method, uri, params, options) + + # make http request + http.start do |agent| + response = agent.request(request) + yield response.body, response, nil + end + rescue => exc + yield nil, response, exc + end + + # + # Aliases for assert_http_send() + # + def assert_get(url, params=nil, options=nil, &block) + assert_http_send("GET", url, params, options, &block) + end + def assert_delete(url, params=nil, options=nil, &block) + assert_http_send("DELETE", url, params, options, &block) + end + def assert_post(url, params=nil, options=nil, &block) + assert_http_send("POST", url, params, options, &block) + end + def assert_put(url, params=nil, options=nil, &block) + assert_http_send("PUT", url, params, options, &block) + end + + # + # calls http_send, yielding results if successful or failing with + # descriptive info otherwise. + # + # options: + # - error_msg: custom error message to display. + # - ok_codes: in addition to 2xx, codes in this array will not produce an error. + # + def assert_http_send(method, url, params=nil, options=nil, &block) + options ||= {} + error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil) + http_send(method, url, params, options) do |body, response, error| + if response + code = response.code.to_i + ok = code >= 200 && code < 300 + if options[:ok_codes] + ok ||= options[:ok_codes].include?(code) + end + if ok + if block + yield(body) if block.arity == 1 + yield(body, response) if block.arity == 2 + yield(body, response, error) if block.arity == 3 + end + else + fail ["Expected success code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n") + end + else + fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error + end + end + end + + # + # only a warning for now, should be a failure in the future + # + def assert_auth_fail(url, params) + uri = URI(url) + get(url, params) do |body, response, error| + unless response.code.to_s == "401" + warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')." + return false + end + end + true + end + + private + + def build_request(method, uri, params, options) + request = case method + when "GET" then Net::HTTP::Get.new(uri.request_uri) + when "DELETE" then Net::HTTP::Delete.new(uri.request_uri) + when "POST" then Net::HTTP::Post.new(uri.request_uri) + when "PUT" then Net::HTTP::Put.new(uri.request_uri) + end + if uri.user + request.basic_auth uri.user, uri.password + end + if params && (method == 'POST' || method == 'PUT') + if options[:format] == :json || options[:format] == 'json' + request["Content-Type"] = "application/json" + request.body = params.to_json + else + request.set_form_data(params) if params + end + end + if options[:headers] + options[:headers].each do |key, value| + request[key] = value + end + end + request + end + +end
\ No newline at end of file diff --git a/tests/helpers/network_helper.rb b/tests/helpers/network_helper.rb new file mode 100644 index 00000000..713d57aa --- /dev/null +++ b/tests/helpers/network_helper.rb @@ -0,0 +1,79 @@ +class LeapTest + + # + # tcp connection helper with timeout + # + def try_tcp_connect(host, port, timeout = 5) + addr = Socket.getaddrinfo(host, nil) + sockaddr = Socket.pack_sockaddr_in(port, addr[0][3]) + + Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket| + socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + begin + socket.connect_nonblock(sockaddr) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Connection timeout" + else + socket.connect_nonblock(sockaddr) + end + end + return socket + end + end + + def try_tcp_write(socket, timeout = 5) + begin + socket.write_nonblock("\0") + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Write timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Write timeout" + else + retry + end + end + end + + def try_tcp_read(socket, timeout = 5) + begin + socket.read_nonblock(1) + rescue IO::WaitReadable + if IO.select([socket], nil, nil, timeout) == nil + raise "Read timeout" + else + retry + end + rescue IO::WaitWritable + if IO.select(nil, [socket], nil, timeout) == nil + raise "Read timeout" + else + retry + end + end + end + + def assert_tcp_socket(host, port, msg=nil) + begin + socket = try_tcp_connect(host, port, 1) + #try_tcp_write(socket,1) + #try_tcp_read(socket,1) + rescue StandardError => exc + fail ["Failed to open socket #{host}:#{port}", exc, msg].compact.join("\n") + ensure + socket.close if socket + end + end + +end
\ No newline at end of file diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb new file mode 100644 index 00000000..9923d5b1 --- /dev/null +++ b/tests/helpers/os_helper.rb @@ -0,0 +1,41 @@ +class LeapTest + + # + # works like pgrep command line + # return an array of hashes like so [{:pid => "1234", :process => "ls"}] + # + def pgrep(match) + output = `pgrep --full --list-name '#{match}'` + output.each_line.map{|line| + pid = line.split(' ')[0] + process = line.gsub(/(#{pid} |\n)/, '') + # filter out pgrep cmd itself + # on wheezy hosts, the "process" var contains the whole cmd including all parameters + # on jessie hosts, it only contains the first cmd (which is the default sheel invoked by 'sh') + if process =~ /^sh/ + nil + else + {:pid => pid, :process => process} + end + }.compact + end + + def assert_running(process, options={}) + processes = pgrep(process) + assert processes.any?, "No running process for #{process}" + if options[:single] + assert processes.length == 1, "More than one process for #{process}" + end + end + + # + # runs the specified command, failing on a non-zero exit status. + # + def assert_run(command) + output = `#{command} 2>&1` + if $?.exitstatus != 0 + fail "Error running `#{command}`:\n#{output}" + end + end + +end
\ No newline at end of file diff --git a/tests/helpers/smtp_helper.rb b/tests/helpers/smtp_helper.rb new file mode 100644 index 00000000..ea7fb9fa --- /dev/null +++ b/tests/helpers/smtp_helper.rb @@ -0,0 +1,45 @@ +require 'net/smtp' + +class LeapTest + + TEST_EMAIL_USER = "test_user_email" + TEST_BAD_USER = "test_user_bad" + + MSG_BODY = %(Since it seems that any heart which beats for freedom has the right only to a +lump of lead, I too claim my share. If you let me live, I shall never stop +crying for revenge and I shall avenge my brothers. I have finished. If you are +not cowards, kill me! + +--Louise Michel) + + def send_email(recipient, options={}) + sender = options[:sender] || recipient + helo_domain = property('domain.full_suffix') + headers = { + "Date" => Time.now.utc, + "From" => sender, + "To" => recipient, + "Subject" => "Test Message", + "X-LEAP-TEST" => "true" + }.merge(options[:headers]||{}) + message = [] + headers.each do |key, value| + message << "#{key}: #{value}" + end + message << "" + message << MSG_BODY + Net::SMTP.start('localhost', 25, helo_domain) do |smtp| + smtp.send_message message.join("\n"), recipient, sender + end + end + + def assert_send_email(recipient, options={}) + begin + send_email(recipient, options) + rescue IOError, Net::OpenTimeout, + Net::ReadTimeout, Net::SMTPError => e + fail "Could not send mail to #{recipient} (#{e})" + end + end + +end
\ No newline at end of file diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py new file mode 100755 index 00000000..f4fc81ae --- /dev/null +++ b/tests/helpers/soledad_sync.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +""" +soledad_sync.py + +This script exercises soledad synchronization. +Its exit code is 0 if the sync took place correctly, 1 otherwise. + +It takes 5 arguments: + + uuid: uuid of the user to sync + token: a valid session token + server: the url of the soledad server we should connect to + cert_file: the file containing the certificate for the CA that signed the + cert for the soledad server. + password: the password for the user to sync + +__author__: kali@leap.se +""" +import os +import shutil +import sys +import tempfile + +# This is needed because the twisted shipped with wheezy is too old +# to do proper ssl verification. +os.environ['SKIP_TWISTED_SSL_CHECK'] = '1' + +from twisted.internet import defer, reactor +from twisted.python import log + +from client_side_db import get_soledad_instance +from leap.common.events import flags + +flags.set_events_enabled(False) + +NUMDOCS = 1 +USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0] + + +def bail(msg, exitcode): + print "[!] %s" % msg + sys.exit(exitcode) + + +def create_docs(soledad): + """ + Populates the soledad database with dummy messages, so we can exercise + sending payloads during the sync. + """ + deferreds = [] + for index in xrange(NUMDOCS): + deferreds.append(soledad.create_doc({'payload': 'dummy'})) + return defer.gatherResults(deferreds) + +# main program + +if __name__ == '__main__': + + tempdir = tempfile.mkdtemp() + + def rm_tempdir(): + shutil.rmtree(tempdir) + + if len(sys.argv) < 6: + bail(USAGE, 2) + + uuid, token, server, cert_file, passphrase = sys.argv[1:] + s = get_soledad_instance( + uuid, passphrase, tempdir, server, cert_file, token) + + def onSyncDone(sync_result): + print "SYNC_RESULT:", sync_result + s.close() + rm_tempdir() + reactor.stop() + + def log_and_exit(f): + log.err(f) + rm_tempdir() + reactor.stop() + + def start_sync(): + d = create_docs(s) + d.addCallback(lambda _: s.sync()) + d.addCallback(onSyncDone) + d.addErrback(log_and_exit) + + reactor.callWhenRunning(start_sync) + reactor.run() diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb new file mode 100644 index 00000000..b30fa768 --- /dev/null +++ b/tests/helpers/srp_helper.rb @@ -0,0 +1,171 @@ +# +# Here are some very stripped down helper methods for SRP, useful only for +# testing the client side. +# + +require 'digest' +require 'openssl' +require 'securerandom' +require 'base64' + +module SRP + + ## + ## UTIL + ## + + module Util + PRIME_N = <<-EOS.split.join.hex +115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3 + EOS + BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N) +eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657 +6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da +d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5 +7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb +06e3 + EOS + GENERATOR = 2 # g + + def hn_xor_hg + byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR)) + end + + # a^n (mod m) + def modpow(a, n, m = BIG_PRIME_N) + r = 1 + while true + r = r * a % m if n[0] == 1 + n >>= 1 + return r if n == 0 + a = a * a % m + end + end + + # Hashes the (long) int args + def sha256_int(*args) + sha256_hex(*args.map{|a| "%02x" % a}) + end + + # Hashes the hex args + def sha256_hex(*args) + h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('') + sha256_str([h].pack('H*')) + end + + def sha256_str(s) + Digest::SHA2.hexdigest(s) + end + + def bigrand(bytes) + OpenSSL::Random.random_bytes(bytes).unpack("H*")[0] + end + + def multiplier + @muliplier ||= calculate_multiplier + end + + protected + + def calculate_multiplier + sha256_int(BIG_PRIME_N, GENERATOR).hex + end + + def byte_xor_hex(a, b) + a = [a].pack('H*') + b = [b].pack('H*') + a.bytes.each_with_index.map do |a_byte, i| + (a_byte ^ (b[i].ord || 0)).chr + end.join + end + end + + ## + ## SESSION + ## + + class Session + include SRP::Util + attr_accessor :user + attr_accessor :bb + + def initialize(user, aa=nil) + @user = user + @a = bigrand(32).hex + end + + def m + @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k) + end + + def aa + @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N) + end + + protected + + # client: K = H( (B - kg^x) ^ (a + ux) ) + def client_secret + base = bb.hex + base -= modpow(GENERATOR, @user.private_key) * multiplier + base = base % BIG_PRIME_N + modpow(base, @user.private_key * u.hex + @a) + end + + def k + @k ||= sha256_int(client_secret) + end + + def n_xor_g_long + @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join + end + + def login_hash + @login_hash ||= sha256_str(@user.username) + end + + def u + @u ||= sha256_hex(aa, bb) + end + end + + ## + ## Dummy USER + ## + + class User + include SRP::Util + + attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted + + def initialize(username=nil) + @username = username || "tmp_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '') + @password = "password_" + SecureRandom.urlsafe_base64(10) + @salt = bigrand(4).hex + @verifier = modpow(GENERATOR, private_key) + @ok = false + @deleted = false + end + + def private_key + @private_key ||= calculate_private_key + end + + def to_params + { + 'user[login]' => @username, + 'user[password_verifier]' => @verifier.to_s(16), + 'user[password_salt]' => @salt.to_s(16) + } + end + + private + + def calculate_private_key + shex = '%x' % [@salt] + inner = sha256_str([@username, @password].join(':')) + sha256_hex(shex, inner).hex + end + end + +end diff --git a/tests/order.rb b/tests/order.rb new file mode 100644 index 00000000..14aad9be --- /dev/null +++ b/tests/order.rb @@ -0,0 +1,22 @@ +class LeapCli::Config::Node + # + # returns a list of node names that should be tested before this node. + # make sure to not return ourselves (please no dependency loops!). + # + # NOTE: this method determines the order that nodes are tested in. To specify + # the order of tests on a particular node, each test can call class method + # LeapTest.depends_on(). + # + def test_dependencies + dependents = LeapCli::Config::ObjectList.new + + # webapp, mx, and soledad depend on couchdb nodes + if services.include?('webapp') || services.include?('mx') || services.include?('soledad') + if !services.include?('couchdb') + dependents.merge! nodes_like_me[:services => 'couchdb'] + end + end + + dependents.keys.delete_if {|name| self.name == name} + end +end
\ No newline at end of file diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb new file mode 100644 index 00000000..85dc6840 --- /dev/null +++ b/tests/white-box/couchdb.rb @@ -0,0 +1,186 @@ +raise SkipTest unless service?(:couchdb) + +require 'json' + +class CouchDB < LeapTest + depends_on "Network" + + def setup + end + + def test_00_Are_daemons_running? + assert_running 'bin/beam' + if multimaster? + assert_running 'bin/epmd' + end + pass + end + + # + # check to make sure we can get welcome response from local couchdb + # + def test_01_Is_CouchDB_running? + assert_get(couchdb_url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Could not get welcome message from #{couchdb_url}. Probably couchdb is not running." + end + pass + end + + # + # compare the configured nodes to the nodes that are actually listed in bigcouch + # + def test_02_Is_cluster_membership_ok? + return unless multimaster? + url = couchdb_backend_url("/nodes/_all_docs") + neighbors = assert_property('couch.bigcouch.neighbors') + neighbors << assert_property('domain.full') + neighbors.sort! + assert_get(url) do |body| + response = JSON.parse(body) + nodes_in_db = response['rows'].collect{|row| row['id'].sub(/^bigcouch@/, '')}.sort + assert_equal neighbors, nodes_in_db, "The couchdb replication node list is wrong (/nodes/_all_docs)" + end + pass + end + + # + # all configured nodes are in 'cluster_nodes' + # all nodes online and communicating are in 'all_nodes' + # + # this seems backward to me, so it might be the other way around. + # + def test_03_Are_configured_nodes_online? + return unless multimaster? + url = couchdb_url("/_membership", :username => 'admin') + assert_get(url) do |body| + response = JSON.parse(body) + nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes'] + nodes_available_but_not_configured = response['all_nodes'] - response['cluster_nodes'] + if nodes_configured_but_not_available.any? + warn "These nodes are configured but not available:", nodes_configured_but_not_available + end + if nodes_available_but_not_configured.any? + warn "These nodes are available but not configured:", nodes_available_but_not_configured + end + if response['cluster_nodes'] == response['all_nodes'] + pass + end + end + end + + def test_04_Do_ACL_users_exist? + acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication'] + url = couchdb_backend_url("/_users/_all_docs", :username => 'admin') + assert_get(url) do |body| + response = JSON.parse(body) + assert_equal acl_users.count, response['total_rows'] + actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') } + assert_equal acl_users.sort, actual_users.sort + end + pass + end + + def test_05_Do_required_databases_exist? + dbs_that_should_exist = ["customers","identities","keycache","shared","tickets","users", "tmp_users"] + dbs_that_should_exist << "tokens_#{rotation_suffix}" + dbs_that_should_exist << "sessions_#{rotation_suffix}" + dbs_that_should_exist.each do |db_name| + url = couchdb_url("/"+db_name, :username => 'admin') + assert_get(url) do |body| + assert response = JSON.parse(body) + assert_equal db_name, response['db_name'] + end + end + pass + end + + # disable ACL enforcement, because it's a known issue with bigcouch + # and will only confuse the user + # see https://leap.se/code/issues/6030 for more details + # + ## for now, this just prints warnings, since we are failing these tests. + ## + + #def test_06_Is_ACL_enforced? + # ok = assert_auth_fail( + # couchdb_url('/users/_all_docs', :username => 'leap_mx'), + # {:limit => 1} + # ) + # ok = assert_auth_fail( + # couchdb_url('/users/_all_docs', :username => 'leap_mx'), + # {:limit => 1} + # ) && ok + # pass if ok + #end + + def test_07_Can_records_be_created? + record = DummyRecord.new + url = couchdb_url("/tokens_#{rotation_suffix}", :username => 'admin') + assert_post(url, record, :format => :json) do |body| + assert response = JSON.parse(body), "POST response should be JSON" + assert response["ok"], "POST response should be OK" + assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body| + assert response = JSON.parse(body), "DELETE response should be JSON" + assert response["ok"], "DELETE response should be OK" + end + end + pass + end + + # + # This is not really a "test", just an attempt to make sure that + # the mx tests that fire off dummy emails don't fill up the + # storage db. + # + # mx tests can't run this because they don't have access to + # the storage db. + # + # This "test" is responsible for both creating the db if it does not + # exist, and destroying if it does. + # + # Yes, this is super hacky. Properly, we should add something to + # the soledad api to support create/delete of user storage dbs. + # + def test_99_Delete_mail_storage_used_in_mx_tests + user = find_user_by_login(TEST_EMAIL_USER) + if user + if user_db_exists?(user["id"]) + # keep the test email db from filling up: + assert_destroy_user_db(user["id"], :username => 'admin') + end + # either way, make sure we leave a db for the mx tests: + assert_create_user_db(user["id"], :username => 'admin') + end + silent_pass + end + + private + + def multimaster? + mode == "multimaster" + end + + def mode + assert_property('couch.mode') + end + + # TODO: admin port is hardcoded for now but should be configurable. + def couchdb_backend_url(path="", options={}) + options = {port: multimaster? && "5986"}.merge options + couchdb_url(path, options) + end + + def rotation_suffix + rotation_suffix = Time.now.utc.to_i / 2592000 # monthly + end + + require 'securerandom' + require 'digest/sha2' + class DummyRecord < Hash + def initialize + self['data'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '') + self['_id'] = Digest::SHA512.hexdigest(self['data']) + end + end + +end diff --git a/tests/white-box/dummy.rb b/tests/white-box/dummy.rb new file mode 100644 index 00000000..a3e8ad68 --- /dev/null +++ b/tests/white-box/dummy.rb @@ -0,0 +1,71 @@ +# only run in the dummy case where there is no hiera.yaml file. +raise SkipTest unless $node["dummy"] + +class Robot + def can_shoot_lasers? + "OHAI!" + end + + def can_fly? + "YES!" + end +end + +class TestDummy < LeapTest + def setup + @robot = Robot.new + end + + def test_lasers + assert_equal "OHAI!", @robot.can_shoot_lasers? + pass + end + + def test_fly + refute_match /^no/i, @robot.can_fly? + pass + end + + def test_fail + fail "fail" + pass + end + + def test_01_will_be_skipped + skip "test this later" + pass + end + + def test_socket_failure + assert_tcp_socket('localhost', 900000) + pass + end + + def test_warn + block_test do + warn "not everything", "is a success or failure" + end + end + + # used to test extracting the proper caller even when in a block + def block_test + yield + end + + def test_socket_success + fork { + Socket.tcp_server_loop('localhost', 12345) do |sock, client_addrinfo| + begin + sock.write('hi') + ensure + sock.close + exit + end + end + } + sleep 0.2 + assert_tcp_socket('localhost', 12345) + pass + end + +end diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb new file mode 100644 index 00000000..e0cb273a --- /dev/null +++ b/tests/white-box/mx.rb @@ -0,0 +1,267 @@ +raise SkipTest unless service?(:mx) + +require 'date' +require 'json' +require 'net/smtp' + +class Mx < LeapTest + depends_on "Network" + depends_on "Webapp" if service?(:webapp) + + def setup + end + + def test_01_Can_contact_couchdb? + dbs = ["identities"] + dbs.each do |db_name| + couchdb_urls("/"+db_name, couch_url_options).each do |url| + assert_get(url) do |body| + assert response = JSON.parse(body) + assert_equal db_name, response['db_name'] + end + end + end + pass + end + + def test_02_Can_contact_couchdb_via_haproxy? + if property('haproxy.couch') + url = couchdb_url_via_haproxy("", couch_url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + end + pass + end + end + + # + # this test picks a random identity document, then queries + # using the by_address view for that same document again. + # + def test_03_Can_query_identities_db? + ident = pick_random_identity + address = ident['address'] + url_base = %(/identities/_design/Identity/_view/by_address) + params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}") + assert_get(couchdb_url(url_base+params, couch_url_options)) do |body| + assert response = JSON.parse(body) + assert record = response['rows'].first + assert_equal address, record['doc']['address'] + pass + end + end + + def test_04_Are_MX_daemons_running? + assert_running '.*/usr/bin/twistd.*mx.tac' + assert_running '^/usr/lib/postfix/master$' + assert_running '^/usr/sbin/postfwd' + assert_running 'postfwd2::cache$' + assert_running 'postfwd2::policy$' + assert_running '^/usr/sbin/unbound$' + assert_running '^/usr/bin/freshclam' + assert_running '^/usr/sbin/opendkim' + if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0 + assert_running '^/usr/sbin/clamd' + assert_running '^/usr/sbin/clamav-milter' + else + skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running.\nDon't worry, mail delivery will work without clamav. The download should finish soon." + end + pass + end + + # + # TODO: test to make sure postmap returned the right result + # + def test_05_Can_postfix_query_leapmx? + ident = pick_random_identity(10, :with_public_key => true) + address = ident["address"] + + # + # virtual alias map: + # + # user@domain => 41c29a80a44f4775513c64ac9cab91b9@deliver.local + # + assert_run("postmap -v -q \"#{address}\" tcp:localhost:4242") + + # + # recipient access map: + # + # user@domain => [OK|REJECT|TEMP_FAIL] + # + # This map is queried by the mail server before delivery to the mail spool + # directory, and should check if the address is able to receive messages. + # Examples of reasons for denying delivery would be that the user is out of + # quota, is user, or have no pgp public key in the server. + # + # NOTE: in the future, when we support quota, we need to make sure that + # we don't randomly pick a user for this test that happens to be over quota. + # + assert_run("postmap -v -q \"#{address}\" tcp:localhost:2244") + + # + # certificate validity map: + # + # fa:2a:70:1f:d8:16:4e:1a:3b:15:c1:67:00:f0 => [200|500] + # + # Determines whether a particular SMTP client cert is authorized + # to relay mail, based on the fingerprint. + # + if ident["cert_fingerprints"] + not_expired = ident["cert_fingerprints"].select {|key, value| + Time.now.utc < DateTime.strptime("2016-01-03", "%F").to_time.utc + } + if not_expired.any? + fingerprint = not_expired.first + assert_run("postmap -v -q #{fingerprint} tcp:localhost:2424") + end + end + + pass + end + + # + # The email sent by this test might get bounced back. + # In this case, the test will pass, but the bounce message will + # get sent to root, so the sysadmin will still figure out pretty + # quickly that something is wrong. + # + def test_06_Can_deliver_email? + addr = [TEST_EMAIL_USER, property('domain.full_suffix')].join('@') + bad_addr = [TEST_BAD_USER, property('domain.full_suffix')].join('@') + + assert !identity_exists?(bad_addr), "the address #{bad_addr} must not exist." + if !identity_exists?(addr) + user = assert_create_user(TEST_EMAIL_USER, :monitor) + upload_public_key(user.id, TEST_EMAIL_PUBLIC_KEY) + end + assert identity_exists?(addr), "The identity #{addr} should have been created, but it doesn't exist yet." + assert_send_email(addr) + assert_raises(Net::SMTPError) do + send_email(bad_addr) + end + pass + end + + private + + def couch_url_options + { + :username => property('couchdb_leap_mx_user.username'), + :password => property('couchdb_leap_mx_user.password') + } + end + + # + # returns a random identity record that also has valid address + # and destination fields. + # + # options: + # + # * :with_public_key -- searches only for identities with public keys + # + # note to self: for debugging, here is the curl you want: + # curl --netrc "127.0.0.1:5984/identities/_design/Identity/_view/by_address?startkey=\"xxxx@leap.se\"&endkey=\"xxxx@leap.se\"&reduce=false&include_docs=true" + # + def pick_random_identity(tries=5, options={}) + assert_get(couchdb_url("/identities", couch_url_options)) do |body| + assert response = JSON.parse(body) + doc_count = response['doc_count'].to_i + if doc_count <= 1 + # the design document counts as one document. + skip "There are no identity documents yet." + else + # try repeatedly to get a valid doc + for i in 1..tries + offset = rand(doc_count) # pick a random document + url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options) + assert_get(url) do |body| + assert response = JSON.parse(body) + record = response['rows'].first + if record['id'] =~ /_design/ + next + elsif record['doc'] && record['doc']['address'] + next if record['doc']['destination'].nil? || record['doc']['destination'].empty? + next if options[:with_public_key] && !record_has_key?(record) + return record['doc'] + else + fail "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}" + end + end + end + if options[:with_public_key] + skip "Could not find an Identity document with a public key for testing." + else + fail "Failed to find a valid Identity document (with address and destination)." + end + end + end + end + + def record_has_key?(record) + !record['doc']['keys'].nil? && + !record['doc']['keys'].empty? && + !record['doc']['keys']['pgp'].nil? && + !record['doc']['keys']['pgp'].empty? + end + + TEST_EMAIL_PUBLIC_KEY=<<HERE +-----BEGIN PGP PUBLIC KEY BLOCK----- +mI0EVvzIKQEEAN4f8FOGntJGTTD+fFUQS6y/ihn6tYLtyGZZbCOd0t/9kHt/raoR +xEUks8rCOPMqHX+yeHsvDBtDyZYTvyhtfuWrBUbYGW+QZ4Pdvo+7NyLHPW0dKsCB +Czrx7pxqpq1oq+LpUFqpSfjJTfYaGVDNXrPK144a7Rox2+MCbgq3twnFABEBAAG0 +EiA8dGVzdF91c2VyX2VtYWlsPoi4BBMBAgAiBQJW/MgpAhsvBgsJCAcDAgYVCAIJ +CgsEFgIDAQIeAQIXgAAKCRAqYf65XmeSk0orBADUXjEiGnjzyBpXqaiVmJr4MyfP +IfKTK4a+4qvR+2fseD7hteF98m26i1YRI5omLp4/MnxGSpgKFKIuWIdkEiLg7IJc +pFZVdoDVufEtzbj9gmOHlnteksbCtuESyB0Hytsba4uS9afcTJdGiPNMHeniI/SY +UKcCcIrQmpNIoOA5OLiNBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD +7PK4xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6Ok +NkhfGfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+c +m3JuZy1k7QARAQABiQE9BBgBAgAJBQJW/MgpAhsuAKgJECph/rleZ5KTnSAEGQEC +AAYFAlb8yCkACgkQsJSYitQUOv4w1wQAn3atI5EsmRyw6iC6UVWWJv/lKi1Priyt +DsrdH5xUmHUgp6VU8Pw9Y6G+sv50KLfbVQ1l+8/3B71TjadsOxh+PBPsEyYpK6WX +TVGy44IDvFWGyOod8tmfcFN9IpU5DmSk/vny9G7RK/nbnta2VnfZOzwm5i3cNkPr +FGPL1z0K3qs0VwP+M7BXdqBRSFDDBpG1J0TrZioEjvKeOsT/Ul8mbVt7HQpcN93I +wTO4uky0Woy2nb7SbTQw6wOpU54u7+5dSQ03ltUHg1owy6Y3CMOeFL+e9ALpAZAU +aMwY7zMFhqlPVZZMfdMLRsdLin67RIM+OJ6A925AM52bEQT1YwkQlP4mvQY= +=qclE +-----END PGP PUBLIC KEY BLOCK----- +HERE + + TEST_EMAIL_PRIVATE_KEY = <<HERE +-----BEGIN PGP PRIVATE KEY BLOCK----- +lQHYBFb8yCkBBADeH/BThp7SRk0w/nxVEEusv4oZ+rWC7chmWWwjndLf/ZB7f62q +EcRFJLPKwjjzKh1/snh7LwwbQ8mWE78obX7lqwVG2BlvkGeD3b6Puzcixz1tHSrA +gQs68e6caqataKvi6VBaqUn4yU32GhlQzV6zyteOGu0aMdvjAm4Kt7cJxQARAQAB +AAP8DTFfcE6UG1AioJDU6KZ9oCaGONHLuxmNaArSofDrR/ODA9rLAUlp22N5LEdJ +46NyOhXrEwHx2aK2k+vbVDbgrP4ZTH7GxIK/2KzmH4zX0fWUNsaRy94Q12lJegXH +sH2Im8Jjxu16YwGgFNTX1fCPqLB6WdQpf1796s6+/3PnCDcCAOXTCul3N7V5Yl+9 +N2Anupn+qNDXKT/kiKIZLHsMbo7EriGWReG3lLj1cOJPC6Nf0uOEri4ErSjFEadR +F2TNITsCAPdsZjc5RGppUXyBfxhQkAnZ0r+UT2meCH3g3EVh3W9SBrXNhwipNpW3 +bPzRjUCDtmA8EOvd93oPCZv4/tb50P8B/jC+QIZ3GncP1CFPSVDoIZ7OUU5M1330 +DP77vG1GxeQvYO/hlxL5/KdtTR6m5zlIuooDxUaNJz1w5/oVjlG3NZKpl7QSIDx0 +ZXN0X3VzZXJfZW1haWw+iLgEEwECACIFAlb8yCkCGy8GCwkIBwMCBhUIAgkKCwQW +AgMBAh4BAheAAAoJECph/rleZ5KTSisEANReMSIaePPIGlepqJWYmvgzJ88h8pMr +hr7iq9H7Z+x4PuG14X3ybbqLVhEjmiYunj8yfEZKmAoUoi5Yh2QSIuDsglykVlV2 +gNW58S3NuP2CY4eWe16SxsK24RLIHQfK2xtri5L1p9xMl0aI80wd6eIj9JhQpwJw +itCak0ig4Dk4nQHYBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD7PK4 +xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6OkNkhf +GfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+cm3Ju +Zy1k7QARAQABAAP9HrUaGvdpqTwVx3cHyXUhId6GzCuuKyaP4mZoGeBCcaQS2vQR +YtiykwBwX/AlfwSFJmmHKB6EErWIA+QyaEFR/fO56cHD2TY3Ql0BGcuHIx3+9pkp +biPBZdiiGz7oa6k6GWsbKSksqwV8poSXV7qbn+Bjm2xCM4VnjNZIrFtL7fkCAMOf +e9yHBFoXfc175bkNXEUXrNS34kv2ODAlx6KyY+PS77D+nprpHpGCnLn77G+xH1Xi +qvX1Dr/iSQU5Tzsd+tcCAPkYZulaC/9itwme7wIT3ur+mdqMHymsCzv9193iLgjJ +9t7fARo18yB845hI9Xv7TwRcoyuSpfvuM05rCMRzydsCAOI1MZeKtZSogXVa9QTX +sVGZeCkrujSVOgsA3w48OLc2OrwZskDfx5QHfeJnumjQLut5qsnZ+1onj9P2dGdn +JaChe4kBPQQYAQIACQUCVvzIKQIbLgCoCRAqYf65XmeSk50gBBkBAgAGBQJW/Mgp +AAoJELCUmIrUFDr+MNcEAJ92rSORLJkcsOogulFVlib/5SotT64srQ7K3R+cVJh1 +IKelVPD8PWOhvrL+dCi321UNZfvP9we9U42nbDsYfjwT7BMmKSull01RsuOCA7xV +hsjqHfLZn3BTfSKVOQ5kpP758vRu0Sv5257WtlZ32Ts8JuYt3DZD6xRjy9c9Ct6r +NFcD/jOwV3agUUhQwwaRtSdE62YqBI7ynjrE/1JfJm1bex0KXDfdyMEzuLpMtFqM +tp2+0m00MOsDqVOeLu/uXUkNN5bVB4NaMMumNwjDnhS/nvQC6QGQFGjMGO8zBYap +T1WWTH3TC0bHS4p+u0SDPjiegPduQDOdmxEE9WMJEJT+Jr0G +=hvJM +-----END PGP PRIVATE KEY BLOCK----- +HERE + +end diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb new file mode 100644 index 00000000..436fc8a8 --- /dev/null +++ b/tests/white-box/network.rb @@ -0,0 +1,90 @@ +require 'socket' +require 'openssl' + +raise SkipTest if $node["dummy"] + +class Network < LeapTest + + def setup + end + + def test_01_Can_connect_to_internet? + assert_get('http://www.google.com/images/srpr/logo11w.png') + pass + end + + # + # example properties: + # + # stunnel: + # ednp_clients: + # elk_9002: + # accept_port: 4003 + # connect: elk.dev.bitmask.i + # connect_port: 19002 + # couch_server: + # accept: 15984 + # connect: "127.0.0.1:5984" + # + def test_02_Is_stunnel_running? + ignore unless $node['stunnel'] + good_stunnel_pids = [] + release = `facter lsbmajdistrelease` + if release.to_i > 7 + # on jessie, there is only one stunnel proc running instead of 6 + expected = 1 + else + expected = 6 + end + $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs| + stunnel_configs.each do |stunnel_name, stunnel_conf| + config_file_name = "/etc/stunnel/#{stunnel_name}.conf" + processes = pgrep(config_file_name) + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" + good_stunnel_pids += processes.map{|ps| ps[:pid]} + assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.' + assert_tcp_socket('localhost', port) + end + end + $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf| + config_file_name = "/etc/stunnel/#{stunnel_name}.conf" + processes = pgrep(config_file_name) + assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`" + good_stunnel_pids += processes.map{|ps| ps[:pid]} + assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', accept_port) + assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`" + assert_tcp_socket('localhost', connect_port, + "The local connect endpoint for stunnel `#{stunnel_name}` is unavailable.\n"+ + "This is probably caused by a daemon that died or failed to start on\n"+ + "port `#{connect_port}`, not stunnel itself.") + end + all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq + assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel" + pass + end + + def test_03_Is_shorewall_running? + ignore unless File.exists?('/sbin/shorewall') + assert_run('/sbin/shorewall status') + pass + end + + THIRTY_DAYS = 60*60*24*30 + + def test_04_Are_server_certificates_valid? + cert_paths = ["/etc/x509/certs/leap_commercial.crt", "/etc/x509/certs/leap.crt"] + cert_paths.each do |cert_path| + if File.exists?(cert_path) + cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) + if Time.now > cert.not_after + fail "The certificate #{cert_path} expired on #{cert.not_after}" + elsif Time.now + THIRTY_DAYS > cert.not_after + fail "The certificate #{cert_path} will expire soon, on #{cert.not_after}" + end + end + end + pass + end + +end diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb new file mode 100644 index 00000000..170d4503 --- /dev/null +++ b/tests/white-box/openvpn.rb @@ -0,0 +1,16 @@ +raise SkipTest unless service?(:openvpn) + +class OpenVPN < LeapTest + depends_on "Network" + + def setup + end + + def test_01_Are_daemons_running? + assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' + assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' + assert_running '^/usr/sbin/unbound$' + pass + end + +end diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb new file mode 100644 index 00000000..d41bee58 --- /dev/null +++ b/tests/white-box/soledad.rb @@ -0,0 +1,17 @@ +raise SkipTest unless service?(:soledad) + +require 'json' + +class Soledad < LeapTest + depends_on "Network" + depends_on "CouchDB" if service?(:couchdb) + + def setup + end + + def test_00_Is_Soledad_running? + assert_running '.*/usr/bin/twistd.*--wsgi=leap.soledad.server.application' + pass + end + +end diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb new file mode 100644 index 00000000..68f3dcd2 --- /dev/null +++ b/tests/white-box/webapp.rb @@ -0,0 +1,134 @@ +raise SkipTest unless service?(:webapp) + +require 'json' + +class Webapp < LeapTest + depends_on "Network" + + def setup + end + + def test_01_Can_contact_couchdb? + url = couchdb_url("", url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + end + pass + end + + def test_02_Can_contact_couchdb_via_haproxy? + if property('haproxy.couch') + url = couchdb_url_via_haproxy("", url_options) + assert_get(url) do |body| + assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." + end + pass + end + end + + def test_03_Are_daemons_running? + assert_running '^/usr/sbin/apache2' + assert_running '^/usr/bin/ruby /usr/bin/nickserver' + pass + end + + # + # this is technically a black-box test. so, move this when we have support + # for black box tests. + # + def test_04_Can_access_webapp? + assert_get('https://' + $node['webapp']['domain'] + '/') + pass + end + + def test_05_Can_create_and_authenticate_and_delete_user_via_API? + if property('webapp.allow_registration') + assert_tmp_user + pass + else + skip "New user registrations are disabled." + end + end + + def test_06_Can_sync_Soledad? + return unless property('webapp.allow_registration') + soledad_config = property('definition_files.soledad_service') + if soledad_config && !soledad_config.empty? + soledad_server = pick_soledad_server(soledad_config) + if soledad_server + assert_tmp_user do |user| + command = File.expand_path "../../helpers/soledad_sync.py", __FILE__ + soledad_url = "https://#{soledad_server}/user-#{user.id}" + soledad_cert = "/usr/local/share/ca-certificates/leap_ca.crt" + assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url} #{soledad_cert} #{user.password}" + assert_user_db_exists(user) + pass + end + end + else + skip 'No soledad service configuration' + end + end + + private + + def url_options + { + :username => property('webapp.couchdb_webapp_user.username'), + :password => property('webapp.couchdb_webapp_user.password') + } + end + + # + # pick a random soledad server. + # I am not sure why, but using IP address directly does not work. + # + def pick_soledad_server(soledad_config_json_str) + soledad_config = JSON.parse(soledad_config_json_str) + host_name = soledad_config['hosts'].keys.shuffle.first + if host_name + hostname = soledad_config['hosts'][host_name]['hostname'] + port = soledad_config['hosts'][host_name]['port'] + return "#{hostname}:#{port}" + else + return nil + end + end + + # + # returns true if the per-user db created by soledad-server exists. + # we try three times, and give up after that. + # + def assert_user_db_exists(user) + db_name = "user-#{user.id}" + repeatedly_try("/#{db_name}") do |body, response, error| + assert false, "Could not find user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}" + end + repeatedly_try("/#{db_name}/_design/docs") do |body, response, error| + assert false, "Could not find design docs for user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}" + end + end + + # + # tries the URL repeatedly, giving up and yield the last response if + # no try returned a 200 http status code. + # + def repeatedly_try(url, &block) + last_body, last_response, last_error = nil + 3.times do + sleep 0.2 + get(couchdb_url(url)) do |body, response, error| + last_body, last_response, last_error = body, response, error + # After moving to couchdb, webapp user is not allowed to Read user dbs, + # but the return code for non-existent databases is 404. See #7674 + if response.code.to_i == 401 + return + end + end + sleep 1 + end + yield last_body, last_response, last_error + return + end + +end diff --git a/vagrant/add-pixelated.sh b/vagrant/add-pixelated.sh new file mode 100755 index 00000000..f9908947 --- /dev/null +++ b/vagrant/add-pixelated.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# adds pixelated-server to the node + +. /vagrant/vagrant/vagrant.config + +cd "$PROVIDERDIR" + +if ! git submodule status files/puppet/modules/pixelated > /dev/null 2>&1; then + git submodule add https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated +fi + +echo '{}' > services/pixelated.json +[ -d files/puppet/modules/custom/manifests ] || mkdir -p files/puppet/modules/custom/manifests +echo 'class custom { include ::pixelated}' > files/puppet/modules/custom/manifests/init.pp + +$LEAP $OPTS -v 2 deploy + +echo '===============================================' +echo 'testing the platform' +echo '===============================================' + +$LEAP $OPTS -v 2 test --continue + + +echo -e '\n===========================================================================================================\n\n' +echo -e 'You are now ready to use your vagrant Pixelated provider.\n' + +echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated Useragent.\n' +echo -e 'The Pixelated Useragent is available at https://localhost:8080\n' + +echo -e 'Please add an exception for both sites in your browser dialog to allow the self-signed certificate.\n' diff --git a/vagrant/configure-leap.sh b/vagrant/configure-leap.sh new file mode 100755 index 00000000..9ddee039 --- /dev/null +++ b/vagrant/configure-leap.sh @@ -0,0 +1,92 @@ +#!/bin/bash + + +. /vagrant/vagrant/vagrant.config + +echo '===============================================' +echo 'configuring leap' +echo '===============================================' + +# purge $PROVIDERDIR so this script can be run multiple times +[ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR + +mkdir -p $PROVIDERDIR +chown ${USER}:${USER} ${PROVIDERDIR} +cd $PROVIDERDIR + +$LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform=/vagrant . +echo -e '\n@log = "./deploy.log"' >> Leapfile + +if [ ! -e /home/${USER}/.ssh/id_rsa ]; then + $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' + [ -d /root/.ssh ] || mkdir /root/.ssh + cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +fi + +$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} +sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" +chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub + +$LEAP $OPTS add-user --self +$LEAP $OPTS cert ca +$LEAP $OPTS cert csr +$LEAP $OPTS node add $NODE ip_address:"$(facter ipaddress)" couch.mode:plain services:"$services" tags:production +echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json + +$LEAP $OPTS compile + +$GIT init +$GIT add . +$GIT commit -m'configured provider' + +$LEAP $OPTS node init $NODE +if [ $? -eq 1 ]; then + echo 'node init failed' + exit 1 +fi + +# couchrest gem does currently not install on jessie +# https://leap.se/code/issues/7754 +# workaround is to install rake as gem +gem install rake + +$LEAP $OPTS -v 2 deploy + +$GIT add . +$GIT commit -m'initialized and deployed provider' + +# Vagrant: leap_mx fails to start on jessie +# https://leap.se/code/issues/7755 +# Workaround: we stop and start leap-mx after deploy and +# before testing + +service leap-mx stop +service leap-mx start + + + +echo '===============================================' +echo 'testing the platform' +echo '===============================================' + +$LEAP $OPTS -v 2 test --continue + +echo '===============================================' +echo 'setting node to demo-mode' +echo '===============================================' +postconf -e default_transport='error: in demo mode' + +# add users: testadmin and testuser with passwords "hallo123" +curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" +curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" + +echo -e '\n===========================================================================================================\n\n' +echo -e 'You are now ready to use your local LEAP provider.\n' +echo 'If you want to use the *Bitmask client* with your provider, please update your /etc/hosts with following dns overrides:' + +$LEAP list --print ip_address,domain.full,dns.aliases | sed 's/^.* //' | sed 's/, null//g' | tr -d '\]\[",' + +echo 'Please see https://leap.se/en/docs/platform/tutorials/vagrant#use-the-bitmask-client-to-do-an-initial-soledad-sync for more details how to use and test your LEAP provider.' +echo -e "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" +echo -e 'The LEAP webapp is now available at https://localhost:4443\n' +echo -e 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' diff --git a/vagrant/install-platform.pp b/vagrant/install-platform.pp new file mode 100755 index 00000000..223853c1 --- /dev/null +++ b/vagrant/install-platform.pp @@ -0,0 +1,15 @@ +class {'apt': } +Exec['update_apt'] -> Package <||> + +# install leap_cli from source, so it will work with the develop +# branch of leap_platform +class { '::leap::cli::install': + source => true, +} + +file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: + ensure => directory +} + +# install prerequisites for configuring the provider +include ::git diff --git a/vagrant/vagrant.config b/vagrant/vagrant.config new file mode 100644 index 00000000..e601488d --- /dev/null +++ b/vagrant/vagrant.config @@ -0,0 +1,22 @@ +# provider config values used by vagrant provision scripts +provider_domain='example.org' +provider_name='Leap Example Provider' +contacts="no-reply@$provider_domain" + +# serivces that get configured +# note that the "openvpn" service does currently *not* work +# in a vagrant setup, +# see https://leap.se/en/docs/platform/troubleshooting/known-issues#Special.Environments +# to speed up things, don't deploy monitor service by default +# services='webapp,mx,couchdb,soledad,monitor' +services='webapp,mx,couchdb,soledad' + +# default vars used by vagrant provision scripts +OPTS='' +USER='vagrant' +NODE='node1' +SUDO="sudo -u ${USER}" +PROVIDERDIR="/home/${USER}/leap/configuration" +LEAP="$SUDO /usr/local/bin/leap" +GIT="$SUDO git" + |
