diff options
Diffstat (limited to 'common')
50 files changed, 0 insertions, 9496 deletions
diff --git a/common/.gitattributes b/common/.gitattributes deleted file mode 100644 index 1d59a395..00000000 --- a/common/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -src/leap/soledad/common/_version.py export-subst diff --git a/common/AUTHORS b/common/AUTHORS deleted file mode 100644 index 9051b8c4..00000000 --- a/common/AUTHORS +++ /dev/null @@ -1,11 +0,0 @@ -drebs <drebs@leap.se> -Tomás Touceda <chiiph@leap.se> -Kali Kaneko <kali@leap.se> -Ivan Alejandro <ivanalejandro0@gmail.com> -Micah Anderson <micah@riseup.net> -Victor Shyba <victor.shyba@gmail.com> -Bruno Wagner <bwgpro@gmail.com> -Ruben Pollan <meskio@sindominio.net> -Duda Dornelles <ddornell@thoughtworks.com> -antialias <antialias@leap.se> -efkin <efkin@riseup.net> diff --git a/common/LICENSE b/common/LICENSE deleted file mode 100644 index bc08fe2e..00000000 --- a/common/LICENSE +++ /dev/null @@ -1,619 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. diff --git a/common/MANIFEST.in b/common/MANIFEST.in deleted file mode 100644 index 93f46bbe..00000000 --- a/common/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include pkg/* -include versioneer.py -include LICENSE -include CHANGELOG - -# What do we want the ddocs folder in the source package for? -- kali -# it should be enough with having the compiled stuff. -recursive-include src/leap/soledad/common/ddocs * -include src/leap/soledad/common/_version.py diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT deleted file mode 100644 index cc00ecf7..00000000 --- a/common/changes/VERSION_COMPAT +++ /dev/null @@ -1,10 +0,0 @@ -################################################# -# This file keeps track of the recent changes -# introduced in internal leap dependencies. -# Add your changes here so we can properly update -# requirements.pip during the release process. -# (leave header when resetting) -################################################# -# -# BEGIN DEPENDENCY LIST ------------------------- -# leap.foo.bar>=x.y.z diff --git a/common/changes/next-changelog.rst b/common/changes/next-changelog.rst deleted file mode 100644 index 64162b7b..00000000 --- a/common/changes/next-changelog.rst +++ /dev/null @@ -1,28 +0,0 @@ -0.8.2 - ... -+++++++++++++++++++++++++++++++ - -Please add lines to this file, they will be moved to the CHANGELOG.rst during -the next release. - -There are two template lines for each category, use them as reference. - -I've added a new category `Misc` so we can track doc/style/packaging stuff. - -Features -~~~~~~~~ -- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234. -- New feature without related issue number. - -Bugfixes -~~~~~~~~ -- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235. -- Bugfix without related issue number. - -Misc -~~~~ -- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236. -- Some change without issue number. - -Known Issues -~~~~~~~~~~~~ -- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236. diff --git a/common/pkg/__init__.py b/common/pkg/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/common/pkg/__init__.py +++ /dev/null diff --git a/common/pkg/generate_wheels.sh b/common/pkg/generate_wheels.sh deleted file mode 100755 index a13e2c7a..00000000 --- a/common/pkg/generate_wheels.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# Generate wheels for dependencies -# Use at your own risk. - -if [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse -fi - -pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip -if [ -f pkg/requirements-testing.pip ]; then - pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip -fi diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh deleted file mode 100755 index f4b5f67a..00000000 --- a/common/pkg/pip_install_requirements.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Update pip and install LEAP base/testing requirements. -# For convenience, $insecure_packages are allowed with insecure flags enabled. -# Use at your own risk. -# See $usage for help - -insecure_packages="" -leap_wheelhouse=https://lizard.leap.se/wheels - -show_help() { - usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n -\t\t\tOtherwise, it will install requirements.pip\n ---use-leap-wheels\tUse wheels from leap.se" - echo -e $usage - - exit 1 -} - -process_arguments() { - testing=false - while [ "$#" -gt 0 ]; do - # From http://stackoverflow.com/a/31443098 - case "$1" in - --help) show_help;; - --testing) testing=true; shift 1;; - --use-leap-wheels) use_leap_wheels=true; shift 1;; - - -h) show_help;; - -*) echo "unknown option: $1" >&2; exit 1;; - esac - done -} - -return_wheelhouse() { - if $use_leap_wheels ; then - WHEELHOUSE=$leap_wheelhouse - elif [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse - fi - - # Tested with bash and zsh - if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then - mkdir $WHEELHOUSE - fi - - echo "$WHEELHOUSE" -} - -return_install_options() { - wheelhouse=`return_wheelhouse` - install_options="-U --find-links=$wheelhouse" - if $use_leap_wheels ; then - install_options="$install_options --trusted-host lizard.leap.se" - fi - - echo $install_options -} - -return_insecure_flags() { - for insecure_package in $insecure_packages; do - flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" - done - - echo $flags -} - -return_packages() { - if $testing ; then - packages="-r pkg/requirements-testing.pip" - else - packages="-r pkg/requirements.pip" - fi - - echo $packages -} - -process_arguments $@ -install_options=`return_install_options` -insecure_flags=`return_insecure_flags` -packages=`return_packages` - -pip install -U wheel -pip install -U pip -pip install $install_options $insecure_flags $packages diff --git a/common/pkg/requirements-latest.pip b/common/pkg/requirements-latest.pip deleted file mode 100644 index 852f2433..00000000 --- a/common/pkg/requirements-latest.pip +++ /dev/null @@ -1,4 +0,0 @@ ---index-url https://pypi.python.org/simple/ - --e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common' --e . diff --git a/common/pkg/requirements-leap.pip b/common/pkg/requirements-leap.pip deleted file mode 100644 index b311859e..00000000 --- a/common/pkg/requirements-leap.pip +++ /dev/null @@ -1 +0,0 @@ -leap.common>=0.4.0 diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip deleted file mode 100644 index 29506cf0..00000000 --- a/common/pkg/requirements.pip +++ /dev/null @@ -1,3 +0,0 @@ -paste -routes -six
\ No newline at end of file diff --git a/common/pkg/tools/get_authors.sh b/common/pkg/tools/get_authors.sh deleted file mode 100755 index 0169bb17..00000000 --- a/common/pkg/tools/get_authors.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- diff --git a/common/pkg/tools/with_venvwrapper.sh b/common/pkg/tools/with_venvwrapper.sh deleted file mode 100755 index b62bc10f..00000000 --- a/common/pkg/tools/with_venvwrapper.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -#Wraps a command in a virtualenwrapper passed as first argument. -#Example: -#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh - -wd=`pwd` -alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"' - -source `which virtualenvwrapper.sh` -echo "Activating virtualenv " $1 -echo "------------------------------------" -workon $1 -cd $wd -echo "running version: " -echo `pyver leap.bitmask` -echo `pyver leap.soledad.common` -echo `pyver leap.keymanager` -$2 $3 $4 $5 diff --git a/common/pkg/utils.py b/common/pkg/utils.py deleted file mode 100644 index d1680102..00000000 --- a/common/pkg/utils.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -# utils.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Utils to help in the setup process -""" -import os -import re -import sys - - -def is_develop_mode(): - """ - Returns True if we're calling the setup script using the argument for - setuptools development mode. - - This avoids messing up with dependency pinning and order, the - responsibility of installing the leap dependencies is left to the - developer. - """ - args = sys.argv - devflags = "setup.py", "develop" - if (args[0], args[1]) == devflags: - return True - return False - - -def get_reqs_from_files(reqfiles): - """ - Returns the contents of the top requirement file listed as a - string list with the lines. - - @param reqfiles: requirement files to parse - @type reqfiles: list of str - """ - for reqfile in reqfiles: - if os.path.isfile(reqfile): - return open(reqfile, 'r').read().split('\n') - - -def parse_requirements(reqfiles=['requirements.txt', - 'requirements.pip', - 'pkg/requirements.pip']): - """ - Parses the requirement files provided. - - The passed reqfiles list is a list of possible locations to try, the - function will return the contents of the first path found. - - Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should - return PySide as a dep or not. Don't set, or set to 0 if you want - to install it through pip. - - @param reqfiles: requirement files to parse - @type reqfiles: list of str - """ - - requirements = [] - skip_pyside = os.getenv("LEAP_VENV_SKIP_PYSIDE", "0") != "0" - for line in get_reqs_from_files(reqfiles): - # -e git://foo.bar/baz/master#egg=foobar - if re.match(r'\s*-e\s+', line): - pass - # do not try to do anything with externals on vcs - # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - # line)) - # http://foo.bar/baz/foobar/zipball/master#egg=foobar - elif re.match(r'\s*https?:', line): - requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', - line)) - # -f lines are for index locations, and don't get used here - elif re.match(r'\s*-f\s+', line): - pass - - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass - elif line == 'PySide' and skip_pyside: - pass - # do not include comments - elif line.lstrip().startswith('#'): - pass - else: - if line != '': - requirements.append(line) - - return requirements diff --git a/common/setup.cfg b/common/setup.cfg deleted file mode 100644 index ed06f60d..00000000 --- a/common/setup.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[aliases] -test = trial - -[pep8] -exclude = versioneer.py,_version.py,ddocs.py,*.egg,build,docs -ignore = E731 - -[flake8] -exclude = versioneer.py,_version.py,ddocs.py,*.egg,build,docs -ignore = E731 - -[versioneer] -VCS = git -style = pep440 -versionfile_source = src/leap/soledad/common/_version.py -versionfile_build = leap/soledad/common/_version.py -tag_prefix = diff --git a/common/setup.py b/common/setup.py deleted file mode 100644 index 6de26034..00000000 --- a/common/setup.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8 -*- -# setup.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -setup file for leap.soledad.common -""" -import re - -from setuptools import setup -from setuptools import find_packages -from setuptools import Command -from setuptools.command.develop import develop as _cmd_develop -import versioneer - -from pkg import utils - - -trove_classifiers = ( - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: " - "GNU General Public License v3 or later (GPLv3+)", - "Environment :: Console", - "Operating System :: OS Independent", - "Operating System :: POSIX", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Topic :: Database :: Front-Ends", - "Topic :: Software Development :: Libraries :: Python Modules" -) - -DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/' - 'archive/%s.tar.gz') -_versions = versioneer.get_versions() -VERSION = _versions['version'] -VERSION_REVISION = _versions['full-revisionid'] -DOWNLOAD_URL = "" - -# get the short version for the download url -_version_short = re.findall('\d+\.\d+\.\d+', VERSION) -if len(_version_short) > 0: - VERSION_SHORT = _version_short[0] - DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT - - -class freeze_debianver(Command): - - """ - Freezes the version in a debian branch. - To be used after merging the development branch onto the debian one. - """ - user_options = [] - template = r""" -# This file was generated by the `freeze_debianver` command in setup.py -# Using 'versioneer.py' (0.16) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json -import sys - -version_json = ''' -{ - "dirty": false, - "error": null, - "full-revisionid": "FULL_REVISIONID", - "version": "VERSION_STRING" -} -''' # END VERSION_JSON - -def get_versions(): - return json.loads(version_json) -""" - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - proceed = str(raw_input( - "This will overwrite the file _version.py. Continue? [y/N] ")) - if proceed != "y": - print("He. You scared. Aborting.") - return - subst_template = self.template.replace( - 'VERSION_STRING', VERSION_SHORT).replace( - 'FULL_REVISIONID', VERSION_REVISION) - versioneer_cfg = versioneer.get_config_from_root('.') - with open(versioneer_cfg.versionfile_source, 'w') as f: - f.write(subst_template) - - -class cmd_develop(_cmd_develop): - def run(self): - # versioneer: - versions = versioneer.get_versions(verbose=True) - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old version - self.distribution.metadata.version = versions["version"] - _cmd_develop.run(self) - - -cmdclass = versioneer.get_cmdclass() -cmdclass["freeze_debianver"] = freeze_debianver -cmdclass["develop"] = cmd_develop - - -# XXX add ref to docs - -requirements = utils.parse_requirements() - -if utils.is_develop_mode(): - print - print("[WARNING] Skipping leap-specific dependencies " - "because development mode is detected.") - print("[WARNING] You can install " - "the latest published versions with " - "'pip install -r pkg/requirements-leap.pip'") - print("[WARNING] Or you can instead do 'python setup.py develop' " - "from the parent folder of each one of them.") - print -else: - requirements += utils.parse_requirements( - reqfiles=["pkg/requirements-leap.pip"]) - -setup( - name='leap.soledad.common', - version=VERSION, - cmdclass=cmdclass, - url='https://leap.se/', - download_url=DOWNLOAD_URL, - license='GPLv3+', - description='Synchronization of locally encrypted data among devices ' - '(common files).', - author='The LEAP Encryption Access Project', - author_email='info@leap.se', - maintainer='Kali Kaneko', - maintainer_email='kali@leap.se', - long_description=( - "Soledad is the part of LEAP that allows application data to be " - "securely shared among devices. It provides, to other parts of the " - "LEAP project, an API for data storage and sync." - ), - classifiers=trove_classifiers, - namespace_packages=["leap", "leap.soledad"], - packages=find_packages('src', exclude=['*.tests', '*.tests.*']), - package_dir={'': 'src'}, - test_suite='leap.soledad.common.tests', - install_requires=requirements, -) diff --git a/common/src/leap/__init__.py b/common/src/leap/__init__.py deleted file mode 100644 index f48ad105..00000000 --- a/common/src/leap/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/common/src/leap/soledad/__init__.py b/common/src/leap/soledad/__init__.py deleted file mode 100644 index f48ad105..00000000 --- a/common/src/leap/soledad/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/common/src/leap/soledad/common/README.txt b/common/src/leap/soledad/common/README.txt deleted file mode 100644 index 0a252650..00000000 --- a/common/src/leap/soledad/common/README.txt +++ /dev/null @@ -1,70 +0,0 @@ -Soledad common package -====================== - -This package contains Soledad bits used by both server and client. - -Couch L2DB Backend ------------------- - -L2DB backends rely on some atomic operations that modify documents contents -and metadata (conflicts, transaction ids and indexes). The only atomic -operation in Couch is a document put, so every u1db atomic operation has to be -mapped to a couch document put. - -The atomic operations in the U1DB SQLite reference backend implementation may -be identified by the use of a context manager to access the underlying -database. A listing of the methods involved in each atomic operation are -depiced below. The top-level elements correpond to the atomic operations that -have to be mapped, and items on deeper levels of the list have to be -implemented in a way that all changes will be pushed with just one operation. - - * _set_replica_uid - * put_doc: - * _get_doc - * _put_and_update_indexes - * insert/update the document - * insert into transaction log - * delete_doc - * _get_doc - * _put_and_update_indexes - * get_doc_conflicts - * _get_conflicts - * _set_replica_gen_and_trans_id - * _do_set_replica_gen_and_trans_id - * _put_doc_if_newer - * _get_doc - * _validate_source (**) - * _get_replica_gen_and_trans_id - * cases: - * is newer: - * _prune_conflicts (**) - * _has_conflicts - * _delete_conflicts - * _put_and_update_indexes - * same content as: - * _put_and_update_indexes - * conflicted: - * _force_doc_sync_conflict - * _prune_conflicts - * _add_conflict - * _put_and_update_indexes - * _do_set_replica_gen_and_trans_id - * resolve_doc - * _get_doc - * cases: - * doc is superseded - * _put_and_update_indexes - * else - * _add_conflict - * _delete_conflicts - * delete_index - * create_index - -Notes: - - * Currently, the couch backend does not implement indexing, so what is - depicted as `_put_and_update_indexes` above will be found as `_put_doc` in - the backend. - - * Conflict updates are part of document put using couch update functions, - and as such are part of the same atomic operation as document put. diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py deleted file mode 100644 index 4948ad20..00000000 --- a/common/src/leap/soledad/common/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from leap.common.check import leap_assert as soledad_assert -from leap.common.check import leap_assert_type as soledad_assert_type - -from ._version import get_versions - -""" -Soledad routines common to client and server. -""" - - -# -# Global constants -# - -SHARED_DB_NAME = 'shared' - - -# -# Global functions -# - -__version__ = get_versions()['version'] -del get_versions - - -__all__ = [ - "soledad_assert", - "soledad_assert_type", - "__version__", -] diff --git a/common/src/leap/soledad/common/_version.py b/common/src/leap/soledad/common/_version.py deleted file mode 100644 index 1168a88d..00000000 --- a/common/src/leap/soledad/common/_version.py +++ /dev/null @@ -1,484 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.16 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - keywords = {"refnames": git_refnames, "full": git_full} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "None" - cfg.versionfile_source = "src/leap/soledad/common/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - return None - return stdout - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree"} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version"} diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py deleted file mode 100644 index 4a29ca87..00000000 --- a/common/src/leap/soledad/common/backend.py +++ /dev/null @@ -1,642 +0,0 @@ -# -*- coding: utf-8 -*- -# backend.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -"""A L2DB generic backend.""" - -import functools - -from leap.soledad.common.document import ServerDocument -from leap.soledad.common.l2db import vectorclock -from leap.soledad.common.l2db.errors import ( - RevisionConflict, - InvalidDocId, - ConflictedDoc, - DocumentDoesNotExist, - DocumentAlreadyDeleted, -) -from leap.soledad.common.l2db.backends import CommonBackend -from leap.soledad.common.l2db.backends import CommonSyncTarget - - -class SoledadBackend(CommonBackend): - BATCH_SUPPORT = False - - """ - A L2DB backend implementation. - """ - - def __init__(self, database, replica_uid=None): - """ - Create a new backend. - - :param database: the database implementation - :type database: Database - :param replica_uid: an optional unique replica identifier - :type replica_uid: str - """ - # save params - self._factory = ServerDocument - self._real_replica_uid = None - self._cache = None - self._dbname = database._dbname - self._database = database - self.batching = False - if replica_uid is not None: - self._set_replica_uid(replica_uid) - - def batch_start(self): - if not self.BATCH_SUPPORT: - return - self.batching = True - self.after_batch_callbacks = {} - self._database.batch_start() - if not self._cache: - # batching needs cache - self._cache = {} - self._get_generation() # warm up gen info - - def batch_end(self): - if not self.BATCH_SUPPORT: - return - self._database.batch_end() - self.batching = False - for name in self.after_batch_callbacks: - self.after_batch_callbacks[name]() - self.after_batch_callbacks = None - - @property - def cache(self): - if self._cache is not None: - return self._cache - else: - return {} - - def init_caching(self, cache): - """ - Start using cache by setting internal _cache attribute. - - :param cache: the cache instance, anything that behaves like a dict - :type cache: dict - """ - self._cache = cache - - def get_sync_target(self): - """ - Return a SyncTarget object, for another u1db to synchronize with. - - :return: The sync target. - :rtype: SoledadSyncTarget - """ - return SoledadSyncTarget(self) - - def delete_database(self): - """ - Delete a U1DB database. - """ - self._database.delete_database() - - def close(self): - """ - Release any resources associated with this database. - - :return: True if db was succesfully closed. - :rtype: bool - """ - self._database.close() - return True - - def __del__(self): - """ - Close the database upon garbage collection. - """ - self.close() - - def _set_replica_uid(self, replica_uid): - """ - Force the replica uid to be set. - - :param replica_uid: The new replica uid. - :type replica_uid: str - """ - self._database.set_replica_uid(replica_uid) - self._real_replica_uid = replica_uid - self.cache['replica_uid'] = self._real_replica_uid - - def _get_replica_uid(self): - """ - Get the replica uid. - - :return: The replica uid. - :rtype: str - """ - if self._real_replica_uid is not None: - self.cache['replica_uid'] = self._real_replica_uid - return self._real_replica_uid - if 'replica_uid' in self.cache: - return self.cache['replica_uid'] - self._real_replica_uid = self._database.get_replica_uid() - self._set_replica_uid(self._real_replica_uid) - return self._real_replica_uid - - _replica_uid = property(_get_replica_uid, _set_replica_uid) - - replica_uid = property(_get_replica_uid) - - def _get_generation(self): - """ - Return the current generation. - - :return: The current generation. - :rtype: int - - :raise SoledadError: Raised by database on operation failure - """ - return self._get_generation_info()[0] - - def _get_generation_info(self): - """ - Return the current generation. - - :return: A tuple containing the current generation and transaction id. - :rtype: (int, str) - - :raise SoledadError: Raised by database on operation failure - """ - cur_gen, newest_trans_id = self._database.get_generation_info() - return (cur_gen, newest_trans_id) - - def _get_trans_id_for_gen(self, generation): - """ - Get the transaction id corresponding to a particular generation. - - :param generation: The generation for which to get the transaction id. - :type generation: int - - :return: The transaction id for C{generation}. - :rtype: str - - :raise InvalidGeneration: Raised when the generation does not exist. - - """ - return self._database.get_trans_id_for_gen(generation) - - def _get_transaction_log(self): - """ - This is only for the test suite, it is not part of the api. - - :return: The complete transaction log. - :rtype: [(str, str)] - - """ - return self._database.get_transaction_log() - - def _get_doc(self, doc_id, check_for_conflicts=False): - """ - Extract the document from storage. - - This can return None if the document doesn't exist. - - :param doc_id: The unique document identifier - :type doc_id: str - :param check_for_conflicts: If set to False, then the conflict check - will be skipped. - :type check_for_conflicts: bool - - :return: The document. - :rtype: ServerDocument - """ - return self._database.get_doc(doc_id, check_for_conflicts) - - def get_doc(self, doc_id, include_deleted=False): - """ - Get the JSON string for the given document. - - :param doc_id: The unique document identifier - :type doc_id: str - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise asking for a deleted - document will return None. - :type include_deleted: bool - - :return: A document object. - :rtype: ServerDocument. - """ - doc = self._get_doc(doc_id, check_for_conflicts=True) - if doc is None: - return None - if doc.is_tombstone() and not include_deleted: - return None - return doc - - def get_all_docs(self, include_deleted=False): - """ - Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :type include_deleted: bool - - :return: (generation, [ServerDocument]) - The current generation of the database, followed by a list of all - the documents in the database. - :rtype: (int, [ServerDocument]) - """ - return self._database.get_all_docs(include_deleted) - - def _put_doc(self, old_doc, doc): - """ - Put the document in the backend database. - - Note that C{old_doc} must have been fetched with the parameter - C{check_for_conflicts} equal to True, so we can properly update the - new document using the conflict information from the old one. - - :param old_doc: The old document version. - :type old_doc: ServerDocument - :param doc: The document to be put. - :type doc: ServerDocument - """ - self._database.save_document(old_doc, doc, - self._allocate_transaction_id()) - - def put_doc(self, doc): - """ - Update a document. - - If the document currently has conflicts, put will fail. - If the database specifies a maximum document size and the document - exceeds it, put will fail and raise a DocumentTooBig exception. - - :param doc: A Document with new content. - :return: new_doc_rev - The new revision identifier for the document. - The Document object will also be updated. - - :raise InvalidDocId: Raised if the document's id is invalid. - :raise DocumentTooBig: Raised if the document size is too big. - :raise ConflictedDoc: Raised if the document has conflicts. - """ - if doc.doc_id is None: - raise InvalidDocId() - self._check_doc_id(doc.doc_id) - self._check_doc_size(doc) - old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) - if old_doc and old_doc.has_conflicts: - raise ConflictedDoc() - if old_doc and doc.rev is None and old_doc.is_tombstone(): - new_rev = self._allocate_doc_rev(old_doc.rev) - else: - if old_doc is not None: - if old_doc.rev != doc.rev: - raise RevisionConflict() - else: - if doc.rev is not None: - raise RevisionConflict() - new_rev = self._allocate_doc_rev(doc.rev) - doc.rev = new_rev - self._put_doc(old_doc, doc) - return new_rev - - def whats_changed(self, old_generation=0): - """ - Return a list of documents that have changed since old_generation. - - :param old_generation: The generation of the database in the old - state. - :type old_generation: int - - :return: (generation, trans_id, [(doc_id, generation, trans_id),...]) - The current generation of the database, its associated - transaction id, and a list of of changed documents since - old_generation, represented by tuples with for each document - its doc_id and the generation and transaction id corresponding - to the last intervening change and sorted by generation (old - changes first) - :rtype: (int, str, [(str, int, str)]) - """ - return self._database.whats_changed(old_generation) - - def delete_doc(self, doc): - """ - Mark a document as deleted. - - Will abort if the current revision doesn't match doc.rev. - This will also set doc.content to None. - - :param doc: The document to mark as deleted. - :type doc: ServerDocument. - - :raise DocumentDoesNotExist: Raised if the document does not - exist. - :raise RevisionConflict: Raised if the revisions do not match. - :raise DocumentAlreadyDeleted: Raised if the document is - already deleted. - :raise ConflictedDoc: Raised if the doc has conflicts. - """ - old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) - if old_doc is None: - raise DocumentDoesNotExist - if old_doc.rev != doc.rev: - raise RevisionConflict() - if old_doc.is_tombstone(): - raise DocumentAlreadyDeleted - if old_doc.has_conflicts: - raise ConflictedDoc() - new_rev = self._allocate_doc_rev(doc.rev) - doc.rev = new_rev - doc.make_tombstone() - self._put_doc(old_doc, doc) - return new_rev - - def get_doc_conflicts(self, doc_id): - """ - Get the conflicted versions of a document. - - :param doc_id: The document id. - :type doc_id: str - - :return: A list of conflicted versions of the document. - :rtype: list - """ - return self._database.get_doc_conflicts(doc_id) - - def _get_replica_gen_and_trans_id(self, other_replica_uid): - """ - Return the last known generation and transaction id for the other db - replica. - - When you do a synchronization with another replica, the Database keeps - track of what generation the other database replica was at, and what - the associated transaction id was. This is used to determine what data - needs to be sent, and if two databases are claiming to be the same - replica. - - :param other_replica_uid: The identifier for the other replica. - :type other_replica_uid: str - - :return: A tuple containing the generation and transaction id we - encountered during synchronization. If we've never - synchronized with the replica, this is (0, ''). - :rtype: (int, str) - """ - if other_replica_uid in self.cache: - return self.cache[other_replica_uid] - gen, trans_id = \ - self._database.get_replica_gen_and_trans_id(other_replica_uid) - self.cache[other_replica_uid] = (gen, trans_id) - return (gen, trans_id) - - def _set_replica_gen_and_trans_id(self, other_replica_uid, - other_generation, other_transaction_id): - """ - Set the last-known generation and transaction id for the other - database replica. - - We have just performed some synchronization, and we want to track what - generation the other replica was at. See also - _get_replica_gen_and_trans_id. - - :param other_replica_uid: The U1DB identifier for the other replica. - :type other_replica_uid: str - :param other_generation: The generation number for the other replica. - :type other_generation: int - :param other_transaction_id: The transaction id associated with the - generation. - :type other_transaction_id: str - """ - if other_replica_uid is not None and other_generation is not None: - self.cache[other_replica_uid] = (other_generation, - other_transaction_id) - self._database.set_replica_gen_and_trans_id(other_replica_uid, - other_generation, - other_transaction_id) - - def _do_set_replica_gen_and_trans_id( - self, other_replica_uid, other_generation, other_transaction_id): - """ - _put_doc_if_newer from super class is calling it. So we declare this. - - :param other_replica_uid: The U1DB identifier for the other replica. - :type other_replica_uid: str - :param other_generation: The generation number for the other replica. - :type other_generation: int - :param other_transaction_id: The transaction id associated with the - generation. - :type other_transaction_id: str - """ - args = [other_replica_uid, other_generation, other_transaction_id] - callback = functools.partial(self._set_replica_gen_and_trans_id, *args) - if self.batching: - self.after_batch_callbacks['set_source_info'] = callback - else: - callback() - - def _force_doc_sync_conflict(self, doc): - """ - Add a conflict and force a document put. - - :param doc: The document to be put. - :type doc: ServerDocument - """ - my_doc = self._get_doc(doc.doc_id) - self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) - doc.add_conflict(self._factory(doc.doc_id, my_doc.rev, - my_doc.get_json())) - doc.has_conflicts = True - self._put_doc(my_doc, doc) - - def resolve_doc(self, doc, conflicted_doc_revs): - """ - Mark a document as no longer conflicted. - - We take the list of revisions that the client knows about that it is - superseding. This may be a different list from the actual current - conflicts, in which case only those are removed as conflicted. This - may fail if the conflict list is significantly different from the - supplied information. (sync could have happened in the background from - the time you GET_DOC_CONFLICTS until the point where you RESOLVE) - - :param doc: A Document with the new content to be inserted. - :type doc: ServerDocument - :param conflicted_doc_revs: A list of revisions that the new content - supersedes. - :type conflicted_doc_revs: [str] - - :raise SoledadError: Raised by database on operation failure - """ - cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) - new_rev = self._ensure_maximal_rev(cur_doc.rev, - conflicted_doc_revs) - superseded_revs = set(conflicted_doc_revs) - doc.rev = new_rev - # this backend stores conflicts as properties of the documents, so we - # have to copy these conflicts over to the document being updated. - if cur_doc.rev in superseded_revs: - # the newer doc version will supersede the one in the database, so - # we copy conflicts before updating the backend. - doc.set_conflicts(cur_doc.get_conflicts()) # copy conflicts over. - doc.delete_conflicts(superseded_revs) - self._put_doc(cur_doc, doc) - else: - # the newer doc version does not supersede the one in the - # database, so we will add a conflict to the database and copy - # those over to the document the user has in her hands. - cur_doc.add_conflict(doc) - cur_doc.delete_conflicts(superseded_revs) - self._put_doc(cur_doc, cur_doc) # just update conflicts - # backend has been updated with current conflicts, now copy them - # to the current document. - doc.set_conflicts(cur_doc.get_conflicts()) - - def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, - replica_trans_id='', number_of_docs=None, - doc_idx=None, sync_id=None): - """ - Insert/update document into the database with a given revision. - - This api is used during synchronization operations. - - If a document would conflict and save_conflict is set to True, the - content will be selected as the 'current' content for doc.doc_id, - even though doc.rev doesn't supersede the currently stored revision. - The currently stored document will be added to the list of conflict - alternatives for the given doc_id. - - This forces the new content to be 'current' so that we get convergence - after synchronizing, even if people don't resolve conflicts. Users can - then notice that their content is out of date, update it, and - synchronize again. (The alternative is that users could synchronize and - think the data has propagated, but their local copy looks fine, and the - remote copy is never updated again.) - - :param doc: A document object - :type doc: ServerDocument - :param save_conflict: If this document is a conflict, do you want to - save it as a conflict, or just ignore it. - :type save_conflict: bool - :param replica_uid: A unique replica identifier. - :type replica_uid: str - :param replica_gen: The generation of the replica corresponding to the - this document. The replica arguments are optional, - but are used during synchronization. - :type replica_gen: int - :param replica_trans_id: The transaction_id associated with the - generation. - :type replica_trans_id: str - :param number_of_docs: The total amount of documents sent on this sync - session. - :type number_of_docs: int - :param doc_idx: The index of the current document being sent. - :type doc_idx: int - :param sync_id: The id of the current sync session. - :type sync_id: str - - :return: (state, at_gen) - If we don't have doc_id already, or if - doc_rev supersedes the existing document revision, then the - content will be inserted, and state is 'inserted'. If - doc_rev is less than or equal to the existing revision, then - the put is ignored and state is respecitvely 'superseded' or - 'converged'. If doc_rev is not strictly superseded or - supersedes, then state is 'conflicted'. The document will not - be inserted if save_conflict is False. For 'inserted' or - 'converged', at_gen is the insertion/current generation. - :rtype: (str, int) - """ - if not isinstance(doc, ServerDocument): - doc = self._factory(doc.doc_id, doc.rev, doc.get_json()) - my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) - if my_doc: - doc.set_conflicts(my_doc.get_conflicts()) - return CommonBackend._put_doc_if_newer(self, doc, save_conflict, - replica_uid, replica_gen, - replica_trans_id) - - def _put_and_update_indexes(self, cur_doc, doc): - self._put_doc(cur_doc, doc) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False, read_content=True): - """ - Get the JSON content for many documents. - - :param doc_ids: A list of document identifiers or None for all. - :type doc_ids: list - :param check_for_conflicts: If set to False, then the conflict check - will be skipped, and 'None' will be - returned instead of True/False. - :type check_for_conflicts: bool - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :return: iterable giving the Document object for each document id - in matching doc_ids order. - :rtype: iterable - """ - return self._database.get_docs(doc_ids, check_for_conflicts, - include_deleted, read_content) - - def _prune_conflicts(self, doc, doc_vcr): - """ - Prune conflicts that are older then the current document's revision, or - whose content match to the current document's content. - Originally in u1db.CommonBackend - - :param doc: The document to have conflicts pruned. - :type doc: ServerDocument - :param doc_vcr: A vector clock representing the current document's - revision. - :type doc_vcr: u1db.vectorclock.VectorClock - """ - if doc.has_conflicts: - autoresolved = False - c_revs_to_prune = [] - for c_doc in doc._conflicts: - c_vcr = vectorclock.VectorClockRev(c_doc.rev) - if doc_vcr.is_newer(c_vcr): - c_revs_to_prune.append(c_doc.rev) - elif doc.same_content_as(c_doc): - c_revs_to_prune.append(c_doc.rev) - doc_vcr.maximize(c_vcr) - autoresolved = True - if autoresolved: - doc_vcr.increment(self._replica_uid) - doc.rev = doc_vcr.as_str() - doc.delete_conflicts(c_revs_to_prune) - - -class SoledadSyncTarget(CommonSyncTarget): - - """ - Functionality for using a SoledadBackend as a synchronization target. - """ - - def get_sync_info(self, source_replica_uid): - source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( - source_replica_uid) - my_gen, my_trans_id = self._db._get_generation_info() - return ( - self._db._replica_uid, my_gen, my_trans_id, source_gen, - source_trans_id) - - def record_sync_info(self, source_replica_uid, source_replica_generation, - source_replica_transaction_id): - if self._trace_hook: - self._trace_hook('record_sync_info') - self._db._set_replica_gen_and_trans_id( - source_replica_uid, source_replica_generation, - source_replica_transaction_id) diff --git a/common/src/leap/soledad/common/command.py b/common/src/leap/soledad/common/command.py deleted file mode 100644 index 66aa6b7a..00000000 --- a/common/src/leap/soledad/common/command.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# command.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Utility to sanitize and run shell commands. -""" - - -import subprocess - - -def exec_validated_cmd(cmd, argument, validator=None): - """ - Executes cmd, validating argument with a validator function. - - :param cmd: command. - :type dbname: str - :param argument: argument. - :type argument: str - :param validator: optional function to validate argument - :type validator: function - - :return: exit code and stdout or stderr (if code != 0) - :rtype: (int, str) - """ - if validator and not validator(argument): - return 1, "invalid argument" - command = cmd.split(' ') - command.append(argument) - try: - process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError as e: - return 1, e - (out, err) = process.communicate() - code = process.wait() - if code is not 0: - return code, err - else: - return code, out diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py deleted file mode 100644 index 2343e849..00000000 --- a/common/src/leap/soledad/common/couch/__init__.py +++ /dev/null @@ -1,812 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -"""A U1DB backend that uses CouchDB as its persistence layer.""" - - -import json -import copy -import re -import uuid -import binascii - -from six import StringIO -from six.moves.urllib.parse import urljoin -from contextlib import contextmanager - - -from couchdb.client import Server, Database -from couchdb.http import ( - ResourceConflict, - ResourceNotFound, - Session, - urljoin as couch_urljoin, - Resource, -) -from leap.soledad.common.l2db.errors import ( - DatabaseDoesNotExist, - InvalidGeneration, - RevisionConflict, -) -from leap.soledad.common.l2db.remote import http_app - - -from .support import MultipartWriter -from leap.soledad.common.errors import InvalidURLError -from leap.soledad.common.document import ServerDocument -from leap.soledad.common.backend import SoledadBackend - - -COUCH_TIMEOUT = 120 # timeout for transfers between Soledad server and Couch - - -def list_users_dbs(couch_url): - """ - Retrieves a list with all databases that starts with 'user-' on CouchDB. - Those databases belongs to users. So, the list will contain all the - database names in the form of 'user-{uuid4}'. - - :param couch_url: The couch url with needed credentials - :type couch_url: str - - :return: The list of all database names from users. - :rtype: [str] - """ - with couch_server(couch_url) as server: - users = [dbname for dbname in server if dbname.startswith('user-')] - return users - - -# monkey-patch the u1db http app to use ServerDocument -http_app.Document = ServerDocument - - -@contextmanager -def couch_server(url): - """ - Provide a connection to a couch server and cleanup after use. - - For database creation and deletion we use an ephemeral connection to the - couch server. That connection has to be properly closed, so we provide it - as a context manager. - - :param url: The URL of the Couch server. - :type url: str - """ - session = Session(timeout=COUCH_TIMEOUT) - server = Server(url=url, full_commit=False, session=session) - yield server - - -def _get_gen_doc_id(gen): - return 'gen-%s' % str(gen).zfill(10) - - -GENERATION_KEY = 'gen' -TRANSACTION_ID_KEY = 'trans_id' -REPLICA_UID_KEY = 'replica_uid' -DOC_ID_KEY = 'doc_id' -SCHEMA_VERSION_KEY = 'schema_version' - -CONFIG_DOC_ID = '_local/config' -SYNC_DOC_ID_PREFIX = '_local/sync_' -SCHEMA_VERSION = 1 - - -class CouchDatabase(object): - """ - Holds CouchDB related code. - This class gives methods to encapsulate database operations and hide - CouchDB details from backend code. - """ - - @classmethod - def open_database(cls, url, create, replica_uid=None, - database_security=None): - """ - Open a U1DB database using CouchDB as backend. - - :param url: the url of the database replica - :type url: str - :param create: should the replica be created if it does not exist? - :type create: bool - :param replica_uid: an optional unique replica identifier - :type replica_uid: str - :param database_security: security rules as CouchDB security doc - :type database_security: dict - - :return: the database instance - :rtype: SoledadBackend - - :raise DatabaseDoesNotExist: Raised if database does not exist. - """ - # get database from url - m = re.match('(^https?://[^/]+)/(.+)$', url) - if not m: - raise InvalidURLError - url = m.group(1) - dbname = m.group(2) - with couch_server(url) as server: - if dbname not in server: - if create: - server.create(dbname) - else: - raise DatabaseDoesNotExist() - db = cls(url, dbname, ensure_security=create, - database_security=database_security) - return SoledadBackend( - db, replica_uid=replica_uid) - - def __init__(self, url, dbname, ensure_security=False, - database_security=None): - """ - :param url: Couch server URL with necessary credentials - :type url: string - :param dbname: Couch database name - :type dbname: string - :param ensure_security: will PUT a _security ddoc if set - :type ensure_security: bool - :param database_security: security rules as CouchDB security doc - :type database_security: dict - """ - self._session = Session(timeout=COUCH_TIMEOUT) - self._url = url - self._dbname = dbname - self._database = self.get_couch_database(url, dbname) - self.batching = False - self.batch_generation = None - self.batch_docs = {} - if ensure_security: - self.ensure_security_ddoc(database_security) - - def batch_start(self): - self.batching = True - self.batch_generation = self.get_generation_info() - ids = set(row.id for row in self._database.view('_all_docs')) - self.batched_ids = ids - - def batch_end(self): - self.batching = False - self.batch_generation = None - self.__perform_batch() - - def get_couch_database(self, url, dbname): - """ - Generate a couchdb.Database instance given a url and dbname. - - :param url: CouchDB's server url with credentials - :type url: str - :param dbname: Database name - :type dbname: str - - :return: couch library database instance - :rtype: couchdb.Database - - :raise DatabaseDoesNotExist: Raised if database does not exist. - """ - try: - return Database( - urljoin(url, dbname), - self._session) - except ResourceNotFound: - raise DatabaseDoesNotExist() - - def ensure_security_ddoc(self, security_config=None): - """ - Make sure that only soledad user is able to access this database as - an unprivileged member, meaning that administration access will - be forbidden even inside an user database. - The goal is to make sure that only the lowest access level is given - to the unprivileged CouchDB user set on the server process. - This is achieved by creating a _security design document, see: - http://docs.couchdb.org/en/latest/api/database/security.html - - :param security_config: security configuration parsed from conf file - :type security_config: dict - """ - security_config = security_config or {} - security = self._database.resource.get_json('_security')[2] - security['members'] = {'names': [], 'roles': []} - security['members']['names'] = security_config.get('members', - ['soledad']) - security['members']['roles'] = security_config.get('members_roles', []) - security['admins'] = {'names': [], 'roles': []} - security['admins']['names'] = security_config.get('admins', []) - security['admins']['roles'] = security_config.get('admins_roles', []) - self._database.resource.put_json('_security', body=security) - - def delete_database(self): - """ - Delete a U1DB CouchDB database. - """ - with couch_server(self._url) as server: - del(server[self._dbname]) - - def set_replica_uid(self, replica_uid): - """ - Force the replica uid to be set. - - :param replica_uid: The new replica uid. - :type replica_uid: str - """ - try: - # set on existent config document - doc = self._database[CONFIG_DOC_ID] - doc[REPLICA_UID_KEY] = replica_uid - except ResourceNotFound: - # or create the config document - doc = { - '_id': CONFIG_DOC_ID, - REPLICA_UID_KEY: replica_uid, - SCHEMA_VERSION_KEY: SCHEMA_VERSION, - } - self._database.save(doc) - - def get_replica_uid(self): - """ - Get the replica uid. - - :return: The replica uid. - :rtype: str - """ - try: - # grab replica_uid from server - doc = self._database[CONFIG_DOC_ID] - replica_uid = doc[REPLICA_UID_KEY] - return replica_uid - except ResourceNotFound: - # create a unique replica_uid - replica_uid = uuid.uuid4().hex - self.set_replica_uid(replica_uid) - return replica_uid - - def close(self): - self._database = None - - def get_all_docs(self, include_deleted=False): - """ - Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :type include_deleted: bool - - :return: (generation, [ServerDocument]) - The current generation of the database, followed by a list of all - the documents in the database. - :rtype: (int, [ServerDocument]) - """ - - generation, _ = self.get_generation_info() - results = list( - self.get_docs(None, True, include_deleted)) - return (generation, results) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False, read_content=True): - """ - Get the JSON content for many documents. - - Use couch's `_all_docs` view to get the documents indicated in - `doc_ids`, - - :param doc_ids: A list of document identifiers or None for all. - :type doc_ids: list - :param check_for_conflicts: If set to False, then the conflict check - will be skipped, and 'None' will be - returned instead of True/False. - :type check_for_conflicts: bool - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - - :return: iterable giving the Document object for each document id - in matching doc_ids order. - :rtype: iterable - """ - params = {'include_docs': 'true', 'attachments': 'false'} - if doc_ids is not None: - params['keys'] = doc_ids - view = self._database.view("_all_docs", **params) - for row in view.rows: - result = copy.deepcopy(row['doc']) - for file_name in result.get('_attachments', {}).keys(): - data = self._database.get_attachment(result, file_name) - if data: - if read_content: - data = data.read() - result['_attachments'][file_name] = {'data': data} - doc = self.__parse_doc_from_couch( - result, result['_id'], - check_for_conflicts=check_for_conflicts, decode=False) - # filter out non-u1db or deleted documents - if not doc or (not include_deleted and doc.is_tombstone()): - continue - yield doc - - def get_doc(self, doc_id, check_for_conflicts=False): - """ - Extract the document from storage. - - This can return None if the document doesn't exist. - - :param doc_id: The unique document identifier - :type doc_id: str - :param check_for_conflicts: If set to False, then the conflict check - will be skipped. - :type check_for_conflicts: bool - - :return: The document. - :rtype: ServerDocument - """ - doc_from_batch = self.__check_batch_before_get(doc_id) - if doc_from_batch: - return doc_from_batch - if self.batching and doc_id not in self.batched_ids: - return None - if doc_id not in self._database: - return None - # get document with all attachments (u1db content and eventual - # conflicts) - result = self.json_from_resource([doc_id], attachments=True) - return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts) - - def __check_batch_before_get(self, doc_id): - """ - If doc_id is staged for batching, then we need to commit the batch - before going ahead. This avoids consistency problems, like trying to - get a document that isn't persisted and processing like it is missing. - - :param doc_id: The unique document identifier - :type doc_id: str - """ - if doc_id in self.batch_docs: - couch_doc = self.batch_docs[doc_id] - rev = self.__perform_batch(doc_id) - couch_doc['_rev'] = rev - self.batched_ids.add(doc_id) - return self.__parse_doc_from_couch(couch_doc, doc_id, True) - return None - - def __perform_batch(self, doc_id=None): - status = self._database.update(self.batch_docs.values()) - rev = None - for ok, stored_doc_id, rev_or_error in status: - if not ok: - error = rev_or_error - if type(error) is ResourceConflict: - raise RevisionConflict - raise error - elif doc_id == stored_doc_id: - rev = rev_or_error - self.batch_docs.clear() - return rev - - def __parse_doc_from_couch(self, result, doc_id, - check_for_conflicts=False, decode=True): - # restrict to u1db documents - if 'u1db_rev' not in result: - return None - doc = ServerDocument(doc_id, result['u1db_rev']) - # set contents or make tombstone - if '_attachments' not in result \ - or 'u1db_content' not in result['_attachments']: - doc.make_tombstone() - elif decode: - doc.content = json.loads( - binascii.a2b_base64( - result['_attachments']['u1db_content']['data'])) - else: - doc._json = result['_attachments']['u1db_content']['data'] - # determine if there are conflicts - if check_for_conflicts \ - and '_attachments' in result \ - and 'u1db_conflicts' in result['_attachments']: - if decode: - conflicts = binascii.a2b_base64( - result['_attachments']['u1db_conflicts']['data']) - else: - conflicts = result['_attachments']['u1db_conflicts']['data'] - conflicts = json.loads(conflicts) - doc.set_conflicts(self._build_conflicts(doc.doc_id, conflicts)) - # store couch revision - doc.couch_rev = result['_rev'] - return doc - - def _build_conflicts(self, doc_id, attached_conflicts): - """ - Build the conflicted documents list from the conflicts attachment - fetched from a couch document. - - :param attached_conflicts: The document's conflicts as fetched from a - couch document attachment. - :type attached_conflicts: dict - """ - conflicts = [] - for doc_rev, content in attached_conflicts: - doc = ServerDocument(doc_id, doc_rev) - if content is None: - doc.make_tombstone() - else: - doc.content = content - conflicts.append(doc) - return conflicts - - def get_trans_id_for_gen(self, generation): - """ - Get the transaction id corresponding to a particular generation. - - :param generation: The generation for which to get the transaction id. - :type generation: int - - :return: The transaction id for C{generation}. - :rtype: str - - :raise InvalidGeneration: Raised when the generation does not exist. - """ - if generation == 0: - return '' - log = self._get_transaction_log(start=generation, end=generation) - if not log: - raise InvalidGeneration - _, _, trans_id = log[0] - return trans_id - - def get_replica_gen_and_trans_id(self, other_replica_uid): - """ - Return the last known generation and transaction id for the other db - replica. - - When you do a synchronization with another replica, the Database keeps - track of what generation the other database replica was at, and what - the associated transaction id was. This is used to determine what data - needs to be sent, and if two databases are claiming to be the same - replica. - - :param other_replica_uid: The identifier for the other replica. - :type other_replica_uid: str - - :return: A tuple containing the generation and transaction id we - encountered during synchronization. If we've never - synchronized with the replica, this is (0, ''). - :rtype: (int, str) - """ - doc_id = '%s%s' % (SYNC_DOC_ID_PREFIX, other_replica_uid) - try: - doc = self._database[doc_id] - except ResourceNotFound: - doc = { - '_id': doc_id, - GENERATION_KEY: 0, - REPLICA_UID_KEY: str(other_replica_uid), - TRANSACTION_ID_KEY: '', - } - self._database.save(doc) - gen, trans_id = doc[GENERATION_KEY], doc[TRANSACTION_ID_KEY] - return gen, trans_id - - def get_doc_conflicts(self, doc_id, couch_rev=None): - """ - Get the conflicted versions of a document. - - If the C{couch_rev} parameter is not None, conflicts for a specific - document's couch revision are returned. - - :param couch_rev: The couch document revision. - :type couch_rev: str - - :return: A list of conflicted versions of the document. - :rtype: list - """ - # request conflicts attachment from server - params = {} - conflicts = [] - if couch_rev is not None: - params['rev'] = couch_rev # restric document's couch revision - else: - # TODO: move into resource logic! - first_entry = self.get_doc(doc_id, check_for_conflicts=True) - conflicts.append(first_entry) - - try: - response = self.json_from_resource([doc_id, 'u1db_conflicts'], - **params) - return conflicts + self._build_conflicts( - doc_id, json.loads(response.read())) - except ResourceNotFound: - return [] - - def set_replica_gen_and_trans_id( - self, other_replica_uid, other_generation, other_transaction_id): - """ - Set the last-known generation and transaction id for the other - database replica. - - We have just performed some synchronization, and we want to track what - generation the other replica was at. See also - get_replica_gen_and_trans_id. - - :param other_replica_uid: The U1DB identifier for the other replica. - :type other_replica_uid: str - :param other_generation: The generation number for the other replica. - :type other_generation: int - :param other_transaction_id: The transaction id associated with the - generation. - :type other_transaction_id: str - """ - doc_id = '%s%s' % (SYNC_DOC_ID_PREFIX, other_replica_uid) - try: - doc = self._database[doc_id] - except ResourceNotFound: - doc = {'_id': doc_id} - doc[GENERATION_KEY] = other_generation - doc[TRANSACTION_ID_KEY] = other_transaction_id - self._database.save(doc) - - def get_transaction_log(self): - """ - This is only for the test suite, it is not part of the api. - - :return: The complete transaction log. - :rtype: [(str, str)] - """ - log = self._get_transaction_log() - return map(lambda i: (i[1], i[2]), log) - - def _get_gen_docs( - self, start=0, end=9999999999, descending=None, limit=None): - params = {} - if descending: - params['descending'] = 'true' - # honor couch way of traversing the view tree in reverse order - start, end = end, start - params['startkey'] = _get_gen_doc_id(start) - params['endkey'] = _get_gen_doc_id(end) - params['include_docs'] = 'true' - if limit: - params['limit'] = limit - view = self._database.view("_all_docs", **params) - return view.rows - - def _get_transaction_log(self, start=0, end=9999999999): - # get current gen and trans_id - rows = self._get_gen_docs(start=start, end=end) - log = [] - for row in rows: - doc = row['doc'] - log.append(( - doc[GENERATION_KEY], - doc[DOC_ID_KEY], - doc[TRANSACTION_ID_KEY])) - return log - - def whats_changed(self, old_generation=0): - """ - Return a list of documents that have changed since old_generation. - - :param old_generation: The generation of the database in the old - state. - :type old_generation: int - - :return: (generation, trans_id, [(doc_id, generation, trans_id),...]) - The current generation of the database, its associated - transaction id, and a list of of changed documents since - old_generation, represented by tuples with for each document - its doc_id and the generation and transaction id corresponding - to the last intervening change and sorted by generation (old - changes first) - :rtype: (int, str, [(str, int, str)]) - """ - changes = [] - cur_generation, last_trans_id = self.get_generation_info() - relevant_tail = self._get_transaction_log(start=old_generation + 1) - seen = set() - for generation, doc_id, trans_id in reversed(relevant_tail): - if doc_id not in seen: - changes.append((doc_id, generation, trans_id)) - seen.add(doc_id) - changes.reverse() - return (cur_generation, last_trans_id, changes) - - def get_generation_info(self): - """ - Return the current generation. - - :return: A tuple containing the current generation and transaction id. - :rtype: (int, str) - """ - if self.batching and self.batch_generation: - return self.batch_generation - rows = self._get_gen_docs(descending=True, limit=1) - if not rows: - return 0, '' - gen_doc = rows.pop()['doc'] - return gen_doc[GENERATION_KEY], gen_doc[TRANSACTION_ID_KEY] - - def json_from_resource(self, doc_path, **kwargs): - """ - Get a resource from it's path and gets a doc's JSON using provided - parameters. - - :param doc_path: The path to resource. - :type doc_path: [str] - - :return: The request's data parsed from JSON to a dict. - :rtype: dict - """ - if doc_path is not None: - resource = self._database.resource(*doc_path) - else: - resource = self._database.resource() - _, _, data = resource.get_json(**kwargs) - return data - - def _allocate_new_generation(self, doc_id, transaction_id, save=True): - """ - Allocate a new generation number for a document modification. - - We need to allocate a new generation to this document modification by - creating a new gen doc. In order to avoid concurrent database updates - from allocating the same new generation, we will try to create the - document until we succeed, meaning that no other piece of code holds - the same generation number as ours. - - The loop below would only be executed more than once if: - - 1. there's more than one thread trying to modify the user's database, - and - - 2. the execution of getting the current generation and saving the gen - doc different threads get interleaved (one of them will succeed - and the others will fail and try again). - - Number 1 only happens when more than one user device is syncing at the - same time. Number 2 depends on not-so-frequent coincidence of - code execution. - - Also, in the race between threads for a generation number there's - always one thread that wins. so if there are N threads in the race, the - expected number of repetitions of the loop for each thread would be - N/2. If N is equal to the number of devices that the user has, the - number of possible repetitions of the loop should always be low. - """ - while True: - try: - # add the gen document - gen, _ = self.get_generation_info() - new_gen = gen + 1 - gen_doc = { - '_id': _get_gen_doc_id(new_gen), - GENERATION_KEY: new_gen, - DOC_ID_KEY: doc_id, - TRANSACTION_ID_KEY: transaction_id, - } - if save: - self._database.save(gen_doc) - break # succeeded allocating a new generation, proceed - except ResourceConflict: - pass # try again! - return gen_doc - - def save_document(self, old_doc, doc, transaction_id): - """ - Put the document in the Couch backend database. - - Note that C{old_doc} must have been fetched with the parameter - C{check_for_conflicts} equal to True, so we can properly update the - new document using the conflict information from the old one. - - :param old_doc: The old document version. - :type old_doc: ServerDocument - :param doc: The document to be put. - :type doc: ServerDocument - - :raise RevisionConflict: Raised when trying to update a document but - couch revisions mismatch. - """ - attachments = {} # we save content and conflicts as attachments - parts = [] # and we put it using couch's multipart PUT - # save content as attachment - if doc.is_tombstone() is False: - content = doc.get_json() - attachments['u1db_content'] = { - 'follows': True, - 'content_type': 'application/octet-stream', - 'length': len(content), - } - parts.append(content) - - # save conflicts as attachment - if doc.has_conflicts is True: - conflicts = json.dumps( - map(lambda cdoc: (cdoc.rev, cdoc.content), - doc.get_conflicts())) - attachments['u1db_conflicts'] = { - 'follows': True, - 'content_type': 'application/octet-stream', - 'length': len(conflicts), - } - parts.append(conflicts) - - # build the couch document - couch_doc = { - '_id': doc.doc_id, - 'u1db_rev': doc.rev, - '_attachments': attachments, - } - # if we are updating a doc we have to add the couch doc revision - if old_doc is not None and hasattr(old_doc, 'couch_rev'): - couch_doc['_rev'] = old_doc.couch_rev - # prepare the multipart PUT - if not self.batching: - buf = StringIO() - envelope = MultipartWriter(buf) - # the order in which attachments are described inside the - # serialization of the couch document must match the order in - # which they are actually written in the multipart structure. - # Because of that, we use `sorted_keys=True` in the json - # serialization (so "u1db_conflicts" comes before - # "u1db_content" on the couch document attachments - # description), and also reverse the order of the parts before - # writing them, so the "conflict" part is written before the - # "content" part. - envelope.add( - 'application/json', - json.dumps(couch_doc, sort_keys=True)) - parts.reverse() - for part in parts: - envelope.add('application/octet-stream', part) - envelope.close() - # try to save and fail if there's a revision conflict - try: - resource = self._new_resource() - resource.put_json( - doc.doc_id, body=str(buf.getvalue()), - headers=envelope.headers) - except ResourceConflict: - raise RevisionConflict() - self._allocate_new_generation(doc.doc_id, transaction_id) - else: - for name, attachment in attachments.items(): - del attachment['follows'] - del attachment['length'] - index = 0 if name is 'u1db_content' else 1 - attachment['data'] = binascii.b2a_base64( - parts[index]).strip() - couch_doc['_attachments'] = attachments - gen_doc = self._allocate_new_generation( - doc.doc_id, transaction_id, save=False) - self.batch_docs[doc.doc_id] = couch_doc - self.batch_docs[gen_doc['_id']] = gen_doc - last_gen, last_trans_id = self.batch_generation - self.batch_generation = (last_gen + 1, transaction_id) - - def _new_resource(self, *path): - """ - Return a new resource for accessing a couch database. - - :return: A resource for accessing a couch database. - :rtype: couchdb.http.Resource - """ - # Workaround for: https://leap.se/code/issues/5448 - url = couch_urljoin(self._database.resource.url, *path) - resource = Resource(url, Session(timeout=COUCH_TIMEOUT)) - resource.credentials = self._database.resource.credentials - resource.headers = self._database.resource.headers.copy() - return resource diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py deleted file mode 100644 index 8cbe0934..00000000 --- a/common/src/leap/soledad/common/couch/state.py +++ /dev/null @@ -1,158 +0,0 @@ -# -*- coding: utf-8 -*- -# state.py -# Copyright (C) 2015,2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Server state using CouchDatabase as backend. -""" -import couchdb -import re - -from six.moves.urllib.parse import urljoin - -from leap.soledad.common.log import getLogger -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch import CONFIG_DOC_ID -from leap.soledad.common.couch import SCHEMA_VERSION -from leap.soledad.common.couch import SCHEMA_VERSION_KEY -from leap.soledad.common.command import exec_validated_cmd -from leap.soledad.common.l2db.remote.server_state import ServerState -from leap.soledad.common.l2db.errors import Unauthorized -from leap.soledad.common.errors import WrongCouchSchemaVersionError -from leap.soledad.common.errors import MissingCouchConfigDocumentError - - -logger = getLogger(__name__) - - -def is_db_name_valid(name): - """ - Validate a user database using a regular expression. - - :param name: database name. - :type name: str - - :return: boolean for name vailidity - :rtype: bool - """ - db_name_regex = "^user-[a-f0-9]+$" - return re.match(db_name_regex, name) is not None - - -class CouchServerState(ServerState): - - """ - Inteface of the WSGI server with the CouchDB backend. - """ - - def __init__(self, couch_url, create_cmd=None, - check_schema_versions=False): - """ - Initialize the couch server state. - - :param couch_url: The URL for the couch database. - :type couch_url: str - :param create_cmd: Command to be executed for user db creation. It will - receive a properly sanitized parameter with user db - name and should access CouchDB with necessary - privileges, which server lacks for security reasons. - :type create_cmd: str - :param check_schema_versions: Whether to check couch schema version of - user dbs. Set to False as this is only - intended to run once during start-up. - :type check_schema_versions: bool - """ - self.couch_url = couch_url - self.create_cmd = create_cmd - if check_schema_versions: - self._check_schema_versions() - - def _check_schema_versions(self): - """ - Check that all user databases use the correct couch schema. - """ - server = couchdb.client.Server(self.couch_url) - for dbname in server: - if not dbname.startswith('user-'): - continue - db = server[dbname] - - # if there are documents, ensure that a config doc exists - config_doc = db.get(CONFIG_DOC_ID) - if config_doc: - if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION: - logger.error( - "Unsupported database schema in database %s" % dbname) - raise WrongCouchSchemaVersionError(dbname) - else: - result = db.view('_all_docs', limit=1) - if result.total_rows != 0: - logger.error( - "Missing couch config document in database %s" - % dbname) - raise MissingCouchConfigDocumentError(dbname) - - def open_database(self, dbname): - """ - Open a couch database. - - :param dbname: The name of the database to open. - :type dbname: str - - :return: The SoledadBackend object. - :rtype: SoledadBackend - """ - url = urljoin(self.couch_url, dbname) - db = CouchDatabase.open_database(url, create=False) - return db - - def ensure_database(self, dbname): - """ - Ensure couch database exists. - - :param dbname: The name of the database to ensure. - :type dbname: str - - :raise Unauthorized: If disabled or other error was raised. - - :return: The SoledadBackend object and its replica_uid. - :rtype: (SoledadBackend, str) - """ - if not self.create_cmd: - raise Unauthorized() - else: - code, out = exec_validated_cmd(self.create_cmd, dbname, - validator=is_db_name_valid) - if code is not 0: - logger.error(""" - Error while creating database (%s) with (%s) command. - Output: %s - Exit code: %d - """ % (dbname, self.create_cmd, out, code)) - raise Unauthorized() - db = self.open_database(dbname) - return db, db.replica_uid - - def delete_database(self, dbname): - """ - Delete couch database. - - :param dbname: The name of the database to delete. - :type dbname: str - - :raise Unauthorized: Always, because Soledad server is not allowed to - delete databases. - """ - raise Unauthorized() diff --git a/common/src/leap/soledad/common/couch/support.py b/common/src/leap/soledad/common/couch/support.py deleted file mode 100644 index bfc4fef6..00000000 --- a/common/src/leap/soledad/common/couch/support.py +++ /dev/null @@ -1,115 +0,0 @@ -# -*- coding: utf-8 -*- -# support.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -import sys - - -""" -Monkey patches and temporary code that may be removed with version changes. -""" - - -# for bigcouch -# TODO: Remove if bigcouch support is dropped -class MultipartWriter(object): - - """ - A multipart writer adapted from python-couchdb's one so we can PUT - documents using couch's multipart PUT. - - This stripped down version does not allow for nested structures, and - contains only the essential things we need to PUT SoledadDocuments to the - couch backend. Also, please note that this is a patch. The couchdb lib has - another implementation that works fine with CouchDB 1.6, but removing this - now will break compatibility with bigcouch. - """ - - CRLF = '\r\n' - - def __init__(self, fileobj, headers=None, boundary=None): - """ - Initialize the multipart writer. - """ - self.fileobj = fileobj - if boundary is None: - boundary = self._make_boundary() - self._boundary = boundary - self._build_headers('related', headers) - - def add(self, mimetype, content, headers={}): - """ - Add a part to the multipart stream. - """ - self.fileobj.write('--') - self.fileobj.write(self._boundary) - self.fileobj.write(self.CRLF) - headers['Content-Type'] = mimetype - self._write_headers(headers) - if content: - # XXX: throw an exception if a boundary appears in the content?? - self.fileobj.write(content) - self.fileobj.write(self.CRLF) - - def close(self): - """ - Close the multipart stream. - """ - self.fileobj.write('--') - self.fileobj.write(self._boundary) - # be careful not to have anything after '--', otherwise old couch - # versions (including bigcouch) will fail. - self.fileobj.write('--') - - def _make_boundary(self): - """ - Create a boundary to discern multi parts. - """ - try: - from uuid import uuid4 - return '==' + uuid4().hex + '==' - except ImportError: - from random import randrange - token = randrange(sys.maxint) - format = '%%0%dd' % len(repr(sys.maxint - 1)) - return '===============' + (format % token) + '==' - - def _write_headers(self, headers): - """ - Write a part header in the buffer stream. - """ - if headers: - for name in sorted(headers.keys()): - value = headers[name] - self.fileobj.write(name) - self.fileobj.write(': ') - self.fileobj.write(value) - self.fileobj.write(self.CRLF) - self.fileobj.write(self.CRLF) - - def _build_headers(self, subtype, headers): - """ - Build the main headers of the multipart stream. - - This is here so we can send headers separete from content using - python-couchdb API. - """ - self.headers = {} - self.headers['Content-Type'] = 'multipart/%s; boundary="%s"' % \ - (subtype, self._boundary) - if headers: - for name in sorted(headers.keys()): - value = headers[name] - self.headers[name] = value diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py deleted file mode 100644 index c13c4aa7..00000000 --- a/common/src/leap/soledad/common/crypto.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- -# crypto.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Soledad common crypto bits. -""" - - -# -# Encryption schemes used for encryption. -# - -class EncryptionSchemes(object): - - """ - Representation of encryption schemes used to encrypt documents. - """ - - NONE = 'none' - SYMKEY = 'symkey' - PUBKEY = 'pubkey' - - -class UnknownEncryptionSchemeError(Exception): - - """ - Raised when trying to decrypt from unknown encryption schemes. - """ - pass - - -class EncryptionMethods(object): - - """ - Representation of encryption methods that can be used. - """ - - AES_256_CTR = 'aes-256-ctr' - - -class UnknownEncryptionMethodError(Exception): - - """ - Raised when trying to encrypt/decrypt with unknown method. - """ - pass - - -class MacMethods(object): - - """ - Representation of MAC methods used to authenticate document's contents. - """ - - HMAC = 'hmac' - - -class UnknownMacMethodError(Exception): - - """ - Raised when trying to authenticate document's content with unknown MAC - mehtod. - """ - pass - - -class WrongMacError(Exception): - - """ - Raised when failing to authenticate document's contents based on MAC. - """ - - -# -# Crypto utilities for a SoledadDocument. -# - -ENC_JSON_KEY = '_enc_json' -ENC_SCHEME_KEY = '_enc_scheme' -ENC_METHOD_KEY = '_enc_method' -ENC_IV_KEY = '_enc_iv' -MAC_KEY = '_mac' -MAC_METHOD_KEY = '_mac_method' diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py deleted file mode 100644 index 6c26a29f..00000000 --- a/common/src/leap/soledad/common/document.py +++ /dev/null @@ -1,180 +0,0 @@ -# -*- coding: utf-8 -*- -# document.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -A Soledad Document is an l2db.Document with lasers. -""" - - -from .l2db import Document - - -# -# SoledadDocument -# - -class SoledadDocument(Document): - - """ - Encryptable and syncable document. - - LEAP Documents can be flagged as syncable or not, so the replicas - might not sync every document. - """ - - def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False, - syncable=True): - """ - Container for handling an encryptable document. - - @param doc_id: The unique document identifier. - @type doc_id: str - @param rev: The revision identifier of the document. - @type rev: str - @param json: The JSON string for this document. - @type json: str - @param has_conflicts: Boolean indicating if this document has conflicts - @type has_conflicts: bool - @param syncable: Should this document be synced with remote replicas? - @type syncable: bool - """ - Document.__init__(self, doc_id, rev, json, has_conflicts) - self._syncable = syncable - - def _get_syncable(self): - """ - Return whether this document is syncable. - - @return: Is this document syncable? - @rtype: bool - """ - return self._syncable - - def _set_syncable(self, syncable=True): - """ - Determine if this document should be synced with remote replicas. - - @param syncable: Should this document be synced with remote replicas? - @type syncable: bool - """ - self._syncable = syncable - - syncable = property( - _get_syncable, - _set_syncable, - doc="Determine if document should be synced with server." - ) - - def _get_rev(self): - """ - Get the document revision. - - Returning the revision as string solves the following exception in - Twisted web: - exceptions.TypeError: Can only pass-through bytes on Python 2 - - @return: The document revision. - @rtype: str - """ - if self._rev is None: - return None - return str(self._rev) - - def _set_rev(self, rev): - """ - Set document revision. - - @param rev: The new document revision. - @type rev: bytes - """ - self._rev = rev - - rev = property( - _get_rev, - _set_rev, - doc="Wrapper to ensure `doc.rev` is always returned as bytes.") - - -class ServerDocument(SoledadDocument): - """ - This is the document used by server to hold conflicts and transactions - on a database. - - The goal is to ensure an atomic and consistent update of the database. - """ - - def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False): - """ - Container for handling a document that stored on server. - - :param doc_id: The unique document identifier. - :type doc_id: str - :param rev: The revision identifier of the document. - :type rev: str - :param json: The JSON string for this document. - :type json: str - :param has_conflicts: Boolean indicating if this document has conflicts - :type has_conflicts: bool - """ - SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts) - self._conflicts = None - - def get_conflicts(self): - """ - Get the conflicted versions of the document. - - :return: The conflicted versions of the document. - :rtype: [ServerDocument] - """ - return self._conflicts or [] - - def set_conflicts(self, conflicts): - """ - Set the conflicted versions of the document. - - :param conflicts: The conflicted versions of the document. - :type conflicts: list - """ - self._conflicts = conflicts - self.has_conflicts = len(self._conflicts) > 0 - - def add_conflict(self, doc): - """ - Add a conflict to this document. - - :param doc: The conflicted version to be added. - :type doc: Document - """ - if self._conflicts is None: - raise Exception("Fetch conflicts first!") - self._conflicts.append(doc) - self.has_conflicts = len(self._conflicts) > 0 - - def delete_conflicts(self, conflict_revs): - """ - Delete conflicted versions of this document. - - :param conflict_revs: The conflicted revisions to be deleted. - :type conflict_revs: [str] - """ - if self._conflicts is None: - raise Exception("Fetch conflicts first!") - self._conflicts = filter( - lambda doc: doc.rev not in conflict_revs, - self._conflicts) - self.has_conflicts = len(self._conflicts) > 0 diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py deleted file mode 100644 index d543a3de..00000000 --- a/common/src/leap/soledad/common/errors.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- -# errors.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Soledad errors. -""" - -from .l2db import errors -from .l2db.remote import http_errors - - -def register_exception(cls): - """ - A small decorator that registers exceptions in u1db maps. - """ - # update u1db "wire description to status" and "wire description to - # exception" maps. - http_errors.wire_description_to_status.update({ - cls.wire_description: cls.status}) - errors.wire_description_to_exc.update({ - cls.wire_description: cls}) - # do not modify the exception - return cls - - -class SoledadError(errors.U1DBError): - - """ - Base Soledad HTTP errors. - """ - pass - - -# -# Authorization errors -# - - -class DatabaseAccessError(Exception): - pass - - -@register_exception -class InvalidAuthTokenError(errors.Unauthorized): - - """ - Exception raised when failing to get authorization for some action because - the provided token either does not exist in the tokens database, has a - distinct structure from the expected one, or is associated with a user - with a distinct uuid than the one provided by the client. - """ - - wire_descrition = "invalid auth token" - status = 401 - - -# -# SoledadBackend errors -# u1db error statuses also have to be updated -http_errors.ERROR_STATUSES = set( - http_errors.wire_description_to_status.values()) - - -class InvalidURLError(Exception): - """ - Exception raised when Soledad encounters a malformed URL. - """ - - -class BackendNotReadyError(SoledadError): - """ - Generic exception raised when the backend is not ready to dispatch a client - request. - """ - wire_description = "backend not ready" - status = 500 - - -class WrongCouchSchemaVersionError(SoledadError): - """ - Raised in case there is a user database with wrong couch schema version. - """ - - -class MissingCouchConfigDocumentError(SoledadError): - """ - Raised if a database has documents but lacks the couch config document. - """ diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py deleted file mode 100644 index 43d61b1d..00000000 --- a/common/src/leap/soledad/common/l2db/__init__.py +++ /dev/null @@ -1,694 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""L2DB""" - -import json - -from leap.soledad.common.l2db.errors import InvalidJSON, InvalidContent - -__version_info__ = (13, 9) -__version__ = '.'.join(map(lambda x: '%02d' % x, __version_info__)) - - -def open(path, create, document_factory=None): - """Open a database at the given location. - - Will raise u1db.errors.DatabaseDoesNotExist if create=False and the - database does not already exist. - - :param path: The filesystem path for the database to open. - :param create: True/False, should the database be created if it doesn't - already exist? - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :return: An instance of Database. - """ - from leap.soledad.client._db import sqlite - return sqlite.SQLiteDatabase.open_database( - path, create=create, document_factory=document_factory) - - -# constraints on database names (relevant for remote access, as regex) -DBNAME_CONSTRAINTS = r"[a-zA-Z0-9][a-zA-Z0-9.-]*" - -# constraints on doc ids (as regex) -# (no slashes, and no characters outside the ascii range) -DOC_ID_CONSTRAINTS = r"[a-zA-Z0-9.%_-]+" - - -class Database(object): - """A JSON Document data store. - - This data store can be synchronized with other u1db.Database instances. - """ - - def set_document_factory(self, factory): - """Set the document factory that will be used to create objects to be - returned as documents by the database. - - :param factory: A function that returns an object which at minimum must - satisfy the same interface as does the class DocumentBase. - Subclassing that class is the easiest way to create such - a function. - """ - raise NotImplementedError(self.set_document_factory) - - def set_document_size_limit(self, limit): - """Set the maximum allowed document size for this database. - - :param limit: Maximum allowed document size in bytes. - """ - raise NotImplementedError(self.set_document_size_limit) - - def whats_changed(self, old_generation=0): - """Return a list of documents that have changed since old_generation. - This allows APPS to only store a db generation before going - 'offline', and then when coming back online they can use this - data to update whatever extra data they are storing. - - :param old_generation: The generation of the database in the old - state. - :return: (generation, trans_id, [(doc_id, generation, trans_id),...]) - The current generation of the database, its associated transaction - id, and a list of of changed documents since old_generation, - represented by tuples with for each document its doc_id and the - generation and transaction id corresponding to the last intervening - change and sorted by generation (old changes first) - """ - raise NotImplementedError(self.whats_changed) - - def get_doc(self, doc_id, include_deleted=False): - """Get the JSON string for the given document. - - :param doc_id: The unique document identifier - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise asking for a deleted - document will return None. - :return: a Document object. - """ - raise NotImplementedError(self.get_doc) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - """Get the JSON content for many documents. - - :param doc_ids: A list of document identifiers. - :param check_for_conflicts: If set to False, then the conflict check - will be skipped, and 'None' will be returned instead of True/False. - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted documents will not - be included in the results. - :return: iterable giving the Document object for each document id - in matching doc_ids order. - """ - raise NotImplementedError(self.get_docs) - - def get_all_docs(self, include_deleted=False): - """Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted documents will not - be included in the results. - :return: (generation, [Document]) - The current generation of the database, followed by a list of all - the documents in the database. - """ - raise NotImplementedError(self.get_all_docs) - - def create_doc(self, content, doc_id=None): - """Create a new document. - - You can optionally specify the document identifier, but the document - must not already exist. See 'put_doc' if you want to override an - existing document. - If the database specifies a maximum document size and the document - exceeds it, create will fail and raise a DocumentTooBig exception. - - :param content: A Python dictionary. - :param doc_id: An optional identifier specifying the document id. - :return: Document - """ - raise NotImplementedError(self.create_doc) - - def create_doc_from_json(self, json, doc_id=None): - """Create a new document. - - You can optionally specify the document identifier, but the document - must not already exist. See 'put_doc' if you want to override an - existing document. - If the database specifies a maximum document size and the document - exceeds it, create will fail and raise a DocumentTooBig exception. - - :param json: The JSON document string - :param doc_id: An optional identifier specifying the document id. - :return: Document - """ - raise NotImplementedError(self.create_doc_from_json) - - def put_doc(self, doc): - """Update a document. - If the document currently has conflicts, put will fail. - If the database specifies a maximum document size and the document - exceeds it, put will fail and raise a DocumentTooBig exception. - - :param doc: A Document with new content. - :return: new_doc_rev - The new revision identifier for the document. - The Document object will also be updated. - """ - raise NotImplementedError(self.put_doc) - - def delete_doc(self, doc): - """Mark a document as deleted. - Will abort if the current revision doesn't match doc.rev. - This will also set doc.content to None. - """ - raise NotImplementedError(self.delete_doc) - - def create_index(self, index_name, *index_expressions): - """Create an named index, which can then be queried for future lookups. - Creating an index which already exists is not an error, and is cheap. - Creating an index which does not match the index_expressions of the - existing index is an error. - Creating an index will block until the expressions have been evaluated - and the index generated. - - :param index_name: A unique name which can be used as a key prefix - :param index_expressions: index expressions defining the index - information. - - Examples: - - "fieldname", or "fieldname.subfieldname" to index alphabetically - sorted on the contents of a field. - - "number(fieldname, width)", "lower(fieldname)" - """ - raise NotImplementedError(self.create_index) - - def delete_index(self, index_name): - """Remove a named index. - - :param index_name: The name of the index we are removing - """ - raise NotImplementedError(self.delete_index) - - def list_indexes(self): - """List the definitions of all known indexes. - - :return: A list of [('index-name', ['field', 'field2'])] definitions. - """ - raise NotImplementedError(self.list_indexes) - - def get_from_index(self, index_name, *key_values): - """Return documents that match the keys supplied. - - You must supply exactly the same number of values as have been defined - in the index. It is possible to do a prefix match by using '*' to - indicate a wildcard match. You can only supply '*' to trailing entries, - (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) - It is also possible to append a '*' to the last supplied value (eg - 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :return: List of [Document] - """ - raise NotImplementedError(self.get_from_index) - - def get_range_from_index(self, index_name, start_value, end_value): - """Return documents that fall within the specified range. - - Both ends of the range are inclusive. For both start_value and - end_value, one must supply exactly the same number of values as have - been defined in the index, or pass None. In case of a single column - index, a string is accepted as an alternative for a tuple with a single - value. It is possible to do a prefix match by using '*' to indicate - a wildcard match. You can only supply '*' to trailing entries, (eg - 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also - possible to append a '*' to the last supplied value (eg 'val*', '*', - '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :param start_values: tuples of values that define the lower bound of - the range. eg, if you have an index with 3 fields then you would - have: (val1, val2, val3) - :param end_values: tuples of values that define the upper bound of the - range. eg, if you have an index with 3 fields then you would have: - (val1, val2, val3) - :return: List of [Document] - """ - raise NotImplementedError(self.get_range_from_index) - - def get_index_keys(self, index_name): - """Return all keys under which documents are indexed in this index. - - :param index_name: The index to query - :return: [] A list of tuples of indexed keys. - """ - raise NotImplementedError(self.get_index_keys) - - def get_doc_conflicts(self, doc_id): - """Get the list of conflicts for the given document. - - The order of the conflicts is such that the first entry is the value - that would be returned by "get_doc". - - :return: [doc] A list of the Document entries that are conflicted. - """ - raise NotImplementedError(self.get_doc_conflicts) - - def resolve_doc(self, doc, conflicted_doc_revs): - """Mark a document as no longer conflicted. - - We take the list of revisions that the client knows about that it is - superseding. This may be a different list from the actual current - conflicts, in which case only those are removed as conflicted. This - may fail if the conflict list is significantly different from the - supplied information. (sync could have happened in the background from - the time you GET_DOC_CONFLICTS until the point where you RESOLVE) - - :param doc: A Document with the new content to be inserted. - :param conflicted_doc_revs: A list of revisions that the new content - supersedes. - """ - raise NotImplementedError(self.resolve_doc) - - def get_sync_target(self): - """Return a SyncTarget object, for another u1db to synchronize with. - - :return: An instance of SyncTarget. - """ - raise NotImplementedError(self.get_sync_target) - - def close(self): - """Release any resources associated with this database.""" - raise NotImplementedError(self.close) - - def sync(self, url, creds=None, autocreate=True): - """Synchronize documents with remote replica exposed at url. - - :param url: the url of the target replica to sync with. - :param creds: optional dictionary giving credentials - to authorize the operation with the server. For using OAuth - the form of creds is: - {'oauth': { - 'consumer_key': ..., - 'consumer_secret': ..., - 'token_key': ..., - 'token_secret': ... - }} - :param autocreate: ask the target to create the db if non-existent. - :return: local_gen_before_sync The local generation before the - synchronisation was performed. This is useful to pass into - whatschanged, if an application wants to know which documents were - affected by a synchronisation. - """ - from u1db.sync import Synchronizer - from u1db.remote.http_target import HTTPSyncTarget - return Synchronizer(self, HTTPSyncTarget(url, creds=creds)).sync( - autocreate=autocreate) - - def _get_replica_gen_and_trans_id(self, other_replica_uid): - """Return the last known generation and transaction id for the other db - replica. - - When you do a synchronization with another replica, the Database keeps - track of what generation the other database replica was at, and what - the associated transaction id was. This is used to determine what data - needs to be sent, and if two databases are claiming to be the same - replica. - - :param other_replica_uid: The identifier for the other replica. - :return: (gen, trans_id) The generation and transaction id we - encountered during synchronization. If we've never synchronized - with the replica, this is (0, ''). - """ - raise NotImplementedError(self._get_replica_gen_and_trans_id) - - def _set_replica_gen_and_trans_id(self, other_replica_uid, - other_generation, other_transaction_id): - """Set the last-known generation and transaction id for the other - database replica. - - We have just performed some synchronization, and we want to track what - generation the other replica was at. See also - _get_replica_gen_and_trans_id. - :param other_replica_uid: The U1DB identifier for the other replica. - :param other_generation: The generation number for the other replica. - :param other_transaction_id: The transaction id associated with the - generation. - """ - raise NotImplementedError(self._set_replica_gen_and_trans_id) - - def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, - replica_trans_id=''): - """Insert/update document into the database with a given revision. - - This api is used during synchronization operations. - - If a document would conflict and save_conflict is set to True, the - content will be selected as the 'current' content for doc.doc_id, - even though doc.rev doesn't supersede the currently stored revision. - The currently stored document will be added to the list of conflict - alternatives for the given doc_id. - - This forces the new content to be 'current' so that we get convergence - after synchronizing, even if people don't resolve conflicts. Users can - then notice that their content is out of date, update it, and - synchronize again. (The alternative is that users could synchronize and - think the data has propagated, but their local copy looks fine, and the - remote copy is never updated again.) - - :param doc: A Document object - :param save_conflict: If this document is a conflict, do you want to - save it as a conflict, or just ignore it. - :param replica_uid: A unique replica identifier. - :param replica_gen: The generation of the replica corresponding to the - this document. The replica arguments are optional, but are used - during synchronization. - :param replica_trans_id: The transaction_id associated with the - generation. - :return: (state, at_gen) - If we don't have doc_id already, - or if doc_rev supersedes the existing document revision, - then the content will be inserted, and state is 'inserted'. - If doc_rev is less than or equal to the existing revision, - then the put is ignored and state is respecitvely 'superseded' - or 'converged'. - If doc_rev is not strictly superseded or supersedes, then - state is 'conflicted'. The document will not be inserted if - save_conflict is False. - For 'inserted' or 'converged', at_gen is the insertion/current - generation. - """ - raise NotImplementedError(self._put_doc_if_newer) - - -class DocumentBase(object): - """Container for handling a single document. - - :ivar doc_id: Unique identifier for this document. - :ivar rev: The revision identifier of the document. - :ivar json_string: The JSON string for this document. - :ivar has_conflicts: Boolean indicating if this document has conflicts - """ - - def __init__(self, doc_id, rev, json_string, has_conflicts=False): - self.doc_id = doc_id - self.rev = rev - if json_string is not None: - try: - value = json.loads(json_string) - except ValueError: - raise InvalidJSON - if not isinstance(value, dict): - raise InvalidJSON - self._json = json_string - self.has_conflicts = has_conflicts - - def same_content_as(self, other): - """Compare the content of two documents.""" - if self._json: - c1 = json.loads(self._json) - else: - c1 = None - if other._json: - c2 = json.loads(other._json) - else: - c2 = None - return c1 == c2 - - def __repr__(self): - if self.has_conflicts: - extra = ', conflicted' - else: - extra = '' - return '%s(%s, %s%s, %r)' % (self.__class__.__name__, self.doc_id, - self.rev, extra, self.get_json()) - - def __hash__(self): - raise NotImplementedError(self.__hash__) - - def __eq__(self, other): - if not isinstance(other, Document): - return NotImplemented - return ( - self.doc_id == other.doc_id and self.rev == other.rev and - self.same_content_as(other) and self.has_conflicts == - other.has_conflicts) - - def __lt__(self, other): - """This is meant for testing, not part of the official api. - - It is implemented so that sorted([Document, Document]) can be used. - It doesn't imply that users would want their documents to be sorted in - this order. - """ - # Since this is just for testing, we don't worry about comparing - # against things that aren't a Document. - return ((self.doc_id, self.rev, self.get_json()) < - (other.doc_id, other.rev, other.get_json())) - - def get_json(self): - """Get the json serialization of this document.""" - if self._json is not None: - return self._json - return None - - def get_size(self): - """Calculate the total size of the document.""" - size = 0 - json = self.get_json() - if json: - size += len(json) - if self.rev: - size += len(self.rev) - if self.doc_id: - size += len(self.doc_id) - return size - - def set_json(self, json_string): - """Set the json serialization of this document.""" - if json_string is not None: - try: - value = json.loads(json_string) - except ValueError: - raise InvalidJSON - if not isinstance(value, dict): - raise InvalidJSON - self._json = json_string - - def make_tombstone(self): - """Make this document into a tombstone.""" - self._json = None - - def is_tombstone(self): - """Return True if the document is a tombstone, False otherwise.""" - if self._json is not None: - return False - return True - - -class Document(DocumentBase): - """Container for handling a single document. - - :ivar doc_id: Unique identifier for this document. - :ivar rev: The revision identifier of the document. - :ivar json: The JSON string for this document. - :ivar has_conflicts: Boolean indicating if this document has conflicts - """ - - # The following part of the API is optional: no implementation is forced to - # have it but if the language supports dictionaries/hashtables, it makes - # Documents a lot more user friendly. - - def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False): - # TODO: We convert the json in the superclass to check its validity so - # we might as well set _content here directly since the price is - # already being paid. - super(Document, self).__init__(doc_id, rev, json, has_conflicts) - self._content = None - - def same_content_as(self, other): - """Compare the content of two documents.""" - if self._json: - c1 = json.loads(self._json) - else: - c1 = self._content - if other._json: - c2 = json.loads(other._json) - else: - c2 = other._content - return c1 == c2 - - def get_json(self): - """Get the json serialization of this document.""" - json_string = super(Document, self).get_json() - if json_string is not None: - return json_string - if self._content is not None: - return json.dumps(self._content) - return None - - def set_json(self, json): - """Set the json serialization of this document.""" - self._content = None - super(Document, self).set_json(json) - - def make_tombstone(self): - """Make this document into a tombstone.""" - self._content = None - super(Document, self).make_tombstone() - - def is_tombstone(self): - """Return True if the document is a tombstone, False otherwise.""" - if self._content is not None: - return False - return super(Document, self).is_tombstone() - - def _get_content(self): - """Get the dictionary representing this document.""" - if self._json is not None: - self._content = json.loads(self._json) - self._json = None - if self._content is not None: - return self._content - return None - - def _set_content(self, content): - """Set the dictionary representing this document.""" - try: - tmp = json.dumps(content) - except TypeError: - raise InvalidContent( - "Can not be converted to JSON: %r" % (content,)) - if not tmp.startswith('{'): - raise InvalidContent( - "Can not be converted to a JSON object: %r." % (content,)) - # We might as well store the JSON at this point since we did the work - # of encoding it, and it doesn't lose any information. - self._json = tmp - self._content = None - - content = property( - _get_content, _set_content, doc="Content of the Document.") - - # End of optional part. - - -class SyncTarget(object): - """Functionality for using a Database as a synchronization target.""" - - def get_sync_info(self, source_replica_uid): - """Return information about known state. - - Return the replica_uid and the current database generation of this - database, and the last-seen database generation for source_replica_uid - - :param source_replica_uid: Another replica which we might have - synchronized with in the past. - :return: (target_replica_uid, target_replica_generation, - target_trans_id, source_replica_last_known_generation, - source_replica_last_known_transaction_id) - """ - raise NotImplementedError(self.get_sync_info) - - def record_sync_info(self, source_replica_uid, source_replica_generation, - source_replica_transaction_id): - """Record tip information for another replica. - - After sync_exchange has been processed, the caller will have - received new content from this replica. This call allows the - source replica instigating the sync to inform us what their - generation became after applying the documents we returned. - - This is used to allow future sync operations to not need to repeat data - that we just talked about. It also means that if this is called at the - wrong time, there can be database records that will never be - synchronized. - - :param source_replica_uid: The identifier for the source replica. - :param source_replica_generation: - The database generation for the source replica. - :param source_replica_transaction_id: The transaction id associated - with the source replica generation. - """ - raise NotImplementedError(self.record_sync_info) - - def sync_exchange(self, docs_by_generation, source_replica_uid, - last_known_generation, last_known_trans_id, - return_doc_cb, ensure_callback=None): - """Incorporate the documents sent from the source replica. - - This is not meant to be called by client code directly, but is used as - part of sync(). - - This adds docs to the local store, and determines documents that need - to be returned to the source replica. - - Documents must be supplied in docs_by_generation paired with - the generation of their latest change in order from the oldest - change to the newest, that means from the oldest generation to - the newest. - - Documents are also returned paired with the generation of - their latest change in order from the oldest change to the - newest. - - :param docs_by_generation: A list of [(Document, generation, - transaction_id)] tuples indicating documents which should be - updated on this replica paired with the generation and transaction - id of their latest change. - :param source_replica_uid: The source replica's identifier - :param last_known_generation: The last generation that the source - replica knows about this target replica - :param last_known_trans_id: The last transaction id that the source - replica knows about this target replica - :param: return_doc_cb(doc, gen): is a callback - used to return documents to the source replica, it will - be invoked in turn with Documents that have changed since - last_known_generation together with the generation of - their last change. - :param: ensure_callback(replica_uid): if set the target may create - the target db if not yet existent, the callback can then - be used to inform of the created db replica uid. - :return: new_generation - After applying docs_by_generation, this is - the current generation for this replica - """ - raise NotImplementedError(self.sync_exchange) - - def _set_trace_hook(self, cb): - """Set a callback that will be invoked to trace database actions. - - The callback will be passed a string indicating the current state, and - the sync target object. Implementations do not have to implement this - api, it is used by the test suite. - - :param cb: A callable that takes cb(state) - """ - raise NotImplementedError(self._set_trace_hook) - - def _set_trace_hook_shallow(self, cb): - """Set a callback that will be invoked to trace database actions. - - Similar to _set_trace_hook, for implementations that don't offer - state changes from the inner working of sync_exchange(). - - :param cb: A callable that takes cb(state) - """ - self._set_trace_hook(cb) diff --git a/common/src/leap/soledad/common/l2db/backends/__init__.py b/common/src/leap/soledad/common/l2db/backends/__init__.py deleted file mode 100644 index c731c3d3..00000000 --- a/common/src/leap/soledad/common/l2db/backends/__init__.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Abstract classes and common implementations for the backends.""" - -import re -import json -import uuid - -from leap.soledad.common import l2db -from leap.soledad.common.l2db import sync as l2db_sync -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db.vectorclock import VectorClockRev - - -check_doc_id_re = re.compile("^" + l2db.DOC_ID_CONSTRAINTS + "$", re.UNICODE) - - -class CommonSyncTarget(l2db_sync.LocalSyncTarget): - pass - - -class CommonBackend(l2db.Database): - - document_size_limit = 0 - - def _allocate_doc_id(self): - """Generate a unique identifier for this document.""" - return 'D-' + uuid.uuid4().hex # 'D-' stands for document - - def _allocate_transaction_id(self): - return 'T-' + uuid.uuid4().hex # 'T-' stands for transaction - - def _allocate_doc_rev(self, old_doc_rev): - vcr = VectorClockRev(old_doc_rev) - vcr.increment(self._replica_uid) - return vcr.as_str() - - def _check_doc_id(self, doc_id): - if not check_doc_id_re.match(doc_id): - raise errors.InvalidDocId() - - def _check_doc_size(self, doc): - if not self.document_size_limit: - return - if doc.get_size() > self.document_size_limit: - raise errors.DocumentTooBig - - def _get_generation(self): - """Return the current generation. - - """ - raise NotImplementedError(self._get_generation) - - def _get_generation_info(self): - """Return the current generation and transaction id. - - """ - raise NotImplementedError(self._get_generation_info) - - def _get_doc(self, doc_id, check_for_conflicts=False): - """Extract the document from storage. - - This can return None if the document doesn't exist. - """ - raise NotImplementedError(self._get_doc) - - def _has_conflicts(self, doc_id): - """Return True if the doc has conflicts, False otherwise.""" - raise NotImplementedError(self._has_conflicts) - - def create_doc(self, content, doc_id=None): - if not isinstance(content, dict): - raise errors.InvalidContent - json_string = json.dumps(content) - return self.create_doc_from_json(json_string, doc_id) - - def create_doc_from_json(self, json, doc_id=None): - if doc_id is None: - doc_id = self._allocate_doc_id() - doc = self._factory(doc_id, None, json) - self.put_doc(doc) - return doc - - def _get_transaction_log(self): - """This is only for the test suite, it is not part of the api.""" - raise NotImplementedError(self._get_transaction_log) - - def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content): - raise NotImplementedError(self._put_and_update_indexes) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - for doc_id in doc_ids: - doc = self._get_doc( - doc_id, check_for_conflicts=check_for_conflicts) - if doc.is_tombstone() and not include_deleted: - continue - yield doc - - def _get_trans_id_for_gen(self, generation): - """Get the transaction id corresponding to a particular generation. - - Raises an InvalidGeneration when the generation does not exist. - - """ - raise NotImplementedError(self._get_trans_id_for_gen) - - def validate_gen_and_trans_id(self, generation, trans_id): - """Validate the generation and transaction id. - - Raises an InvalidGeneration when the generation does not exist, and an - InvalidTransactionId when it does but with a different transaction id. - - """ - if generation == 0: - return - known_trans_id = self._get_trans_id_for_gen(generation) - if known_trans_id != trans_id: - raise errors.InvalidTransactionId - - def _validate_source(self, other_replica_uid, other_generation, - other_transaction_id): - """Validate the new generation and transaction id. - - other_generation must be greater than what we have stored for this - replica, *or* it must be the same and the transaction_id must be the - same as well. - """ - (old_generation, - old_transaction_id) = self._get_replica_gen_and_trans_id( - other_replica_uid) - if other_generation < old_generation: - raise errors.InvalidGeneration - if other_generation > old_generation: - return - if other_transaction_id == old_transaction_id: - return - raise errors.InvalidTransactionId - - def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, - replica_trans_id=''): - cur_doc = self._get_doc(doc.doc_id) - doc_vcr = VectorClockRev(doc.rev) - if cur_doc is None: - cur_vcr = VectorClockRev(None) - else: - cur_vcr = VectorClockRev(cur_doc.rev) - self._validate_source(replica_uid, replica_gen, replica_trans_id) - if doc_vcr.is_newer(cur_vcr): - rev = doc.rev - self._prune_conflicts(doc, doc_vcr) - if doc.rev != rev: - # conflicts have been autoresolved - state = 'superseded' - else: - state = 'inserted' - self._put_and_update_indexes(cur_doc, doc) - elif doc.rev == cur_doc.rev: - # magical convergence - state = 'converged' - elif cur_vcr.is_newer(doc_vcr): - # Don't add this to seen_ids, because we have something newer, - # so we should send it back, and we should not generate a - # conflict - state = 'superseded' - elif cur_doc.same_content_as(doc): - # the documents have been edited to the same thing at both ends - doc_vcr.maximize(cur_vcr) - doc_vcr.increment(self._replica_uid) - doc.rev = doc_vcr.as_str() - self._put_and_update_indexes(cur_doc, doc) - state = 'superseded' - else: - state = 'conflicted' - if save_conflict: - self._force_doc_sync_conflict(doc) - if replica_uid is not None and replica_gen is not None: - self._do_set_replica_gen_and_trans_id( - replica_uid, replica_gen, replica_trans_id) - return state, self._get_generation() - - def _ensure_maximal_rev(self, cur_rev, extra_revs): - vcr = VectorClockRev(cur_rev) - for rev in extra_revs: - vcr.maximize(VectorClockRev(rev)) - vcr.increment(self._replica_uid) - return vcr.as_str() - - def set_document_size_limit(self, limit): - self.document_size_limit = limit diff --git a/common/src/leap/soledad/common/l2db/backends/inmemory.py b/common/src/leap/soledad/common/l2db/backends/inmemory.py deleted file mode 100644 index 6fd251af..00000000 --- a/common/src/leap/soledad/common/l2db/backends/inmemory.py +++ /dev/null @@ -1,466 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""The in-memory Database class for U1DB.""" - -import json - -from leap.soledad.common.l2db import ( - Document, errors, - query_parser, vectorclock) -from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget - - -def get_prefix(value): - key_prefix = '\x01'.join(value) - return key_prefix.rstrip('*') - - -class InMemoryDatabase(CommonBackend): - """A database that only stores the data internally.""" - - def __init__(self, replica_uid, document_factory=None): - self._transaction_log = [] - self._docs = {} - # Map from doc_id => [(doc_rev, doc)] conflicts beyond 'winner' - self._conflicts = {} - self._other_generations = {} - self._indexes = {} - self._replica_uid = replica_uid - self._factory = document_factory or Document - - def _set_replica_uid(self, replica_uid): - """Force the replica_uid to be set.""" - self._replica_uid = replica_uid - - def set_document_factory(self, factory): - self._factory = factory - - def close(self): - # This is a no-op, We don't want to free the data because one client - # may be closing it, while another wants to inspect the results. - pass - - def _get_replica_gen_and_trans_id(self, other_replica_uid): - return self._other_generations.get(other_replica_uid, (0, '')) - - def _set_replica_gen_and_trans_id(self, other_replica_uid, - other_generation, other_transaction_id): - self._do_set_replica_gen_and_trans_id( - other_replica_uid, other_generation, other_transaction_id) - - def _do_set_replica_gen_and_trans_id(self, other_replica_uid, - other_generation, - other_transaction_id): - # TODO: to handle race conditions, we may want to check if the current - # value is greater than this new value. - self._other_generations[other_replica_uid] = (other_generation, - other_transaction_id) - - def get_sync_target(self): - return InMemorySyncTarget(self) - - def _get_transaction_log(self): - # snapshot! - return self._transaction_log[:] - - def _get_generation(self): - return len(self._transaction_log) - - def _get_generation_info(self): - if not self._transaction_log: - return 0, '' - return len(self._transaction_log), self._transaction_log[-1][1] - - def _get_trans_id_for_gen(self, generation): - if generation == 0: - return '' - if generation > len(self._transaction_log): - raise errors.InvalidGeneration - return self._transaction_log[generation - 1][1] - - def put_doc(self, doc): - if doc.doc_id is None: - raise errors.InvalidDocId() - self._check_doc_id(doc.doc_id) - self._check_doc_size(doc) - old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) - if old_doc and old_doc.has_conflicts: - raise errors.ConflictedDoc() - if old_doc and doc.rev is None and old_doc.is_tombstone(): - new_rev = self._allocate_doc_rev(old_doc.rev) - else: - if old_doc is not None: - if old_doc.rev != doc.rev: - raise errors.RevisionConflict() - else: - if doc.rev is not None: - raise errors.RevisionConflict() - new_rev = self._allocate_doc_rev(doc.rev) - doc.rev = new_rev - self._put_and_update_indexes(old_doc, doc) - return new_rev - - def _put_and_update_indexes(self, old_doc, doc): - for index in self._indexes.itervalues(): - if old_doc is not None and not old_doc.is_tombstone(): - index.remove_json(old_doc.doc_id, old_doc.get_json()) - if not doc.is_tombstone(): - index.add_json(doc.doc_id, doc.get_json()) - trans_id = self._allocate_transaction_id() - self._docs[doc.doc_id] = (doc.rev, doc.get_json()) - self._transaction_log.append((doc.doc_id, trans_id)) - - def _get_doc(self, doc_id, check_for_conflicts=False): - try: - doc_rev, content = self._docs[doc_id] - except KeyError: - return None - doc = self._factory(doc_id, doc_rev, content) - if check_for_conflicts: - doc.has_conflicts = (doc.doc_id in self._conflicts) - return doc - - def _has_conflicts(self, doc_id): - return doc_id in self._conflicts - - def get_doc(self, doc_id, include_deleted=False): - doc = self._get_doc(doc_id, check_for_conflicts=True) - if doc is None: - return None - if doc.is_tombstone() and not include_deleted: - return None - return doc - - def get_all_docs(self, include_deleted=False): - """Return all documents in the database.""" - generation = self._get_generation() - results = [] - for doc_id, (doc_rev, content) in self._docs.items(): - if content is None and not include_deleted: - continue - doc = self._factory(doc_id, doc_rev, content) - doc.has_conflicts = self._has_conflicts(doc_id) - results.append(doc) - return (generation, results) - - def get_doc_conflicts(self, doc_id): - if doc_id not in self._conflicts: - return [] - result = [self._get_doc(doc_id)] - result[0].has_conflicts = True - result.extend([self._factory(doc_id, rev, content) - for rev, content in self._conflicts[doc_id]]) - return result - - def _replace_conflicts(self, doc, conflicts): - if not conflicts: - del self._conflicts[doc.doc_id] - else: - self._conflicts[doc.doc_id] = conflicts - doc.has_conflicts = bool(conflicts) - - def _prune_conflicts(self, doc, doc_vcr): - if self._has_conflicts(doc.doc_id): - autoresolved = False - remaining_conflicts = [] - cur_conflicts = self._conflicts[doc.doc_id] - for c_rev, c_doc in cur_conflicts: - c_vcr = vectorclock.VectorClockRev(c_rev) - if doc_vcr.is_newer(c_vcr): - continue - if doc.same_content_as(Document(doc.doc_id, c_rev, c_doc)): - doc_vcr.maximize(c_vcr) - autoresolved = True - continue - remaining_conflicts.append((c_rev, c_doc)) - if autoresolved: - doc_vcr.increment(self._replica_uid) - doc.rev = doc_vcr.as_str() - self._replace_conflicts(doc, remaining_conflicts) - - def resolve_doc(self, doc, conflicted_doc_revs): - cur_doc = self._get_doc(doc.doc_id) - if cur_doc is None: - cur_rev = None - else: - cur_rev = cur_doc.rev - new_rev = self._ensure_maximal_rev(cur_rev, conflicted_doc_revs) - superseded_revs = set(conflicted_doc_revs) - remaining_conflicts = [] - cur_conflicts = self._conflicts[doc.doc_id] - for c_rev, c_doc in cur_conflicts: - if c_rev in superseded_revs: - continue - remaining_conflicts.append((c_rev, c_doc)) - doc.rev = new_rev - if cur_rev in superseded_revs: - self._put_and_update_indexes(cur_doc, doc) - else: - remaining_conflicts.append((new_rev, doc.get_json())) - self._replace_conflicts(doc, remaining_conflicts) - - def delete_doc(self, doc): - if doc.doc_id not in self._docs: - raise errors.DocumentDoesNotExist - if self._docs[doc.doc_id][1] in ('null', None): - raise errors.DocumentAlreadyDeleted - doc.make_tombstone() - self.put_doc(doc) - - def create_index(self, index_name, *index_expressions): - if index_name in self._indexes: - if self._indexes[index_name]._definition == list( - index_expressions): - return - raise errors.IndexNameTakenError - index = InMemoryIndex(index_name, list(index_expressions)) - for doc_id, (doc_rev, doc) in self._docs.iteritems(): - if doc is not None: - index.add_json(doc_id, doc) - self._indexes[index_name] = index - - def delete_index(self, index_name): - try: - del self._indexes[index_name] - except KeyError: - pass - - def list_indexes(self): - definitions = [] - for idx in self._indexes.itervalues(): - definitions.append((idx._name, idx._definition)) - return definitions - - def get_from_index(self, index_name, *key_values): - try: - index = self._indexes[index_name] - except KeyError: - raise errors.IndexDoesNotExist - doc_ids = index.lookup(key_values) - result = [] - for doc_id in doc_ids: - result.append(self._get_doc(doc_id, check_for_conflicts=True)) - return result - - def get_range_from_index(self, index_name, start_value=None, - end_value=None): - """Return all documents with key values in the specified range.""" - try: - index = self._indexes[index_name] - except KeyError: - raise errors.IndexDoesNotExist - if isinstance(start_value, basestring): - start_value = (start_value,) - if isinstance(end_value, basestring): - end_value = (end_value,) - doc_ids = index.lookup_range(start_value, end_value) - result = [] - for doc_id in doc_ids: - result.append(self._get_doc(doc_id, check_for_conflicts=True)) - return result - - def get_index_keys(self, index_name): - try: - index = self._indexes[index_name] - except KeyError: - raise errors.IndexDoesNotExist - keys = index.keys() - # XXX inefficiency warning - return list(set([tuple(key.split('\x01')) for key in keys])) - - def whats_changed(self, old_generation=0): - changes = [] - relevant_tail = self._transaction_log[old_generation:] - # We don't use len(self._transaction_log) because _transaction_log may - # get mutated by a concurrent operation. - cur_generation = old_generation + len(relevant_tail) - last_trans_id = '' - if relevant_tail: - last_trans_id = relevant_tail[-1][1] - elif self._transaction_log: - last_trans_id = self._transaction_log[-1][1] - seen = set() - generation = cur_generation - for doc_id, trans_id in reversed(relevant_tail): - if doc_id not in seen: - changes.append((doc_id, generation, trans_id)) - seen.add(doc_id) - generation -= 1 - changes.reverse() - return (cur_generation, last_trans_id, changes) - - def _force_doc_sync_conflict(self, doc): - my_doc = self._get_doc(doc.doc_id) - self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) - self._conflicts.setdefault(doc.doc_id, []).append( - (my_doc.rev, my_doc.get_json())) - doc.has_conflicts = True - self._put_and_update_indexes(my_doc, doc) - - -class InMemoryIndex(object): - """Interface for managing an Index.""" - - def __init__(self, index_name, index_definition): - self._name = index_name - self._definition = index_definition - self._values = {} - parser = query_parser.Parser() - self._getters = parser.parse_all(self._definition) - - def evaluate_json(self, doc): - """Determine the 'key' after applying this index to the doc.""" - raw = json.loads(doc) - return self.evaluate(raw) - - def evaluate(self, obj): - """Evaluate a dict object, applying this definition.""" - all_rows = [[]] - for getter in self._getters: - new_rows = [] - keys = getter.get(obj) - if not keys: - return [] - for key in keys: - new_rows.extend([row + [key] for row in all_rows]) - all_rows = new_rows - all_rows = ['\x01'.join(row) for row in all_rows] - return all_rows - - def add_json(self, doc_id, doc): - """Add this json doc to the index.""" - keys = self.evaluate_json(doc) - if not keys: - return - for key in keys: - self._values.setdefault(key, []).append(doc_id) - - def remove_json(self, doc_id, doc): - """Remove this json doc from the index.""" - keys = self.evaluate_json(doc) - if keys: - for key in keys: - doc_ids = self._values[key] - doc_ids.remove(doc_id) - if not doc_ids: - del self._values[key] - - def _find_non_wildcards(self, values): - """Check if this should be a wildcard match. - - Further, this will raise an exception if the syntax is improperly - defined. - - :return: The offset of the last value we need to match against. - """ - if len(values) != len(self._definition): - raise errors.InvalidValueForIndex() - is_wildcard = False - last = 0 - for idx, val in enumerate(values): - if val.endswith('*'): - if val != '*': - # We have an 'x*' style wildcard - if is_wildcard: - # We were already in wildcard mode, so this is invalid - raise errors.InvalidGlobbing - last = idx + 1 - is_wildcard = True - else: - if is_wildcard: - # We were in wildcard mode, we can't follow that with - # non-wildcard - raise errors.InvalidGlobbing - last = idx + 1 - if not is_wildcard: - return -1 - return last - - def lookup(self, values): - """Find docs that match the values.""" - last = self._find_non_wildcards(values) - if last == -1: - return self._lookup_exact(values) - else: - return self._lookup_prefix(values[:last]) - - def lookup_range(self, start_values, end_values): - """Find docs within the range.""" - # TODO: Wildly inefficient, which is unlikely to be a problem for the - # inmemory implementation. - if start_values: - self._find_non_wildcards(start_values) - start_values = get_prefix(start_values) - if end_values: - if self._find_non_wildcards(end_values) == -1: - exact = True - else: - exact = False - end_values = get_prefix(end_values) - found = [] - for key, doc_ids in sorted(self._values.iteritems()): - if start_values and start_values > key: - continue - if end_values and end_values < key: - if exact: - break - else: - if not key.startswith(end_values): - break - found.extend(doc_ids) - return found - - def keys(self): - """Find the indexed keys.""" - return self._values.keys() - - def _lookup_prefix(self, value): - """Find docs that match the prefix string in values.""" - # TODO: We need a different data structure to make prefix style fast, - # some sort of sorted list would work, but a plain dict doesn't. - key_prefix = get_prefix(value) - all_doc_ids = [] - for key, doc_ids in sorted(self._values.iteritems()): - if key.startswith(key_prefix): - all_doc_ids.extend(doc_ids) - return all_doc_ids - - def _lookup_exact(self, value): - """Find docs that match exactly.""" - key = '\x01'.join(value) - if key in self._values: - return self._values[key] - return () - - -class InMemorySyncTarget(CommonSyncTarget): - - def get_sync_info(self, source_replica_uid): - source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( - source_replica_uid) - my_gen, my_trans_id = self._db._get_generation_info() - return ( - self._db._replica_uid, my_gen, my_trans_id, source_gen, - source_trans_id) - - def record_sync_info(self, source_replica_uid, source_replica_generation, - source_transaction_id): - if self._trace_hook: - self._trace_hook('record_sync_info') - self._db._set_replica_gen_and_trans_id( - source_replica_uid, source_replica_generation, - source_transaction_id) diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py deleted file mode 100644 index b502fc2d..00000000 --- a/common/src/leap/soledad/common/l2db/errors.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""A list of errors that u1db can raise.""" - - -class U1DBError(Exception): - """Generic base class for U1DB errors.""" - - # description/tag for identifying the error during transmission (http,...) - wire_description = "error" - - def __init__(self, message=None): - self.message = message - - -class RevisionConflict(U1DBError): - """The document revisions supplied does not match the current version.""" - - wire_description = "revision conflict" - - -class InvalidJSON(U1DBError): - """Content was not valid json.""" - - -class InvalidContent(U1DBError): - """Content was not a python dictionary.""" - - -class InvalidDocId(U1DBError): - """A document was requested with an invalid document identifier.""" - - wire_description = "invalid document id" - - -class MissingDocIds(U1DBError): - """Needs document ids.""" - - wire_description = "missing document ids" - - -class DocumentTooBig(U1DBError): - """Document exceeds the maximum document size for this database.""" - - wire_description = "document too big" - - -class UserQuotaExceeded(U1DBError): - """Document exceeds the maximum document size for this database.""" - - wire_description = "user quota exceeded" - - -class SubscriptionNeeded(U1DBError): - """User needs a subscription to be able to use this replica..""" - - wire_description = "user needs subscription" - - -class InvalidTransactionId(U1DBError): - """Invalid transaction for generation.""" - - wire_description = "invalid transaction id" - - -class InvalidGeneration(U1DBError): - """Generation was previously synced with a different transaction id.""" - - wire_description = "invalid generation" - - -class InvalidReplicaUID(U1DBError): - """Attempting to sync a database with itself.""" - - wire_description = "invalid replica uid" - - -class ConflictedDoc(U1DBError): - """The document is conflicted, you must call resolve before put()""" - - -class InvalidValueForIndex(U1DBError): - """The values supplied does not match the index definition.""" - - -class InvalidGlobbing(U1DBError): - """Raised if wildcard matches are not strictly at the tail of the request. - """ - - -class DocumentDoesNotExist(U1DBError): - """The document does not exist.""" - - wire_description = "document does not exist" - - -class DocumentAlreadyDeleted(U1DBError): - """The document was already deleted.""" - - wire_description = "document already deleted" - - -class DatabaseDoesNotExist(U1DBError): - """The database does not exist.""" - - wire_description = "database does not exist" - - -class IndexNameTakenError(U1DBError): - """The given index name is already taken.""" - - -class IndexDefinitionParseError(U1DBError): - """The index definition cannot be parsed.""" - - -class IndexDoesNotExist(U1DBError): - """No index of that name exists.""" - - -class Unauthorized(U1DBError): - """Request wasn't authorized properly.""" - - wire_description = "unauthorized" - - -class HTTPError(U1DBError): - """Unspecific HTTP errror.""" - - wire_description = None - - def __init__(self, status, message=None, headers={}): - self.status = status - self.message = message - self.headers = headers - - def __str__(self): - if not self.message: - return "HTTPError(%d)" % self.status - else: - return "HTTPError(%d, %r)" % (self.status, self.message) - - -class Unavailable(HTTPError): - """Server not available not serve request.""" - - wire_description = "unavailable" - - def __init__(self, message=None, headers={}): - super(Unavailable, self).__init__(503, message, headers) - - def __str__(self): - if not self.message: - return "Unavailable()" - else: - return "Unavailable(%r)" % self.message - - -class BrokenSyncStream(U1DBError): - """Unterminated or otherwise broken sync exchange stream.""" - - wire_description = None - - -class UnknownAuthMethod(U1DBError): - """Unknown auhorization method.""" - - wire_description = None - - -# mapping wire (transimission) descriptions/tags for errors to the exceptions -wire_description_to_exc = dict( - (x.wire_description, x) for x in globals().values() - if getattr(x, 'wire_description', None) not in (None, "error")) -wire_description_to_exc["error"] = U1DBError - - -# -# wire error descriptions not corresponding to an exception -DOCUMENT_DELETED = "document deleted" diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py deleted file mode 100644 index 15a9ac80..00000000 --- a/common/src/leap/soledad/common/l2db/query_parser.py +++ /dev/null @@ -1,371 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. -""" -Code for parsing Index definitions. -""" - -import re - -from leap.soledad.common.l2db import errors - - -class Getter(object): - """Get values from a document based on a specification.""" - - def get(self, raw_doc): - """Get a value from the document. - - :param raw_doc: a python dictionary to get the value from. - :return: A list of values that match the description. - """ - raise NotImplementedError(self.get) - - -class StaticGetter(Getter): - """A getter that returns a defined value (independent of the doc).""" - - def __init__(self, value): - """Create a StaticGetter. - - :param value: the value to return when get is called. - """ - if value is None: - self.value = [] - elif isinstance(value, list): - self.value = value - else: - self.value = [value] - - def get(self, raw_doc): - return self.value - - -def extract_field(raw_doc, subfields, index=0): - if not isinstance(raw_doc, dict): - return [] - val = raw_doc.get(subfields[index]) - if val is None: - return [] - if index < len(subfields) - 1: - if isinstance(val, list): - results = [] - for item in val: - results.extend(extract_field(item, subfields, index + 1)) - return results - if isinstance(val, dict): - return extract_field(val, subfields, index + 1) - return [] - if isinstance(val, dict): - return [] - if isinstance(val, list): - # Strip anything in the list that isn't a simple type - return [v for v in val if not isinstance(v, (dict, list))] - return [val] - - -class ExtractField(Getter): - """Extract a field from the document.""" - - def __init__(self, field): - """Create an ExtractField object. - - When a document is passed to get() this will return a value - from the document based on the field specifier passed to - the constructor. - - None will be returned if the field is nonexistant, or refers to an - object, rather than a simple type or list of simple types. - - :param field: a specifier for the field to return. - This is either a field name, or a dotted field name. - """ - self.field = field.split('.') - - def get(self, raw_doc): - return extract_field(raw_doc, self.field) - - -class Transformation(Getter): - """A transformation on a value from another Getter.""" - - name = None - arity = 1 - args = ['expression'] - - def __init__(self, inner): - """Create a transformation. - - :param inner: the argument(s) to the transformation. - """ - self.inner = inner - - def get(self, raw_doc): - inner_values = self.inner.get(raw_doc) - assert isinstance(inner_values, list),\ - 'get() should always return a list' - return self.transform(inner_values) - - def transform(self, values): - """Transform the values. - - This should be implemented by subclasses to transform the - value when get() is called. - - :param values: the values from the other Getter - :return: the transformed values. - """ - raise NotImplementedError(self.transform) - - -class Lower(Transformation): - """Lowercase a string. - - This transformation will return None for non-string inputs. However, - it will lowercase any strings in a list, dropping any elements - that are not strings. - """ - - name = "lower" - - def _can_transform(self, val): - return isinstance(val, basestring) - - def transform(self, values): - if not values: - return [] - return [val.lower() for val in values if self._can_transform(val)] - - -class Number(Transformation): - """Convert an integer to a zero padded string. - - This transformation will return None for non-integer inputs. However, it - will transform any integers in a list, dropping any elements that are not - integers. - """ - - name = 'number' - arity = 2 - args = ['expression', int] - - def __init__(self, inner, number): - super(Number, self).__init__(inner) - self.padding = "%%0%sd" % number - - def _can_transform(self, val): - return isinstance(val, int) and not isinstance(val, bool) - - def transform(self, values): - """Transform any integers in values into zero padded strings.""" - if not values: - return [] - return [self.padding % (v,) for v in values if self._can_transform(v)] - - -class Bool(Transformation): - """Convert bool to string.""" - - name = "bool" - args = ['expression'] - - def _can_transform(self, val): - return isinstance(val, bool) - - def transform(self, values): - """Transform any booleans in values into strings.""" - if not values: - return [] - return [('1' if v else '0') for v in values if self._can_transform(v)] - - -class SplitWords(Transformation): - """Split a string on whitespace. - - This Getter will return [] for non-string inputs. It will however - split any strings in an input list, discarding any elements that - are not strings. - """ - - name = "split_words" - - def _can_transform(self, val): - return isinstance(val, basestring) - - def transform(self, values): - if not values: - return [] - result = set() - for value in values: - if self._can_transform(value): - for word in value.split(): - result.add(word) - return list(result) - - -class Combine(Transformation): - """Combine multiple expressions into a single index.""" - - name = "combine" - # variable number of args - arity = -1 - - def __init__(self, *inner): - super(Combine, self).__init__(inner) - - def get(self, raw_doc): - inner_values = [] - for inner in self.inner: - inner_values.extend(inner.get(raw_doc)) - return self.transform(inner_values) - - def transform(self, values): - return values - - -class IsNull(Transformation): - """Indicate whether the input is None. - - This Getter returns a bool indicating whether the input is nil. - """ - - name = "is_null" - - def transform(self, values): - return [len(values) == 0] - - -def check_fieldname(fieldname): - if fieldname.endswith('.'): - raise errors.IndexDefinitionParseError( - "Fieldname cannot end in '.':%s^" % (fieldname,)) - - -class Parser(object): - """Parse an index expression into a sequence of transformations.""" - - _transformations = {} - _delimiters = re.compile("\(|\)|,") - - def __init__(self): - self._tokens = [] - - def _set_expression(self, expression): - self._open_parens = 0 - self._tokens = [] - expression = expression.strip() - while expression: - delimiter = self._delimiters.search(expression) - if delimiter: - idx = delimiter.start() - if idx == 0: - result, expression = (expression[:1], expression[1:]) - self._tokens.append(result) - else: - result, expression = (expression[:idx], expression[idx:]) - result = result.strip() - if result: - self._tokens.append(result) - else: - expression = expression.strip() - if expression: - self._tokens.append(expression) - expression = None - - def _get_token(self): - if self._tokens: - return self._tokens.pop(0) - - def _peek_token(self): - if self._tokens: - return self._tokens[0] - - @staticmethod - def _to_getter(term): - if isinstance(term, Getter): - return term - check_fieldname(term) - return ExtractField(term) - - def _parse_op(self, op_name): - self._get_token() # '(' - op = self._transformations.get(op_name, None) - if op is None: - raise errors.IndexDefinitionParseError( - "Unknown operation: %s" % op_name) - args = [] - while True: - args.append(self._parse_term()) - sep = self._get_token() - if sep == ')': - break - if sep != ',': - raise errors.IndexDefinitionParseError( - "Unexpected token '%s' in parentheses." % (sep,)) - parsed = [] - for i, arg in enumerate(args): - arg_type = op.args[i % len(op.args)] - if arg_type == 'expression': - inner = self._to_getter(arg) - else: - try: - inner = arg_type(arg) - except ValueError as e: - raise errors.IndexDefinitionParseError( - "Invalid value %r for argument type %r " - "(%r)." % (arg, arg_type, e)) - parsed.append(inner) - return op(*parsed) - - def _parse_term(self): - term = self._get_token() - if term is None: - raise errors.IndexDefinitionParseError( - "Unexpected end of index definition.") - if term in (',', ')', '('): - raise errors.IndexDefinitionParseError( - "Unexpected token '%s' at start of expression." % (term,)) - next_token = self._peek_token() - if next_token == '(': - return self._parse_op(term) - return term - - def parse(self, expression): - self._set_expression(expression) - term = self._to_getter(self._parse_term()) - if self._peek_token(): - raise errors.IndexDefinitionParseError( - "Unexpected token '%s' after end of expression." - % (self._peek_token(),)) - return term - - def parse_all(self, fields): - return [self.parse(field) for field in fields] - - @classmethod - def register_transormation(cls, transform): - assert transform.name not in cls._transformations, ( - "Transform %s already registered for %s" - % (transform.name, cls._transformations[transform.name])) - cls._transformations[transform.name] = transform - - -Parser.register_transormation(SplitWords) -Parser.register_transormation(Lower) -Parser.register_transormation(Number) -Parser.register_transormation(Bool) -Parser.register_transormation(IsNull) -Parser.register_transormation(Combine) diff --git a/common/src/leap/soledad/common/l2db/remote/__init__.py b/common/src/leap/soledad/common/l2db/remote/__init__.py deleted file mode 100644 index 3f32e381..00000000 --- a/common/src/leap/soledad/common/l2db/remote/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py deleted file mode 100644 index a4eddb36..00000000 --- a/common/src/leap/soledad/common/l2db/remote/http_app.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -""" -HTTP Application exposing U1DB. -""" -# TODO -- deprecate, use twisted/txaio. - -import functools -import six.moves.http_client as httplib -import inspect -import json -import sys -import six.moves.urllib.parse as urlparse - -import routes.mapper - -from leap.soledad.common.l2db import ( - __version__ as _u1db_version, - DBNAME_CONSTRAINTS, Document, - errors, sync) -from leap.soledad.common.l2db.remote import http_errors, utils - - -def parse_bool(expression): - """Parse boolean querystring parameter.""" - if expression == 'true': - return True - return False - - -def parse_list(expression): - if not expression: - return [] - return [t.strip() for t in expression.split(',')] - - -def none_or_str(expression): - if expression is None: - return None - return str(expression) - - -class BadRequest(Exception): - """Bad request.""" - - -class _FencedReader(object): - """Read and get lines from a file but not past a given length.""" - - MAXCHUNK = 8192 - - def __init__(self, rfile, total, max_entry_size): - self.rfile = rfile - self.remaining = total - self.max_entry_size = max_entry_size - self._kept = None - - def read_chunk(self, atmost): - if self._kept is not None: - # ignore atmost, kept data should be a subchunk anyway - kept, self._kept = self._kept, None - return kept - if self.remaining == 0: - return '' - data = self.rfile.read(min(self.remaining, atmost)) - self.remaining -= len(data) - return data - - def getline(self): - line_parts = [] - size = 0 - while True: - chunk = self.read_chunk(self.MAXCHUNK) - if chunk == '': - break - nl = chunk.find("\n") - if nl != -1: - size += nl + 1 - if size > self.max_entry_size: - raise BadRequest - line_parts.append(chunk[:nl + 1]) - rest = chunk[nl + 1:] - self._kept = rest or None - break - else: - size += len(chunk) - if size > self.max_entry_size: - raise BadRequest - line_parts.append(chunk) - return ''.join(line_parts) - - -def http_method(**control): - """Decoration for handling of query arguments and content for a HTTP - method. - - args and content here are the query arguments and body of the incoming - HTTP requests. - - Match query arguments to python method arguments: - w = http_method()(f) - w(self, args, content) => args["content"]=content; - f(self, **args) - - JSON deserialize content to arguments: - w = http_method(content_as_args=True,...)(f) - w(self, args, content) => args.update(json.loads(content)); - f(self, **args) - - Support conversions (e.g int): - w = http_method(Arg=Conv,...)(f) - w(self, args, content) => args["Arg"]=Conv(args["Arg"]); - f(self, **args) - - Enforce no use of query arguments: - w = http_method(no_query=True,...)(f) - w(self, args, content) raises BadRequest if args is not empty - - Argument mismatches, deserialisation failures produce BadRequest. - """ - content_as_args = control.pop('content_as_args', False) - no_query = control.pop('no_query', False) - conversions = control.items() - - def wrap(f): - argspec = inspect.getargspec(f) - assert argspec.args[0] == "self" - nargs = len(argspec.args) - ndefaults = len(argspec.defaults or ()) - required_args = set(argspec.args[1:nargs - ndefaults]) - all_args = set(argspec.args) - - @functools.wraps(f) - def wrapper(self, args, content): - if no_query and args: - raise BadRequest() - if content is not None: - if content_as_args: - try: - args.update(json.loads(content)) - except ValueError: - raise BadRequest() - else: - args["content"] = content - if not (required_args <= set(args) <= all_args): - raise BadRequest("Missing required arguments.") - for name, conv in conversions: - if name not in args: - continue - try: - args[name] = conv(args[name]) - except ValueError: - raise BadRequest() - return f(self, **args) - - return wrapper - - return wrap - - -class URLToResource(object): - """Mappings from URLs to resources.""" - - def __init__(self): - self._map = routes.mapper.Mapper(controller_scan=None) - - def register(self, resource_cls): - # register - self._map.connect(None, resource_cls.url_pattern, - resource_cls=resource_cls, - requirements={"dbname": DBNAME_CONSTRAINTS}) - self._map.create_regs() - return resource_cls - - def match(self, path): - params = self._map.match(path) - if params is None: - return None, None - resource_cls = params.pop('resource_cls') - return resource_cls, params - - -url_to_resource = URLToResource() - - -@url_to_resource.register -class GlobalResource(object): - """Global (root) resource.""" - - url_pattern = "/" - - def __init__(self, state, responder): - self.state = state - self.responder = responder - - @http_method() - def get(self): - info = self.state.global_info() - info['version'] = _u1db_version - self.responder.send_response_json(**info) - - -@url_to_resource.register -class DatabaseResource(object): - """Database resource.""" - - url_pattern = "/{dbname}" - - def __init__(self, dbname, state, responder): - self.dbname = dbname - self.state = state - self.responder = responder - - @http_method() - def get(self): - self.state.check_database(self.dbname) - self.responder.send_response_json(200) - - @http_method(content_as_args=True) - def put(self): - self.state.ensure_database(self.dbname) - self.responder.send_response_json(200, ok=True) - - @http_method() - def delete(self): - self.state.delete_database(self.dbname) - self.responder.send_response_json(200, ok=True) - - -@url_to_resource.register -class DocsResource(object): - """Documents resource.""" - - url_pattern = "/{dbname}/docs" - - def __init__(self, dbname, state, responder): - self.responder = responder - self.db = state.open_database(dbname) - - @http_method(doc_ids=parse_list, check_for_conflicts=parse_bool, - include_deleted=parse_bool) - def get(self, doc_ids=None, check_for_conflicts=True, - include_deleted=False): - if doc_ids is None: - raise errors.MissingDocIds - docs = self.db.get_docs(doc_ids, include_deleted=include_deleted) - self.responder.content_type = 'application/json' - self.responder.start_response(200) - self.responder.start_stream(), - for doc in docs: - entry = dict( - doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), - has_conflicts=doc.has_conflicts) - self.responder.stream_entry(entry) - self.responder.end_stream() - self.responder.finish_response() - - -@url_to_resource.register -class AllDocsResource(object): - """All Documents resource.""" - - url_pattern = "/{dbname}/all-docs" - - def __init__(self, dbname, state, responder): - self.responder = responder - self.db = state.open_database(dbname) - - @http_method(include_deleted=parse_bool) - def get(self, include_deleted=False): - gen, docs = self.db.get_all_docs(include_deleted=include_deleted) - self.responder.content_type = 'application/json' - # returning a x-u1db-generation header is optional - # HTTPDatabase will fallback to return -1 if it's missing - self.responder.start_response(200, - headers={'x-u1db-generation': str(gen)}) - self.responder.start_stream(), - for doc in docs: - entry = dict( - doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), - has_conflicts=doc.has_conflicts) - self.responder.stream_entry(entry) - self.responder.end_stream() - self.responder.finish_response() - - -@url_to_resource.register -class DocResource(object): - """Document resource.""" - - url_pattern = "/{dbname}/doc/{id:.*}" - - def __init__(self, dbname, id, state, responder): - self.id = id - self.responder = responder - self.db = state.open_database(dbname) - - @http_method(old_rev=str) - def put(self, content, old_rev=None): - doc = Document(self.id, old_rev, content) - doc_rev = self.db.put_doc(doc) - if old_rev is None: - status = 201 # created - else: - status = 200 - self.responder.send_response_json(status, rev=doc_rev) - - @http_method(old_rev=str) - def delete(self, old_rev=None): - doc = Document(self.id, old_rev, None) - self.db.delete_doc(doc) - self.responder.send_response_json(200, rev=doc.rev) - - @http_method(include_deleted=parse_bool) - def get(self, include_deleted=False): - doc = self.db.get_doc(self.id, include_deleted=include_deleted) - if doc is None: - wire_descr = errors.DocumentDoesNotExist.wire_description - self.responder.send_response_json( - http_errors.wire_description_to_status[wire_descr], - error=wire_descr, - headers={ - 'x-u1db-rev': '', - 'x-u1db-has-conflicts': 'false' - }) - return - headers = { - 'x-u1db-rev': doc.rev, - 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts) - } - if doc.is_tombstone(): - self.responder.send_response_json( - http_errors.wire_description_to_status[ - errors.DOCUMENT_DELETED], - error=errors.DOCUMENT_DELETED, - headers=headers) - else: - self.responder.send_response_content( - doc.get_json(), headers=headers) - - -@url_to_resource.register -class SyncResource(object): - """Sync endpoint resource.""" - - # maximum allowed request body size - max_request_size = 15 * 1024 * 1024 # 15Mb - # maximum allowed entry/line size in request body - max_entry_size = 10 * 1024 * 1024 # 10Mb - - url_pattern = "/{dbname}/sync-from/{source_replica_uid}" - - # pluggable - sync_exchange_class = sync.SyncExchange - - def __init__(self, dbname, source_replica_uid, state, responder): - self.source_replica_uid = source_replica_uid - self.responder = responder - self.state = state - self.dbname = dbname - self.replica_uid = None - - def get_target(self): - return self.state.open_database(self.dbname).get_sync_target() - - @http_method() - def get(self): - result = self.get_target().get_sync_info(self.source_replica_uid) - self.responder.send_response_json( - target_replica_uid=result[0], target_replica_generation=result[1], - target_replica_transaction_id=result[2], - source_replica_uid=self.source_replica_uid, - source_replica_generation=result[3], - source_transaction_id=result[4]) - - @http_method(generation=int, - content_as_args=True, no_query=True) - def put(self, generation, transaction_id): - self.get_target().record_sync_info(self.source_replica_uid, - generation, - transaction_id) - self.responder.send_response_json(ok=True) - - # Implements the same logic as LocalSyncTarget.sync_exchange - - @http_method(last_known_generation=int, last_known_trans_id=none_or_str, - content_as_args=True) - def post_args(self, last_known_generation, last_known_trans_id=None, - ensure=False): - if ensure: - db, self.replica_uid = self.state.ensure_database(self.dbname) - else: - db = self.state.open_database(self.dbname) - db.validate_gen_and_trans_id( - last_known_generation, last_known_trans_id) - self.sync_exch = self.sync_exchange_class( - db, self.source_replica_uid, last_known_generation) - - @http_method(content_as_args=True) - def post_stream_entry(self, id, rev, content, gen, trans_id): - doc = Document(id, rev, content) - self.sync_exch.insert_doc_from_source(doc, gen, trans_id) - - def post_end(self): - - def send_doc(doc, gen, trans_id): - entry = dict(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), - gen=gen, trans_id=trans_id) - self.responder.stream_entry(entry) - - new_gen = self.sync_exch.find_changes_to_return() - self.responder.content_type = 'application/x-u1db-sync-stream' - self.responder.start_response(200) - self.responder.start_stream(), - header = {"new_generation": new_gen, - "new_transaction_id": self.sync_exch.new_trans_id} - if self.replica_uid is not None: - header['replica_uid'] = self.replica_uid - self.responder.stream_entry(header) - self.sync_exch.return_docs(send_doc) - self.responder.end_stream() - self.responder.finish_response() - - -class HTTPResponder(object): - """Encode responses from the server back to the client.""" - - # a multi document response will put args and documents - # each on one line of the response body - - def __init__(self, start_response): - self._started = False - self._stream_state = -1 - self._no_initial_obj = True - self.sent_response = False - self._start_response = start_response - self._write = None - self.content_type = 'application/json' - self.content = [] - - def start_response(self, status, obj_dic=None, headers={}): - """start sending response with optional first json object.""" - if self._started: - return - self._started = True - status_text = httplib.responses[status] - self._write = self._start_response( - '%d %s' % (status, status_text), - [('content-type', self.content_type), - ('cache-control', 'no-cache')] + - headers.items()) - # xxx version in headers - if obj_dic is not None: - self._no_initial_obj = False - self._write(json.dumps(obj_dic) + "\r\n") - - def finish_response(self): - """finish sending response.""" - self.sent_response = True - - def send_response_json(self, status=200, headers={}, **kwargs): - """send and finish response with json object body from keyword args.""" - content = json.dumps(kwargs) + "\r\n" - self.send_response_content(content, headers=headers, status=status) - - def send_response_content(self, content, status=200, headers={}): - """send and finish response with content""" - headers['content-length'] = str(len(content)) - self.start_response(status, headers=headers) - if self._stream_state == 1: - self.content = [',\r\n', content] - else: - self.content = [content] - self.finish_response() - - def start_stream(self): - "start stream (array) as part of the response." - assert self._started and self._no_initial_obj - self._stream_state = 0 - self._write("[") - - def stream_entry(self, entry): - "send stream entry as part of the response." - assert self._stream_state != -1 - if self._stream_state == 0: - self._stream_state = 1 - self._write('\r\n') - else: - self._write(',\r\n') - if type(entry) == dict: - entry = json.dumps(entry) - self._write(entry) - - def end_stream(self): - "end stream (array)." - assert self._stream_state != -1 - self._write("\r\n]\r\n") - - -class HTTPInvocationByMethodWithBody(object): - """Invoke methods on a resource.""" - - def __init__(self, resource, environ, parameters): - self.resource = resource - self.environ = environ - self.max_request_size = getattr( - resource, 'max_request_size', parameters.max_request_size) - self.max_entry_size = getattr( - resource, 'max_entry_size', parameters.max_entry_size) - - def _lookup(self, method): - try: - return getattr(self.resource, method) - except AttributeError: - raise BadRequest() - - def __call__(self): - args = urlparse.parse_qsl(self.environ['QUERY_STRING'], - strict_parsing=False) - try: - args = dict( - (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) - except ValueError: - raise BadRequest() - method = self.environ['REQUEST_METHOD'].lower() - if method in ('get', 'delete'): - meth = self._lookup(method) - return meth(args, None) - else: - # we expect content-length > 0, reconsider if we move - # to support chunked enconding - try: - content_length = int(self.environ['CONTENT_LENGTH']) - except (ValueError, KeyError): - raise BadRequest - if content_length <= 0: - raise BadRequest - if content_length > self.max_request_size: - raise BadRequest - reader = _FencedReader(self.environ['wsgi.input'], content_length, - self.max_entry_size) - content_type = self.environ.get('CONTENT_TYPE', '') - content_type = content_type.split(';', 1)[0].strip() - if content_type == 'application/json': - meth = self._lookup(method) - body = reader.read_chunk(sys.maxint) - return meth(args, body) - elif content_type == 'application/x-u1db-sync-stream': - meth_args = self._lookup('%s_args' % method) - meth_entry = self._lookup('%s_stream_entry' % method) - meth_end = self._lookup('%s_end' % method) - body_getline = reader.getline - if body_getline().strip() != '[': - raise BadRequest() - line = body_getline() - line, comma = utils.check_and_strip_comma(line.strip()) - meth_args(args, line) - while True: - line = body_getline() - entry = line.strip() - if entry == ']': - break - if not entry or not comma: # empty or no prec comma - raise BadRequest - entry, comma = utils.check_and_strip_comma(entry) - meth_entry({}, entry) - if comma or body_getline(): # extra comma or data - raise BadRequest - return meth_end() - else: - raise BadRequest() - - -class HTTPApp(object): - - # maximum allowed request body size - max_request_size = 15 * 1024 * 1024 # 15Mb - # maximum allowed entry/line size in request body - max_entry_size = 10 * 1024 * 1024 # 10Mb - - def __init__(self, state): - self.state = state - - def _lookup_resource(self, environ, responder): - resource_cls, params = url_to_resource.match(environ['PATH_INFO']) - if resource_cls is None: - raise BadRequest # 404 instead? - resource = resource_cls( - state=self.state, responder=responder, **params) - return resource - - def __call__(self, environ, start_response): - responder = HTTPResponder(start_response) - self.request_begin(environ) - try: - resource = self._lookup_resource(environ, responder) - HTTPInvocationByMethodWithBody(resource, environ, self)() - except errors.U1DBError as e: - self.request_u1db_error(environ, e) - status = http_errors.wire_description_to_status.get( - e.wire_description, 500) - responder.send_response_json(status, error=e.wire_description) - except BadRequest: - self.request_bad_request(environ) - responder.send_response_json(400, error="bad request") - except KeyboardInterrupt: - raise - except: - self.request_failed(environ) - raise - else: - self.request_done(environ) - return responder.content - - # hooks for tracing requests - - def request_begin(self, environ): - """Hook called at the beginning of processing a request.""" - pass - - def request_done(self, environ): - """Hook called when done processing a request.""" - pass - - def request_u1db_error(self, environ, exc): - """Hook called when processing a request resulted in a U1DBError. - - U1DBError passed as exc. - """ - pass - - def request_bad_request(self, environ): - """Hook called when processing a bad request. - - No actual processing was done. - """ - pass - - def request_failed(self, environ): - """Hook called when processing a request failed unexpectedly. - - Invoked from an except block, so there's interpreter exception - information available. - """ - pass diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py deleted file mode 100644 index 1124b038..00000000 --- a/common/src/leap/soledad/common/l2db/remote/http_client.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Base class to make requests to a remote HTTP server.""" - -import json -import socket -import ssl -import sys -import urllib -import six.moves.urllib.parse as urlparse -import six.moves.http_client as httplib -from time import sleep -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db.remote import http_errors - -from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname - -# Ubuntu/debian -# XXX other... -CA_CERTS = "/etc/ssl/certs/ca-certificates.crt" - - -def _encode_query_parameter(value): - """Encode query parameter.""" - if isinstance(value, bool): - if value: - value = 'true' - else: - value = 'false' - return unicode(value).encode('utf-8') - - -class _VerifiedHTTPSConnection(httplib.HTTPSConnection): - """HTTPSConnection verifying server side certificates.""" - # derived from httplib.py - - def connect(self): - "Connect to a host on a given (SSL) port." - - sock = socket.create_connection((self.host, self.port), - self.timeout, self.source_address) - if self._tunnel_host: - self.sock = sock - self._tunnel() - if sys.platform.startswith('linux'): - cert_opts = { - 'cert_reqs': ssl.CERT_REQUIRED, - 'ca_certs': CA_CERTS - } - else: - # XXX no cert verification implemented elsewhere for now - cert_opts = {} - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, - ssl_version=ssl.PROTOCOL_SSLv3, - **cert_opts - ) - if cert_opts: - match_hostname(self.sock.getpeercert(), self.host) - - -class HTTPClientBase(object): - """Base class to make requests to a remote HTTP server.""" - - # Will use these delays to retry on 503 befor finally giving up. The final - # 0 is there to not wait after the final try fails. - _delays = (1, 1, 2, 4, 0) - - def __init__(self, url, creds=None): - self._url = urlparse.urlsplit(url) - self._conn = None - self._creds = {} - if creds is not None: - if len(creds) != 1: - raise errors.UnknownAuthMethod() - auth_meth, credentials = creds.items()[0] - try: - set_creds = getattr(self, 'set_%s_credentials' % auth_meth) - except AttributeError: - raise errors.UnknownAuthMethod(auth_meth) - set_creds(**credentials) - - def _ensure_connection(self): - if self._conn is not None: - return - if self._url.scheme == 'https': - connClass = _VerifiedHTTPSConnection - else: - connClass = httplib.HTTPConnection - self._conn = connClass(self._url.hostname, self._url.port) - - def close(self): - if self._conn: - self._conn.close() - self._conn = None - - # xxx retry mechanism? - - def _error(self, respdic): - descr = respdic.get("error") - exc_cls = errors.wire_description_to_exc.get(descr) - if exc_cls is not None: - message = respdic.get("message") - raise exc_cls(message) - - def _response(self): - resp = self._conn.getresponse() - body = resp.read() - headers = dict(resp.getheaders()) - if resp.status in (200, 201): - return body, headers - elif resp.status in http_errors.ERROR_STATUSES: - try: - respdic = json.loads(body) - except ValueError: - pass - else: - self._error(respdic) - # special case - if resp.status == 503: - raise errors.Unavailable(body, headers) - raise errors.HTTPError(resp.status, body, headers) - - def _sign_request(self, method, url_query, params): - raise NotImplementedError - - def _request(self, method, url_parts, params=None, body=None, - content_type=None): - self._ensure_connection() - unquoted_url = url_query = self._url.path - if url_parts: - if not url_query.endswith('/'): - url_query += '/' - unquoted_url = url_query - url_query += '/'.join(urllib.quote(part, safe='') - for part in url_parts) - # oauth performs its own quoting - unquoted_url += '/'.join(url_parts) - encoded_params = {} - if params: - for key, value in params.items(): - key = unicode(key).encode('utf-8') - encoded_params[key] = _encode_query_parameter(value) - url_query += ('?' + urllib.urlencode(encoded_params)) - if body is not None and not isinstance(body, basestring): - body = json.dumps(body) - content_type = 'application/json' - headers = {} - if content_type: - headers['content-type'] = content_type - headers.update( - self._sign_request(method, unquoted_url, encoded_params)) - for delay in self._delays: - try: - self._conn.request(method, url_query, body, headers) - return self._response() - except errors.Unavailable as e: - sleep(delay) - raise e - - def _request_json(self, method, url_parts, params=None, body=None, - content_type=None): - res, headers = self._request(method, url_parts, params, body, - content_type) - return json.loads(res), headers diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py deleted file mode 100644 index 7e61e5a4..00000000 --- a/common/src/leap/soledad/common/l2db/remote/http_database.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""HTTPDatabase to access a remote db over the HTTP API.""" - -import json -import uuid - -from leap.soledad.common.l2db import ( - Database, - Document, - errors) -from leap.soledad.common.l2db.remote import ( - http_client, - http_errors, - http_target) - - -DOCUMENT_DELETED_STATUS = http_errors.wire_description_to_status[ - errors.DOCUMENT_DELETED] - - -class HTTPDatabase(http_client.HTTPClientBase, Database): - """Implement the Database API to a remote HTTP server.""" - - def __init__(self, url, document_factory=None, creds=None): - super(HTTPDatabase, self).__init__(url, creds=creds) - self._factory = document_factory or Document - - def set_document_factory(self, factory): - self._factory = factory - - @staticmethod - def open_database(url, create): - db = HTTPDatabase(url) - db.open(create) - return db - - @staticmethod - def delete_database(url): - db = HTTPDatabase(url) - db._delete() - db.close() - - def open(self, create): - if create: - self._ensure() - else: - self._check() - - def _check(self): - return self._request_json('GET', [])[0] - - def _ensure(self): - self._request_json('PUT', [], {}, {}) - - def _delete(self): - self._request_json('DELETE', [], {}, {}) - - def put_doc(self, doc): - if doc.doc_id is None: - raise errors.InvalidDocId() - params = {} - if doc.rev is not None: - params['old_rev'] = doc.rev - res, headers = self._request_json('PUT', ['doc', doc.doc_id], params, - doc.get_json(), 'application/json') - doc.rev = res['rev'] - return res['rev'] - - def get_doc(self, doc_id, include_deleted=False): - try: - res, headers = self._request( - 'GET', ['doc', doc_id], {"include_deleted": include_deleted}) - except errors.DocumentDoesNotExist: - return None - except errors.HTTPError as e: - if (e.status == DOCUMENT_DELETED_STATUS and - 'x-u1db-rev' in e.headers): - res = None - headers = e.headers - else: - raise - doc_rev = headers['x-u1db-rev'] - has_conflicts = json.loads(headers['x-u1db-has-conflicts']) - doc = self._factory(doc_id, doc_rev, res) - doc.has_conflicts = has_conflicts - return doc - - def _build_docs(self, res): - for doc_dict in json.loads(res): - doc = self._factory( - doc_dict['doc_id'], doc_dict['doc_rev'], doc_dict['content']) - doc.has_conflicts = doc_dict['has_conflicts'] - yield doc - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - if not doc_ids: - return [] - doc_ids = ','.join(doc_ids) - res, headers = self._request( - 'GET', ['docs'], { - "doc_ids": doc_ids, "include_deleted": include_deleted, - "check_for_conflicts": check_for_conflicts}) - return self._build_docs(res) - - def get_all_docs(self, include_deleted=False): - res, headers = self._request( - 'GET', ['all-docs'], {"include_deleted": include_deleted}) - gen = -1 - if 'x-u1db-generation' in headers: - gen = int(headers['x-u1db-generation']) - return gen, list(self._build_docs(res)) - - def _allocate_doc_id(self): - return 'D-%s' % (uuid.uuid4().hex,) - - def create_doc(self, content, doc_id=None): - if not isinstance(content, dict): - raise errors.InvalidContent - json_string = json.dumps(content) - return self.create_doc_from_json(json_string, doc_id) - - def create_doc_from_json(self, content, doc_id=None): - if doc_id is None: - doc_id = self._allocate_doc_id() - res, headers = self._request_json('PUT', ['doc', doc_id], {}, - content, 'application/json') - new_doc = self._factory(doc_id, res['rev'], content) - return new_doc - - def delete_doc(self, doc): - if doc.doc_id is None: - raise errors.InvalidDocId() - params = {'old_rev': doc.rev} - res, headers = self._request_json( - 'DELETE', ['doc', doc.doc_id], params) - doc.make_tombstone() - doc.rev = res['rev'] - - def get_sync_target(self): - st = http_target.HTTPSyncTarget(self._url.geturl()) - st._creds = self._creds - return st diff --git a/common/src/leap/soledad/common/l2db/remote/http_errors.py b/common/src/leap/soledad/common/l2db/remote/http_errors.py deleted file mode 100644 index ee4cfefa..00000000 --- a/common/src/leap/soledad/common/l2db/remote/http_errors.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -""" -Information about the encoding of errors over HTTP. -""" - -from leap.soledad.common.l2db import errors - - -# error wire descriptions mapping to HTTP status codes -wire_description_to_status = dict([ - (errors.InvalidDocId.wire_description, 400), - (errors.MissingDocIds.wire_description, 400), - (errors.Unauthorized.wire_description, 401), - (errors.DocumentTooBig.wire_description, 403), - (errors.UserQuotaExceeded.wire_description, 403), - (errors.SubscriptionNeeded.wire_description, 403), - (errors.DatabaseDoesNotExist.wire_description, 404), - (errors.DocumentDoesNotExist.wire_description, 404), - (errors.DocumentAlreadyDeleted.wire_description, 404), - (errors.RevisionConflict.wire_description, 409), - (errors.InvalidGeneration.wire_description, 409), - (errors.InvalidReplicaUID.wire_description, 409), - (errors.InvalidTransactionId.wire_description, 409), - (errors.Unavailable.wire_description, 503), - # without matching exception - (errors.DOCUMENT_DELETED, 404) -]) - - -ERROR_STATUSES = set(wire_description_to_status.values()) -# 400 included explicitly for tests -ERROR_STATUSES.add(400) diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py deleted file mode 100644 index 38804f01..00000000 --- a/common/src/leap/soledad/common/l2db/remote/http_target.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""SyncTarget API implementation to a remote HTTP server.""" - -import json - -from leap.soledad.common.l2db import Document, SyncTarget -from leap.soledad.common.l2db.errors import BrokenSyncStream -from leap.soledad.common.l2db.remote import ( - http_client, utils) - - -class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): - """Implement the SyncTarget api to a remote HTTP server.""" - - @staticmethod - def connect(url): - return HTTPSyncTarget(url) - - def get_sync_info(self, source_replica_uid): - self._ensure_connection() - res, _ = self._request_json('GET', ['sync-from', source_replica_uid]) - return (res['target_replica_uid'], res['target_replica_generation'], - res['target_replica_transaction_id'], - res['source_replica_generation'], res['source_transaction_id']) - - def record_sync_info(self, source_replica_uid, source_replica_generation, - source_transaction_id): - self._ensure_connection() - if self._trace_hook: # for tests - self._trace_hook('record_sync_info') - self._request_json('PUT', ['sync-from', source_replica_uid], {}, - {'generation': source_replica_generation, - 'transaction_id': source_transaction_id}) - - def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): - parts = data.splitlines() # one at a time - if not parts or parts[0] != '[': - raise BrokenSyncStream - data = parts[1:-1] - comma = False - if data: - line, comma = utils.check_and_strip_comma(data[0]) - res = json.loads(line) - if ensure_callback and 'replica_uid' in res: - ensure_callback(res['replica_uid']) - for entry in data[1:]: - if not comma: # missing in between comma - raise BrokenSyncStream - line, comma = utils.check_and_strip_comma(entry) - entry = json.loads(line) - doc = Document(entry['id'], entry['rev'], entry['content']) - return_doc_cb(doc, entry['gen'], entry['trans_id']) - if parts[-1] != ']': - try: - partdic = json.loads(parts[-1]) - except ValueError: - pass - else: - if isinstance(partdic, dict): - self._error(partdic) - raise BrokenSyncStream - if not data or comma: # no entries or bad extra comma - raise BrokenSyncStream - return res - - def sync_exchange(self, docs_by_generations, source_replica_uid, - last_known_generation, last_known_trans_id, - return_doc_cb, ensure_callback=None): - self._ensure_connection() - if self._trace_hook: # for tests - self._trace_hook('sync_exchange') - url = '%s/sync-from/%s' % (self._url.path, source_replica_uid) - self._conn.putrequest('POST', url) - self._conn.putheader('content-type', 'application/x-u1db-sync-stream') - for header_name, header_value in self._sign_request('POST', url, {}): - self._conn.putheader(header_name, header_value) - entries = ['['] - size = 1 - - def prepare(**dic): - entry = comma + '\r\n' + json.dumps(dic) - entries.append(entry) - return len(entry) - - comma = '' - size += prepare( - last_known_generation=last_known_generation, - last_known_trans_id=last_known_trans_id, - ensure=ensure_callback is not None) - comma = ',' - for doc, gen, trans_id in docs_by_generations: - size += prepare(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), - gen=gen, trans_id=trans_id) - entries.append('\r\n]') - size += len(entries[-1]) - self._conn.putheader('content-length', str(size)) - self._conn.endheaders() - for entry in entries: - self._conn.send(entry) - entries = None - data, _ = self._response() - res = self._parse_sync_stream(data, return_doc_cb, ensure_callback) - data = None - return res['new_generation'], res['new_transaction_id'] - - # for tests - _trace_hook = None - - def _set_trace_hook_shallow(self, cb): - self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py deleted file mode 100644 index d4c3c45f..00000000 --- a/common/src/leap/soledad/common/l2db/remote/server_state.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""State for servers exposing a set of U1DB databases.""" - - -class ServerState(object): - """Passed to a Request when it is instantiated. - - This is used to track server-side state, such as working-directory, open - databases, etc. - """ - - def __init__(self): - self._workingdir = None - - def set_workingdir(self, path): - self._workingdir = path - - def global_info(self): - """Return global information about the server.""" - return {} - - def _relpath(self, relpath): - # Note: We don't want to allow absolute paths here, because we - # don't want to expose the filesystem. We should also check that - # relpath doesn't have '..' in it, etc. - return self._workingdir + '/' + relpath - - def open_database(self, path): - """Open a database at the given location.""" - from leap.soledad.client._db import sqlite - full_path = self._relpath(path) - return sqlite.SQLiteDatabase.open_database(full_path, create=False) - - def check_database(self, path): - """Check if the database at the given location exists. - - Simply returns if it does or raises DatabaseDoesNotExist. - """ - db = self.open_database(path) - db.close() - - def ensure_database(self, path): - """Ensure database at the given location.""" - from leap.soledad.client._db import sqlite - full_path = self._relpath(path) - db = sqlite.SQLiteDatabase.open_database(full_path, create=True) - return db, db._replica_uid - - def delete_database(self, path): - """Delete database at the given location.""" - from leap.soledad.client._db import sqlite - full_path = self._relpath(path) - sqlite.SQLiteDatabase.delete_database(full_path) diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py deleted file mode 100644 index ce82f1b2..00000000 --- a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py +++ /dev/null @@ -1,65 +0,0 @@ -"""The match_hostname() function from Python 3.2, essential when using SSL.""" -# XXX put it here until it's packaged - -import re - -__version__ = '3.2a3' - - -class CertificateError(ValueError): - pass - - -def _dnsname_to_pat(dn): - pats = [] - for frag in dn.split(r'.'): - if frag == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - else: - # Otherwise, '*' matches any dotless fragment. - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - - -def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules - are mostly followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not san: - # The subject is only checked when subjectAltName is empty - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError( - "hostname %r doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError( - "hostname %r doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise CertificateError( - "no appropriate commonName or " - "subjectAltName fields were found") diff --git a/common/src/leap/soledad/common/l2db/remote/utils.py b/common/src/leap/soledad/common/l2db/remote/utils.py deleted file mode 100644 index 14cedea9..00000000 --- a/common/src/leap/soledad/common/l2db/remote/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Utilities for details of the procotol.""" - - -def check_and_strip_comma(line): - if line and line[-1] == ',': - return line[:-1], True - return line, False diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py deleted file mode 100644 index 32281f30..00000000 --- a/common/src/leap/soledad/common/l2db/sync.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""The synchronization utilities for U1DB.""" -from six.moves import zip as izip - -from leap.soledad.common import l2db -from leap.soledad.common.l2db import errors - - -class Synchronizer(object): - """Collect the state around synchronizing 2 U1DB replicas. - - Synchronization is bi-directional, in that new items in the source are sent - to the target, and new items in the target are returned to the source. - However, it still recognizes that one side is initiating the request. Also, - at the moment, conflicts are only created in the source. - """ - - def __init__(self, source, sync_target): - """Create a new Synchronization object. - - :param source: A Database - :param sync_target: A SyncTarget - """ - self.source = source - self.sync_target = sync_target - self.target_replica_uid = None - self.num_inserted = 0 - - def _insert_doc_from_target(self, doc, replica_gen, trans_id): - """Try to insert synced document from target. - - Implements TAKE OTHER semantics: any document from the target - that is in conflict will be taken as the new official value, - while the current conflicting value will be stored alongside - as a conflict. In the process indexes will be updated etc. - - :return: None - """ - # Increases self.num_inserted depending whether the document - # was effectively inserted. - state, _ = self.source._put_doc_if_newer( - doc, save_conflict=True, - replica_uid=self.target_replica_uid, replica_gen=replica_gen, - replica_trans_id=trans_id) - if state == 'inserted': - self.num_inserted += 1 - elif state == 'converged': - # magical convergence - pass - elif state == 'superseded': - # we have something newer, will be taken care of at the next sync - pass - else: - assert state == 'conflicted' - # The doc was saved as a conflict, so the database was updated - self.num_inserted += 1 - - def _record_sync_info_with_the_target(self, start_generation): - """Record our new after sync generation with the target if gapless. - - Any documents received from the target will cause the local - database to increment its generation. We do not want to send - them back to the target in a future sync. However, there could - also be concurrent updates from another process doing eg - 'put_doc' while the sync was running. And we do want to - synchronize those documents. We can tell if there was a - concurrent update by comparing our new generation number - versus the generation we started, and how many documents we - inserted from the target. If it matches exactly, then we can - record with the target that they are fully up to date with our - new generation. - """ - cur_gen, trans_id = self.source._get_generation_info() - last_gen = start_generation + self.num_inserted - if (cur_gen == last_gen and self.num_inserted > 0): - self.sync_target.record_sync_info( - self.source._replica_uid, cur_gen, trans_id) - - def sync(self, callback=None, autocreate=False): - """Synchronize documents between source and target.""" - sync_target = self.sync_target - # get target identifier, its current generation, - # and its last-seen database generation for this source - try: - (self.target_replica_uid, target_gen, target_trans_id, - target_my_gen, target_my_trans_id) = sync_target.get_sync_info( - self.source._replica_uid) - except errors.DatabaseDoesNotExist: - if not autocreate: - raise - # will try to ask sync_exchange() to create the db - self.target_replica_uid = None - target_gen, target_trans_id = 0, '' - target_my_gen, target_my_trans_id = 0, '' - - def ensure_callback(replica_uid): - self.target_replica_uid = replica_uid - - else: - ensure_callback = None - if self.target_replica_uid == self.source._replica_uid: - raise errors.InvalidReplicaUID - # validate the generation and transaction id the target knows about us - self.source.validate_gen_and_trans_id( - target_my_gen, target_my_trans_id) - # what's changed since that generation and this current gen - my_gen, _, changes = self.source.whats_changed(target_my_gen) - - # this source last-seen database generation for the target - if self.target_replica_uid is None: - target_last_known_gen, target_last_known_trans_id = 0, '' - else: - target_last_known_gen, target_last_known_trans_id = ( - self.source._get_replica_gen_and_trans_id( # nopep8 - self.target_replica_uid)) - if not changes and target_last_known_gen == target_gen: - if target_trans_id != target_last_known_trans_id: - raise errors.InvalidTransactionId - return my_gen - changed_doc_ids = [doc_id for doc_id, _, _ in changes] - # prepare to send all the changed docs - docs_to_send = self.source.get_docs( - changed_doc_ids, - check_for_conflicts=False, include_deleted=True) - # TODO: there must be a way to not iterate twice - docs_by_generation = zip( - docs_to_send, (gen for _, gen, _ in changes), - (trans for _, _, trans in changes)) - - # exchange documents and try to insert the returned ones with - # the target, return target synced-up-to gen - new_gen, new_trans_id = sync_target.sync_exchange( - docs_by_generation, self.source._replica_uid, - target_last_known_gen, target_last_known_trans_id, - self._insert_doc_from_target, ensure_callback=ensure_callback) - # record target synced-up-to generation including applying what we sent - self.source._set_replica_gen_and_trans_id( - self.target_replica_uid, new_gen, new_trans_id) - - # if gapless record current reached generation with target - self._record_sync_info_with_the_target(my_gen) - - return my_gen - - -class SyncExchange(object): - """Steps and state for carrying through a sync exchange on a target.""" - - def __init__(self, db, source_replica_uid, last_known_generation): - self._db = db - self.source_replica_uid = source_replica_uid - self.source_last_known_generation = last_known_generation - self.seen_ids = {} # incoming ids not superseded - self.changes_to_return = None - self.new_gen = None - self.new_trans_id = None - # for tests - self._incoming_trace = [] - self._trace_hook = None - self._db._last_exchange_log = { - 'receive': {'docs': self._incoming_trace}, - 'return': None - } - - def _set_trace_hook(self, cb): - self._trace_hook = cb - - def _trace(self, state): - if not self._trace_hook: - return - self._trace_hook(state) - - def insert_doc_from_source(self, doc, source_gen, trans_id): - """Try to insert synced document from source. - - Conflicting documents are not inserted but will be sent over - to the sync source. - - It keeps track of progress by storing the document source - generation as well. - - The 1st step of a sync exchange is to call this repeatedly to - try insert all incoming documents from the source. - - :param doc: A Document object. - :param source_gen: The source generation of doc. - :return: None - """ - state, at_gen = self._db._put_doc_if_newer( - doc, save_conflict=False, - replica_uid=self.source_replica_uid, replica_gen=source_gen, - replica_trans_id=trans_id) - if state == 'inserted': - self.seen_ids[doc.doc_id] = at_gen - elif state == 'converged': - # magical convergence - self.seen_ids[doc.doc_id] = at_gen - elif state == 'superseded': - # we have something newer that we will return - pass - else: - # conflict that we will returne - assert state == 'conflicted' - # for tests - self._incoming_trace.append((doc.doc_id, doc.rev)) - self._db._last_exchange_log['receive'].update({ - 'source_uid': self.source_replica_uid, - 'source_gen': source_gen - }) - - def find_changes_to_return(self): - """Find changes to return. - - Find changes since last_known_generation in db generation - order using whats_changed. It excludes documents ids that have - already been considered (superseded by the sender, etc). - - :return: new_generation - the generation of this database - which the caller can consider themselves to be synchronized after - processing the returned documents. - """ - self._db._last_exchange_log['receive'].update({ # for tests - 'last_known_gen': self.source_last_known_generation - }) - self._trace('before whats_changed') - gen, trans_id, changes = self._db.whats_changed( - self.source_last_known_generation) - self._trace('after whats_changed') - self.new_gen = gen - self.new_trans_id = trans_id - seen_ids = self.seen_ids - # changed docs that weren't superseded by or converged with - self.changes_to_return = [ - (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes if - # there was a subsequent update - doc_id not in seen_ids or seen_ids.get(doc_id) < gen] - return self.new_gen - - def return_docs(self, return_doc_cb): - """Return the changed documents and their last change generation - repeatedly invoking the callback return_doc_cb. - - The final step of a sync exchange. - - :param: return_doc_cb(doc, gen, trans_id): is a callback - used to return the documents with their last change generation - to the target replica. - :return: None - """ - changes_to_return = self.changes_to_return - # return docs, including conflicts - changed_doc_ids = [doc_id for doc_id, _, _ in changes_to_return] - self._trace('before get_docs') - docs = self._db.get_docs( - changed_doc_ids, check_for_conflicts=False, include_deleted=True) - - docs_by_gen = izip( - docs, (gen for _, gen, _ in changes_to_return), - (trans_id for _, _, trans_id in changes_to_return)) - _outgoing_trace = [] # for tests - for doc, gen, trans_id in docs_by_gen: - return_doc_cb(doc, gen, trans_id) - _outgoing_trace.append((doc.doc_id, doc.rev)) - # for tests - self._db._last_exchange_log['return'] = { - 'docs': _outgoing_trace, - 'last_gen': self.new_gen} - - -class LocalSyncTarget(l2db.SyncTarget): - """Common sync target implementation logic for all local sync targets.""" - - def __init__(self, db): - self._db = db - self._trace_hook = None - - def sync_exchange(self, docs_by_generations, source_replica_uid, - last_known_generation, last_known_trans_id, - return_doc_cb, ensure_callback=None): - self._db.validate_gen_and_trans_id( - last_known_generation, last_known_trans_id) - sync_exch = SyncExchange( - self._db, source_replica_uid, last_known_generation) - if self._trace_hook: - sync_exch._set_trace_hook(self._trace_hook) - # 1st step: try to insert incoming docs and record progress - for doc, doc_gen, trans_id in docs_by_generations: - sync_exch.insert_doc_from_source(doc, doc_gen, trans_id) - # 2nd step: find changed documents (including conflicts) to return - new_gen = sync_exch.find_changes_to_return() - # final step: return docs and record source replica sync point - sync_exch.return_docs(return_doc_cb) - return new_gen, sync_exch.new_trans_id - - def _set_trace_hook(self, cb): - self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/vectorclock.py b/common/src/leap/soledad/common/l2db/vectorclock.py deleted file mode 100644 index 42bceaa8..00000000 --- a/common/src/leap/soledad/common/l2db/vectorclock.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""VectorClockRev helper class.""" - - -class VectorClockRev(object): - """Track vector clocks for multiple replica ids. - - This allows simple comparison to determine if one VectorClockRev is - newer/older/in-conflict-with another VectorClockRev without having to - examine history. Every replica has a strictly increasing revision. When - creating a new revision, they include all revisions for all other replicas - which the new revision dominates, and increment their own revision to - something greater than the current value. - """ - - def __init__(self, value): - self._values = self._expand(value) - - def __repr__(self): - s = self.as_str() - return '%s(%s)' % (self.__class__.__name__, s) - - def as_str(self): - s = '|'.join(['%s:%d' % (m, r) for m, r - in sorted(self._values.items())]) - return s - - def _expand(self, value): - result = {} - if value is None: - return result - for replica_info in value.split('|'): - replica_uid, counter = replica_info.split(':') - counter = int(counter) - result[replica_uid] = counter - return result - - def is_newer(self, other): - """Is this VectorClockRev strictly newer than other. - """ - if not self._values: - return False - if not other._values: - return True - this_is_newer = False - other_expand = dict(other._values) - for key, value in self._values.iteritems(): - if key in other_expand: - other_value = other_expand.pop(key) - if other_value > value: - return False - elif other_value < value: - this_is_newer = True - else: - this_is_newer = True - if other_expand: - return False - return this_is_newer - - def increment(self, replica_uid): - """Increase the 'replica_uid' section of this vector clock. - - :return: A string representing the new vector clock value - """ - self._values[replica_uid] = self._values.get(replica_uid, 0) + 1 - - def maximize(self, other_vcr): - for replica_uid, counter in other_vcr._values.iteritems(): - if replica_uid not in self._values: - self._values[replica_uid] = counter - else: - this_counter = self._values[replica_uid] - if this_counter < counter: - self._values[replica_uid] = counter diff --git a/common/src/leap/soledad/common/log.py b/common/src/leap/soledad/common/log.py deleted file mode 100644 index 59a47726..00000000 --- a/common/src/leap/soledad/common/log.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# log.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -This module centralizes logging facilities and allows for different behaviours, -as using the python logging module instead of twisted logger, and to print logs -to stdout, mainly for development purposes. -""" - - -import os -import sys -import time - -from twisted.logger import Logger -from twisted.logger import textFileLogObserver -from twisted.logger import LogLevel -from twisted.logger import InvalidLogLevelError -from twisted.python.failure import Failure - - -# What follows is a patched class to correctly log namespace and level when -# using the default formatter and --syslog option in twistd. This seems to be a -# known bug but it has not been reported to upstream yet. - -class SyslogLogger(Logger): - - def emit(self, level, format=None, **kwargs): - if level not in LogLevel.iterconstants(): - self.failure( - "Got invalid log level {invalidLevel!r} in {logger}.emit().", - Failure(InvalidLogLevelError(level)), - invalidLevel=level, - logger=self, - ) - return - - event = kwargs - event.update( - log_logger=self, log_level=level, log_namespace=self.namespace, - log_source=self.source, log_format=format, log_time=time.time(), - ) - - # ---------------------------------8<--------------------------------- - # this is a workaround for the mess between twisted's legacy log system - # and twistd's --syslog option. - event["system"] = "%s#%s" % (self.namespace, level.name) - # ---------------------------------8<--------------------------------- - - if "log_trace" in event: - event["log_trace"].append((self, self.observer)) - - self.observer(event) - - -def getLogger(*args, **kwargs): - - if os.environ.get('SOLEDAD_USE_PYTHON_LOGGING'): - import logging - return logging.getLogger(__name__) - - if os.environ.get('SOLEDAD_LOG_TO_STDOUT'): - kwargs({'observer': textFileLogObserver(sys.stdout)}) - - return SyslogLogger(*args, **kwargs) - - -__all__ = ['getLogger'] diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py deleted file mode 100644 index acebb77b..00000000 --- a/common/src/leap/soledad/common/tests/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Tests to make sure Soledad provides U1DB functionality and more. -""" - - -import os - - -def load_tests(): - """ - Build a test suite that includes all tests in leap.soledad.common.tests - but does not include tests in the u1db_tests/ subfolder. The reason for - not including those tests are: - - 1. they by themselves only test u1db functionality in the u1db module - (despite we use them as basis for testing soledad functionalities). - - 2. they would fail because we monkey patch u1db's remote http server - to add soledad functionality we need. - """ - import unittest - import glob - import imp - tests_prefix = os.path.join( - '.', 'src', 'leap', 'soledad', 'common', 'tests') - suite = unittest.TestSuite() - for testcase in glob.glob(os.path.join(tests_prefix, 'test_*.py')): - modname = os.path.basename(os.path.splitext(testcase)[0]) - f, pathname, description = imp.find_module(modname, [tests_prefix]) - module = imp.load_module(modname, f, pathname, description) - suite.addTest(unittest.TestLoader().loadTestsFromModule(module)) - return suite diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py deleted file mode 100644 index 2136bb8f..00000000 --- a/common/src/leap/soledad/common/tests/test_command.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -# test_command.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for command execution using a validator function for arguments. -""" -from twisted.trial import unittest -from leap.soledad.common.command import exec_validated_cmd - - -def validator(arg): - return True if arg is 'valid' else False - - -class ExecuteValidatedCommandTest(unittest.TestCase): - - def test_argument_validation(self): - status, out = exec_validated_cmd("command", "invalid arg", validator) - self.assertEquals(status, 1) - self.assertEquals(out, "invalid argument") - status, out = exec_validated_cmd("echo", "valid", validator) - self.assertEquals(status, 0) - self.assertEquals(out, "valid\n") - - def test_return_status_code_success(self): - status, out = exec_validated_cmd("echo", "arg") - self.assertEquals(status, 0) - self.assertEquals(out, "arg\n") - - def test_handle_command_with_spaces(self): - status, out = exec_validated_cmd("echo I am", "an argument") - self.assertEquals(status, 0, out) - self.assertEquals(out, "I am an argument\n") - - def test_handle_oserror_on_invalid_command(self): - status, out = exec_validated_cmd("inexistent command with", "args") - self.assertEquals(status, 1) - self.assertIn("No such file or directory", out) - - def test_return_status_code_number_on_failure(self): - status, out = exec_validated_cmd("ls", "user-bebacafe") - self.assertNotEquals(status, 0) - self.assertIn('No such file or directory\n', out) diff --git a/common/versioneer.py b/common/versioneer.py deleted file mode 100644 index 58339251..00000000 --- a/common/versioneer.py +++ /dev/null @@ -1,1774 +0,0 @@ - -# Version: 0.16 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -First, decide on values for the following configuration variables: - -* `VCS`: the version control system you use. Currently accepts "git". - -* `style`: the style of version string to be produced. See "Styles" below for - details. Defaults to "pep440", which looks like - `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. - -* `versionfile_source`: - - A project-relative pathname into which the generated version strings should - be written. This is usually a `_version.py` next to your project's main - `__init__.py` file, so it can be imported at runtime. If your project uses - `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. - This file should be checked in to your VCS as usual: the copy created below - by `setup.py setup_versioneer` will include code that parses expanded VCS - keywords in generated tarballs. The 'build' and 'sdist' commands will - replace it with a copy that has just the calculated version string. - - This must be set even if your project does not have any modules (and will - therefore never import `_version.py`), since "setup.py sdist" -based trees - still need somewhere to record the pre-calculated version strings. Anywhere - in the source tree should do. If there is a `__init__.py` next to your - `_version.py`, the `setup.py setup_versioneer` command (described below) - will append some `__version__`-setting assignments, if they aren't already - present. - -* `versionfile_build`: - - Like `versionfile_source`, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, - then you will probably have `versionfile_build='myproject/_version.py'` and - `versionfile_source='src/myproject/_version.py'`. - - If this is set to None, then `setup.py build` will not attempt to rewrite - any `_version.py` in the built tree. If your project does not have any - libraries (e.g. if it only builds a script), then you should use - `versionfile_build = None`. To actually use the computed version string, - your `setup.py` will need to override `distutils.command.build_scripts` - with a subclass that explicitly inserts a copy of - `versioneer.get_version()` into your script file. See - `test/demoapp-script-only/setup.py` for an example. - -* `tag_prefix`: - - a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. - If your tags look like 'myproject-1.2.0', then you should use - tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this - should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. - -* `parentdir_prefix`: - - a optional string, frequently the same as tag_prefix, which appears at the - start of all unpacked tarball filenames. If your tarball unpacks into - 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, - just omit the field from your `setup.cfg`. - -This tool provides one script, named `versioneer`. That script has one mode, -"install", which writes a copy of `versioneer.py` into the current directory -and runs `versioneer.py setup` to finish the installation. - -To versioneer-enable your project: - -* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and - populating it with the configuration values you decided earlier (note that - the option names are not case-sensitive): - - ```` - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - ```` - -* 2: Run `versioneer install`. This will do the following: - - * copy `versioneer.py` into the top of your source tree - * create `_version.py` in the right place (`versionfile_source`) - * modify your `__init__.py` (if one exists next to `_version.py`) to define - `__version__` (by calling a function from `_version.py`) - * modify your `MANIFEST.in` to include both `versioneer.py` and the - generated `_version.py` in sdist tarballs - - `versioneer install` will complain about any problems it finds with your - `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all - the problems. - -* 3: add a `import versioneer` to your setup.py, and add the following - arguments to the setup() call: - - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - -* 4: commit these changes to your VCS. To make sure you won't forget, - `versioneer install` will mark everything it touched for addition using - `git add`. Don't forget to add `setup.py` and `setup.cfg` too. - -## Post-Installation Usage - -Once established, all uses of your tree from a VCS checkout should get the -current version string. All generated tarballs should include an embedded -version string (so users who unpack them will not need a VCS tool installed). - -If you distribute your project through PyPI, then the release process should -boil down to two steps: - -* 1: git tag 1.0 -* 2: python setup.py register sdist upload - -If you distribute it through github (i.e. users use github to generate -tarballs with `git archive`), the process is: - -* 1: git tag 1.0 -* 2: git push; git push --tags - -Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at -least one tag in its history. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See details.md in the Versioneer source tree for -descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -### Upgrading to 0.16 - -Nothing special. - -### Upgrading to 0.15 - -Starting with this version, Versioneer is configured with a `[versioneer]` -section in your `setup.cfg` file. Earlier versions required the `setup.py` to -set attributes on the `versioneer` module immediately after import. The new -version will refuse to run (raising an exception during import) until you -have provided the necessary `setup.cfg` section. - -In addition, the Versioneer package provides an executable named -`versioneer`, and the installation process is driven by running `versioneer -install`. In 0.14 and earlier, the executable was named -`versioneer-installer` and was run without an argument. - -### Upgrading to 0.14 - -0.14 changes the format of the version string. 0.13 and earlier used -hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a -plus-separated "local version" section strings, with dot-separated -components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old -format, but should be ok with the new one. - -### Upgrading from 0.11 to 0.12 - -Nothing special. - -### Upgrading from 0.10 to 0.11 - -You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running -`setup.py setup_versioneer`. This will enable the use of additional -version-control systems (SVN, etc) in the future. - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - return None - return stdout -LONG_VERSION_PY['git'] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.16 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - return None - return stdout - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with " - "prefix '%%s'" %% (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs-tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, "..", ".git")): - if verbose: - print("no .git in %%s" %% root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree"} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version"} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, "..", ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-time keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.16) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json -import sys - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version"} - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - if vers["error"]: - print(" error: %s" % vers["error"]) - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-time keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) |