summaryrefslogtreecommitdiff
path: root/common
diff options
context:
space:
mode:
Diffstat (limited to 'common')
-rw-r--r--common/.gitattributes1
-rw-r--r--common/AUTHORS11
-rw-r--r--common/LICENSE619
-rw-r--r--common/MANIFEST.in9
-rw-r--r--common/changes/VERSION_COMPAT10
-rw-r--r--common/changes/next-changelog.rst28
-rw-r--r--common/pkg/__init__.py0
-rwxr-xr-xcommon/pkg/generate_wheels.sh13
-rwxr-xr-xcommon/pkg/pip_install_requirements.sh84
-rw-r--r--common/pkg/requirements-latest.pip4
-rw-r--r--common/pkg/requirements-leap.pip1
-rw-r--r--common/pkg/requirements.pip3
-rwxr-xr-xcommon/pkg/tools/get_authors.sh2
-rwxr-xr-xcommon/pkg/tools/with_venvwrapper.sh19
-rw-r--r--common/pkg/utils.py101
-rw-r--r--common/setup.cfg17
-rw-r--r--common/setup.py166
-rw-r--r--common/src/leap/__init__.py6
-rw-r--r--common/src/leap/soledad/__init__.py6
-rw-r--r--common/src/leap/soledad/common/README.txt70
-rw-r--r--common/src/leap/soledad/common/__init__.py47
-rw-r--r--common/src/leap/soledad/common/_version.py484
-rw-r--r--common/src/leap/soledad/common/backend.py642
-rw-r--r--common/src/leap/soledad/common/command.py55
-rw-r--r--common/src/leap/soledad/common/couch/__init__.py812
-rw-r--r--common/src/leap/soledad/common/couch/state.py158
-rw-r--r--common/src/leap/soledad/common/couch/support.py115
-rw-r--r--common/src/leap/soledad/common/crypto.py98
-rw-r--r--common/src/leap/soledad/common/document.py180
-rw-r--r--common/src/leap/soledad/common/errors.py103
-rw-r--r--common/src/leap/soledad/common/l2db/__init__.py694
-rw-r--r--common/src/leap/soledad/common/l2db/backends/__init__.py204
-rw-r--r--common/src/leap/soledad/common/l2db/backends/inmemory.py466
-rw-r--r--common/src/leap/soledad/common/l2db/errors.py194
-rw-r--r--common/src/leap/soledad/common/l2db/query_parser.py371
-rw-r--r--common/src/leap/soledad/common/l2db/remote/__init__.py15
-rw-r--r--common/src/leap/soledad/common/l2db/remote/http_app.py660
-rw-r--r--common/src/leap/soledad/common/l2db/remote/http_client.py178
-rw-r--r--common/src/leap/soledad/common/l2db/remote/http_database.py158
-rw-r--r--common/src/leap/soledad/common/l2db/remote/http_errors.py48
-rw-r--r--common/src/leap/soledad/common/l2db/remote/http_target.py125
-rw-r--r--common/src/leap/soledad/common/l2db/remote/server_state.py68
-rw-r--r--common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py65
-rw-r--r--common/src/leap/soledad/common/l2db/remote/utils.py23
-rw-r--r--common/src/leap/soledad/common/l2db/sync.py311
-rw-r--r--common/src/leap/soledad/common/l2db/vectorclock.py89
-rw-r--r--common/src/leap/soledad/common/log.py83
-rw-r--r--common/src/leap/soledad/common/tests/__init__.py50
-rw-r--r--common/src/leap/soledad/common/tests/test_command.py56
-rw-r--r--common/versioneer.py1774
50 files changed, 0 insertions, 9496 deletions
diff --git a/common/.gitattributes b/common/.gitattributes
deleted file mode 100644
index 1d59a395..00000000
--- a/common/.gitattributes
+++ /dev/null
@@ -1 +0,0 @@
-src/leap/soledad/common/_version.py export-subst
diff --git a/common/AUTHORS b/common/AUTHORS
deleted file mode 100644
index 9051b8c4..00000000
--- a/common/AUTHORS
+++ /dev/null
@@ -1,11 +0,0 @@
-drebs <drebs@leap.se>
-Tomás Touceda <chiiph@leap.se>
-Kali Kaneko <kali@leap.se>
-Ivan Alejandro <ivanalejandro0@gmail.com>
-Micah Anderson <micah@riseup.net>
-Victor Shyba <victor.shyba@gmail.com>
-Bruno Wagner <bwgpro@gmail.com>
-Ruben Pollan <meskio@sindominio.net>
-Duda Dornelles <ddornell@thoughtworks.com>
-antialias <antialias@leap.se>
-efkin <efkin@riseup.net>
diff --git a/common/LICENSE b/common/LICENSE
deleted file mode 100644
index bc08fe2e..00000000
--- a/common/LICENSE
+++ /dev/null
@@ -1,619 +0,0 @@
- GNU GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users. We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors. You can apply it to
-your programs, too.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
- For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
- Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
- For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
- Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so. This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software. The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable. Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products. If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
- Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary. To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Use with the GNU Affero General Public License.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time. Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
diff --git a/common/MANIFEST.in b/common/MANIFEST.in
deleted file mode 100644
index 93f46bbe..00000000
--- a/common/MANIFEST.in
+++ /dev/null
@@ -1,9 +0,0 @@
-include pkg/*
-include versioneer.py
-include LICENSE
-include CHANGELOG
-
-# What do we want the ddocs folder in the source package for? -- kali
-# it should be enough with having the compiled stuff.
-recursive-include src/leap/soledad/common/ddocs *
-include src/leap/soledad/common/_version.py
diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT
deleted file mode 100644
index cc00ecf7..00000000
--- a/common/changes/VERSION_COMPAT
+++ /dev/null
@@ -1,10 +0,0 @@
-#################################################
-# This file keeps track of the recent changes
-# introduced in internal leap dependencies.
-# Add your changes here so we can properly update
-# requirements.pip during the release process.
-# (leave header when resetting)
-#################################################
-#
-# BEGIN DEPENDENCY LIST -------------------------
-# leap.foo.bar>=x.y.z
diff --git a/common/changes/next-changelog.rst b/common/changes/next-changelog.rst
deleted file mode 100644
index 64162b7b..00000000
--- a/common/changes/next-changelog.rst
+++ /dev/null
@@ -1,28 +0,0 @@
-0.8.2 - ...
-+++++++++++++++++++++++++++++++
-
-Please add lines to this file, they will be moved to the CHANGELOG.rst during
-the next release.
-
-There are two template lines for each category, use them as reference.
-
-I've added a new category `Misc` so we can track doc/style/packaging stuff.
-
-Features
-~~~~~~~~
-- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
-- New feature without related issue number.
-
-Bugfixes
-~~~~~~~~
-- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
-- Bugfix without related issue number.
-
-Misc
-~~~~
-- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
-- Some change without issue number.
-
-Known Issues
-~~~~~~~~~~~~
-- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236.
diff --git a/common/pkg/__init__.py b/common/pkg/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/common/pkg/__init__.py
+++ /dev/null
diff --git a/common/pkg/generate_wheels.sh b/common/pkg/generate_wheels.sh
deleted file mode 100755
index a13e2c7a..00000000
--- a/common/pkg/generate_wheels.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-# Generate wheels for dependencies
-# Use at your own risk.
-
-if [ "$WHEELHOUSE" = "" ]; then
- WHEELHOUSE=$HOME/wheelhouse
-fi
-
-pip wheel --wheel-dir $WHEELHOUSE pip
-pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip
-if [ -f pkg/requirements-testing.pip ]; then
- pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip
-fi
diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh
deleted file mode 100755
index f4b5f67a..00000000
--- a/common/pkg/pip_install_requirements.sh
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/bin/bash
-# Update pip and install LEAP base/testing requirements.
-# For convenience, $insecure_packages are allowed with insecure flags enabled.
-# Use at your own risk.
-# See $usage for help
-
-insecure_packages=""
-leap_wheelhouse=https://lizard.leap.se/wheels
-
-show_help() {
- usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n
-\t\t\tOtherwise, it will install requirements.pip\n
---use-leap-wheels\tUse wheels from leap.se"
- echo -e $usage
-
- exit 1
-}
-
-process_arguments() {
- testing=false
- while [ "$#" -gt 0 ]; do
- # From http://stackoverflow.com/a/31443098
- case "$1" in
- --help) show_help;;
- --testing) testing=true; shift 1;;
- --use-leap-wheels) use_leap_wheels=true; shift 1;;
-
- -h) show_help;;
- -*) echo "unknown option: $1" >&2; exit 1;;
- esac
- done
-}
-
-return_wheelhouse() {
- if $use_leap_wheels ; then
- WHEELHOUSE=$leap_wheelhouse
- elif [ "$WHEELHOUSE" = "" ]; then
- WHEELHOUSE=$HOME/wheelhouse
- fi
-
- # Tested with bash and zsh
- if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then
- mkdir $WHEELHOUSE
- fi
-
- echo "$WHEELHOUSE"
-}
-
-return_install_options() {
- wheelhouse=`return_wheelhouse`
- install_options="-U --find-links=$wheelhouse"
- if $use_leap_wheels ; then
- install_options="$install_options --trusted-host lizard.leap.se"
- fi
-
- echo $install_options
-}
-
-return_insecure_flags() {
- for insecure_package in $insecure_packages; do
- flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package"
- done
-
- echo $flags
-}
-
-return_packages() {
- if $testing ; then
- packages="-r pkg/requirements-testing.pip"
- else
- packages="-r pkg/requirements.pip"
- fi
-
- echo $packages
-}
-
-process_arguments $@
-install_options=`return_install_options`
-insecure_flags=`return_insecure_flags`
-packages=`return_packages`
-
-pip install -U wheel
-pip install -U pip
-pip install $install_options $insecure_flags $packages
diff --git a/common/pkg/requirements-latest.pip b/common/pkg/requirements-latest.pip
deleted file mode 100644
index 852f2433..00000000
--- a/common/pkg/requirements-latest.pip
+++ /dev/null
@@ -1,4 +0,0 @@
---index-url https://pypi.python.org/simple/
-
--e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common'
--e .
diff --git a/common/pkg/requirements-leap.pip b/common/pkg/requirements-leap.pip
deleted file mode 100644
index b311859e..00000000
--- a/common/pkg/requirements-leap.pip
+++ /dev/null
@@ -1 +0,0 @@
-leap.common>=0.4.0
diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip
deleted file mode 100644
index 29506cf0..00000000
--- a/common/pkg/requirements.pip
+++ /dev/null
@@ -1,3 +0,0 @@
-paste
-routes
-six \ No newline at end of file
diff --git a/common/pkg/tools/get_authors.sh b/common/pkg/tools/get_authors.sh
deleted file mode 100755
index 0169bb17..00000000
--- a/common/pkg/tools/get_authors.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2-
diff --git a/common/pkg/tools/with_venvwrapper.sh b/common/pkg/tools/with_venvwrapper.sh
deleted file mode 100755
index b62bc10f..00000000
--- a/common/pkg/tools/with_venvwrapper.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-#Wraps a command in a virtualenwrapper passed as first argument.
-#Example:
-#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh
-
-wd=`pwd`
-alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"'
-
-source `which virtualenvwrapper.sh`
-echo "Activating virtualenv " $1
-echo "------------------------------------"
-workon $1
-cd $wd
-echo "running version: "
-echo `pyver leap.bitmask`
-echo `pyver leap.soledad.common`
-echo `pyver leap.keymanager`
-$2 $3 $4 $5
diff --git a/common/pkg/utils.py b/common/pkg/utils.py
deleted file mode 100644
index d1680102..00000000
--- a/common/pkg/utils.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-# utils.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Utils to help in the setup process
-"""
-import os
-import re
-import sys
-
-
-def is_develop_mode():
- """
- Returns True if we're calling the setup script using the argument for
- setuptools development mode.
-
- This avoids messing up with dependency pinning and order, the
- responsibility of installing the leap dependencies is left to the
- developer.
- """
- args = sys.argv
- devflags = "setup.py", "develop"
- if (args[0], args[1]) == devflags:
- return True
- return False
-
-
-def get_reqs_from_files(reqfiles):
- """
- Returns the contents of the top requirement file listed as a
- string list with the lines.
-
- @param reqfiles: requirement files to parse
- @type reqfiles: list of str
- """
- for reqfile in reqfiles:
- if os.path.isfile(reqfile):
- return open(reqfile, 'r').read().split('\n')
-
-
-def parse_requirements(reqfiles=['requirements.txt',
- 'requirements.pip',
- 'pkg/requirements.pip']):
- """
- Parses the requirement files provided.
-
- The passed reqfiles list is a list of possible locations to try, the
- function will return the contents of the first path found.
-
- Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should
- return PySide as a dep or not. Don't set, or set to 0 if you want
- to install it through pip.
-
- @param reqfiles: requirement files to parse
- @type reqfiles: list of str
- """
-
- requirements = []
- skip_pyside = os.getenv("LEAP_VENV_SKIP_PYSIDE", "0") != "0"
- for line in get_reqs_from_files(reqfiles):
- # -e git://foo.bar/baz/master#egg=foobar
- if re.match(r'\s*-e\s+', line):
- pass
- # do not try to do anything with externals on vcs
- # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
- # line))
- # http://foo.bar/baz/foobar/zipball/master#egg=foobar
- elif re.match(r'\s*https?:', line):
- requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
- line))
- # -f lines are for index locations, and don't get used here
- elif re.match(r'\s*-f\s+', line):
- pass
-
- # argparse is part of the standard library starting with 2.7
- # adding it to the requirements list screws distro installs
- elif line == 'argparse' and sys.version_info >= (2, 7):
- pass
- elif line == 'PySide' and skip_pyside:
- pass
- # do not include comments
- elif line.lstrip().startswith('#'):
- pass
- else:
- if line != '':
- requirements.append(line)
-
- return requirements
diff --git a/common/setup.cfg b/common/setup.cfg
deleted file mode 100644
index ed06f60d..00000000
--- a/common/setup.cfg
+++ /dev/null
@@ -1,17 +0,0 @@
-[aliases]
-test = trial
-
-[pep8]
-exclude = versioneer.py,_version.py,ddocs.py,*.egg,build,docs
-ignore = E731
-
-[flake8]
-exclude = versioneer.py,_version.py,ddocs.py,*.egg,build,docs
-ignore = E731
-
-[versioneer]
-VCS = git
-style = pep440
-versionfile_source = src/leap/soledad/common/_version.py
-versionfile_build = leap/soledad/common/_version.py
-tag_prefix =
diff --git a/common/setup.py b/common/setup.py
deleted file mode 100644
index 6de26034..00000000
--- a/common/setup.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# -*- coding: utf-8 -*-
-# setup.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-setup file for leap.soledad.common
-"""
-import re
-
-from setuptools import setup
-from setuptools import find_packages
-from setuptools import Command
-from setuptools.command.develop import develop as _cmd_develop
-import versioneer
-
-from pkg import utils
-
-
-trove_classifiers = (
- "Development Status :: 3 - Alpha",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: "
- "GNU General Public License v3 or later (GPLv3+)",
- "Environment :: Console",
- "Operating System :: OS Independent",
- "Operating System :: POSIX",
- "Programming Language :: Python :: 2.6",
- "Programming Language :: Python :: 2.7",
- "Topic :: Database :: Front-Ends",
- "Topic :: Software Development :: Libraries :: Python Modules"
-)
-
-DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/'
- 'archive/%s.tar.gz')
-_versions = versioneer.get_versions()
-VERSION = _versions['version']
-VERSION_REVISION = _versions['full-revisionid']
-DOWNLOAD_URL = ""
-
-# get the short version for the download url
-_version_short = re.findall('\d+\.\d+\.\d+', VERSION)
-if len(_version_short) > 0:
- VERSION_SHORT = _version_short[0]
- DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
-
-
-class freeze_debianver(Command):
-
- """
- Freezes the version in a debian branch.
- To be used after merging the development branch onto the debian one.
- """
- user_options = []
- template = r"""
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.16) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-import json
-import sys
-
-version_json = '''
-{
- "dirty": false,
- "error": null,
- "full-revisionid": "FULL_REVISIONID",
- "version": "VERSION_STRING"
-}
-''' # END VERSION_JSON
-
-def get_versions():
- return json.loads(version_json)
-"""
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- proceed = str(raw_input(
- "This will overwrite the file _version.py. Continue? [y/N] "))
- if proceed != "y":
- print("He. You scared. Aborting.")
- return
- subst_template = self.template.replace(
- 'VERSION_STRING', VERSION_SHORT).replace(
- 'FULL_REVISIONID', VERSION_REVISION)
- versioneer_cfg = versioneer.get_config_from_root('.')
- with open(versioneer_cfg.versionfile_source, 'w') as f:
- f.write(subst_template)
-
-
-class cmd_develop(_cmd_develop):
- def run(self):
- # versioneer:
- versions = versioneer.get_versions(verbose=True)
- self._versioneer_generated_versions = versions
- # unless we update this, the command will keep using the old version
- self.distribution.metadata.version = versions["version"]
- _cmd_develop.run(self)
-
-
-cmdclass = versioneer.get_cmdclass()
-cmdclass["freeze_debianver"] = freeze_debianver
-cmdclass["develop"] = cmd_develop
-
-
-# XXX add ref to docs
-
-requirements = utils.parse_requirements()
-
-if utils.is_develop_mode():
- print
- print("[WARNING] Skipping leap-specific dependencies "
- "because development mode is detected.")
- print("[WARNING] You can install "
- "the latest published versions with "
- "'pip install -r pkg/requirements-leap.pip'")
- print("[WARNING] Or you can instead do 'python setup.py develop' "
- "from the parent folder of each one of them.")
- print
-else:
- requirements += utils.parse_requirements(
- reqfiles=["pkg/requirements-leap.pip"])
-
-setup(
- name='leap.soledad.common',
- version=VERSION,
- cmdclass=cmdclass,
- url='https://leap.se/',
- download_url=DOWNLOAD_URL,
- license='GPLv3+',
- description='Synchronization of locally encrypted data among devices '
- '(common files).',
- author='The LEAP Encryption Access Project',
- author_email='info@leap.se',
- maintainer='Kali Kaneko',
- maintainer_email='kali@leap.se',
- long_description=(
- "Soledad is the part of LEAP that allows application data to be "
- "securely shared among devices. It provides, to other parts of the "
- "LEAP project, an API for data storage and sync."
- ),
- classifiers=trove_classifiers,
- namespace_packages=["leap", "leap.soledad"],
- packages=find_packages('src', exclude=['*.tests', '*.tests.*']),
- package_dir={'': 'src'},
- test_suite='leap.soledad.common.tests',
- install_requires=requirements,
-)
diff --git a/common/src/leap/__init__.py b/common/src/leap/__init__.py
deleted file mode 100644
index f48ad105..00000000
--- a/common/src/leap/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
-try:
- __import__('pkg_resources').declare_namespace(__name__)
-except ImportError:
- from pkgutil import extend_path
- __path__ = extend_path(__path__, __name__)
diff --git a/common/src/leap/soledad/__init__.py b/common/src/leap/soledad/__init__.py
deleted file mode 100644
index f48ad105..00000000
--- a/common/src/leap/soledad/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
-try:
- __import__('pkg_resources').declare_namespace(__name__)
-except ImportError:
- from pkgutil import extend_path
- __path__ = extend_path(__path__, __name__)
diff --git a/common/src/leap/soledad/common/README.txt b/common/src/leap/soledad/common/README.txt
deleted file mode 100644
index 0a252650..00000000
--- a/common/src/leap/soledad/common/README.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-Soledad common package
-======================
-
-This package contains Soledad bits used by both server and client.
-
-Couch L2DB Backend
-------------------
-
-L2DB backends rely on some atomic operations that modify documents contents
-and metadata (conflicts, transaction ids and indexes). The only atomic
-operation in Couch is a document put, so every u1db atomic operation has to be
-mapped to a couch document put.
-
-The atomic operations in the U1DB SQLite reference backend implementation may
-be identified by the use of a context manager to access the underlying
-database. A listing of the methods involved in each atomic operation are
-depiced below. The top-level elements correpond to the atomic operations that
-have to be mapped, and items on deeper levels of the list have to be
-implemented in a way that all changes will be pushed with just one operation.
-
- * _set_replica_uid
- * put_doc:
- * _get_doc
- * _put_and_update_indexes
- * insert/update the document
- * insert into transaction log
- * delete_doc
- * _get_doc
- * _put_and_update_indexes
- * get_doc_conflicts
- * _get_conflicts
- * _set_replica_gen_and_trans_id
- * _do_set_replica_gen_and_trans_id
- * _put_doc_if_newer
- * _get_doc
- * _validate_source (**)
- * _get_replica_gen_and_trans_id
- * cases:
- * is newer:
- * _prune_conflicts (**)
- * _has_conflicts
- * _delete_conflicts
- * _put_and_update_indexes
- * same content as:
- * _put_and_update_indexes
- * conflicted:
- * _force_doc_sync_conflict
- * _prune_conflicts
- * _add_conflict
- * _put_and_update_indexes
- * _do_set_replica_gen_and_trans_id
- * resolve_doc
- * _get_doc
- * cases:
- * doc is superseded
- * _put_and_update_indexes
- * else
- * _add_conflict
- * _delete_conflicts
- * delete_index
- * create_index
-
-Notes:
-
- * Currently, the couch backend does not implement indexing, so what is
- depicted as `_put_and_update_indexes` above will be found as `_put_doc` in
- the backend.
-
- * Conflict updates are part of document put using couch update functions,
- and as such are part of the same atomic operation as document put.
diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py
deleted file mode 100644
index 4948ad20..00000000
--- a/common/src/leap/soledad/common/__init__.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from leap.common.check import leap_assert as soledad_assert
-from leap.common.check import leap_assert_type as soledad_assert_type
-
-from ._version import get_versions
-
-"""
-Soledad routines common to client and server.
-"""
-
-
-#
-# Global constants
-#
-
-SHARED_DB_NAME = 'shared'
-
-
-#
-# Global functions
-#
-
-__version__ = get_versions()['version']
-del get_versions
-
-
-__all__ = [
- "soledad_assert",
- "soledad_assert_type",
- "__version__",
-]
diff --git a/common/src/leap/soledad/common/_version.py b/common/src/leap/soledad/common/_version.py
deleted file mode 100644
index 1168a88d..00000000
--- a/common/src/leap/soledad/common/_version.py
+++ /dev/null
@@ -1,484 +0,0 @@
-
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.16 (https://github.com/warner/python-versioneer)
-
-"""Git implementation of _version.py."""
-
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
- """Get the keywords needed to look up the version information."""
- # these strings will be replaced by git during git-archive.
- # setup.py/versioneer.py will grep for the variable names, so they must
- # each be defined on a line of their own. _version.py will just call
- # get_keywords().
- git_refnames = "$Format:%d$"
- git_full = "$Format:%H$"
- keywords = {"refnames": git_refnames, "full": git_full}
- return keywords
-
-
-class VersioneerConfig:
- """Container for Versioneer configuration parameters."""
-
-
-def get_config():
- """Create, populate and return the VersioneerConfig() object."""
- # these strings are filled in when 'setup.py versioneer' creates
- # _version.py
- cfg = VersioneerConfig()
- cfg.VCS = "git"
- cfg.style = "pep440"
- cfg.tag_prefix = ""
- cfg.parentdir_prefix = "None"
- cfg.versionfile_source = "src/leap/soledad/common/_version.py"
- cfg.verbose = False
- return cfg
-
-
-class NotThisMethod(Exception):
- """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method): # decorator
- """Decorator to mark a method as the handler for a particular VCS."""
- def decorate(f):
- """Store f in HANDLERS[vcs][method]."""
- if vcs not in HANDLERS:
- HANDLERS[vcs] = {}
- HANDLERS[vcs][method] = f
- return f
- return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
- """Call the given command(s)."""
- assert isinstance(commands, list)
- p = None
- for c in commands:
- try:
- dispcmd = str([c] + args)
- # remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
- stderr=(subprocess.PIPE if hide_stderr
- else None))
- break
- except EnvironmentError:
- e = sys.exc_info()[1]
- if e.errno == errno.ENOENT:
- continue
- if verbose:
- print("unable to run %s" % dispcmd)
- print(e)
- return None
- else:
- if verbose:
- print("unable to find command, tried %s" % (commands,))
- return None
- stdout = p.communicate()[0].strip()
- if sys.version_info[0] >= 3:
- stdout = stdout.decode()
- if p.returncode != 0:
- if verbose:
- print("unable to run %s (error)" % dispcmd)
- return None
- return stdout
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
- """Try to determine the version from the parent directory name.
-
- Source tarballs conventionally unpack into a directory that includes
- both the project name and a version string.
- """
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%s', but '%s' doesn't start with "
- "prefix '%s'" % (root, dirname, parentdir_prefix))
- raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
- return {"version": dirname[len(parentdir_prefix):],
- "full-revisionid": None,
- "dirty": False, "error": None}
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
- """Extract version information from the given file."""
- # the code embedded in _version.py can just fetch the value of these
- # keywords. When used from setup.py, we don't want to import _version.py,
- # so we do it with a regexp instead. This function is not used from
- # _version.py.
- keywords = {}
- try:
- f = open(versionfile_abs, "r")
- for line in f.readlines():
- if line.strip().startswith("git_refnames ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["refnames"] = mo.group(1)
- if line.strip().startswith("git_full ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["full"] = mo.group(1)
- f.close()
- except EnvironmentError:
- pass
- return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
- """Get version information from git keywords."""
- if not keywords:
- raise NotThisMethod("no keywords at all, weird")
- refnames = keywords["refnames"].strip()
- if refnames.startswith("$Format"):
- if verbose:
- print("keywords are unexpanded, not using")
- raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
- refs = set([r.strip() for r in refnames.strip("()").split(",")])
- # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
- # just "foo-1.0". If we see a "tag: " prefix, prefer those.
- TAG = "tag: "
- tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
- if not tags:
- # Either we're using git < 1.8.3, or there really are no tags. We use
- # a heuristic: assume all version tags have a digit. The old git %d
- # expansion behaves like git log --decorate=short and strips out the
- # refs/heads/ and refs/tags/ prefixes that would let us distinguish
- # between branches and tags. By ignoring refnames without digits, we
- # filter out many common branch names like "release" and
- # "stabilization", as well as "HEAD" and "master".
- tags = set([r for r in refs if re.search(r'\d', r)])
- if verbose:
- print("discarding '%s', no digits" % ",".join(refs-tags))
- if verbose:
- print("likely tags: %s" % ",".join(sorted(tags)))
- for ref in sorted(tags):
- # sorting will prefer e.g. "2.0" over "2.0rc1"
- if ref.startswith(tag_prefix):
- r = ref[len(tag_prefix):]
- if verbose:
- print("picking %s" % r)
- return {"version": r,
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": None
- }
- # no suitable tags, so version is "0+unknown", but full hex is still there
- if verbose:
- print("no suitable tags, using unknown + full revision id")
- return {"version": "0+unknown",
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": "no suitable tags"}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
- """Get version from 'git describe' in the root of the source tree.
-
- This only gets called if the git-archive 'subst' keywords were *not*
- expanded, and _version.py hasn't already been rewritten with a short
- version string, meaning we're inside a checked out source tree.
- """
- if not os.path.exists(os.path.join(root, ".git")):
- if verbose:
- print("no .git in %s" % root)
- raise NotThisMethod("no .git directory")
-
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
- # if there isn't one, this yields HEX[-dirty] (no NUM)
- describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long",
- "--match", "%s*" % tag_prefix],
- cwd=root)
- # --long was added in git-1.5.5
- if describe_out is None:
- raise NotThisMethod("'git describe' failed")
- describe_out = describe_out.strip()
- full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if full_out is None:
- raise NotThisMethod("'git rev-parse' failed")
- full_out = full_out.strip()
-
- pieces = {}
- pieces["long"] = full_out
- pieces["short"] = full_out[:7] # maybe improved later
- pieces["error"] = None
-
- # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
- # TAG might have hyphens.
- git_describe = describe_out
-
- # look for -dirty suffix
- dirty = git_describe.endswith("-dirty")
- pieces["dirty"] = dirty
- if dirty:
- git_describe = git_describe[:git_describe.rindex("-dirty")]
-
- # now we have TAG-NUM-gHEX or HEX
-
- if "-" in git_describe:
- # TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- pieces["error"] = ("unable to parse git-describe output: '%s'"
- % describe_out)
- return pieces
-
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%s' doesn't start with prefix '%s'"
- print(fmt % (full_tag, tag_prefix))
- pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
- % (full_tag, tag_prefix))
- return pieces
- pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
- # distance: number of commits since tag
- pieces["distance"] = int(mo.group(2))
-
- # commit: short hex revision ID
- pieces["short"] = mo.group(3)
-
- else:
- # HEX: no tags
- pieces["closest-tag"] = None
- count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
- cwd=root)
- pieces["distance"] = int(count_out) # total number of commits
-
- return pieces
-
-
-def plus_or_dot(pieces):
- """Return a + if we don't already have one, else return a ."""
- if "+" in pieces.get("closest-tag", ""):
- return "."
- return "+"
-
-
-def render_pep440(pieces):
- """Build up version string, with post-release "local version identifier".
-
- Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
- get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
- Exceptions:
- 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += plus_or_dot(pieces)
- rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- else:
- # exception #1
- rendered = "0+untagged.%d.g%s" % (pieces["distance"],
- pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- return rendered
-
-
-def render_pep440_pre(pieces):
- """TAG[.post.devDISTANCE] -- No -dirty.
-
- Exceptions:
- 1: no tags. 0.post.devDISTANCE
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += ".post.dev%d" % pieces["distance"]
- else:
- # exception #1
- rendered = "0.post.dev%d" % pieces["distance"]
- return rendered
-
-
-def render_pep440_post(pieces):
- """TAG[.postDISTANCE[.dev0]+gHEX] .
-
- The ".dev0" means dirty. Note that .dev0 sorts backwards
- (a dirty tree will appear "older" than the corresponding clean one),
- but you shouldn't be releasing software with -dirty anyways.
-
- Exceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += plus_or_dot(pieces)
- rendered += "g%s" % pieces["short"]
- else:
- # exception #1
- rendered = "0.post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += "+g%s" % pieces["short"]
- return rendered
-
-
-def render_pep440_old(pieces):
- """TAG[.postDISTANCE[.dev0]] .
-
- The ".dev0" means dirty.
-
- Eexceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- else:
- # exception #1
- rendered = "0.post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- return rendered
-
-
-def render_git_describe(pieces):
- """TAG[-DISTANCE-gHEX][-dirty].
-
- Like 'git describe --tags --dirty --always'.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render_git_describe_long(pieces):
- """TAG-DISTANCE-gHEX[-dirty].
-
- Like 'git describe --tags --dirty --always -long'.
- The distance/hash is unconditional.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render(pieces, style):
- """Render the given version pieces into the requested style."""
- if pieces["error"]:
- return {"version": "unknown",
- "full-revisionid": pieces.get("long"),
- "dirty": None,
- "error": pieces["error"]}
-
- if not style or style == "default":
- style = "pep440" # the default
-
- if style == "pep440":
- rendered = render_pep440(pieces)
- elif style == "pep440-pre":
- rendered = render_pep440_pre(pieces)
- elif style == "pep440-post":
- rendered = render_pep440_post(pieces)
- elif style == "pep440-old":
- rendered = render_pep440_old(pieces)
- elif style == "git-describe":
- rendered = render_git_describe(pieces)
- elif style == "git-describe-long":
- rendered = render_git_describe_long(pieces)
- else:
- raise ValueError("unknown style '%s'" % style)
-
- return {"version": rendered, "full-revisionid": pieces["long"],
- "dirty": pieces["dirty"], "error": None}
-
-
-def get_versions():
- """Get version information or return default if unable to do so."""
- # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
- # __file__, we can work backwards from there to the root. Some
- # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
- # case we can only use expanded keywords.
-
- cfg = get_config()
- verbose = cfg.verbose
-
- try:
- return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
- verbose)
- except NotThisMethod:
- pass
-
- try:
- root = os.path.realpath(__file__)
- # versionfile_source is the relative path from the top of the source
- # tree (where the .git directory might live) to this file. Invert
- # this to find the root from __file__.
- for i in cfg.versionfile_source.split('/'):
- root = os.path.dirname(root)
- except NameError:
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to find root of source tree"}
-
- try:
- pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
- return render(pieces, cfg.style)
- except NotThisMethod:
- pass
-
- try:
- if cfg.parentdir_prefix:
- return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
- except NotThisMethod:
- pass
-
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to compute version"}
diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
deleted file mode 100644
index 4a29ca87..00000000
--- a/common/src/leap/soledad/common/backend.py
+++ /dev/null
@@ -1,642 +0,0 @@
-# -*- coding: utf-8 -*-
-# backend.py
-# Copyright (C) 2015 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""A L2DB generic backend."""
-
-import functools
-
-from leap.soledad.common.document import ServerDocument
-from leap.soledad.common.l2db import vectorclock
-from leap.soledad.common.l2db.errors import (
- RevisionConflict,
- InvalidDocId,
- ConflictedDoc,
- DocumentDoesNotExist,
- DocumentAlreadyDeleted,
-)
-from leap.soledad.common.l2db.backends import CommonBackend
-from leap.soledad.common.l2db.backends import CommonSyncTarget
-
-
-class SoledadBackend(CommonBackend):
- BATCH_SUPPORT = False
-
- """
- A L2DB backend implementation.
- """
-
- def __init__(self, database, replica_uid=None):
- """
- Create a new backend.
-
- :param database: the database implementation
- :type database: Database
- :param replica_uid: an optional unique replica identifier
- :type replica_uid: str
- """
- # save params
- self._factory = ServerDocument
- self._real_replica_uid = None
- self._cache = None
- self._dbname = database._dbname
- self._database = database
- self.batching = False
- if replica_uid is not None:
- self._set_replica_uid(replica_uid)
-
- def batch_start(self):
- if not self.BATCH_SUPPORT:
- return
- self.batching = True
- self.after_batch_callbacks = {}
- self._database.batch_start()
- if not self._cache:
- # batching needs cache
- self._cache = {}
- self._get_generation() # warm up gen info
-
- def batch_end(self):
- if not self.BATCH_SUPPORT:
- return
- self._database.batch_end()
- self.batching = False
- for name in self.after_batch_callbacks:
- self.after_batch_callbacks[name]()
- self.after_batch_callbacks = None
-
- @property
- def cache(self):
- if self._cache is not None:
- return self._cache
- else:
- return {}
-
- def init_caching(self, cache):
- """
- Start using cache by setting internal _cache attribute.
-
- :param cache: the cache instance, anything that behaves like a dict
- :type cache: dict
- """
- self._cache = cache
-
- def get_sync_target(self):
- """
- Return a SyncTarget object, for another u1db to synchronize with.
-
- :return: The sync target.
- :rtype: SoledadSyncTarget
- """
- return SoledadSyncTarget(self)
-
- def delete_database(self):
- """
- Delete a U1DB database.
- """
- self._database.delete_database()
-
- def close(self):
- """
- Release any resources associated with this database.
-
- :return: True if db was succesfully closed.
- :rtype: bool
- """
- self._database.close()
- return True
-
- def __del__(self):
- """
- Close the database upon garbage collection.
- """
- self.close()
-
- def _set_replica_uid(self, replica_uid):
- """
- Force the replica uid to be set.
-
- :param replica_uid: The new replica uid.
- :type replica_uid: str
- """
- self._database.set_replica_uid(replica_uid)
- self._real_replica_uid = replica_uid
- self.cache['replica_uid'] = self._real_replica_uid
-
- def _get_replica_uid(self):
- """
- Get the replica uid.
-
- :return: The replica uid.
- :rtype: str
- """
- if self._real_replica_uid is not None:
- self.cache['replica_uid'] = self._real_replica_uid
- return self._real_replica_uid
- if 'replica_uid' in self.cache:
- return self.cache['replica_uid']
- self._real_replica_uid = self._database.get_replica_uid()
- self._set_replica_uid(self._real_replica_uid)
- return self._real_replica_uid
-
- _replica_uid = property(_get_replica_uid, _set_replica_uid)
-
- replica_uid = property(_get_replica_uid)
-
- def _get_generation(self):
- """
- Return the current generation.
-
- :return: The current generation.
- :rtype: int
-
- :raise SoledadError: Raised by database on operation failure
- """
- return self._get_generation_info()[0]
-
- def _get_generation_info(self):
- """
- Return the current generation.
-
- :return: A tuple containing the current generation and transaction id.
- :rtype: (int, str)
-
- :raise SoledadError: Raised by database on operation failure
- """
- cur_gen, newest_trans_id = self._database.get_generation_info()
- return (cur_gen, newest_trans_id)
-
- def _get_trans_id_for_gen(self, generation):
- """
- Get the transaction id corresponding to a particular generation.
-
- :param generation: The generation for which to get the transaction id.
- :type generation: int
-
- :return: The transaction id for C{generation}.
- :rtype: str
-
- :raise InvalidGeneration: Raised when the generation does not exist.
-
- """
- return self._database.get_trans_id_for_gen(generation)
-
- def _get_transaction_log(self):
- """
- This is only for the test suite, it is not part of the api.
-
- :return: The complete transaction log.
- :rtype: [(str, str)]
-
- """
- return self._database.get_transaction_log()
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- """
- Extract the document from storage.
-
- This can return None if the document doesn't exist.
-
- :param doc_id: The unique document identifier
- :type doc_id: str
- :param check_for_conflicts: If set to False, then the conflict check
- will be skipped.
- :type check_for_conflicts: bool
-
- :return: The document.
- :rtype: ServerDocument
- """
- return self._database.get_doc(doc_id, check_for_conflicts)
-
- def get_doc(self, doc_id, include_deleted=False):
- """
- Get the JSON string for the given document.
-
- :param doc_id: The unique document identifier
- :type doc_id: str
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- :type include_deleted: bool
-
- :return: A document object.
- :rtype: ServerDocument.
- """
- doc = self._get_doc(doc_id, check_for_conflicts=True)
- if doc is None:
- return None
- if doc.is_tombstone() and not include_deleted:
- return None
- return doc
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted
- documents will not be included in the results.
- :type include_deleted: bool
-
- :return: (generation, [ServerDocument])
- The current generation of the database, followed by a list of all
- the documents in the database.
- :rtype: (int, [ServerDocument])
- """
- return self._database.get_all_docs(include_deleted)
-
- def _put_doc(self, old_doc, doc):
- """
- Put the document in the backend database.
-
- Note that C{old_doc} must have been fetched with the parameter
- C{check_for_conflicts} equal to True, so we can properly update the
- new document using the conflict information from the old one.
-
- :param old_doc: The old document version.
- :type old_doc: ServerDocument
- :param doc: The document to be put.
- :type doc: ServerDocument
- """
- self._database.save_document(old_doc, doc,
- self._allocate_transaction_id())
-
- def put_doc(self, doc):
- """
- Update a document.
-
- If the document currently has conflicts, put will fail.
- If the database specifies a maximum document size and the document
- exceeds it, put will fail and raise a DocumentTooBig exception.
-
- :param doc: A Document with new content.
- :return: new_doc_rev - The new revision identifier for the document.
- The Document object will also be updated.
-
- :raise InvalidDocId: Raised if the document's id is invalid.
- :raise DocumentTooBig: Raised if the document size is too big.
- :raise ConflictedDoc: Raised if the document has conflicts.
- """
- if doc.doc_id is None:
- raise InvalidDocId()
- self._check_doc_id(doc.doc_id)
- self._check_doc_size(doc)
- old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if old_doc and old_doc.has_conflicts:
- raise ConflictedDoc()
- if old_doc and doc.rev is None and old_doc.is_tombstone():
- new_rev = self._allocate_doc_rev(old_doc.rev)
- else:
- if old_doc is not None:
- if old_doc.rev != doc.rev:
- raise RevisionConflict()
- else:
- if doc.rev is not None:
- raise RevisionConflict()
- new_rev = self._allocate_doc_rev(doc.rev)
- doc.rev = new_rev
- self._put_doc(old_doc, doc)
- return new_rev
-
- def whats_changed(self, old_generation=0):
- """
- Return a list of documents that have changed since old_generation.
-
- :param old_generation: The generation of the database in the old
- state.
- :type old_generation: int
-
- :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
- The current generation of the database, its associated
- transaction id, and a list of of changed documents since
- old_generation, represented by tuples with for each document
- its doc_id and the generation and transaction id corresponding
- to the last intervening change and sorted by generation (old
- changes first)
- :rtype: (int, str, [(str, int, str)])
- """
- return self._database.whats_changed(old_generation)
-
- def delete_doc(self, doc):
- """
- Mark a document as deleted.
-
- Will abort if the current revision doesn't match doc.rev.
- This will also set doc.content to None.
-
- :param doc: The document to mark as deleted.
- :type doc: ServerDocument.
-
- :raise DocumentDoesNotExist: Raised if the document does not
- exist.
- :raise RevisionConflict: Raised if the revisions do not match.
- :raise DocumentAlreadyDeleted: Raised if the document is
- already deleted.
- :raise ConflictedDoc: Raised if the doc has conflicts.
- """
- old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if old_doc is None:
- raise DocumentDoesNotExist
- if old_doc.rev != doc.rev:
- raise RevisionConflict()
- if old_doc.is_tombstone():
- raise DocumentAlreadyDeleted
- if old_doc.has_conflicts:
- raise ConflictedDoc()
- new_rev = self._allocate_doc_rev(doc.rev)
- doc.rev = new_rev
- doc.make_tombstone()
- self._put_doc(old_doc, doc)
- return new_rev
-
- def get_doc_conflicts(self, doc_id):
- """
- Get the conflicted versions of a document.
-
- :param doc_id: The document id.
- :type doc_id: str
-
- :return: A list of conflicted versions of the document.
- :rtype: list
- """
- return self._database.get_doc_conflicts(doc_id)
-
- def _get_replica_gen_and_trans_id(self, other_replica_uid):
- """
- Return the last known generation and transaction id for the other db
- replica.
-
- When you do a synchronization with another replica, the Database keeps
- track of what generation the other database replica was at, and what
- the associated transaction id was. This is used to determine what data
- needs to be sent, and if two databases are claiming to be the same
- replica.
-
- :param other_replica_uid: The identifier for the other replica.
- :type other_replica_uid: str
-
- :return: A tuple containing the generation and transaction id we
- encountered during synchronization. If we've never
- synchronized with the replica, this is (0, '').
- :rtype: (int, str)
- """
- if other_replica_uid in self.cache:
- return self.cache[other_replica_uid]
- gen, trans_id = \
- self._database.get_replica_gen_and_trans_id(other_replica_uid)
- self.cache[other_replica_uid] = (gen, trans_id)
- return (gen, trans_id)
-
- def _set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation, other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- We have just performed some synchronization, and we want to track what
- generation the other replica was at. See also
- _get_replica_gen_and_trans_id.
-
- :param other_replica_uid: The U1DB identifier for the other replica.
- :type other_replica_uid: str
- :param other_generation: The generation number for the other replica.
- :type other_generation: int
- :param other_transaction_id: The transaction id associated with the
- generation.
- :type other_transaction_id: str
- """
- if other_replica_uid is not None and other_generation is not None:
- self.cache[other_replica_uid] = (other_generation,
- other_transaction_id)
- self._database.set_replica_gen_and_trans_id(other_replica_uid,
- other_generation,
- other_transaction_id)
-
- def _do_set_replica_gen_and_trans_id(
- self, other_replica_uid, other_generation, other_transaction_id):
- """
- _put_doc_if_newer from super class is calling it. So we declare this.
-
- :param other_replica_uid: The U1DB identifier for the other replica.
- :type other_replica_uid: str
- :param other_generation: The generation number for the other replica.
- :type other_generation: int
- :param other_transaction_id: The transaction id associated with the
- generation.
- :type other_transaction_id: str
- """
- args = [other_replica_uid, other_generation, other_transaction_id]
- callback = functools.partial(self._set_replica_gen_and_trans_id, *args)
- if self.batching:
- self.after_batch_callbacks['set_source_info'] = callback
- else:
- callback()
-
- def _force_doc_sync_conflict(self, doc):
- """
- Add a conflict and force a document put.
-
- :param doc: The document to be put.
- :type doc: ServerDocument
- """
- my_doc = self._get_doc(doc.doc_id)
- self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
- doc.add_conflict(self._factory(doc.doc_id, my_doc.rev,
- my_doc.get_json()))
- doc.has_conflicts = True
- self._put_doc(my_doc, doc)
-
- def resolve_doc(self, doc, conflicted_doc_revs):
- """
- Mark a document as no longer conflicted.
-
- We take the list of revisions that the client knows about that it is
- superseding. This may be a different list from the actual current
- conflicts, in which case only those are removed as conflicted. This
- may fail if the conflict list is significantly different from the
- supplied information. (sync could have happened in the background from
- the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
-
- :param doc: A Document with the new content to be inserted.
- :type doc: ServerDocument
- :param conflicted_doc_revs: A list of revisions that the new content
- supersedes.
- :type conflicted_doc_revs: [str]
-
- :raise SoledadError: Raised by database on operation failure
- """
- cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- new_rev = self._ensure_maximal_rev(cur_doc.rev,
- conflicted_doc_revs)
- superseded_revs = set(conflicted_doc_revs)
- doc.rev = new_rev
- # this backend stores conflicts as properties of the documents, so we
- # have to copy these conflicts over to the document being updated.
- if cur_doc.rev in superseded_revs:
- # the newer doc version will supersede the one in the database, so
- # we copy conflicts before updating the backend.
- doc.set_conflicts(cur_doc.get_conflicts()) # copy conflicts over.
- doc.delete_conflicts(superseded_revs)
- self._put_doc(cur_doc, doc)
- else:
- # the newer doc version does not supersede the one in the
- # database, so we will add a conflict to the database and copy
- # those over to the document the user has in her hands.
- cur_doc.add_conflict(doc)
- cur_doc.delete_conflicts(superseded_revs)
- self._put_doc(cur_doc, cur_doc) # just update conflicts
- # backend has been updated with current conflicts, now copy them
- # to the current document.
- doc.set_conflicts(cur_doc.get_conflicts())
-
- def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
- replica_trans_id='', number_of_docs=None,
- doc_idx=None, sync_id=None):
- """
- Insert/update document into the database with a given revision.
-
- This api is used during synchronization operations.
-
- If a document would conflict and save_conflict is set to True, the
- content will be selected as the 'current' content for doc.doc_id,
- even though doc.rev doesn't supersede the currently stored revision.
- The currently stored document will be added to the list of conflict
- alternatives for the given doc_id.
-
- This forces the new content to be 'current' so that we get convergence
- after synchronizing, even if people don't resolve conflicts. Users can
- then notice that their content is out of date, update it, and
- synchronize again. (The alternative is that users could synchronize and
- think the data has propagated, but their local copy looks fine, and the
- remote copy is never updated again.)
-
- :param doc: A document object
- :type doc: ServerDocument
- :param save_conflict: If this document is a conflict, do you want to
- save it as a conflict, or just ignore it.
- :type save_conflict: bool
- :param replica_uid: A unique replica identifier.
- :type replica_uid: str
- :param replica_gen: The generation of the replica corresponding to the
- this document. The replica arguments are optional,
- but are used during synchronization.
- :type replica_gen: int
- :param replica_trans_id: The transaction_id associated with the
- generation.
- :type replica_trans_id: str
- :param number_of_docs: The total amount of documents sent on this sync
- session.
- :type number_of_docs: int
- :param doc_idx: The index of the current document being sent.
- :type doc_idx: int
- :param sync_id: The id of the current sync session.
- :type sync_id: str
-
- :return: (state, at_gen) - If we don't have doc_id already, or if
- doc_rev supersedes the existing document revision, then the
- content will be inserted, and state is 'inserted'. If
- doc_rev is less than or equal to the existing revision, then
- the put is ignored and state is respecitvely 'superseded' or
- 'converged'. If doc_rev is not strictly superseded or
- supersedes, then state is 'conflicted'. The document will not
- be inserted if save_conflict is False. For 'inserted' or
- 'converged', at_gen is the insertion/current generation.
- :rtype: (str, int)
- """
- if not isinstance(doc, ServerDocument):
- doc = self._factory(doc.doc_id, doc.rev, doc.get_json())
- my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if my_doc:
- doc.set_conflicts(my_doc.get_conflicts())
- return CommonBackend._put_doc_if_newer(self, doc, save_conflict,
- replica_uid, replica_gen,
- replica_trans_id)
-
- def _put_and_update_indexes(self, cur_doc, doc):
- self._put_doc(cur_doc, doc)
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False, read_content=True):
- """
- Get the JSON content for many documents.
-
- :param doc_ids: A list of document identifiers or None for all.
- :type doc_ids: list
- :param check_for_conflicts: If set to False, then the conflict check
- will be skipped, and 'None' will be
- returned instead of True/False.
- :type check_for_conflicts: bool
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted
- documents will not be included in the results.
- :return: iterable giving the Document object for each document id
- in matching doc_ids order.
- :rtype: iterable
- """
- return self._database.get_docs(doc_ids, check_for_conflicts,
- include_deleted, read_content)
-
- def _prune_conflicts(self, doc, doc_vcr):
- """
- Prune conflicts that are older then the current document's revision, or
- whose content match to the current document's content.
- Originally in u1db.CommonBackend
-
- :param doc: The document to have conflicts pruned.
- :type doc: ServerDocument
- :param doc_vcr: A vector clock representing the current document's
- revision.
- :type doc_vcr: u1db.vectorclock.VectorClock
- """
- if doc.has_conflicts:
- autoresolved = False
- c_revs_to_prune = []
- for c_doc in doc._conflicts:
- c_vcr = vectorclock.VectorClockRev(c_doc.rev)
- if doc_vcr.is_newer(c_vcr):
- c_revs_to_prune.append(c_doc.rev)
- elif doc.same_content_as(c_doc):
- c_revs_to_prune.append(c_doc.rev)
- doc_vcr.maximize(c_vcr)
- autoresolved = True
- if autoresolved:
- doc_vcr.increment(self._replica_uid)
- doc.rev = doc_vcr.as_str()
- doc.delete_conflicts(c_revs_to_prune)
-
-
-class SoledadSyncTarget(CommonSyncTarget):
-
- """
- Functionality for using a SoledadBackend as a synchronization target.
- """
-
- def get_sync_info(self, source_replica_uid):
- source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id(
- source_replica_uid)
- my_gen, my_trans_id = self._db._get_generation_info()
- return (
- self._db._replica_uid, my_gen, my_trans_id, source_gen,
- source_trans_id)
-
- def record_sync_info(self, source_replica_uid, source_replica_generation,
- source_replica_transaction_id):
- if self._trace_hook:
- self._trace_hook('record_sync_info')
- self._db._set_replica_gen_and_trans_id(
- source_replica_uid, source_replica_generation,
- source_replica_transaction_id)
diff --git a/common/src/leap/soledad/common/command.py b/common/src/leap/soledad/common/command.py
deleted file mode 100644
index 66aa6b7a..00000000
--- a/common/src/leap/soledad/common/command.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# -*- coding: utf-8 -*-
-# command.py
-# Copyright (C) 2015 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Utility to sanitize and run shell commands.
-"""
-
-
-import subprocess
-
-
-def exec_validated_cmd(cmd, argument, validator=None):
- """
- Executes cmd, validating argument with a validator function.
-
- :param cmd: command.
- :type dbname: str
- :param argument: argument.
- :type argument: str
- :param validator: optional function to validate argument
- :type validator: function
-
- :return: exit code and stdout or stderr (if code != 0)
- :rtype: (int, str)
- """
- if validator and not validator(argument):
- return 1, "invalid argument"
- command = cmd.split(' ')
- command.append(argument)
- try:
- process = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- except OSError as e:
- return 1, e
- (out, err) = process.communicate()
- code = process.wait()
- if code is not 0:
- return code, err
- else:
- return code, out
diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
deleted file mode 100644
index 2343e849..00000000
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ /dev/null
@@ -1,812 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2015 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""A U1DB backend that uses CouchDB as its persistence layer."""
-
-
-import json
-import copy
-import re
-import uuid
-import binascii
-
-from six import StringIO
-from six.moves.urllib.parse import urljoin
-from contextlib import contextmanager
-
-
-from couchdb.client import Server, Database
-from couchdb.http import (
- ResourceConflict,
- ResourceNotFound,
- Session,
- urljoin as couch_urljoin,
- Resource,
-)
-from leap.soledad.common.l2db.errors import (
- DatabaseDoesNotExist,
- InvalidGeneration,
- RevisionConflict,
-)
-from leap.soledad.common.l2db.remote import http_app
-
-
-from .support import MultipartWriter
-from leap.soledad.common.errors import InvalidURLError
-from leap.soledad.common.document import ServerDocument
-from leap.soledad.common.backend import SoledadBackend
-
-
-COUCH_TIMEOUT = 120 # timeout for transfers between Soledad server and Couch
-
-
-def list_users_dbs(couch_url):
- """
- Retrieves a list with all databases that starts with 'user-' on CouchDB.
- Those databases belongs to users. So, the list will contain all the
- database names in the form of 'user-{uuid4}'.
-
- :param couch_url: The couch url with needed credentials
- :type couch_url: str
-
- :return: The list of all database names from users.
- :rtype: [str]
- """
- with couch_server(couch_url) as server:
- users = [dbname for dbname in server if dbname.startswith('user-')]
- return users
-
-
-# monkey-patch the u1db http app to use ServerDocument
-http_app.Document = ServerDocument
-
-
-@contextmanager
-def couch_server(url):
- """
- Provide a connection to a couch server and cleanup after use.
-
- For database creation and deletion we use an ephemeral connection to the
- couch server. That connection has to be properly closed, so we provide it
- as a context manager.
-
- :param url: The URL of the Couch server.
- :type url: str
- """
- session = Session(timeout=COUCH_TIMEOUT)
- server = Server(url=url, full_commit=False, session=session)
- yield server
-
-
-def _get_gen_doc_id(gen):
- return 'gen-%s' % str(gen).zfill(10)
-
-
-GENERATION_KEY = 'gen'
-TRANSACTION_ID_KEY = 'trans_id'
-REPLICA_UID_KEY = 'replica_uid'
-DOC_ID_KEY = 'doc_id'
-SCHEMA_VERSION_KEY = 'schema_version'
-
-CONFIG_DOC_ID = '_local/config'
-SYNC_DOC_ID_PREFIX = '_local/sync_'
-SCHEMA_VERSION = 1
-
-
-class CouchDatabase(object):
- """
- Holds CouchDB related code.
- This class gives methods to encapsulate database operations and hide
- CouchDB details from backend code.
- """
-
- @classmethod
- def open_database(cls, url, create, replica_uid=None,
- database_security=None):
- """
- Open a U1DB database using CouchDB as backend.
-
- :param url: the url of the database replica
- :type url: str
- :param create: should the replica be created if it does not exist?
- :type create: bool
- :param replica_uid: an optional unique replica identifier
- :type replica_uid: str
- :param database_security: security rules as CouchDB security doc
- :type database_security: dict
-
- :return: the database instance
- :rtype: SoledadBackend
-
- :raise DatabaseDoesNotExist: Raised if database does not exist.
- """
- # get database from url
- m = re.match('(^https?://[^/]+)/(.+)$', url)
- if not m:
- raise InvalidURLError
- url = m.group(1)
- dbname = m.group(2)
- with couch_server(url) as server:
- if dbname not in server:
- if create:
- server.create(dbname)
- else:
- raise DatabaseDoesNotExist()
- db = cls(url, dbname, ensure_security=create,
- database_security=database_security)
- return SoledadBackend(
- db, replica_uid=replica_uid)
-
- def __init__(self, url, dbname, ensure_security=False,
- database_security=None):
- """
- :param url: Couch server URL with necessary credentials
- :type url: string
- :param dbname: Couch database name
- :type dbname: string
- :param ensure_security: will PUT a _security ddoc if set
- :type ensure_security: bool
- :param database_security: security rules as CouchDB security doc
- :type database_security: dict
- """
- self._session = Session(timeout=COUCH_TIMEOUT)
- self._url = url
- self._dbname = dbname
- self._database = self.get_couch_database(url, dbname)
- self.batching = False
- self.batch_generation = None
- self.batch_docs = {}
- if ensure_security:
- self.ensure_security_ddoc(database_security)
-
- def batch_start(self):
- self.batching = True
- self.batch_generation = self.get_generation_info()
- ids = set(row.id for row in self._database.view('_all_docs'))
- self.batched_ids = ids
-
- def batch_end(self):
- self.batching = False
- self.batch_generation = None
- self.__perform_batch()
-
- def get_couch_database(self, url, dbname):
- """
- Generate a couchdb.Database instance given a url and dbname.
-
- :param url: CouchDB's server url with credentials
- :type url: str
- :param dbname: Database name
- :type dbname: str
-
- :return: couch library database instance
- :rtype: couchdb.Database
-
- :raise DatabaseDoesNotExist: Raised if database does not exist.
- """
- try:
- return Database(
- urljoin(url, dbname),
- self._session)
- except ResourceNotFound:
- raise DatabaseDoesNotExist()
-
- def ensure_security_ddoc(self, security_config=None):
- """
- Make sure that only soledad user is able to access this database as
- an unprivileged member, meaning that administration access will
- be forbidden even inside an user database.
- The goal is to make sure that only the lowest access level is given
- to the unprivileged CouchDB user set on the server process.
- This is achieved by creating a _security design document, see:
- http://docs.couchdb.org/en/latest/api/database/security.html
-
- :param security_config: security configuration parsed from conf file
- :type security_config: dict
- """
- security_config = security_config or {}
- security = self._database.resource.get_json('_security')[2]
- security['members'] = {'names': [], 'roles': []}
- security['members']['names'] = security_config.get('members',
- ['soledad'])
- security['members']['roles'] = security_config.get('members_roles', [])
- security['admins'] = {'names': [], 'roles': []}
- security['admins']['names'] = security_config.get('admins', [])
- security['admins']['roles'] = security_config.get('admins_roles', [])
- self._database.resource.put_json('_security', body=security)
-
- def delete_database(self):
- """
- Delete a U1DB CouchDB database.
- """
- with couch_server(self._url) as server:
- del(server[self._dbname])
-
- def set_replica_uid(self, replica_uid):
- """
- Force the replica uid to be set.
-
- :param replica_uid: The new replica uid.
- :type replica_uid: str
- """
- try:
- # set on existent config document
- doc = self._database[CONFIG_DOC_ID]
- doc[REPLICA_UID_KEY] = replica_uid
- except ResourceNotFound:
- # or create the config document
- doc = {
- '_id': CONFIG_DOC_ID,
- REPLICA_UID_KEY: replica_uid,
- SCHEMA_VERSION_KEY: SCHEMA_VERSION,
- }
- self._database.save(doc)
-
- def get_replica_uid(self):
- """
- Get the replica uid.
-
- :return: The replica uid.
- :rtype: str
- """
- try:
- # grab replica_uid from server
- doc = self._database[CONFIG_DOC_ID]
- replica_uid = doc[REPLICA_UID_KEY]
- return replica_uid
- except ResourceNotFound:
- # create a unique replica_uid
- replica_uid = uuid.uuid4().hex
- self.set_replica_uid(replica_uid)
- return replica_uid
-
- def close(self):
- self._database = None
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted
- documents will not be included in the results.
- :type include_deleted: bool
-
- :return: (generation, [ServerDocument])
- The current generation of the database, followed by a list of all
- the documents in the database.
- :rtype: (int, [ServerDocument])
- """
-
- generation, _ = self.get_generation_info()
- results = list(
- self.get_docs(None, True, include_deleted))
- return (generation, results)
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False, read_content=True):
- """
- Get the JSON content for many documents.
-
- Use couch's `_all_docs` view to get the documents indicated in
- `doc_ids`,
-
- :param doc_ids: A list of document identifiers or None for all.
- :type doc_ids: list
- :param check_for_conflicts: If set to False, then the conflict check
- will be skipped, and 'None' will be
- returned instead of True/False.
- :type check_for_conflicts: bool
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted
- documents will not be included in the results.
-
- :return: iterable giving the Document object for each document id
- in matching doc_ids order.
- :rtype: iterable
- """
- params = {'include_docs': 'true', 'attachments': 'false'}
- if doc_ids is not None:
- params['keys'] = doc_ids
- view = self._database.view("_all_docs", **params)
- for row in view.rows:
- result = copy.deepcopy(row['doc'])
- for file_name in result.get('_attachments', {}).keys():
- data = self._database.get_attachment(result, file_name)
- if data:
- if read_content:
- data = data.read()
- result['_attachments'][file_name] = {'data': data}
- doc = self.__parse_doc_from_couch(
- result, result['_id'],
- check_for_conflicts=check_for_conflicts, decode=False)
- # filter out non-u1db or deleted documents
- if not doc or (not include_deleted and doc.is_tombstone()):
- continue
- yield doc
-
- def get_doc(self, doc_id, check_for_conflicts=False):
- """
- Extract the document from storage.
-
- This can return None if the document doesn't exist.
-
- :param doc_id: The unique document identifier
- :type doc_id: str
- :param check_for_conflicts: If set to False, then the conflict check
- will be skipped.
- :type check_for_conflicts: bool
-
- :return: The document.
- :rtype: ServerDocument
- """
- doc_from_batch = self.__check_batch_before_get(doc_id)
- if doc_from_batch:
- return doc_from_batch
- if self.batching and doc_id not in self.batched_ids:
- return None
- if doc_id not in self._database:
- return None
- # get document with all attachments (u1db content and eventual
- # conflicts)
- result = self.json_from_resource([doc_id], attachments=True)
- return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
-
- def __check_batch_before_get(self, doc_id):
- """
- If doc_id is staged for batching, then we need to commit the batch
- before going ahead. This avoids consistency problems, like trying to
- get a document that isn't persisted and processing like it is missing.
-
- :param doc_id: The unique document identifier
- :type doc_id: str
- """
- if doc_id in self.batch_docs:
- couch_doc = self.batch_docs[doc_id]
- rev = self.__perform_batch(doc_id)
- couch_doc['_rev'] = rev
- self.batched_ids.add(doc_id)
- return self.__parse_doc_from_couch(couch_doc, doc_id, True)
- return None
-
- def __perform_batch(self, doc_id=None):
- status = self._database.update(self.batch_docs.values())
- rev = None
- for ok, stored_doc_id, rev_or_error in status:
- if not ok:
- error = rev_or_error
- if type(error) is ResourceConflict:
- raise RevisionConflict
- raise error
- elif doc_id == stored_doc_id:
- rev = rev_or_error
- self.batch_docs.clear()
- return rev
-
- def __parse_doc_from_couch(self, result, doc_id,
- check_for_conflicts=False, decode=True):
- # restrict to u1db documents
- if 'u1db_rev' not in result:
- return None
- doc = ServerDocument(doc_id, result['u1db_rev'])
- # set contents or make tombstone
- if '_attachments' not in result \
- or 'u1db_content' not in result['_attachments']:
- doc.make_tombstone()
- elif decode:
- doc.content = json.loads(
- binascii.a2b_base64(
- result['_attachments']['u1db_content']['data']))
- else:
- doc._json = result['_attachments']['u1db_content']['data']
- # determine if there are conflicts
- if check_for_conflicts \
- and '_attachments' in result \
- and 'u1db_conflicts' in result['_attachments']:
- if decode:
- conflicts = binascii.a2b_base64(
- result['_attachments']['u1db_conflicts']['data'])
- else:
- conflicts = result['_attachments']['u1db_conflicts']['data']
- conflicts = json.loads(conflicts)
- doc.set_conflicts(self._build_conflicts(doc.doc_id, conflicts))
- # store couch revision
- doc.couch_rev = result['_rev']
- return doc
-
- def _build_conflicts(self, doc_id, attached_conflicts):
- """
- Build the conflicted documents list from the conflicts attachment
- fetched from a couch document.
-
- :param attached_conflicts: The document's conflicts as fetched from a
- couch document attachment.
- :type attached_conflicts: dict
- """
- conflicts = []
- for doc_rev, content in attached_conflicts:
- doc = ServerDocument(doc_id, doc_rev)
- if content is None:
- doc.make_tombstone()
- else:
- doc.content = content
- conflicts.append(doc)
- return conflicts
-
- def get_trans_id_for_gen(self, generation):
- """
- Get the transaction id corresponding to a particular generation.
-
- :param generation: The generation for which to get the transaction id.
- :type generation: int
-
- :return: The transaction id for C{generation}.
- :rtype: str
-
- :raise InvalidGeneration: Raised when the generation does not exist.
- """
- if generation == 0:
- return ''
- log = self._get_transaction_log(start=generation, end=generation)
- if not log:
- raise InvalidGeneration
- _, _, trans_id = log[0]
- return trans_id
-
- def get_replica_gen_and_trans_id(self, other_replica_uid):
- """
- Return the last known generation and transaction id for the other db
- replica.
-
- When you do a synchronization with another replica, the Database keeps
- track of what generation the other database replica was at, and what
- the associated transaction id was. This is used to determine what data
- needs to be sent, and if two databases are claiming to be the same
- replica.
-
- :param other_replica_uid: The identifier for the other replica.
- :type other_replica_uid: str
-
- :return: A tuple containing the generation and transaction id we
- encountered during synchronization. If we've never
- synchronized with the replica, this is (0, '').
- :rtype: (int, str)
- """
- doc_id = '%s%s' % (SYNC_DOC_ID_PREFIX, other_replica_uid)
- try:
- doc = self._database[doc_id]
- except ResourceNotFound:
- doc = {
- '_id': doc_id,
- GENERATION_KEY: 0,
- REPLICA_UID_KEY: str(other_replica_uid),
- TRANSACTION_ID_KEY: '',
- }
- self._database.save(doc)
- gen, trans_id = doc[GENERATION_KEY], doc[TRANSACTION_ID_KEY]
- return gen, trans_id
-
- def get_doc_conflicts(self, doc_id, couch_rev=None):
- """
- Get the conflicted versions of a document.
-
- If the C{couch_rev} parameter is not None, conflicts for a specific
- document's couch revision are returned.
-
- :param couch_rev: The couch document revision.
- :type couch_rev: str
-
- :return: A list of conflicted versions of the document.
- :rtype: list
- """
- # request conflicts attachment from server
- params = {}
- conflicts = []
- if couch_rev is not None:
- params['rev'] = couch_rev # restric document's couch revision
- else:
- # TODO: move into resource logic!
- first_entry = self.get_doc(doc_id, check_for_conflicts=True)
- conflicts.append(first_entry)
-
- try:
- response = self.json_from_resource([doc_id, 'u1db_conflicts'],
- **params)
- return conflicts + self._build_conflicts(
- doc_id, json.loads(response.read()))
- except ResourceNotFound:
- return []
-
- def set_replica_gen_and_trans_id(
- self, other_replica_uid, other_generation, other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- We have just performed some synchronization, and we want to track what
- generation the other replica was at. See also
- get_replica_gen_and_trans_id.
-
- :param other_replica_uid: The U1DB identifier for the other replica.
- :type other_replica_uid: str
- :param other_generation: The generation number for the other replica.
- :type other_generation: int
- :param other_transaction_id: The transaction id associated with the
- generation.
- :type other_transaction_id: str
- """
- doc_id = '%s%s' % (SYNC_DOC_ID_PREFIX, other_replica_uid)
- try:
- doc = self._database[doc_id]
- except ResourceNotFound:
- doc = {'_id': doc_id}
- doc[GENERATION_KEY] = other_generation
- doc[TRANSACTION_ID_KEY] = other_transaction_id
- self._database.save(doc)
-
- def get_transaction_log(self):
- """
- This is only for the test suite, it is not part of the api.
-
- :return: The complete transaction log.
- :rtype: [(str, str)]
- """
- log = self._get_transaction_log()
- return map(lambda i: (i[1], i[2]), log)
-
- def _get_gen_docs(
- self, start=0, end=9999999999, descending=None, limit=None):
- params = {}
- if descending:
- params['descending'] = 'true'
- # honor couch way of traversing the view tree in reverse order
- start, end = end, start
- params['startkey'] = _get_gen_doc_id(start)
- params['endkey'] = _get_gen_doc_id(end)
- params['include_docs'] = 'true'
- if limit:
- params['limit'] = limit
- view = self._database.view("_all_docs", **params)
- return view.rows
-
- def _get_transaction_log(self, start=0, end=9999999999):
- # get current gen and trans_id
- rows = self._get_gen_docs(start=start, end=end)
- log = []
- for row in rows:
- doc = row['doc']
- log.append((
- doc[GENERATION_KEY],
- doc[DOC_ID_KEY],
- doc[TRANSACTION_ID_KEY]))
- return log
-
- def whats_changed(self, old_generation=0):
- """
- Return a list of documents that have changed since old_generation.
-
- :param old_generation: The generation of the database in the old
- state.
- :type old_generation: int
-
- :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
- The current generation of the database, its associated
- transaction id, and a list of of changed documents since
- old_generation, represented by tuples with for each document
- its doc_id and the generation and transaction id corresponding
- to the last intervening change and sorted by generation (old
- changes first)
- :rtype: (int, str, [(str, int, str)])
- """
- changes = []
- cur_generation, last_trans_id = self.get_generation_info()
- relevant_tail = self._get_transaction_log(start=old_generation + 1)
- seen = set()
- for generation, doc_id, trans_id in reversed(relevant_tail):
- if doc_id not in seen:
- changes.append((doc_id, generation, trans_id))
- seen.add(doc_id)
- changes.reverse()
- return (cur_generation, last_trans_id, changes)
-
- def get_generation_info(self):
- """
- Return the current generation.
-
- :return: A tuple containing the current generation and transaction id.
- :rtype: (int, str)
- """
- if self.batching and self.batch_generation:
- return self.batch_generation
- rows = self._get_gen_docs(descending=True, limit=1)
- if not rows:
- return 0, ''
- gen_doc = rows.pop()['doc']
- return gen_doc[GENERATION_KEY], gen_doc[TRANSACTION_ID_KEY]
-
- def json_from_resource(self, doc_path, **kwargs):
- """
- Get a resource from it's path and gets a doc's JSON using provided
- parameters.
-
- :param doc_path: The path to resource.
- :type doc_path: [str]
-
- :return: The request's data parsed from JSON to a dict.
- :rtype: dict
- """
- if doc_path is not None:
- resource = self._database.resource(*doc_path)
- else:
- resource = self._database.resource()
- _, _, data = resource.get_json(**kwargs)
- return data
-
- def _allocate_new_generation(self, doc_id, transaction_id, save=True):
- """
- Allocate a new generation number for a document modification.
-
- We need to allocate a new generation to this document modification by
- creating a new gen doc. In order to avoid concurrent database updates
- from allocating the same new generation, we will try to create the
- document until we succeed, meaning that no other piece of code holds
- the same generation number as ours.
-
- The loop below would only be executed more than once if:
-
- 1. there's more than one thread trying to modify the user's database,
- and
-
- 2. the execution of getting the current generation and saving the gen
- doc different threads get interleaved (one of them will succeed
- and the others will fail and try again).
-
- Number 1 only happens when more than one user device is syncing at the
- same time. Number 2 depends on not-so-frequent coincidence of
- code execution.
-
- Also, in the race between threads for a generation number there's
- always one thread that wins. so if there are N threads in the race, the
- expected number of repetitions of the loop for each thread would be
- N/2. If N is equal to the number of devices that the user has, the
- number of possible repetitions of the loop should always be low.
- """
- while True:
- try:
- # add the gen document
- gen, _ = self.get_generation_info()
- new_gen = gen + 1
- gen_doc = {
- '_id': _get_gen_doc_id(new_gen),
- GENERATION_KEY: new_gen,
- DOC_ID_KEY: doc_id,
- TRANSACTION_ID_KEY: transaction_id,
- }
- if save:
- self._database.save(gen_doc)
- break # succeeded allocating a new generation, proceed
- except ResourceConflict:
- pass # try again!
- return gen_doc
-
- def save_document(self, old_doc, doc, transaction_id):
- """
- Put the document in the Couch backend database.
-
- Note that C{old_doc} must have been fetched with the parameter
- C{check_for_conflicts} equal to True, so we can properly update the
- new document using the conflict information from the old one.
-
- :param old_doc: The old document version.
- :type old_doc: ServerDocument
- :param doc: The document to be put.
- :type doc: ServerDocument
-
- :raise RevisionConflict: Raised when trying to update a document but
- couch revisions mismatch.
- """
- attachments = {} # we save content and conflicts as attachments
- parts = [] # and we put it using couch's multipart PUT
- # save content as attachment
- if doc.is_tombstone() is False:
- content = doc.get_json()
- attachments['u1db_content'] = {
- 'follows': True,
- 'content_type': 'application/octet-stream',
- 'length': len(content),
- }
- parts.append(content)
-
- # save conflicts as attachment
- if doc.has_conflicts is True:
- conflicts = json.dumps(
- map(lambda cdoc: (cdoc.rev, cdoc.content),
- doc.get_conflicts()))
- attachments['u1db_conflicts'] = {
- 'follows': True,
- 'content_type': 'application/octet-stream',
- 'length': len(conflicts),
- }
- parts.append(conflicts)
-
- # build the couch document
- couch_doc = {
- '_id': doc.doc_id,
- 'u1db_rev': doc.rev,
- '_attachments': attachments,
- }
- # if we are updating a doc we have to add the couch doc revision
- if old_doc is not None and hasattr(old_doc, 'couch_rev'):
- couch_doc['_rev'] = old_doc.couch_rev
- # prepare the multipart PUT
- if not self.batching:
- buf = StringIO()
- envelope = MultipartWriter(buf)
- # the order in which attachments are described inside the
- # serialization of the couch document must match the order in
- # which they are actually written in the multipart structure.
- # Because of that, we use `sorted_keys=True` in the json
- # serialization (so "u1db_conflicts" comes before
- # "u1db_content" on the couch document attachments
- # description), and also reverse the order of the parts before
- # writing them, so the "conflict" part is written before the
- # "content" part.
- envelope.add(
- 'application/json',
- json.dumps(couch_doc, sort_keys=True))
- parts.reverse()
- for part in parts:
- envelope.add('application/octet-stream', part)
- envelope.close()
- # try to save and fail if there's a revision conflict
- try:
- resource = self._new_resource()
- resource.put_json(
- doc.doc_id, body=str(buf.getvalue()),
- headers=envelope.headers)
- except ResourceConflict:
- raise RevisionConflict()
- self._allocate_new_generation(doc.doc_id, transaction_id)
- else:
- for name, attachment in attachments.items():
- del attachment['follows']
- del attachment['length']
- index = 0 if name is 'u1db_content' else 1
- attachment['data'] = binascii.b2a_base64(
- parts[index]).strip()
- couch_doc['_attachments'] = attachments
- gen_doc = self._allocate_new_generation(
- doc.doc_id, transaction_id, save=False)
- self.batch_docs[doc.doc_id] = couch_doc
- self.batch_docs[gen_doc['_id']] = gen_doc
- last_gen, last_trans_id = self.batch_generation
- self.batch_generation = (last_gen + 1, transaction_id)
-
- def _new_resource(self, *path):
- """
- Return a new resource for accessing a couch database.
-
- :return: A resource for accessing a couch database.
- :rtype: couchdb.http.Resource
- """
- # Workaround for: https://leap.se/code/issues/5448
- url = couch_urljoin(self._database.resource.url, *path)
- resource = Resource(url, Session(timeout=COUCH_TIMEOUT))
- resource.credentials = self._database.resource.credentials
- resource.headers = self._database.resource.headers.copy()
- return resource
diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py
deleted file mode 100644
index 8cbe0934..00000000
--- a/common/src/leap/soledad/common/couch/state.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# -*- coding: utf-8 -*-
-# state.py
-# Copyright (C) 2015,2016 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Server state using CouchDatabase as backend.
-"""
-import couchdb
-import re
-
-from six.moves.urllib.parse import urljoin
-
-from leap.soledad.common.log import getLogger
-from leap.soledad.common.couch import CouchDatabase
-from leap.soledad.common.couch import CONFIG_DOC_ID
-from leap.soledad.common.couch import SCHEMA_VERSION
-from leap.soledad.common.couch import SCHEMA_VERSION_KEY
-from leap.soledad.common.command import exec_validated_cmd
-from leap.soledad.common.l2db.remote.server_state import ServerState
-from leap.soledad.common.l2db.errors import Unauthorized
-from leap.soledad.common.errors import WrongCouchSchemaVersionError
-from leap.soledad.common.errors import MissingCouchConfigDocumentError
-
-
-logger = getLogger(__name__)
-
-
-def is_db_name_valid(name):
- """
- Validate a user database using a regular expression.
-
- :param name: database name.
- :type name: str
-
- :return: boolean for name vailidity
- :rtype: bool
- """
- db_name_regex = "^user-[a-f0-9]+$"
- return re.match(db_name_regex, name) is not None
-
-
-class CouchServerState(ServerState):
-
- """
- Inteface of the WSGI server with the CouchDB backend.
- """
-
- def __init__(self, couch_url, create_cmd=None,
- check_schema_versions=False):
- """
- Initialize the couch server state.
-
- :param couch_url: The URL for the couch database.
- :type couch_url: str
- :param create_cmd: Command to be executed for user db creation. It will
- receive a properly sanitized parameter with user db
- name and should access CouchDB with necessary
- privileges, which server lacks for security reasons.
- :type create_cmd: str
- :param check_schema_versions: Whether to check couch schema version of
- user dbs. Set to False as this is only
- intended to run once during start-up.
- :type check_schema_versions: bool
- """
- self.couch_url = couch_url
- self.create_cmd = create_cmd
- if check_schema_versions:
- self._check_schema_versions()
-
- def _check_schema_versions(self):
- """
- Check that all user databases use the correct couch schema.
- """
- server = couchdb.client.Server(self.couch_url)
- for dbname in server:
- if not dbname.startswith('user-'):
- continue
- db = server[dbname]
-
- # if there are documents, ensure that a config doc exists
- config_doc = db.get(CONFIG_DOC_ID)
- if config_doc:
- if config_doc[SCHEMA_VERSION_KEY] != SCHEMA_VERSION:
- logger.error(
- "Unsupported database schema in database %s" % dbname)
- raise WrongCouchSchemaVersionError(dbname)
- else:
- result = db.view('_all_docs', limit=1)
- if result.total_rows != 0:
- logger.error(
- "Missing couch config document in database %s"
- % dbname)
- raise MissingCouchConfigDocumentError(dbname)
-
- def open_database(self, dbname):
- """
- Open a couch database.
-
- :param dbname: The name of the database to open.
- :type dbname: str
-
- :return: The SoledadBackend object.
- :rtype: SoledadBackend
- """
- url = urljoin(self.couch_url, dbname)
- db = CouchDatabase.open_database(url, create=False)
- return db
-
- def ensure_database(self, dbname):
- """
- Ensure couch database exists.
-
- :param dbname: The name of the database to ensure.
- :type dbname: str
-
- :raise Unauthorized: If disabled or other error was raised.
-
- :return: The SoledadBackend object and its replica_uid.
- :rtype: (SoledadBackend, str)
- """
- if not self.create_cmd:
- raise Unauthorized()
- else:
- code, out = exec_validated_cmd(self.create_cmd, dbname,
- validator=is_db_name_valid)
- if code is not 0:
- logger.error("""
- Error while creating database (%s) with (%s) command.
- Output: %s
- Exit code: %d
- """ % (dbname, self.create_cmd, out, code))
- raise Unauthorized()
- db = self.open_database(dbname)
- return db, db.replica_uid
-
- def delete_database(self, dbname):
- """
- Delete couch database.
-
- :param dbname: The name of the database to delete.
- :type dbname: str
-
- :raise Unauthorized: Always, because Soledad server is not allowed to
- delete databases.
- """
- raise Unauthorized()
diff --git a/common/src/leap/soledad/common/couch/support.py b/common/src/leap/soledad/common/couch/support.py
deleted file mode 100644
index bfc4fef6..00000000
--- a/common/src/leap/soledad/common/couch/support.py
+++ /dev/null
@@ -1,115 +0,0 @@
-# -*- coding: utf-8 -*-
-# support.py
-# Copyright (C) 2015 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import sys
-
-
-"""
-Monkey patches and temporary code that may be removed with version changes.
-"""
-
-
-# for bigcouch
-# TODO: Remove if bigcouch support is dropped
-class MultipartWriter(object):
-
- """
- A multipart writer adapted from python-couchdb's one so we can PUT
- documents using couch's multipart PUT.
-
- This stripped down version does not allow for nested structures, and
- contains only the essential things we need to PUT SoledadDocuments to the
- couch backend. Also, please note that this is a patch. The couchdb lib has
- another implementation that works fine with CouchDB 1.6, but removing this
- now will break compatibility with bigcouch.
- """
-
- CRLF = '\r\n'
-
- def __init__(self, fileobj, headers=None, boundary=None):
- """
- Initialize the multipart writer.
- """
- self.fileobj = fileobj
- if boundary is None:
- boundary = self._make_boundary()
- self._boundary = boundary
- self._build_headers('related', headers)
-
- def add(self, mimetype, content, headers={}):
- """
- Add a part to the multipart stream.
- """
- self.fileobj.write('--')
- self.fileobj.write(self._boundary)
- self.fileobj.write(self.CRLF)
- headers['Content-Type'] = mimetype
- self._write_headers(headers)
- if content:
- # XXX: throw an exception if a boundary appears in the content??
- self.fileobj.write(content)
- self.fileobj.write(self.CRLF)
-
- def close(self):
- """
- Close the multipart stream.
- """
- self.fileobj.write('--')
- self.fileobj.write(self._boundary)
- # be careful not to have anything after '--', otherwise old couch
- # versions (including bigcouch) will fail.
- self.fileobj.write('--')
-
- def _make_boundary(self):
- """
- Create a boundary to discern multi parts.
- """
- try:
- from uuid import uuid4
- return '==' + uuid4().hex + '=='
- except ImportError:
- from random import randrange
- token = randrange(sys.maxint)
- format = '%%0%dd' % len(repr(sys.maxint - 1))
- return '===============' + (format % token) + '=='
-
- def _write_headers(self, headers):
- """
- Write a part header in the buffer stream.
- """
- if headers:
- for name in sorted(headers.keys()):
- value = headers[name]
- self.fileobj.write(name)
- self.fileobj.write(': ')
- self.fileobj.write(value)
- self.fileobj.write(self.CRLF)
- self.fileobj.write(self.CRLF)
-
- def _build_headers(self, subtype, headers):
- """
- Build the main headers of the multipart stream.
-
- This is here so we can send headers separete from content using
- python-couchdb API.
- """
- self.headers = {}
- self.headers['Content-Type'] = 'multipart/%s; boundary="%s"' % \
- (subtype, self._boundary)
- if headers:
- for name in sorted(headers.keys()):
- value = headers[name]
- self.headers[name] = value
diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py
deleted file mode 100644
index c13c4aa7..00000000
--- a/common/src/leap/soledad/common/crypto.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# -*- coding: utf-8 -*-
-# crypto.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Soledad common crypto bits.
-"""
-
-
-#
-# Encryption schemes used for encryption.
-#
-
-class EncryptionSchemes(object):
-
- """
- Representation of encryption schemes used to encrypt documents.
- """
-
- NONE = 'none'
- SYMKEY = 'symkey'
- PUBKEY = 'pubkey'
-
-
-class UnknownEncryptionSchemeError(Exception):
-
- """
- Raised when trying to decrypt from unknown encryption schemes.
- """
- pass
-
-
-class EncryptionMethods(object):
-
- """
- Representation of encryption methods that can be used.
- """
-
- AES_256_CTR = 'aes-256-ctr'
-
-
-class UnknownEncryptionMethodError(Exception):
-
- """
- Raised when trying to encrypt/decrypt with unknown method.
- """
- pass
-
-
-class MacMethods(object):
-
- """
- Representation of MAC methods used to authenticate document's contents.
- """
-
- HMAC = 'hmac'
-
-
-class UnknownMacMethodError(Exception):
-
- """
- Raised when trying to authenticate document's content with unknown MAC
- mehtod.
- """
- pass
-
-
-class WrongMacError(Exception):
-
- """
- Raised when failing to authenticate document's contents based on MAC.
- """
-
-
-#
-# Crypto utilities for a SoledadDocument.
-#
-
-ENC_JSON_KEY = '_enc_json'
-ENC_SCHEME_KEY = '_enc_scheme'
-ENC_METHOD_KEY = '_enc_method'
-ENC_IV_KEY = '_enc_iv'
-MAC_KEY = '_mac'
-MAC_METHOD_KEY = '_mac_method'
diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py
deleted file mode 100644
index 6c26a29f..00000000
--- a/common/src/leap/soledad/common/document.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# -*- coding: utf-8 -*-
-# document.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-A Soledad Document is an l2db.Document with lasers.
-"""
-
-
-from .l2db import Document
-
-
-#
-# SoledadDocument
-#
-
-class SoledadDocument(Document):
-
- """
- Encryptable and syncable document.
-
- LEAP Documents can be flagged as syncable or not, so the replicas
- might not sync every document.
- """
-
- def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False,
- syncable=True):
- """
- Container for handling an encryptable document.
-
- @param doc_id: The unique document identifier.
- @type doc_id: str
- @param rev: The revision identifier of the document.
- @type rev: str
- @param json: The JSON string for this document.
- @type json: str
- @param has_conflicts: Boolean indicating if this document has conflicts
- @type has_conflicts: bool
- @param syncable: Should this document be synced with remote replicas?
- @type syncable: bool
- """
- Document.__init__(self, doc_id, rev, json, has_conflicts)
- self._syncable = syncable
-
- def _get_syncable(self):
- """
- Return whether this document is syncable.
-
- @return: Is this document syncable?
- @rtype: bool
- """
- return self._syncable
-
- def _set_syncable(self, syncable=True):
- """
- Determine if this document should be synced with remote replicas.
-
- @param syncable: Should this document be synced with remote replicas?
- @type syncable: bool
- """
- self._syncable = syncable
-
- syncable = property(
- _get_syncable,
- _set_syncable,
- doc="Determine if document should be synced with server."
- )
-
- def _get_rev(self):
- """
- Get the document revision.
-
- Returning the revision as string solves the following exception in
- Twisted web:
- exceptions.TypeError: Can only pass-through bytes on Python 2
-
- @return: The document revision.
- @rtype: str
- """
- if self._rev is None:
- return None
- return str(self._rev)
-
- def _set_rev(self, rev):
- """
- Set document revision.
-
- @param rev: The new document revision.
- @type rev: bytes
- """
- self._rev = rev
-
- rev = property(
- _get_rev,
- _set_rev,
- doc="Wrapper to ensure `doc.rev` is always returned as bytes.")
-
-
-class ServerDocument(SoledadDocument):
- """
- This is the document used by server to hold conflicts and transactions
- on a database.
-
- The goal is to ensure an atomic and consistent update of the database.
- """
-
- def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False):
- """
- Container for handling a document that stored on server.
-
- :param doc_id: The unique document identifier.
- :type doc_id: str
- :param rev: The revision identifier of the document.
- :type rev: str
- :param json: The JSON string for this document.
- :type json: str
- :param has_conflicts: Boolean indicating if this document has conflicts
- :type has_conflicts: bool
- """
- SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
- self._conflicts = None
-
- def get_conflicts(self):
- """
- Get the conflicted versions of the document.
-
- :return: The conflicted versions of the document.
- :rtype: [ServerDocument]
- """
- return self._conflicts or []
-
- def set_conflicts(self, conflicts):
- """
- Set the conflicted versions of the document.
-
- :param conflicts: The conflicted versions of the document.
- :type conflicts: list
- """
- self._conflicts = conflicts
- self.has_conflicts = len(self._conflicts) > 0
-
- def add_conflict(self, doc):
- """
- Add a conflict to this document.
-
- :param doc: The conflicted version to be added.
- :type doc: Document
- """
- if self._conflicts is None:
- raise Exception("Fetch conflicts first!")
- self._conflicts.append(doc)
- self.has_conflicts = len(self._conflicts) > 0
-
- def delete_conflicts(self, conflict_revs):
- """
- Delete conflicted versions of this document.
-
- :param conflict_revs: The conflicted revisions to be deleted.
- :type conflict_revs: [str]
- """
- if self._conflicts is None:
- raise Exception("Fetch conflicts first!")
- self._conflicts = filter(
- lambda doc: doc.rev not in conflict_revs,
- self._conflicts)
- self.has_conflicts = len(self._conflicts) > 0
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
deleted file mode 100644
index d543a3de..00000000
--- a/common/src/leap/soledad/common/errors.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# -*- coding: utf-8 -*-
-# errors.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Soledad errors.
-"""
-
-from .l2db import errors
-from .l2db.remote import http_errors
-
-
-def register_exception(cls):
- """
- A small decorator that registers exceptions in u1db maps.
- """
- # update u1db "wire description to status" and "wire description to
- # exception" maps.
- http_errors.wire_description_to_status.update({
- cls.wire_description: cls.status})
- errors.wire_description_to_exc.update({
- cls.wire_description: cls})
- # do not modify the exception
- return cls
-
-
-class SoledadError(errors.U1DBError):
-
- """
- Base Soledad HTTP errors.
- """
- pass
-
-
-#
-# Authorization errors
-#
-
-
-class DatabaseAccessError(Exception):
- pass
-
-
-@register_exception
-class InvalidAuthTokenError(errors.Unauthorized):
-
- """
- Exception raised when failing to get authorization for some action because
- the provided token either does not exist in the tokens database, has a
- distinct structure from the expected one, or is associated with a user
- with a distinct uuid than the one provided by the client.
- """
-
- wire_descrition = "invalid auth token"
- status = 401
-
-
-#
-# SoledadBackend errors
-# u1db error statuses also have to be updated
-http_errors.ERROR_STATUSES = set(
- http_errors.wire_description_to_status.values())
-
-
-class InvalidURLError(Exception):
- """
- Exception raised when Soledad encounters a malformed URL.
- """
-
-
-class BackendNotReadyError(SoledadError):
- """
- Generic exception raised when the backend is not ready to dispatch a client
- request.
- """
- wire_description = "backend not ready"
- status = 500
-
-
-class WrongCouchSchemaVersionError(SoledadError):
- """
- Raised in case there is a user database with wrong couch schema version.
- """
-
-
-class MissingCouchConfigDocumentError(SoledadError):
- """
- Raised if a database has documents but lacks the couch config document.
- """
diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py
deleted file mode 100644
index 43d61b1d..00000000
--- a/common/src/leap/soledad/common/l2db/__init__.py
+++ /dev/null
@@ -1,694 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""L2DB"""
-
-import json
-
-from leap.soledad.common.l2db.errors import InvalidJSON, InvalidContent
-
-__version_info__ = (13, 9)
-__version__ = '.'.join(map(lambda x: '%02d' % x, __version_info__))
-
-
-def open(path, create, document_factory=None):
- """Open a database at the given location.
-
- Will raise u1db.errors.DatabaseDoesNotExist if create=False and the
- database does not already exist.
-
- :param path: The filesystem path for the database to open.
- :param create: True/False, should the database be created if it doesn't
- already exist?
- :param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- :return: An instance of Database.
- """
- from leap.soledad.client._db import sqlite
- return sqlite.SQLiteDatabase.open_database(
- path, create=create, document_factory=document_factory)
-
-
-# constraints on database names (relevant for remote access, as regex)
-DBNAME_CONSTRAINTS = r"[a-zA-Z0-9][a-zA-Z0-9.-]*"
-
-# constraints on doc ids (as regex)
-# (no slashes, and no characters outside the ascii range)
-DOC_ID_CONSTRAINTS = r"[a-zA-Z0-9.%_-]+"
-
-
-class Database(object):
- """A JSON Document data store.
-
- This data store can be synchronized with other u1db.Database instances.
- """
-
- def set_document_factory(self, factory):
- """Set the document factory that will be used to create objects to be
- returned as documents by the database.
-
- :param factory: A function that returns an object which at minimum must
- satisfy the same interface as does the class DocumentBase.
- Subclassing that class is the easiest way to create such
- a function.
- """
- raise NotImplementedError(self.set_document_factory)
-
- def set_document_size_limit(self, limit):
- """Set the maximum allowed document size for this database.
-
- :param limit: Maximum allowed document size in bytes.
- """
- raise NotImplementedError(self.set_document_size_limit)
-
- def whats_changed(self, old_generation=0):
- """Return a list of documents that have changed since old_generation.
- This allows APPS to only store a db generation before going
- 'offline', and then when coming back online they can use this
- data to update whatever extra data they are storing.
-
- :param old_generation: The generation of the database in the old
- state.
- :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
- The current generation of the database, its associated transaction
- id, and a list of of changed documents since old_generation,
- represented by tuples with for each document its doc_id and the
- generation and transaction id corresponding to the last intervening
- change and sorted by generation (old changes first)
- """
- raise NotImplementedError(self.whats_changed)
-
- def get_doc(self, doc_id, include_deleted=False):
- """Get the JSON string for the given document.
-
- :param doc_id: The unique document identifier
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- :return: a Document object.
- """
- raise NotImplementedError(self.get_doc)
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False):
- """Get the JSON content for many documents.
-
- :param doc_ids: A list of document identifiers.
- :param check_for_conflicts: If set to False, then the conflict check
- will be skipped, and 'None' will be returned instead of True/False.
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- :return: iterable giving the Document object for each document id
- in matching doc_ids order.
- """
- raise NotImplementedError(self.get_docs)
-
- def get_all_docs(self, include_deleted=False):
- """Get the JSON content for all documents in the database.
-
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- :return: (generation, [Document])
- The current generation of the database, followed by a list of all
- the documents in the database.
- """
- raise NotImplementedError(self.get_all_docs)
-
- def create_doc(self, content, doc_id=None):
- """Create a new document.
-
- You can optionally specify the document identifier, but the document
- must not already exist. See 'put_doc' if you want to override an
- existing document.
- If the database specifies a maximum document size and the document
- exceeds it, create will fail and raise a DocumentTooBig exception.
-
- :param content: A Python dictionary.
- :param doc_id: An optional identifier specifying the document id.
- :return: Document
- """
- raise NotImplementedError(self.create_doc)
-
- def create_doc_from_json(self, json, doc_id=None):
- """Create a new document.
-
- You can optionally specify the document identifier, but the document
- must not already exist. See 'put_doc' if you want to override an
- existing document.
- If the database specifies a maximum document size and the document
- exceeds it, create will fail and raise a DocumentTooBig exception.
-
- :param json: The JSON document string
- :param doc_id: An optional identifier specifying the document id.
- :return: Document
- """
- raise NotImplementedError(self.create_doc_from_json)
-
- def put_doc(self, doc):
- """Update a document.
- If the document currently has conflicts, put will fail.
- If the database specifies a maximum document size and the document
- exceeds it, put will fail and raise a DocumentTooBig exception.
-
- :param doc: A Document with new content.
- :return: new_doc_rev - The new revision identifier for the document.
- The Document object will also be updated.
- """
- raise NotImplementedError(self.put_doc)
-
- def delete_doc(self, doc):
- """Mark a document as deleted.
- Will abort if the current revision doesn't match doc.rev.
- This will also set doc.content to None.
- """
- raise NotImplementedError(self.delete_doc)
-
- def create_index(self, index_name, *index_expressions):
- """Create an named index, which can then be queried for future lookups.
- Creating an index which already exists is not an error, and is cheap.
- Creating an index which does not match the index_expressions of the
- existing index is an error.
- Creating an index will block until the expressions have been evaluated
- and the index generated.
-
- :param index_name: A unique name which can be used as a key prefix
- :param index_expressions: index expressions defining the index
- information.
-
- Examples:
-
- "fieldname", or "fieldname.subfieldname" to index alphabetically
- sorted on the contents of a field.
-
- "number(fieldname, width)", "lower(fieldname)"
- """
- raise NotImplementedError(self.create_index)
-
- def delete_index(self, index_name):
- """Remove a named index.
-
- :param index_name: The name of the index we are removing
- """
- raise NotImplementedError(self.delete_index)
-
- def list_indexes(self):
- """List the definitions of all known indexes.
-
- :return: A list of [('index-name', ['field', 'field2'])] definitions.
- """
- raise NotImplementedError(self.list_indexes)
-
- def get_from_index(self, index_name, *key_values):
- """Return documents that match the keys supplied.
-
- You must supply exactly the same number of values as have been defined
- in the index. It is possible to do a prefix match by using '*' to
- indicate a wildcard match. You can only supply '*' to trailing entries,
- (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.)
- It is also possible to append a '*' to the last supplied value (eg
- 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
-
- :param index_name: The index to query
- :param key_values: values to match. eg, if you have
- an index with 3 fields then you would have:
- get_from_index(index_name, val1, val2, val3)
- :return: List of [Document]
- """
- raise NotImplementedError(self.get_from_index)
-
- def get_range_from_index(self, index_name, start_value, end_value):
- """Return documents that fall within the specified range.
-
- Both ends of the range are inclusive. For both start_value and
- end_value, one must supply exactly the same number of values as have
- been defined in the index, or pass None. In case of a single column
- index, a string is accepted as an alternative for a tuple with a single
- value. It is possible to do a prefix match by using '*' to indicate
- a wildcard match. You can only supply '*' to trailing entries, (eg
- 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also
- possible to append a '*' to the last supplied value (eg 'val*', '*',
- '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
-
- :param index_name: The index to query
- :param start_values: tuples of values that define the lower bound of
- the range. eg, if you have an index with 3 fields then you would
- have: (val1, val2, val3)
- :param end_values: tuples of values that define the upper bound of the
- range. eg, if you have an index with 3 fields then you would have:
- (val1, val2, val3)
- :return: List of [Document]
- """
- raise NotImplementedError(self.get_range_from_index)
-
- def get_index_keys(self, index_name):
- """Return all keys under which documents are indexed in this index.
-
- :param index_name: The index to query
- :return: [] A list of tuples of indexed keys.
- """
- raise NotImplementedError(self.get_index_keys)
-
- def get_doc_conflicts(self, doc_id):
- """Get the list of conflicts for the given document.
-
- The order of the conflicts is such that the first entry is the value
- that would be returned by "get_doc".
-
- :return: [doc] A list of the Document entries that are conflicted.
- """
- raise NotImplementedError(self.get_doc_conflicts)
-
- def resolve_doc(self, doc, conflicted_doc_revs):
- """Mark a document as no longer conflicted.
-
- We take the list of revisions that the client knows about that it is
- superseding. This may be a different list from the actual current
- conflicts, in which case only those are removed as conflicted. This
- may fail if the conflict list is significantly different from the
- supplied information. (sync could have happened in the background from
- the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
-
- :param doc: A Document with the new content to be inserted.
- :param conflicted_doc_revs: A list of revisions that the new content
- supersedes.
- """
- raise NotImplementedError(self.resolve_doc)
-
- def get_sync_target(self):
- """Return a SyncTarget object, for another u1db to synchronize with.
-
- :return: An instance of SyncTarget.
- """
- raise NotImplementedError(self.get_sync_target)
-
- def close(self):
- """Release any resources associated with this database."""
- raise NotImplementedError(self.close)
-
- def sync(self, url, creds=None, autocreate=True):
- """Synchronize documents with remote replica exposed at url.
-
- :param url: the url of the target replica to sync with.
- :param creds: optional dictionary giving credentials
- to authorize the operation with the server. For using OAuth
- the form of creds is:
- {'oauth': {
- 'consumer_key': ...,
- 'consumer_secret': ...,
- 'token_key': ...,
- 'token_secret': ...
- }}
- :param autocreate: ask the target to create the db if non-existent.
- :return: local_gen_before_sync The local generation before the
- synchronisation was performed. This is useful to pass into
- whatschanged, if an application wants to know which documents were
- affected by a synchronisation.
- """
- from u1db.sync import Synchronizer
- from u1db.remote.http_target import HTTPSyncTarget
- return Synchronizer(self, HTTPSyncTarget(url, creds=creds)).sync(
- autocreate=autocreate)
-
- def _get_replica_gen_and_trans_id(self, other_replica_uid):
- """Return the last known generation and transaction id for the other db
- replica.
-
- When you do a synchronization with another replica, the Database keeps
- track of what generation the other database replica was at, and what
- the associated transaction id was. This is used to determine what data
- needs to be sent, and if two databases are claiming to be the same
- replica.
-
- :param other_replica_uid: The identifier for the other replica.
- :return: (gen, trans_id) The generation and transaction id we
- encountered during synchronization. If we've never synchronized
- with the replica, this is (0, '').
- """
- raise NotImplementedError(self._get_replica_gen_and_trans_id)
-
- def _set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation, other_transaction_id):
- """Set the last-known generation and transaction id for the other
- database replica.
-
- We have just performed some synchronization, and we want to track what
- generation the other replica was at. See also
- _get_replica_gen_and_trans_id.
- :param other_replica_uid: The U1DB identifier for the other replica.
- :param other_generation: The generation number for the other replica.
- :param other_transaction_id: The transaction id associated with the
- generation.
- """
- raise NotImplementedError(self._set_replica_gen_and_trans_id)
-
- def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
- replica_trans_id=''):
- """Insert/update document into the database with a given revision.
-
- This api is used during synchronization operations.
-
- If a document would conflict and save_conflict is set to True, the
- content will be selected as the 'current' content for doc.doc_id,
- even though doc.rev doesn't supersede the currently stored revision.
- The currently stored document will be added to the list of conflict
- alternatives for the given doc_id.
-
- This forces the new content to be 'current' so that we get convergence
- after synchronizing, even if people don't resolve conflicts. Users can
- then notice that their content is out of date, update it, and
- synchronize again. (The alternative is that users could synchronize and
- think the data has propagated, but their local copy looks fine, and the
- remote copy is never updated again.)
-
- :param doc: A Document object
- :param save_conflict: If this document is a conflict, do you want to
- save it as a conflict, or just ignore it.
- :param replica_uid: A unique replica identifier.
- :param replica_gen: The generation of the replica corresponding to the
- this document. The replica arguments are optional, but are used
- during synchronization.
- :param replica_trans_id: The transaction_id associated with the
- generation.
- :return: (state, at_gen) - If we don't have doc_id already,
- or if doc_rev supersedes the existing document revision,
- then the content will be inserted, and state is 'inserted'.
- If doc_rev is less than or equal to the existing revision,
- then the put is ignored and state is respecitvely 'superseded'
- or 'converged'.
- If doc_rev is not strictly superseded or supersedes, then
- state is 'conflicted'. The document will not be inserted if
- save_conflict is False.
- For 'inserted' or 'converged', at_gen is the insertion/current
- generation.
- """
- raise NotImplementedError(self._put_doc_if_newer)
-
-
-class DocumentBase(object):
- """Container for handling a single document.
-
- :ivar doc_id: Unique identifier for this document.
- :ivar rev: The revision identifier of the document.
- :ivar json_string: The JSON string for this document.
- :ivar has_conflicts: Boolean indicating if this document has conflicts
- """
-
- def __init__(self, doc_id, rev, json_string, has_conflicts=False):
- self.doc_id = doc_id
- self.rev = rev
- if json_string is not None:
- try:
- value = json.loads(json_string)
- except ValueError:
- raise InvalidJSON
- if not isinstance(value, dict):
- raise InvalidJSON
- self._json = json_string
- self.has_conflicts = has_conflicts
-
- def same_content_as(self, other):
- """Compare the content of two documents."""
- if self._json:
- c1 = json.loads(self._json)
- else:
- c1 = None
- if other._json:
- c2 = json.loads(other._json)
- else:
- c2 = None
- return c1 == c2
-
- def __repr__(self):
- if self.has_conflicts:
- extra = ', conflicted'
- else:
- extra = ''
- return '%s(%s, %s%s, %r)' % (self.__class__.__name__, self.doc_id,
- self.rev, extra, self.get_json())
-
- def __hash__(self):
- raise NotImplementedError(self.__hash__)
-
- def __eq__(self, other):
- if not isinstance(other, Document):
- return NotImplemented
- return (
- self.doc_id == other.doc_id and self.rev == other.rev and
- self.same_content_as(other) and self.has_conflicts ==
- other.has_conflicts)
-
- def __lt__(self, other):
- """This is meant for testing, not part of the official api.
-
- It is implemented so that sorted([Document, Document]) can be used.
- It doesn't imply that users would want their documents to be sorted in
- this order.
- """
- # Since this is just for testing, we don't worry about comparing
- # against things that aren't a Document.
- return ((self.doc_id, self.rev, self.get_json()) <
- (other.doc_id, other.rev, other.get_json()))
-
- def get_json(self):
- """Get the json serialization of this document."""
- if self._json is not None:
- return self._json
- return None
-
- def get_size(self):
- """Calculate the total size of the document."""
- size = 0
- json = self.get_json()
- if json:
- size += len(json)
- if self.rev:
- size += len(self.rev)
- if self.doc_id:
- size += len(self.doc_id)
- return size
-
- def set_json(self, json_string):
- """Set the json serialization of this document."""
- if json_string is not None:
- try:
- value = json.loads(json_string)
- except ValueError:
- raise InvalidJSON
- if not isinstance(value, dict):
- raise InvalidJSON
- self._json = json_string
-
- def make_tombstone(self):
- """Make this document into a tombstone."""
- self._json = None
-
- def is_tombstone(self):
- """Return True if the document is a tombstone, False otherwise."""
- if self._json is not None:
- return False
- return True
-
-
-class Document(DocumentBase):
- """Container for handling a single document.
-
- :ivar doc_id: Unique identifier for this document.
- :ivar rev: The revision identifier of the document.
- :ivar json: The JSON string for this document.
- :ivar has_conflicts: Boolean indicating if this document has conflicts
- """
-
- # The following part of the API is optional: no implementation is forced to
- # have it but if the language supports dictionaries/hashtables, it makes
- # Documents a lot more user friendly.
-
- def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False):
- # TODO: We convert the json in the superclass to check its validity so
- # we might as well set _content here directly since the price is
- # already being paid.
- super(Document, self).__init__(doc_id, rev, json, has_conflicts)
- self._content = None
-
- def same_content_as(self, other):
- """Compare the content of two documents."""
- if self._json:
- c1 = json.loads(self._json)
- else:
- c1 = self._content
- if other._json:
- c2 = json.loads(other._json)
- else:
- c2 = other._content
- return c1 == c2
-
- def get_json(self):
- """Get the json serialization of this document."""
- json_string = super(Document, self).get_json()
- if json_string is not None:
- return json_string
- if self._content is not None:
- return json.dumps(self._content)
- return None
-
- def set_json(self, json):
- """Set the json serialization of this document."""
- self._content = None
- super(Document, self).set_json(json)
-
- def make_tombstone(self):
- """Make this document into a tombstone."""
- self._content = None
- super(Document, self).make_tombstone()
-
- def is_tombstone(self):
- """Return True if the document is a tombstone, False otherwise."""
- if self._content is not None:
- return False
- return super(Document, self).is_tombstone()
-
- def _get_content(self):
- """Get the dictionary representing this document."""
- if self._json is not None:
- self._content = json.loads(self._json)
- self._json = None
- if self._content is not None:
- return self._content
- return None
-
- def _set_content(self, content):
- """Set the dictionary representing this document."""
- try:
- tmp = json.dumps(content)
- except TypeError:
- raise InvalidContent(
- "Can not be converted to JSON: %r" % (content,))
- if not tmp.startswith('{'):
- raise InvalidContent(
- "Can not be converted to a JSON object: %r." % (content,))
- # We might as well store the JSON at this point since we did the work
- # of encoding it, and it doesn't lose any information.
- self._json = tmp
- self._content = None
-
- content = property(
- _get_content, _set_content, doc="Content of the Document.")
-
- # End of optional part.
-
-
-class SyncTarget(object):
- """Functionality for using a Database as a synchronization target."""
-
- def get_sync_info(self, source_replica_uid):
- """Return information about known state.
-
- Return the replica_uid and the current database generation of this
- database, and the last-seen database generation for source_replica_uid
-
- :param source_replica_uid: Another replica which we might have
- synchronized with in the past.
- :return: (target_replica_uid, target_replica_generation,
- target_trans_id, source_replica_last_known_generation,
- source_replica_last_known_transaction_id)
- """
- raise NotImplementedError(self.get_sync_info)
-
- def record_sync_info(self, source_replica_uid, source_replica_generation,
- source_replica_transaction_id):
- """Record tip information for another replica.
-
- After sync_exchange has been processed, the caller will have
- received new content from this replica. This call allows the
- source replica instigating the sync to inform us what their
- generation became after applying the documents we returned.
-
- This is used to allow future sync operations to not need to repeat data
- that we just talked about. It also means that if this is called at the
- wrong time, there can be database records that will never be
- synchronized.
-
- :param source_replica_uid: The identifier for the source replica.
- :param source_replica_generation:
- The database generation for the source replica.
- :param source_replica_transaction_id: The transaction id associated
- with the source replica generation.
- """
- raise NotImplementedError(self.record_sync_info)
-
- def sync_exchange(self, docs_by_generation, source_replica_uid,
- last_known_generation, last_known_trans_id,
- return_doc_cb, ensure_callback=None):
- """Incorporate the documents sent from the source replica.
-
- This is not meant to be called by client code directly, but is used as
- part of sync().
-
- This adds docs to the local store, and determines documents that need
- to be returned to the source replica.
-
- Documents must be supplied in docs_by_generation paired with
- the generation of their latest change in order from the oldest
- change to the newest, that means from the oldest generation to
- the newest.
-
- Documents are also returned paired with the generation of
- their latest change in order from the oldest change to the
- newest.
-
- :param docs_by_generation: A list of [(Document, generation,
- transaction_id)] tuples indicating documents which should be
- updated on this replica paired with the generation and transaction
- id of their latest change.
- :param source_replica_uid: The source replica's identifier
- :param last_known_generation: The last generation that the source
- replica knows about this target replica
- :param last_known_trans_id: The last transaction id that the source
- replica knows about this target replica
- :param: return_doc_cb(doc, gen): is a callback
- used to return documents to the source replica, it will
- be invoked in turn with Documents that have changed since
- last_known_generation together with the generation of
- their last change.
- :param: ensure_callback(replica_uid): if set the target may create
- the target db if not yet existent, the callback can then
- be used to inform of the created db replica uid.
- :return: new_generation - After applying docs_by_generation, this is
- the current generation for this replica
- """
- raise NotImplementedError(self.sync_exchange)
-
- def _set_trace_hook(self, cb):
- """Set a callback that will be invoked to trace database actions.
-
- The callback will be passed a string indicating the current state, and
- the sync target object. Implementations do not have to implement this
- api, it is used by the test suite.
-
- :param cb: A callable that takes cb(state)
- """
- raise NotImplementedError(self._set_trace_hook)
-
- def _set_trace_hook_shallow(self, cb):
- """Set a callback that will be invoked to trace database actions.
-
- Similar to _set_trace_hook, for implementations that don't offer
- state changes from the inner working of sync_exchange().
-
- :param cb: A callable that takes cb(state)
- """
- self._set_trace_hook(cb)
diff --git a/common/src/leap/soledad/common/l2db/backends/__init__.py b/common/src/leap/soledad/common/l2db/backends/__init__.py
deleted file mode 100644
index c731c3d3..00000000
--- a/common/src/leap/soledad/common/l2db/backends/__init__.py
+++ /dev/null
@@ -1,204 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""Abstract classes and common implementations for the backends."""
-
-import re
-import json
-import uuid
-
-from leap.soledad.common import l2db
-from leap.soledad.common.l2db import sync as l2db_sync
-from leap.soledad.common.l2db import errors
-from leap.soledad.common.l2db.vectorclock import VectorClockRev
-
-
-check_doc_id_re = re.compile("^" + l2db.DOC_ID_CONSTRAINTS + "$", re.UNICODE)
-
-
-class CommonSyncTarget(l2db_sync.LocalSyncTarget):
- pass
-
-
-class CommonBackend(l2db.Database):
-
- document_size_limit = 0
-
- def _allocate_doc_id(self):
- """Generate a unique identifier for this document."""
- return 'D-' + uuid.uuid4().hex # 'D-' stands for document
-
- def _allocate_transaction_id(self):
- return 'T-' + uuid.uuid4().hex # 'T-' stands for transaction
-
- def _allocate_doc_rev(self, old_doc_rev):
- vcr = VectorClockRev(old_doc_rev)
- vcr.increment(self._replica_uid)
- return vcr.as_str()
-
- def _check_doc_id(self, doc_id):
- if not check_doc_id_re.match(doc_id):
- raise errors.InvalidDocId()
-
- def _check_doc_size(self, doc):
- if not self.document_size_limit:
- return
- if doc.get_size() > self.document_size_limit:
- raise errors.DocumentTooBig
-
- def _get_generation(self):
- """Return the current generation.
-
- """
- raise NotImplementedError(self._get_generation)
-
- def _get_generation_info(self):
- """Return the current generation and transaction id.
-
- """
- raise NotImplementedError(self._get_generation_info)
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- """Extract the document from storage.
-
- This can return None if the document doesn't exist.
- """
- raise NotImplementedError(self._get_doc)
-
- def _has_conflicts(self, doc_id):
- """Return True if the doc has conflicts, False otherwise."""
- raise NotImplementedError(self._has_conflicts)
-
- def create_doc(self, content, doc_id=None):
- if not isinstance(content, dict):
- raise errors.InvalidContent
- json_string = json.dumps(content)
- return self.create_doc_from_json(json_string, doc_id)
-
- def create_doc_from_json(self, json, doc_id=None):
- if doc_id is None:
- doc_id = self._allocate_doc_id()
- doc = self._factory(doc_id, None, json)
- self.put_doc(doc)
- return doc
-
- def _get_transaction_log(self):
- """This is only for the test suite, it is not part of the api."""
- raise NotImplementedError(self._get_transaction_log)
-
- def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content):
- raise NotImplementedError(self._put_and_update_indexes)
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False):
- for doc_id in doc_ids:
- doc = self._get_doc(
- doc_id, check_for_conflicts=check_for_conflicts)
- if doc.is_tombstone() and not include_deleted:
- continue
- yield doc
-
- def _get_trans_id_for_gen(self, generation):
- """Get the transaction id corresponding to a particular generation.
-
- Raises an InvalidGeneration when the generation does not exist.
-
- """
- raise NotImplementedError(self._get_trans_id_for_gen)
-
- def validate_gen_and_trans_id(self, generation, trans_id):
- """Validate the generation and transaction id.
-
- Raises an InvalidGeneration when the generation does not exist, and an
- InvalidTransactionId when it does but with a different transaction id.
-
- """
- if generation == 0:
- return
- known_trans_id = self._get_trans_id_for_gen(generation)
- if known_trans_id != trans_id:
- raise errors.InvalidTransactionId
-
- def _validate_source(self, other_replica_uid, other_generation,
- other_transaction_id):
- """Validate the new generation and transaction id.
-
- other_generation must be greater than what we have stored for this
- replica, *or* it must be the same and the transaction_id must be the
- same as well.
- """
- (old_generation,
- old_transaction_id) = self._get_replica_gen_and_trans_id(
- other_replica_uid)
- if other_generation < old_generation:
- raise errors.InvalidGeneration
- if other_generation > old_generation:
- return
- if other_transaction_id == old_transaction_id:
- return
- raise errors.InvalidTransactionId
-
- def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
- replica_trans_id=''):
- cur_doc = self._get_doc(doc.doc_id)
- doc_vcr = VectorClockRev(doc.rev)
- if cur_doc is None:
- cur_vcr = VectorClockRev(None)
- else:
- cur_vcr = VectorClockRev(cur_doc.rev)
- self._validate_source(replica_uid, replica_gen, replica_trans_id)
- if doc_vcr.is_newer(cur_vcr):
- rev = doc.rev
- self._prune_conflicts(doc, doc_vcr)
- if doc.rev != rev:
- # conflicts have been autoresolved
- state = 'superseded'
- else:
- state = 'inserted'
- self._put_and_update_indexes(cur_doc, doc)
- elif doc.rev == cur_doc.rev:
- # magical convergence
- state = 'converged'
- elif cur_vcr.is_newer(doc_vcr):
- # Don't add this to seen_ids, because we have something newer,
- # so we should send it back, and we should not generate a
- # conflict
- state = 'superseded'
- elif cur_doc.same_content_as(doc):
- # the documents have been edited to the same thing at both ends
- doc_vcr.maximize(cur_vcr)
- doc_vcr.increment(self._replica_uid)
- doc.rev = doc_vcr.as_str()
- self._put_and_update_indexes(cur_doc, doc)
- state = 'superseded'
- else:
- state = 'conflicted'
- if save_conflict:
- self._force_doc_sync_conflict(doc)
- if replica_uid is not None and replica_gen is not None:
- self._do_set_replica_gen_and_trans_id(
- replica_uid, replica_gen, replica_trans_id)
- return state, self._get_generation()
-
- def _ensure_maximal_rev(self, cur_rev, extra_revs):
- vcr = VectorClockRev(cur_rev)
- for rev in extra_revs:
- vcr.maximize(VectorClockRev(rev))
- vcr.increment(self._replica_uid)
- return vcr.as_str()
-
- def set_document_size_limit(self, limit):
- self.document_size_limit = limit
diff --git a/common/src/leap/soledad/common/l2db/backends/inmemory.py b/common/src/leap/soledad/common/l2db/backends/inmemory.py
deleted file mode 100644
index 6fd251af..00000000
--- a/common/src/leap/soledad/common/l2db/backends/inmemory.py
+++ /dev/null
@@ -1,466 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""The in-memory Database class for U1DB."""
-
-import json
-
-from leap.soledad.common.l2db import (
- Document, errors,
- query_parser, vectorclock)
-from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget
-
-
-def get_prefix(value):
- key_prefix = '\x01'.join(value)
- return key_prefix.rstrip('*')
-
-
-class InMemoryDatabase(CommonBackend):
- """A database that only stores the data internally."""
-
- def __init__(self, replica_uid, document_factory=None):
- self._transaction_log = []
- self._docs = {}
- # Map from doc_id => [(doc_rev, doc)] conflicts beyond 'winner'
- self._conflicts = {}
- self._other_generations = {}
- self._indexes = {}
- self._replica_uid = replica_uid
- self._factory = document_factory or Document
-
- def _set_replica_uid(self, replica_uid):
- """Force the replica_uid to be set."""
- self._replica_uid = replica_uid
-
- def set_document_factory(self, factory):
- self._factory = factory
-
- def close(self):
- # This is a no-op, We don't want to free the data because one client
- # may be closing it, while another wants to inspect the results.
- pass
-
- def _get_replica_gen_and_trans_id(self, other_replica_uid):
- return self._other_generations.get(other_replica_uid, (0, ''))
-
- def _set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation, other_transaction_id):
- self._do_set_replica_gen_and_trans_id(
- other_replica_uid, other_generation, other_transaction_id)
-
- def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation,
- other_transaction_id):
- # TODO: to handle race conditions, we may want to check if the current
- # value is greater than this new value.
- self._other_generations[other_replica_uid] = (other_generation,
- other_transaction_id)
-
- def get_sync_target(self):
- return InMemorySyncTarget(self)
-
- def _get_transaction_log(self):
- # snapshot!
- return self._transaction_log[:]
-
- def _get_generation(self):
- return len(self._transaction_log)
-
- def _get_generation_info(self):
- if not self._transaction_log:
- return 0, ''
- return len(self._transaction_log), self._transaction_log[-1][1]
-
- def _get_trans_id_for_gen(self, generation):
- if generation == 0:
- return ''
- if generation > len(self._transaction_log):
- raise errors.InvalidGeneration
- return self._transaction_log[generation - 1][1]
-
- def put_doc(self, doc):
- if doc.doc_id is None:
- raise errors.InvalidDocId()
- self._check_doc_id(doc.doc_id)
- self._check_doc_size(doc)
- old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if old_doc and old_doc.has_conflicts:
- raise errors.ConflictedDoc()
- if old_doc and doc.rev is None and old_doc.is_tombstone():
- new_rev = self._allocate_doc_rev(old_doc.rev)
- else:
- if old_doc is not None:
- if old_doc.rev != doc.rev:
- raise errors.RevisionConflict()
- else:
- if doc.rev is not None:
- raise errors.RevisionConflict()
- new_rev = self._allocate_doc_rev(doc.rev)
- doc.rev = new_rev
- self._put_and_update_indexes(old_doc, doc)
- return new_rev
-
- def _put_and_update_indexes(self, old_doc, doc):
- for index in self._indexes.itervalues():
- if old_doc is not None and not old_doc.is_tombstone():
- index.remove_json(old_doc.doc_id, old_doc.get_json())
- if not doc.is_tombstone():
- index.add_json(doc.doc_id, doc.get_json())
- trans_id = self._allocate_transaction_id()
- self._docs[doc.doc_id] = (doc.rev, doc.get_json())
- self._transaction_log.append((doc.doc_id, trans_id))
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- try:
- doc_rev, content = self._docs[doc_id]
- except KeyError:
- return None
- doc = self._factory(doc_id, doc_rev, content)
- if check_for_conflicts:
- doc.has_conflicts = (doc.doc_id in self._conflicts)
- return doc
-
- def _has_conflicts(self, doc_id):
- return doc_id in self._conflicts
-
- def get_doc(self, doc_id, include_deleted=False):
- doc = self._get_doc(doc_id, check_for_conflicts=True)
- if doc is None:
- return None
- if doc.is_tombstone() and not include_deleted:
- return None
- return doc
-
- def get_all_docs(self, include_deleted=False):
- """Return all documents in the database."""
- generation = self._get_generation()
- results = []
- for doc_id, (doc_rev, content) in self._docs.items():
- if content is None and not include_deleted:
- continue
- doc = self._factory(doc_id, doc_rev, content)
- doc.has_conflicts = self._has_conflicts(doc_id)
- results.append(doc)
- return (generation, results)
-
- def get_doc_conflicts(self, doc_id):
- if doc_id not in self._conflicts:
- return []
- result = [self._get_doc(doc_id)]
- result[0].has_conflicts = True
- result.extend([self._factory(doc_id, rev, content)
- for rev, content in self._conflicts[doc_id]])
- return result
-
- def _replace_conflicts(self, doc, conflicts):
- if not conflicts:
- del self._conflicts[doc.doc_id]
- else:
- self._conflicts[doc.doc_id] = conflicts
- doc.has_conflicts = bool(conflicts)
-
- def _prune_conflicts(self, doc, doc_vcr):
- if self._has_conflicts(doc.doc_id):
- autoresolved = False
- remaining_conflicts = []
- cur_conflicts = self._conflicts[doc.doc_id]
- for c_rev, c_doc in cur_conflicts:
- c_vcr = vectorclock.VectorClockRev(c_rev)
- if doc_vcr.is_newer(c_vcr):
- continue
- if doc.same_content_as(Document(doc.doc_id, c_rev, c_doc)):
- doc_vcr.maximize(c_vcr)
- autoresolved = True
- continue
- remaining_conflicts.append((c_rev, c_doc))
- if autoresolved:
- doc_vcr.increment(self._replica_uid)
- doc.rev = doc_vcr.as_str()
- self._replace_conflicts(doc, remaining_conflicts)
-
- def resolve_doc(self, doc, conflicted_doc_revs):
- cur_doc = self._get_doc(doc.doc_id)
- if cur_doc is None:
- cur_rev = None
- else:
- cur_rev = cur_doc.rev
- new_rev = self._ensure_maximal_rev(cur_rev, conflicted_doc_revs)
- superseded_revs = set(conflicted_doc_revs)
- remaining_conflicts = []
- cur_conflicts = self._conflicts[doc.doc_id]
- for c_rev, c_doc in cur_conflicts:
- if c_rev in superseded_revs:
- continue
- remaining_conflicts.append((c_rev, c_doc))
- doc.rev = new_rev
- if cur_rev in superseded_revs:
- self._put_and_update_indexes(cur_doc, doc)
- else:
- remaining_conflicts.append((new_rev, doc.get_json()))
- self._replace_conflicts(doc, remaining_conflicts)
-
- def delete_doc(self, doc):
- if doc.doc_id not in self._docs:
- raise errors.DocumentDoesNotExist
- if self._docs[doc.doc_id][1] in ('null', None):
- raise errors.DocumentAlreadyDeleted
- doc.make_tombstone()
- self.put_doc(doc)
-
- def create_index(self, index_name, *index_expressions):
- if index_name in self._indexes:
- if self._indexes[index_name]._definition == list(
- index_expressions):
- return
- raise errors.IndexNameTakenError
- index = InMemoryIndex(index_name, list(index_expressions))
- for doc_id, (doc_rev, doc) in self._docs.iteritems():
- if doc is not None:
- index.add_json(doc_id, doc)
- self._indexes[index_name] = index
-
- def delete_index(self, index_name):
- try:
- del self._indexes[index_name]
- except KeyError:
- pass
-
- def list_indexes(self):
- definitions = []
- for idx in self._indexes.itervalues():
- definitions.append((idx._name, idx._definition))
- return definitions
-
- def get_from_index(self, index_name, *key_values):
- try:
- index = self._indexes[index_name]
- except KeyError:
- raise errors.IndexDoesNotExist
- doc_ids = index.lookup(key_values)
- result = []
- for doc_id in doc_ids:
- result.append(self._get_doc(doc_id, check_for_conflicts=True))
- return result
-
- def get_range_from_index(self, index_name, start_value=None,
- end_value=None):
- """Return all documents with key values in the specified range."""
- try:
- index = self._indexes[index_name]
- except KeyError:
- raise errors.IndexDoesNotExist
- if isinstance(start_value, basestring):
- start_value = (start_value,)
- if isinstance(end_value, basestring):
- end_value = (end_value,)
- doc_ids = index.lookup_range(start_value, end_value)
- result = []
- for doc_id in doc_ids:
- result.append(self._get_doc(doc_id, check_for_conflicts=True))
- return result
-
- def get_index_keys(self, index_name):
- try:
- index = self._indexes[index_name]
- except KeyError:
- raise errors.IndexDoesNotExist
- keys = index.keys()
- # XXX inefficiency warning
- return list(set([tuple(key.split('\x01')) for key in keys]))
-
- def whats_changed(self, old_generation=0):
- changes = []
- relevant_tail = self._transaction_log[old_generation:]
- # We don't use len(self._transaction_log) because _transaction_log may
- # get mutated by a concurrent operation.
- cur_generation = old_generation + len(relevant_tail)
- last_trans_id = ''
- if relevant_tail:
- last_trans_id = relevant_tail[-1][1]
- elif self._transaction_log:
- last_trans_id = self._transaction_log[-1][1]
- seen = set()
- generation = cur_generation
- for doc_id, trans_id in reversed(relevant_tail):
- if doc_id not in seen:
- changes.append((doc_id, generation, trans_id))
- seen.add(doc_id)
- generation -= 1
- changes.reverse()
- return (cur_generation, last_trans_id, changes)
-
- def _force_doc_sync_conflict(self, doc):
- my_doc = self._get_doc(doc.doc_id)
- self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
- self._conflicts.setdefault(doc.doc_id, []).append(
- (my_doc.rev, my_doc.get_json()))
- doc.has_conflicts = True
- self._put_and_update_indexes(my_doc, doc)
-
-
-class InMemoryIndex(object):
- """Interface for managing an Index."""
-
- def __init__(self, index_name, index_definition):
- self._name = index_name
- self._definition = index_definition
- self._values = {}
- parser = query_parser.Parser()
- self._getters = parser.parse_all(self._definition)
-
- def evaluate_json(self, doc):
- """Determine the 'key' after applying this index to the doc."""
- raw = json.loads(doc)
- return self.evaluate(raw)
-
- def evaluate(self, obj):
- """Evaluate a dict object, applying this definition."""
- all_rows = [[]]
- for getter in self._getters:
- new_rows = []
- keys = getter.get(obj)
- if not keys:
- return []
- for key in keys:
- new_rows.extend([row + [key] for row in all_rows])
- all_rows = new_rows
- all_rows = ['\x01'.join(row) for row in all_rows]
- return all_rows
-
- def add_json(self, doc_id, doc):
- """Add this json doc to the index."""
- keys = self.evaluate_json(doc)
- if not keys:
- return
- for key in keys:
- self._values.setdefault(key, []).append(doc_id)
-
- def remove_json(self, doc_id, doc):
- """Remove this json doc from the index."""
- keys = self.evaluate_json(doc)
- if keys:
- for key in keys:
- doc_ids = self._values[key]
- doc_ids.remove(doc_id)
- if not doc_ids:
- del self._values[key]
-
- def _find_non_wildcards(self, values):
- """Check if this should be a wildcard match.
-
- Further, this will raise an exception if the syntax is improperly
- defined.
-
- :return: The offset of the last value we need to match against.
- """
- if len(values) != len(self._definition):
- raise errors.InvalidValueForIndex()
- is_wildcard = False
- last = 0
- for idx, val in enumerate(values):
- if val.endswith('*'):
- if val != '*':
- # We have an 'x*' style wildcard
- if is_wildcard:
- # We were already in wildcard mode, so this is invalid
- raise errors.InvalidGlobbing
- last = idx + 1
- is_wildcard = True
- else:
- if is_wildcard:
- # We were in wildcard mode, we can't follow that with
- # non-wildcard
- raise errors.InvalidGlobbing
- last = idx + 1
- if not is_wildcard:
- return -1
- return last
-
- def lookup(self, values):
- """Find docs that match the values."""
- last = self._find_non_wildcards(values)
- if last == -1:
- return self._lookup_exact(values)
- else:
- return self._lookup_prefix(values[:last])
-
- def lookup_range(self, start_values, end_values):
- """Find docs within the range."""
- # TODO: Wildly inefficient, which is unlikely to be a problem for the
- # inmemory implementation.
- if start_values:
- self._find_non_wildcards(start_values)
- start_values = get_prefix(start_values)
- if end_values:
- if self._find_non_wildcards(end_values) == -1:
- exact = True
- else:
- exact = False
- end_values = get_prefix(end_values)
- found = []
- for key, doc_ids in sorted(self._values.iteritems()):
- if start_values and start_values > key:
- continue
- if end_values and end_values < key:
- if exact:
- break
- else:
- if not key.startswith(end_values):
- break
- found.extend(doc_ids)
- return found
-
- def keys(self):
- """Find the indexed keys."""
- return self._values.keys()
-
- def _lookup_prefix(self, value):
- """Find docs that match the prefix string in values."""
- # TODO: We need a different data structure to make prefix style fast,
- # some sort of sorted list would work, but a plain dict doesn't.
- key_prefix = get_prefix(value)
- all_doc_ids = []
- for key, doc_ids in sorted(self._values.iteritems()):
- if key.startswith(key_prefix):
- all_doc_ids.extend(doc_ids)
- return all_doc_ids
-
- def _lookup_exact(self, value):
- """Find docs that match exactly."""
- key = '\x01'.join(value)
- if key in self._values:
- return self._values[key]
- return ()
-
-
-class InMemorySyncTarget(CommonSyncTarget):
-
- def get_sync_info(self, source_replica_uid):
- source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id(
- source_replica_uid)
- my_gen, my_trans_id = self._db._get_generation_info()
- return (
- self._db._replica_uid, my_gen, my_trans_id, source_gen,
- source_trans_id)
-
- def record_sync_info(self, source_replica_uid, source_replica_generation,
- source_transaction_id):
- if self._trace_hook:
- self._trace_hook('record_sync_info')
- self._db._set_replica_gen_and_trans_id(
- source_replica_uid, source_replica_generation,
- source_transaction_id)
diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py
deleted file mode 100644
index b502fc2d..00000000
--- a/common/src/leap/soledad/common/l2db/errors.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""A list of errors that u1db can raise."""
-
-
-class U1DBError(Exception):
- """Generic base class for U1DB errors."""
-
- # description/tag for identifying the error during transmission (http,...)
- wire_description = "error"
-
- def __init__(self, message=None):
- self.message = message
-
-
-class RevisionConflict(U1DBError):
- """The document revisions supplied does not match the current version."""
-
- wire_description = "revision conflict"
-
-
-class InvalidJSON(U1DBError):
- """Content was not valid json."""
-
-
-class InvalidContent(U1DBError):
- """Content was not a python dictionary."""
-
-
-class InvalidDocId(U1DBError):
- """A document was requested with an invalid document identifier."""
-
- wire_description = "invalid document id"
-
-
-class MissingDocIds(U1DBError):
- """Needs document ids."""
-
- wire_description = "missing document ids"
-
-
-class DocumentTooBig(U1DBError):
- """Document exceeds the maximum document size for this database."""
-
- wire_description = "document too big"
-
-
-class UserQuotaExceeded(U1DBError):
- """Document exceeds the maximum document size for this database."""
-
- wire_description = "user quota exceeded"
-
-
-class SubscriptionNeeded(U1DBError):
- """User needs a subscription to be able to use this replica.."""
-
- wire_description = "user needs subscription"
-
-
-class InvalidTransactionId(U1DBError):
- """Invalid transaction for generation."""
-
- wire_description = "invalid transaction id"
-
-
-class InvalidGeneration(U1DBError):
- """Generation was previously synced with a different transaction id."""
-
- wire_description = "invalid generation"
-
-
-class InvalidReplicaUID(U1DBError):
- """Attempting to sync a database with itself."""
-
- wire_description = "invalid replica uid"
-
-
-class ConflictedDoc(U1DBError):
- """The document is conflicted, you must call resolve before put()"""
-
-
-class InvalidValueForIndex(U1DBError):
- """The values supplied does not match the index definition."""
-
-
-class InvalidGlobbing(U1DBError):
- """Raised if wildcard matches are not strictly at the tail of the request.
- """
-
-
-class DocumentDoesNotExist(U1DBError):
- """The document does not exist."""
-
- wire_description = "document does not exist"
-
-
-class DocumentAlreadyDeleted(U1DBError):
- """The document was already deleted."""
-
- wire_description = "document already deleted"
-
-
-class DatabaseDoesNotExist(U1DBError):
- """The database does not exist."""
-
- wire_description = "database does not exist"
-
-
-class IndexNameTakenError(U1DBError):
- """The given index name is already taken."""
-
-
-class IndexDefinitionParseError(U1DBError):
- """The index definition cannot be parsed."""
-
-
-class IndexDoesNotExist(U1DBError):
- """No index of that name exists."""
-
-
-class Unauthorized(U1DBError):
- """Request wasn't authorized properly."""
-
- wire_description = "unauthorized"
-
-
-class HTTPError(U1DBError):
- """Unspecific HTTP errror."""
-
- wire_description = None
-
- def __init__(self, status, message=None, headers={}):
- self.status = status
- self.message = message
- self.headers = headers
-
- def __str__(self):
- if not self.message:
- return "HTTPError(%d)" % self.status
- else:
- return "HTTPError(%d, %r)" % (self.status, self.message)
-
-
-class Unavailable(HTTPError):
- """Server not available not serve request."""
-
- wire_description = "unavailable"
-
- def __init__(self, message=None, headers={}):
- super(Unavailable, self).__init__(503, message, headers)
-
- def __str__(self):
- if not self.message:
- return "Unavailable()"
- else:
- return "Unavailable(%r)" % self.message
-
-
-class BrokenSyncStream(U1DBError):
- """Unterminated or otherwise broken sync exchange stream."""
-
- wire_description = None
-
-
-class UnknownAuthMethod(U1DBError):
- """Unknown auhorization method."""
-
- wire_description = None
-
-
-# mapping wire (transimission) descriptions/tags for errors to the exceptions
-wire_description_to_exc = dict(
- (x.wire_description, x) for x in globals().values()
- if getattr(x, 'wire_description', None) not in (None, "error"))
-wire_description_to_exc["error"] = U1DBError
-
-
-#
-# wire error descriptions not corresponding to an exception
-DOCUMENT_DELETED = "document deleted"
diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py
deleted file mode 100644
index 15a9ac80..00000000
--- a/common/src/leap/soledad/common/l2db/query_parser.py
+++ /dev/null
@@ -1,371 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-# Copyright 2016 LEAP Encryption Access Project
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-"""
-Code for parsing Index definitions.
-"""
-
-import re
-
-from leap.soledad.common.l2db import errors
-
-
-class Getter(object):
- """Get values from a document based on a specification."""
-
- def get(self, raw_doc):
- """Get a value from the document.
-
- :param raw_doc: a python dictionary to get the value from.
- :return: A list of values that match the description.
- """
- raise NotImplementedError(self.get)
-
-
-class StaticGetter(Getter):
- """A getter that returns a defined value (independent of the doc)."""
-
- def __init__(self, value):
- """Create a StaticGetter.
-
- :param value: the value to return when get is called.
- """
- if value is None:
- self.value = []
- elif isinstance(value, list):
- self.value = value
- else:
- self.value = [value]
-
- def get(self, raw_doc):
- return self.value
-
-
-def extract_field(raw_doc, subfields, index=0):
- if not isinstance(raw_doc, dict):
- return []
- val = raw_doc.get(subfields[index])
- if val is None:
- return []
- if index < len(subfields) - 1:
- if isinstance(val, list):
- results = []
- for item in val:
- results.extend(extract_field(item, subfields, index + 1))
- return results
- if isinstance(val, dict):
- return extract_field(val, subfields, index + 1)
- return []
- if isinstance(val, dict):
- return []
- if isinstance(val, list):
- # Strip anything in the list that isn't a simple type
- return [v for v in val if not isinstance(v, (dict, list))]
- return [val]
-
-
-class ExtractField(Getter):
- """Extract a field from the document."""
-
- def __init__(self, field):
- """Create an ExtractField object.
-
- When a document is passed to get() this will return a value
- from the document based on the field specifier passed to
- the constructor.
-
- None will be returned if the field is nonexistant, or refers to an
- object, rather than a simple type or list of simple types.
-
- :param field: a specifier for the field to return.
- This is either a field name, or a dotted field name.
- """
- self.field = field.split('.')
-
- def get(self, raw_doc):
- return extract_field(raw_doc, self.field)
-
-
-class Transformation(Getter):
- """A transformation on a value from another Getter."""
-
- name = None
- arity = 1
- args = ['expression']
-
- def __init__(self, inner):
- """Create a transformation.
-
- :param inner: the argument(s) to the transformation.
- """
- self.inner = inner
-
- def get(self, raw_doc):
- inner_values = self.inner.get(raw_doc)
- assert isinstance(inner_values, list),\
- 'get() should always return a list'
- return self.transform(inner_values)
-
- def transform(self, values):
- """Transform the values.
-
- This should be implemented by subclasses to transform the
- value when get() is called.
-
- :param values: the values from the other Getter
- :return: the transformed values.
- """
- raise NotImplementedError(self.transform)
-
-
-class Lower(Transformation):
- """Lowercase a string.
-
- This transformation will return None for non-string inputs. However,
- it will lowercase any strings in a list, dropping any elements
- that are not strings.
- """
-
- name = "lower"
-
- def _can_transform(self, val):
- return isinstance(val, basestring)
-
- def transform(self, values):
- if not values:
- return []
- return [val.lower() for val in values if self._can_transform(val)]
-
-
-class Number(Transformation):
- """Convert an integer to a zero padded string.
-
- This transformation will return None for non-integer inputs. However, it
- will transform any integers in a list, dropping any elements that are not
- integers.
- """
-
- name = 'number'
- arity = 2
- args = ['expression', int]
-
- def __init__(self, inner, number):
- super(Number, self).__init__(inner)
- self.padding = "%%0%sd" % number
-
- def _can_transform(self, val):
- return isinstance(val, int) and not isinstance(val, bool)
-
- def transform(self, values):
- """Transform any integers in values into zero padded strings."""
- if not values:
- return []
- return [self.padding % (v,) for v in values if self._can_transform(v)]
-
-
-class Bool(Transformation):
- """Convert bool to string."""
-
- name = "bool"
- args = ['expression']
-
- def _can_transform(self, val):
- return isinstance(val, bool)
-
- def transform(self, values):
- """Transform any booleans in values into strings."""
- if not values:
- return []
- return [('1' if v else '0') for v in values if self._can_transform(v)]
-
-
-class SplitWords(Transformation):
- """Split a string on whitespace.
-
- This Getter will return [] for non-string inputs. It will however
- split any strings in an input list, discarding any elements that
- are not strings.
- """
-
- name = "split_words"
-
- def _can_transform(self, val):
- return isinstance(val, basestring)
-
- def transform(self, values):
- if not values:
- return []
- result = set()
- for value in values:
- if self._can_transform(value):
- for word in value.split():
- result.add(word)
- return list(result)
-
-
-class Combine(Transformation):
- """Combine multiple expressions into a single index."""
-
- name = "combine"
- # variable number of args
- arity = -1
-
- def __init__(self, *inner):
- super(Combine, self).__init__(inner)
-
- def get(self, raw_doc):
- inner_values = []
- for inner in self.inner:
- inner_values.extend(inner.get(raw_doc))
- return self.transform(inner_values)
-
- def transform(self, values):
- return values
-
-
-class IsNull(Transformation):
- """Indicate whether the input is None.
-
- This Getter returns a bool indicating whether the input is nil.
- """
-
- name = "is_null"
-
- def transform(self, values):
- return [len(values) == 0]
-
-
-def check_fieldname(fieldname):
- if fieldname.endswith('.'):
- raise errors.IndexDefinitionParseError(
- "Fieldname cannot end in '.':%s^" % (fieldname,))
-
-
-class Parser(object):
- """Parse an index expression into a sequence of transformations."""
-
- _transformations = {}
- _delimiters = re.compile("\(|\)|,")
-
- def __init__(self):
- self._tokens = []
-
- def _set_expression(self, expression):
- self._open_parens = 0
- self._tokens = []
- expression = expression.strip()
- while expression:
- delimiter = self._delimiters.search(expression)
- if delimiter:
- idx = delimiter.start()
- if idx == 0:
- result, expression = (expression[:1], expression[1:])
- self._tokens.append(result)
- else:
- result, expression = (expression[:idx], expression[idx:])
- result = result.strip()
- if result:
- self._tokens.append(result)
- else:
- expression = expression.strip()
- if expression:
- self._tokens.append(expression)
- expression = None
-
- def _get_token(self):
- if self._tokens:
- return self._tokens.pop(0)
-
- def _peek_token(self):
- if self._tokens:
- return self._tokens[0]
-
- @staticmethod
- def _to_getter(term):
- if isinstance(term, Getter):
- return term
- check_fieldname(term)
- return ExtractField(term)
-
- def _parse_op(self, op_name):
- self._get_token() # '('
- op = self._transformations.get(op_name, None)
- if op is None:
- raise errors.IndexDefinitionParseError(
- "Unknown operation: %s" % op_name)
- args = []
- while True:
- args.append(self._parse_term())
- sep = self._get_token()
- if sep == ')':
- break
- if sep != ',':
- raise errors.IndexDefinitionParseError(
- "Unexpected token '%s' in parentheses." % (sep,))
- parsed = []
- for i, arg in enumerate(args):
- arg_type = op.args[i % len(op.args)]
- if arg_type == 'expression':
- inner = self._to_getter(arg)
- else:
- try:
- inner = arg_type(arg)
- except ValueError as e:
- raise errors.IndexDefinitionParseError(
- "Invalid value %r for argument type %r "
- "(%r)." % (arg, arg_type, e))
- parsed.append(inner)
- return op(*parsed)
-
- def _parse_term(self):
- term = self._get_token()
- if term is None:
- raise errors.IndexDefinitionParseError(
- "Unexpected end of index definition.")
- if term in (',', ')', '('):
- raise errors.IndexDefinitionParseError(
- "Unexpected token '%s' at start of expression." % (term,))
- next_token = self._peek_token()
- if next_token == '(':
- return self._parse_op(term)
- return term
-
- def parse(self, expression):
- self._set_expression(expression)
- term = self._to_getter(self._parse_term())
- if self._peek_token():
- raise errors.IndexDefinitionParseError(
- "Unexpected token '%s' after end of expression."
- % (self._peek_token(),))
- return term
-
- def parse_all(self, fields):
- return [self.parse(field) for field in fields]
-
- @classmethod
- def register_transormation(cls, transform):
- assert transform.name not in cls._transformations, (
- "Transform %s already registered for %s"
- % (transform.name, cls._transformations[transform.name]))
- cls._transformations[transform.name] = transform
-
-
-Parser.register_transormation(SplitWords)
-Parser.register_transormation(Lower)
-Parser.register_transormation(Number)
-Parser.register_transormation(Bool)
-Parser.register_transormation(IsNull)
-Parser.register_transormation(Combine)
diff --git a/common/src/leap/soledad/common/l2db/remote/__init__.py b/common/src/leap/soledad/common/l2db/remote/__init__.py
deleted file mode 100644
index 3f32e381..00000000
--- a/common/src/leap/soledad/common/l2db/remote/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py
deleted file mode 100644
index a4eddb36..00000000
--- a/common/src/leap/soledad/common/l2db/remote/http_app.py
+++ /dev/null
@@ -1,660 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd.
-# Copyright 2016 LEAP Encryption Access Project
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""
-HTTP Application exposing U1DB.
-"""
-# TODO -- deprecate, use twisted/txaio.
-
-import functools
-import six.moves.http_client as httplib
-import inspect
-import json
-import sys
-import six.moves.urllib.parse as urlparse
-
-import routes.mapper
-
-from leap.soledad.common.l2db import (
- __version__ as _u1db_version,
- DBNAME_CONSTRAINTS, Document,
- errors, sync)
-from leap.soledad.common.l2db.remote import http_errors, utils
-
-
-def parse_bool(expression):
- """Parse boolean querystring parameter."""
- if expression == 'true':
- return True
- return False
-
-
-def parse_list(expression):
- if not expression:
- return []
- return [t.strip() for t in expression.split(',')]
-
-
-def none_or_str(expression):
- if expression is None:
- return None
- return str(expression)
-
-
-class BadRequest(Exception):
- """Bad request."""
-
-
-class _FencedReader(object):
- """Read and get lines from a file but not past a given length."""
-
- MAXCHUNK = 8192
-
- def __init__(self, rfile, total, max_entry_size):
- self.rfile = rfile
- self.remaining = total
- self.max_entry_size = max_entry_size
- self._kept = None
-
- def read_chunk(self, atmost):
- if self._kept is not None:
- # ignore atmost, kept data should be a subchunk anyway
- kept, self._kept = self._kept, None
- return kept
- if self.remaining == 0:
- return ''
- data = self.rfile.read(min(self.remaining, atmost))
- self.remaining -= len(data)
- return data
-
- def getline(self):
- line_parts = []
- size = 0
- while True:
- chunk = self.read_chunk(self.MAXCHUNK)
- if chunk == '':
- break
- nl = chunk.find("\n")
- if nl != -1:
- size += nl + 1
- if size > self.max_entry_size:
- raise BadRequest
- line_parts.append(chunk[:nl + 1])
- rest = chunk[nl + 1:]
- self._kept = rest or None
- break
- else:
- size += len(chunk)
- if size > self.max_entry_size:
- raise BadRequest
- line_parts.append(chunk)
- return ''.join(line_parts)
-
-
-def http_method(**control):
- """Decoration for handling of query arguments and content for a HTTP
- method.
-
- args and content here are the query arguments and body of the incoming
- HTTP requests.
-
- Match query arguments to python method arguments:
- w = http_method()(f)
- w(self, args, content) => args["content"]=content;
- f(self, **args)
-
- JSON deserialize content to arguments:
- w = http_method(content_as_args=True,...)(f)
- w(self, args, content) => args.update(json.loads(content));
- f(self, **args)
-
- Support conversions (e.g int):
- w = http_method(Arg=Conv,...)(f)
- w(self, args, content) => args["Arg"]=Conv(args["Arg"]);
- f(self, **args)
-
- Enforce no use of query arguments:
- w = http_method(no_query=True,...)(f)
- w(self, args, content) raises BadRequest if args is not empty
-
- Argument mismatches, deserialisation failures produce BadRequest.
- """
- content_as_args = control.pop('content_as_args', False)
- no_query = control.pop('no_query', False)
- conversions = control.items()
-
- def wrap(f):
- argspec = inspect.getargspec(f)
- assert argspec.args[0] == "self"
- nargs = len(argspec.args)
- ndefaults = len(argspec.defaults or ())
- required_args = set(argspec.args[1:nargs - ndefaults])
- all_args = set(argspec.args)
-
- @functools.wraps(f)
- def wrapper(self, args, content):
- if no_query and args:
- raise BadRequest()
- if content is not None:
- if content_as_args:
- try:
- args.update(json.loads(content))
- except ValueError:
- raise BadRequest()
- else:
- args["content"] = content
- if not (required_args <= set(args) <= all_args):
- raise BadRequest("Missing required arguments.")
- for name, conv in conversions:
- if name not in args:
- continue
- try:
- args[name] = conv(args[name])
- except ValueError:
- raise BadRequest()
- return f(self, **args)
-
- return wrapper
-
- return wrap
-
-
-class URLToResource(object):
- """Mappings from URLs to resources."""
-
- def __init__(self):
- self._map = routes.mapper.Mapper(controller_scan=None)
-
- def register(self, resource_cls):
- # register
- self._map.connect(None, resource_cls.url_pattern,
- resource_cls=resource_cls,
- requirements={"dbname": DBNAME_CONSTRAINTS})
- self._map.create_regs()
- return resource_cls
-
- def match(self, path):
- params = self._map.match(path)
- if params is None:
- return None, None
- resource_cls = params.pop('resource_cls')
- return resource_cls, params
-
-
-url_to_resource = URLToResource()
-
-
-@url_to_resource.register
-class GlobalResource(object):
- """Global (root) resource."""
-
- url_pattern = "/"
-
- def __init__(self, state, responder):
- self.state = state
- self.responder = responder
-
- @http_method()
- def get(self):
- info = self.state.global_info()
- info['version'] = _u1db_version
- self.responder.send_response_json(**info)
-
-
-@url_to_resource.register
-class DatabaseResource(object):
- """Database resource."""
-
- url_pattern = "/{dbname}"
-
- def __init__(self, dbname, state, responder):
- self.dbname = dbname
- self.state = state
- self.responder = responder
-
- @http_method()
- def get(self):
- self.state.check_database(self.dbname)
- self.responder.send_response_json(200)
-
- @http_method(content_as_args=True)
- def put(self):
- self.state.ensure_database(self.dbname)
- self.responder.send_response_json(200, ok=True)
-
- @http_method()
- def delete(self):
- self.state.delete_database(self.dbname)
- self.responder.send_response_json(200, ok=True)
-
-
-@url_to_resource.register
-class DocsResource(object):
- """Documents resource."""
-
- url_pattern = "/{dbname}/docs"
-
- def __init__(self, dbname, state, responder):
- self.responder = responder
- self.db = state.open_database(dbname)
-
- @http_method(doc_ids=parse_list, check_for_conflicts=parse_bool,
- include_deleted=parse_bool)
- def get(self, doc_ids=None, check_for_conflicts=True,
- include_deleted=False):
- if doc_ids is None:
- raise errors.MissingDocIds
- docs = self.db.get_docs(doc_ids, include_deleted=include_deleted)
- self.responder.content_type = 'application/json'
- self.responder.start_response(200)
- self.responder.start_stream(),
- for doc in docs:
- entry = dict(
- doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(),
- has_conflicts=doc.has_conflicts)
- self.responder.stream_entry(entry)
- self.responder.end_stream()
- self.responder.finish_response()
-
-
-@url_to_resource.register
-class AllDocsResource(object):
- """All Documents resource."""
-
- url_pattern = "/{dbname}/all-docs"
-
- def __init__(self, dbname, state, responder):
- self.responder = responder
- self.db = state.open_database(dbname)
-
- @http_method(include_deleted=parse_bool)
- def get(self, include_deleted=False):
- gen, docs = self.db.get_all_docs(include_deleted=include_deleted)
- self.responder.content_type = 'application/json'
- # returning a x-u1db-generation header is optional
- # HTTPDatabase will fallback to return -1 if it's missing
- self.responder.start_response(200,
- headers={'x-u1db-generation': str(gen)})
- self.responder.start_stream(),
- for doc in docs:
- entry = dict(
- doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(),
- has_conflicts=doc.has_conflicts)
- self.responder.stream_entry(entry)
- self.responder.end_stream()
- self.responder.finish_response()
-
-
-@url_to_resource.register
-class DocResource(object):
- """Document resource."""
-
- url_pattern = "/{dbname}/doc/{id:.*}"
-
- def __init__(self, dbname, id, state, responder):
- self.id = id
- self.responder = responder
- self.db = state.open_database(dbname)
-
- @http_method(old_rev=str)
- def put(self, content, old_rev=None):
- doc = Document(self.id, old_rev, content)
- doc_rev = self.db.put_doc(doc)
- if old_rev is None:
- status = 201 # created
- else:
- status = 200
- self.responder.send_response_json(status, rev=doc_rev)
-
- @http_method(old_rev=str)
- def delete(self, old_rev=None):
- doc = Document(self.id, old_rev, None)
- self.db.delete_doc(doc)
- self.responder.send_response_json(200, rev=doc.rev)
-
- @http_method(include_deleted=parse_bool)
- def get(self, include_deleted=False):
- doc = self.db.get_doc(self.id, include_deleted=include_deleted)
- if doc is None:
- wire_descr = errors.DocumentDoesNotExist.wire_description
- self.responder.send_response_json(
- http_errors.wire_description_to_status[wire_descr],
- error=wire_descr,
- headers={
- 'x-u1db-rev': '',
- 'x-u1db-has-conflicts': 'false'
- })
- return
- headers = {
- 'x-u1db-rev': doc.rev,
- 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts)
- }
- if doc.is_tombstone():
- self.responder.send_response_json(
- http_errors.wire_description_to_status[
- errors.DOCUMENT_DELETED],
- error=errors.DOCUMENT_DELETED,
- headers=headers)
- else:
- self.responder.send_response_content(
- doc.get_json(), headers=headers)
-
-
-@url_to_resource.register
-class SyncResource(object):
- """Sync endpoint resource."""
-
- # maximum allowed request body size
- max_request_size = 15 * 1024 * 1024 # 15Mb
- # maximum allowed entry/line size in request body
- max_entry_size = 10 * 1024 * 1024 # 10Mb
-
- url_pattern = "/{dbname}/sync-from/{source_replica_uid}"
-
- # pluggable
- sync_exchange_class = sync.SyncExchange
-
- def __init__(self, dbname, source_replica_uid, state, responder):
- self.source_replica_uid = source_replica_uid
- self.responder = responder
- self.state = state
- self.dbname = dbname
- self.replica_uid = None
-
- def get_target(self):
- return self.state.open_database(self.dbname).get_sync_target()
-
- @http_method()
- def get(self):
- result = self.get_target().get_sync_info(self.source_replica_uid)
- self.responder.send_response_json(
- target_replica_uid=result[0], target_replica_generation=result[1],
- target_replica_transaction_id=result[2],
- source_replica_uid=self.source_replica_uid,
- source_replica_generation=result[3],
- source_transaction_id=result[4])
-
- @http_method(generation=int,
- content_as_args=True, no_query=True)
- def put(self, generation, transaction_id):
- self.get_target().record_sync_info(self.source_replica_uid,
- generation,
- transaction_id)
- self.responder.send_response_json(ok=True)
-
- # Implements the same logic as LocalSyncTarget.sync_exchange
-
- @http_method(last_known_generation=int, last_known_trans_id=none_or_str,
- content_as_args=True)
- def post_args(self, last_known_generation, last_known_trans_id=None,
- ensure=False):
- if ensure:
- db, self.replica_uid = self.state.ensure_database(self.dbname)
- else:
- db = self.state.open_database(self.dbname)
- db.validate_gen_and_trans_id(
- last_known_generation, last_known_trans_id)
- self.sync_exch = self.sync_exchange_class(
- db, self.source_replica_uid, last_known_generation)
-
- @http_method(content_as_args=True)
- def post_stream_entry(self, id, rev, content, gen, trans_id):
- doc = Document(id, rev, content)
- self.sync_exch.insert_doc_from_source(doc, gen, trans_id)
-
- def post_end(self):
-
- def send_doc(doc, gen, trans_id):
- entry = dict(id=doc.doc_id, rev=doc.rev, content=doc.get_json(),
- gen=gen, trans_id=trans_id)
- self.responder.stream_entry(entry)
-
- new_gen = self.sync_exch.find_changes_to_return()
- self.responder.content_type = 'application/x-u1db-sync-stream'
- self.responder.start_response(200)
- self.responder.start_stream(),
- header = {"new_generation": new_gen,
- "new_transaction_id": self.sync_exch.new_trans_id}
- if self.replica_uid is not None:
- header['replica_uid'] = self.replica_uid
- self.responder.stream_entry(header)
- self.sync_exch.return_docs(send_doc)
- self.responder.end_stream()
- self.responder.finish_response()
-
-
-class HTTPResponder(object):
- """Encode responses from the server back to the client."""
-
- # a multi document response will put args and documents
- # each on one line of the response body
-
- def __init__(self, start_response):
- self._started = False
- self._stream_state = -1
- self._no_initial_obj = True
- self.sent_response = False
- self._start_response = start_response
- self._write = None
- self.content_type = 'application/json'
- self.content = []
-
- def start_response(self, status, obj_dic=None, headers={}):
- """start sending response with optional first json object."""
- if self._started:
- return
- self._started = True
- status_text = httplib.responses[status]
- self._write = self._start_response(
- '%d %s' % (status, status_text),
- [('content-type', self.content_type),
- ('cache-control', 'no-cache')] +
- headers.items())
- # xxx version in headers
- if obj_dic is not None:
- self._no_initial_obj = False
- self._write(json.dumps(obj_dic) + "\r\n")
-
- def finish_response(self):
- """finish sending response."""
- self.sent_response = True
-
- def send_response_json(self, status=200, headers={}, **kwargs):
- """send and finish response with json object body from keyword args."""
- content = json.dumps(kwargs) + "\r\n"
- self.send_response_content(content, headers=headers, status=status)
-
- def send_response_content(self, content, status=200, headers={}):
- """send and finish response with content"""
- headers['content-length'] = str(len(content))
- self.start_response(status, headers=headers)
- if self._stream_state == 1:
- self.content = [',\r\n', content]
- else:
- self.content = [content]
- self.finish_response()
-
- def start_stream(self):
- "start stream (array) as part of the response."
- assert self._started and self._no_initial_obj
- self._stream_state = 0
- self._write("[")
-
- def stream_entry(self, entry):
- "send stream entry as part of the response."
- assert self._stream_state != -1
- if self._stream_state == 0:
- self._stream_state = 1
- self._write('\r\n')
- else:
- self._write(',\r\n')
- if type(entry) == dict:
- entry = json.dumps(entry)
- self._write(entry)
-
- def end_stream(self):
- "end stream (array)."
- assert self._stream_state != -1
- self._write("\r\n]\r\n")
-
-
-class HTTPInvocationByMethodWithBody(object):
- """Invoke methods on a resource."""
-
- def __init__(self, resource, environ, parameters):
- self.resource = resource
- self.environ = environ
- self.max_request_size = getattr(
- resource, 'max_request_size', parameters.max_request_size)
- self.max_entry_size = getattr(
- resource, 'max_entry_size', parameters.max_entry_size)
-
- def _lookup(self, method):
- try:
- return getattr(self.resource, method)
- except AttributeError:
- raise BadRequest()
-
- def __call__(self):
- args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
- strict_parsing=False)
- try:
- args = dict(
- (k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
- except ValueError:
- raise BadRequest()
- method = self.environ['REQUEST_METHOD'].lower()
- if method in ('get', 'delete'):
- meth = self._lookup(method)
- return meth(args, None)
- else:
- # we expect content-length > 0, reconsider if we move
- # to support chunked enconding
- try:
- content_length = int(self.environ['CONTENT_LENGTH'])
- except (ValueError, KeyError):
- raise BadRequest
- if content_length <= 0:
- raise BadRequest
- if content_length > self.max_request_size:
- raise BadRequest
- reader = _FencedReader(self.environ['wsgi.input'], content_length,
- self.max_entry_size)
- content_type = self.environ.get('CONTENT_TYPE', '')
- content_type = content_type.split(';', 1)[0].strip()
- if content_type == 'application/json':
- meth = self._lookup(method)
- body = reader.read_chunk(sys.maxint)
- return meth(args, body)
- elif content_type == 'application/x-u1db-sync-stream':
- meth_args = self._lookup('%s_args' % method)
- meth_entry = self._lookup('%s_stream_entry' % method)
- meth_end = self._lookup('%s_end' % method)
- body_getline = reader.getline
- if body_getline().strip() != '[':
- raise BadRequest()
- line = body_getline()
- line, comma = utils.check_and_strip_comma(line.strip())
- meth_args(args, line)
- while True:
- line = body_getline()
- entry = line.strip()
- if entry == ']':
- break
- if not entry or not comma: # empty or no prec comma
- raise BadRequest
- entry, comma = utils.check_and_strip_comma(entry)
- meth_entry({}, entry)
- if comma or body_getline(): # extra comma or data
- raise BadRequest
- return meth_end()
- else:
- raise BadRequest()
-
-
-class HTTPApp(object):
-
- # maximum allowed request body size
- max_request_size = 15 * 1024 * 1024 # 15Mb
- # maximum allowed entry/line size in request body
- max_entry_size = 10 * 1024 * 1024 # 10Mb
-
- def __init__(self, state):
- self.state = state
-
- def _lookup_resource(self, environ, responder):
- resource_cls, params = url_to_resource.match(environ['PATH_INFO'])
- if resource_cls is None:
- raise BadRequest # 404 instead?
- resource = resource_cls(
- state=self.state, responder=responder, **params)
- return resource
-
- def __call__(self, environ, start_response):
- responder = HTTPResponder(start_response)
- self.request_begin(environ)
- try:
- resource = self._lookup_resource(environ, responder)
- HTTPInvocationByMethodWithBody(resource, environ, self)()
- except errors.U1DBError as e:
- self.request_u1db_error(environ, e)
- status = http_errors.wire_description_to_status.get(
- e.wire_description, 500)
- responder.send_response_json(status, error=e.wire_description)
- except BadRequest:
- self.request_bad_request(environ)
- responder.send_response_json(400, error="bad request")
- except KeyboardInterrupt:
- raise
- except:
- self.request_failed(environ)
- raise
- else:
- self.request_done(environ)
- return responder.content
-
- # hooks for tracing requests
-
- def request_begin(self, environ):
- """Hook called at the beginning of processing a request."""
- pass
-
- def request_done(self, environ):
- """Hook called when done processing a request."""
- pass
-
- def request_u1db_error(self, environ, exc):
- """Hook called when processing a request resulted in a U1DBError.
-
- U1DBError passed as exc.
- """
- pass
-
- def request_bad_request(self, environ):
- """Hook called when processing a bad request.
-
- No actual processing was done.
- """
- pass
-
- def request_failed(self, environ):
- """Hook called when processing a request failed unexpectedly.
-
- Invoked from an except block, so there's interpreter exception
- information available.
- """
- pass
diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py
deleted file mode 100644
index 1124b038..00000000
--- a/common/src/leap/soledad/common/l2db/remote/http_client.py
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""Base class to make requests to a remote HTTP server."""
-
-import json
-import socket
-import ssl
-import sys
-import urllib
-import six.moves.urllib.parse as urlparse
-import six.moves.http_client as httplib
-from time import sleep
-from leap.soledad.common.l2db import errors
-from leap.soledad.common.l2db.remote import http_errors
-
-from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname
-
-# Ubuntu/debian
-# XXX other...
-CA_CERTS = "/etc/ssl/certs/ca-certificates.crt"
-
-
-def _encode_query_parameter(value):
- """Encode query parameter."""
- if isinstance(value, bool):
- if value:
- value = 'true'
- else:
- value = 'false'
- return unicode(value).encode('utf-8')
-
-
-class _VerifiedHTTPSConnection(httplib.HTTPSConnection):
- """HTTPSConnection verifying server side certificates."""
- # derived from httplib.py
-
- def connect(self):
- "Connect to a host on a given (SSL) port."
-
- sock = socket.create_connection((self.host, self.port),
- self.timeout, self.source_address)
- if self._tunnel_host:
- self.sock = sock
- self._tunnel()
- if sys.platform.startswith('linux'):
- cert_opts = {
- 'cert_reqs': ssl.CERT_REQUIRED,
- 'ca_certs': CA_CERTS
- }
- else:
- # XXX no cert verification implemented elsewhere for now
- cert_opts = {}
- self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
- ssl_version=ssl.PROTOCOL_SSLv3,
- **cert_opts
- )
- if cert_opts:
- match_hostname(self.sock.getpeercert(), self.host)
-
-
-class HTTPClientBase(object):
- """Base class to make requests to a remote HTTP server."""
-
- # Will use these delays to retry on 503 befor finally giving up. The final
- # 0 is there to not wait after the final try fails.
- _delays = (1, 1, 2, 4, 0)
-
- def __init__(self, url, creds=None):
- self._url = urlparse.urlsplit(url)
- self._conn = None
- self._creds = {}
- if creds is not None:
- if len(creds) != 1:
- raise errors.UnknownAuthMethod()
- auth_meth, credentials = creds.items()[0]
- try:
- set_creds = getattr(self, 'set_%s_credentials' % auth_meth)
- except AttributeError:
- raise errors.UnknownAuthMethod(auth_meth)
- set_creds(**credentials)
-
- def _ensure_connection(self):
- if self._conn is not None:
- return
- if self._url.scheme == 'https':
- connClass = _VerifiedHTTPSConnection
- else:
- connClass = httplib.HTTPConnection
- self._conn = connClass(self._url.hostname, self._url.port)
-
- def close(self):
- if self._conn:
- self._conn.close()
- self._conn = None
-
- # xxx retry mechanism?
-
- def _error(self, respdic):
- descr = respdic.get("error")
- exc_cls = errors.wire_description_to_exc.get(descr)
- if exc_cls is not None:
- message = respdic.get("message")
- raise exc_cls(message)
-
- def _response(self):
- resp = self._conn.getresponse()
- body = resp.read()
- headers = dict(resp.getheaders())
- if resp.status in (200, 201):
- return body, headers
- elif resp.status in http_errors.ERROR_STATUSES:
- try:
- respdic = json.loads(body)
- except ValueError:
- pass
- else:
- self._error(respdic)
- # special case
- if resp.status == 503:
- raise errors.Unavailable(body, headers)
- raise errors.HTTPError(resp.status, body, headers)
-
- def _sign_request(self, method, url_query, params):
- raise NotImplementedError
-
- def _request(self, method, url_parts, params=None, body=None,
- content_type=None):
- self._ensure_connection()
- unquoted_url = url_query = self._url.path
- if url_parts:
- if not url_query.endswith('/'):
- url_query += '/'
- unquoted_url = url_query
- url_query += '/'.join(urllib.quote(part, safe='')
- for part in url_parts)
- # oauth performs its own quoting
- unquoted_url += '/'.join(url_parts)
- encoded_params = {}
- if params:
- for key, value in params.items():
- key = unicode(key).encode('utf-8')
- encoded_params[key] = _encode_query_parameter(value)
- url_query += ('?' + urllib.urlencode(encoded_params))
- if body is not None and not isinstance(body, basestring):
- body = json.dumps(body)
- content_type = 'application/json'
- headers = {}
- if content_type:
- headers['content-type'] = content_type
- headers.update(
- self._sign_request(method, unquoted_url, encoded_params))
- for delay in self._delays:
- try:
- self._conn.request(method, url_query, body, headers)
- return self._response()
- except errors.Unavailable as e:
- sleep(delay)
- raise e
-
- def _request_json(self, method, url_parts, params=None, body=None,
- content_type=None):
- res, headers = self._request(method, url_parts, params, body,
- content_type)
- return json.loads(res), headers
diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py
deleted file mode 100644
index 7e61e5a4..00000000
--- a/common/src/leap/soledad/common/l2db/remote/http_database.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""HTTPDatabase to access a remote db over the HTTP API."""
-
-import json
-import uuid
-
-from leap.soledad.common.l2db import (
- Database,
- Document,
- errors)
-from leap.soledad.common.l2db.remote import (
- http_client,
- http_errors,
- http_target)
-
-
-DOCUMENT_DELETED_STATUS = http_errors.wire_description_to_status[
- errors.DOCUMENT_DELETED]
-
-
-class HTTPDatabase(http_client.HTTPClientBase, Database):
- """Implement the Database API to a remote HTTP server."""
-
- def __init__(self, url, document_factory=None, creds=None):
- super(HTTPDatabase, self).__init__(url, creds=creds)
- self._factory = document_factory or Document
-
- def set_document_factory(self, factory):
- self._factory = factory
-
- @staticmethod
- def open_database(url, create):
- db = HTTPDatabase(url)
- db.open(create)
- return db
-
- @staticmethod
- def delete_database(url):
- db = HTTPDatabase(url)
- db._delete()
- db.close()
-
- def open(self, create):
- if create:
- self._ensure()
- else:
- self._check()
-
- def _check(self):
- return self._request_json('GET', [])[0]
-
- def _ensure(self):
- self._request_json('PUT', [], {}, {})
-
- def _delete(self):
- self._request_json('DELETE', [], {}, {})
-
- def put_doc(self, doc):
- if doc.doc_id is None:
- raise errors.InvalidDocId()
- params = {}
- if doc.rev is not None:
- params['old_rev'] = doc.rev
- res, headers = self._request_json('PUT', ['doc', doc.doc_id], params,
- doc.get_json(), 'application/json')
- doc.rev = res['rev']
- return res['rev']
-
- def get_doc(self, doc_id, include_deleted=False):
- try:
- res, headers = self._request(
- 'GET', ['doc', doc_id], {"include_deleted": include_deleted})
- except errors.DocumentDoesNotExist:
- return None
- except errors.HTTPError as e:
- if (e.status == DOCUMENT_DELETED_STATUS and
- 'x-u1db-rev' in e.headers):
- res = None
- headers = e.headers
- else:
- raise
- doc_rev = headers['x-u1db-rev']
- has_conflicts = json.loads(headers['x-u1db-has-conflicts'])
- doc = self._factory(doc_id, doc_rev, res)
- doc.has_conflicts = has_conflicts
- return doc
-
- def _build_docs(self, res):
- for doc_dict in json.loads(res):
- doc = self._factory(
- doc_dict['doc_id'], doc_dict['doc_rev'], doc_dict['content'])
- doc.has_conflicts = doc_dict['has_conflicts']
- yield doc
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False):
- if not doc_ids:
- return []
- doc_ids = ','.join(doc_ids)
- res, headers = self._request(
- 'GET', ['docs'], {
- "doc_ids": doc_ids, "include_deleted": include_deleted,
- "check_for_conflicts": check_for_conflicts})
- return self._build_docs(res)
-
- def get_all_docs(self, include_deleted=False):
- res, headers = self._request(
- 'GET', ['all-docs'], {"include_deleted": include_deleted})
- gen = -1
- if 'x-u1db-generation' in headers:
- gen = int(headers['x-u1db-generation'])
- return gen, list(self._build_docs(res))
-
- def _allocate_doc_id(self):
- return 'D-%s' % (uuid.uuid4().hex,)
-
- def create_doc(self, content, doc_id=None):
- if not isinstance(content, dict):
- raise errors.InvalidContent
- json_string = json.dumps(content)
- return self.create_doc_from_json(json_string, doc_id)
-
- def create_doc_from_json(self, content, doc_id=None):
- if doc_id is None:
- doc_id = self._allocate_doc_id()
- res, headers = self._request_json('PUT', ['doc', doc_id], {},
- content, 'application/json')
- new_doc = self._factory(doc_id, res['rev'], content)
- return new_doc
-
- def delete_doc(self, doc):
- if doc.doc_id is None:
- raise errors.InvalidDocId()
- params = {'old_rev': doc.rev}
- res, headers = self._request_json(
- 'DELETE', ['doc', doc.doc_id], params)
- doc.make_tombstone()
- doc.rev = res['rev']
-
- def get_sync_target(self):
- st = http_target.HTTPSyncTarget(self._url.geturl())
- st._creds = self._creds
- return st
diff --git a/common/src/leap/soledad/common/l2db/remote/http_errors.py b/common/src/leap/soledad/common/l2db/remote/http_errors.py
deleted file mode 100644
index ee4cfefa..00000000
--- a/common/src/leap/soledad/common/l2db/remote/http_errors.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd.
-# Copyright 2016 LEAP Encryption Access Project
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""
-Information about the encoding of errors over HTTP.
-"""
-
-from leap.soledad.common.l2db import errors
-
-
-# error wire descriptions mapping to HTTP status codes
-wire_description_to_status = dict([
- (errors.InvalidDocId.wire_description, 400),
- (errors.MissingDocIds.wire_description, 400),
- (errors.Unauthorized.wire_description, 401),
- (errors.DocumentTooBig.wire_description, 403),
- (errors.UserQuotaExceeded.wire_description, 403),
- (errors.SubscriptionNeeded.wire_description, 403),
- (errors.DatabaseDoesNotExist.wire_description, 404),
- (errors.DocumentDoesNotExist.wire_description, 404),
- (errors.DocumentAlreadyDeleted.wire_description, 404),
- (errors.RevisionConflict.wire_description, 409),
- (errors.InvalidGeneration.wire_description, 409),
- (errors.InvalidReplicaUID.wire_description, 409),
- (errors.InvalidTransactionId.wire_description, 409),
- (errors.Unavailable.wire_description, 503),
- # without matching exception
- (errors.DOCUMENT_DELETED, 404)
-])
-
-
-ERROR_STATUSES = set(wire_description_to_status.values())
-# 400 included explicitly for tests
-ERROR_STATUSES.add(400)
diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py
deleted file mode 100644
index 38804f01..00000000
--- a/common/src/leap/soledad/common/l2db/remote/http_target.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Copyright 2011-2012 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""SyncTarget API implementation to a remote HTTP server."""
-
-import json
-
-from leap.soledad.common.l2db import Document, SyncTarget
-from leap.soledad.common.l2db.errors import BrokenSyncStream
-from leap.soledad.common.l2db.remote import (
- http_client, utils)
-
-
-class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget):
- """Implement the SyncTarget api to a remote HTTP server."""
-
- @staticmethod
- def connect(url):
- return HTTPSyncTarget(url)
-
- def get_sync_info(self, source_replica_uid):
- self._ensure_connection()
- res, _ = self._request_json('GET', ['sync-from', source_replica_uid])
- return (res['target_replica_uid'], res['target_replica_generation'],
- res['target_replica_transaction_id'],
- res['source_replica_generation'], res['source_transaction_id'])
-
- def record_sync_info(self, source_replica_uid, source_replica_generation,
- source_transaction_id):
- self._ensure_connection()
- if self._trace_hook: # for tests
- self._trace_hook('record_sync_info')
- self._request_json('PUT', ['sync-from', source_replica_uid], {},
- {'generation': source_replica_generation,
- 'transaction_id': source_transaction_id})
-
- def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
- parts = data.splitlines() # one at a time
- if not parts or parts[0] != '[':
- raise BrokenSyncStream
- data = parts[1:-1]
- comma = False
- if data:
- line, comma = utils.check_and_strip_comma(data[0])
- res = json.loads(line)
- if ensure_callback and 'replica_uid' in res:
- ensure_callback(res['replica_uid'])
- for entry in data[1:]:
- if not comma: # missing in between comma
- raise BrokenSyncStream
- line, comma = utils.check_and_strip_comma(entry)
- entry = json.loads(line)
- doc = Document(entry['id'], entry['rev'], entry['content'])
- return_doc_cb(doc, entry['gen'], entry['trans_id'])
- if parts[-1] != ']':
- try:
- partdic = json.loads(parts[-1])
- except ValueError:
- pass
- else:
- if isinstance(partdic, dict):
- self._error(partdic)
- raise BrokenSyncStream
- if not data or comma: # no entries or bad extra comma
- raise BrokenSyncStream
- return res
-
- def sync_exchange(self, docs_by_generations, source_replica_uid,
- last_known_generation, last_known_trans_id,
- return_doc_cb, ensure_callback=None):
- self._ensure_connection()
- if self._trace_hook: # for tests
- self._trace_hook('sync_exchange')
- url = '%s/sync-from/%s' % (self._url.path, source_replica_uid)
- self._conn.putrequest('POST', url)
- self._conn.putheader('content-type', 'application/x-u1db-sync-stream')
- for header_name, header_value in self._sign_request('POST', url, {}):
- self._conn.putheader(header_name, header_value)
- entries = ['[']
- size = 1
-
- def prepare(**dic):
- entry = comma + '\r\n' + json.dumps(dic)
- entries.append(entry)
- return len(entry)
-
- comma = ''
- size += prepare(
- last_known_generation=last_known_generation,
- last_known_trans_id=last_known_trans_id,
- ensure=ensure_callback is not None)
- comma = ','
- for doc, gen, trans_id in docs_by_generations:
- size += prepare(id=doc.doc_id, rev=doc.rev, content=doc.get_json(),
- gen=gen, trans_id=trans_id)
- entries.append('\r\n]')
- size += len(entries[-1])
- self._conn.putheader('content-length', str(size))
- self._conn.endheaders()
- for entry in entries:
- self._conn.send(entry)
- entries = None
- data, _ = self._response()
- res = self._parse_sync_stream(data, return_doc_cb, ensure_callback)
- data = None
- return res['new_generation'], res['new_transaction_id']
-
- # for tests
- _trace_hook = None
-
- def _set_trace_hook_shallow(self, cb):
- self._trace_hook = cb
diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py
deleted file mode 100644
index d4c3c45f..00000000
--- a/common/src/leap/soledad/common/l2db/remote/server_state.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""State for servers exposing a set of U1DB databases."""
-
-
-class ServerState(object):
- """Passed to a Request when it is instantiated.
-
- This is used to track server-side state, such as working-directory, open
- databases, etc.
- """
-
- def __init__(self):
- self._workingdir = None
-
- def set_workingdir(self, path):
- self._workingdir = path
-
- def global_info(self):
- """Return global information about the server."""
- return {}
-
- def _relpath(self, relpath):
- # Note: We don't want to allow absolute paths here, because we
- # don't want to expose the filesystem. We should also check that
- # relpath doesn't have '..' in it, etc.
- return self._workingdir + '/' + relpath
-
- def open_database(self, path):
- """Open a database at the given location."""
- from leap.soledad.client._db import sqlite
- full_path = self._relpath(path)
- return sqlite.SQLiteDatabase.open_database(full_path, create=False)
-
- def check_database(self, path):
- """Check if the database at the given location exists.
-
- Simply returns if it does or raises DatabaseDoesNotExist.
- """
- db = self.open_database(path)
- db.close()
-
- def ensure_database(self, path):
- """Ensure database at the given location."""
- from leap.soledad.client._db import sqlite
- full_path = self._relpath(path)
- db = sqlite.SQLiteDatabase.open_database(full_path, create=True)
- return db, db._replica_uid
-
- def delete_database(self, path):
- """Delete database at the given location."""
- from leap.soledad.client._db import sqlite
- full_path = self._relpath(path)
- sqlite.SQLiteDatabase.delete_database(full_path)
diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py
deleted file mode 100644
index ce82f1b2..00000000
--- a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""The match_hostname() function from Python 3.2, essential when using SSL."""
-# XXX put it here until it's packaged
-
-import re
-
-__version__ = '3.2a3'
-
-
-class CertificateError(ValueError):
- pass
-
-
-def _dnsname_to_pat(dn):
- pats = []
- for frag in dn.split(r'.'):
- if frag == '*':
- # When '*' is a fragment by itself, it matches a non-empty dotless
- # fragment.
- pats.append('[^.]+')
- else:
- # Otherwise, '*' matches any dotless fragment.
- frag = re.escape(frag)
- pats.append(frag.replace(r'\*', '[^.]*'))
- return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
-
-
-def match_hostname(cert, hostname):
- """Verify that *cert* (in decoded format as returned by
- SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules
- are mostly followed, but IP addresses are not accepted for *hostname*.
-
- CertificateError is raised on failure. On success, the function
- returns nothing.
- """
- if not cert:
- raise ValueError("empty or no certificate")
- dnsnames = []
- san = cert.get('subjectAltName', ())
- for key, value in san:
- if key == 'DNS':
- if _dnsname_to_pat(value).match(hostname):
- return
- dnsnames.append(value)
- if not san:
- # The subject is only checked when subjectAltName is empty
- for sub in cert.get('subject', ()):
- for key, value in sub:
- # XXX according to RFC 2818, the most specific Common Name
- # must be used.
- if key == 'commonName':
- if _dnsname_to_pat(value).match(hostname):
- return
- dnsnames.append(value)
- if len(dnsnames) > 1:
- raise CertificateError(
- "hostname %r doesn't match either of %s"
- % (hostname, ', '.join(map(repr, dnsnames))))
- elif len(dnsnames) == 1:
- raise CertificateError(
- "hostname %r doesn't match %r"
- % (hostname, dnsnames[0]))
- else:
- raise CertificateError(
- "no appropriate commonName or "
- "subjectAltName fields were found")
diff --git a/common/src/leap/soledad/common/l2db/remote/utils.py b/common/src/leap/soledad/common/l2db/remote/utils.py
deleted file mode 100644
index 14cedea9..00000000
--- a/common/src/leap/soledad/common/l2db/remote/utils.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2012 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""Utilities for details of the procotol."""
-
-
-def check_and_strip_comma(line):
- if line and line[-1] == ',':
- return line[:-1], True
- return line, False
diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py
deleted file mode 100644
index 32281f30..00000000
--- a/common/src/leap/soledad/common/l2db/sync.py
+++ /dev/null
@@ -1,311 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""The synchronization utilities for U1DB."""
-from six.moves import zip as izip
-
-from leap.soledad.common import l2db
-from leap.soledad.common.l2db import errors
-
-
-class Synchronizer(object):
- """Collect the state around synchronizing 2 U1DB replicas.
-
- Synchronization is bi-directional, in that new items in the source are sent
- to the target, and new items in the target are returned to the source.
- However, it still recognizes that one side is initiating the request. Also,
- at the moment, conflicts are only created in the source.
- """
-
- def __init__(self, source, sync_target):
- """Create a new Synchronization object.
-
- :param source: A Database
- :param sync_target: A SyncTarget
- """
- self.source = source
- self.sync_target = sync_target
- self.target_replica_uid = None
- self.num_inserted = 0
-
- def _insert_doc_from_target(self, doc, replica_gen, trans_id):
- """Try to insert synced document from target.
-
- Implements TAKE OTHER semantics: any document from the target
- that is in conflict will be taken as the new official value,
- while the current conflicting value will be stored alongside
- as a conflict. In the process indexes will be updated etc.
-
- :return: None
- """
- # Increases self.num_inserted depending whether the document
- # was effectively inserted.
- state, _ = self.source._put_doc_if_newer(
- doc, save_conflict=True,
- replica_uid=self.target_replica_uid, replica_gen=replica_gen,
- replica_trans_id=trans_id)
- if state == 'inserted':
- self.num_inserted += 1
- elif state == 'converged':
- # magical convergence
- pass
- elif state == 'superseded':
- # we have something newer, will be taken care of at the next sync
- pass
- else:
- assert state == 'conflicted'
- # The doc was saved as a conflict, so the database was updated
- self.num_inserted += 1
-
- def _record_sync_info_with_the_target(self, start_generation):
- """Record our new after sync generation with the target if gapless.
-
- Any documents received from the target will cause the local
- database to increment its generation. We do not want to send
- them back to the target in a future sync. However, there could
- also be concurrent updates from another process doing eg
- 'put_doc' while the sync was running. And we do want to
- synchronize those documents. We can tell if there was a
- concurrent update by comparing our new generation number
- versus the generation we started, and how many documents we
- inserted from the target. If it matches exactly, then we can
- record with the target that they are fully up to date with our
- new generation.
- """
- cur_gen, trans_id = self.source._get_generation_info()
- last_gen = start_generation + self.num_inserted
- if (cur_gen == last_gen and self.num_inserted > 0):
- self.sync_target.record_sync_info(
- self.source._replica_uid, cur_gen, trans_id)
-
- def sync(self, callback=None, autocreate=False):
- """Synchronize documents between source and target."""
- sync_target = self.sync_target
- # get target identifier, its current generation,
- # and its last-seen database generation for this source
- try:
- (self.target_replica_uid, target_gen, target_trans_id,
- target_my_gen, target_my_trans_id) = sync_target.get_sync_info(
- self.source._replica_uid)
- except errors.DatabaseDoesNotExist:
- if not autocreate:
- raise
- # will try to ask sync_exchange() to create the db
- self.target_replica_uid = None
- target_gen, target_trans_id = 0, ''
- target_my_gen, target_my_trans_id = 0, ''
-
- def ensure_callback(replica_uid):
- self.target_replica_uid = replica_uid
-
- else:
- ensure_callback = None
- if self.target_replica_uid == self.source._replica_uid:
- raise errors.InvalidReplicaUID
- # validate the generation and transaction id the target knows about us
- self.source.validate_gen_and_trans_id(
- target_my_gen, target_my_trans_id)
- # what's changed since that generation and this current gen
- my_gen, _, changes = self.source.whats_changed(target_my_gen)
-
- # this source last-seen database generation for the target
- if self.target_replica_uid is None:
- target_last_known_gen, target_last_known_trans_id = 0, ''
- else:
- target_last_known_gen, target_last_known_trans_id = (
- self.source._get_replica_gen_and_trans_id( # nopep8
- self.target_replica_uid))
- if not changes and target_last_known_gen == target_gen:
- if target_trans_id != target_last_known_trans_id:
- raise errors.InvalidTransactionId
- return my_gen
- changed_doc_ids = [doc_id for doc_id, _, _ in changes]
- # prepare to send all the changed docs
- docs_to_send = self.source.get_docs(
- changed_doc_ids,
- check_for_conflicts=False, include_deleted=True)
- # TODO: there must be a way to not iterate twice
- docs_by_generation = zip(
- docs_to_send, (gen for _, gen, _ in changes),
- (trans for _, _, trans in changes))
-
- # exchange documents and try to insert the returned ones with
- # the target, return target synced-up-to gen
- new_gen, new_trans_id = sync_target.sync_exchange(
- docs_by_generation, self.source._replica_uid,
- target_last_known_gen, target_last_known_trans_id,
- self._insert_doc_from_target, ensure_callback=ensure_callback)
- # record target synced-up-to generation including applying what we sent
- self.source._set_replica_gen_and_trans_id(
- self.target_replica_uid, new_gen, new_trans_id)
-
- # if gapless record current reached generation with target
- self._record_sync_info_with_the_target(my_gen)
-
- return my_gen
-
-
-class SyncExchange(object):
- """Steps and state for carrying through a sync exchange on a target."""
-
- def __init__(self, db, source_replica_uid, last_known_generation):
- self._db = db
- self.source_replica_uid = source_replica_uid
- self.source_last_known_generation = last_known_generation
- self.seen_ids = {} # incoming ids not superseded
- self.changes_to_return = None
- self.new_gen = None
- self.new_trans_id = None
- # for tests
- self._incoming_trace = []
- self._trace_hook = None
- self._db._last_exchange_log = {
- 'receive': {'docs': self._incoming_trace},
- 'return': None
- }
-
- def _set_trace_hook(self, cb):
- self._trace_hook = cb
-
- def _trace(self, state):
- if not self._trace_hook:
- return
- self._trace_hook(state)
-
- def insert_doc_from_source(self, doc, source_gen, trans_id):
- """Try to insert synced document from source.
-
- Conflicting documents are not inserted but will be sent over
- to the sync source.
-
- It keeps track of progress by storing the document source
- generation as well.
-
- The 1st step of a sync exchange is to call this repeatedly to
- try insert all incoming documents from the source.
-
- :param doc: A Document object.
- :param source_gen: The source generation of doc.
- :return: None
- """
- state, at_gen = self._db._put_doc_if_newer(
- doc, save_conflict=False,
- replica_uid=self.source_replica_uid, replica_gen=source_gen,
- replica_trans_id=trans_id)
- if state == 'inserted':
- self.seen_ids[doc.doc_id] = at_gen
- elif state == 'converged':
- # magical convergence
- self.seen_ids[doc.doc_id] = at_gen
- elif state == 'superseded':
- # we have something newer that we will return
- pass
- else:
- # conflict that we will returne
- assert state == 'conflicted'
- # for tests
- self._incoming_trace.append((doc.doc_id, doc.rev))
- self._db._last_exchange_log['receive'].update({
- 'source_uid': self.source_replica_uid,
- 'source_gen': source_gen
- })
-
- def find_changes_to_return(self):
- """Find changes to return.
-
- Find changes since last_known_generation in db generation
- order using whats_changed. It excludes documents ids that have
- already been considered (superseded by the sender, etc).
-
- :return: new_generation - the generation of this database
- which the caller can consider themselves to be synchronized after
- processing the returned documents.
- """
- self._db._last_exchange_log['receive'].update({ # for tests
- 'last_known_gen': self.source_last_known_generation
- })
- self._trace('before whats_changed')
- gen, trans_id, changes = self._db.whats_changed(
- self.source_last_known_generation)
- self._trace('after whats_changed')
- self.new_gen = gen
- self.new_trans_id = trans_id
- seen_ids = self.seen_ids
- # changed docs that weren't superseded by or converged with
- self.changes_to_return = [
- (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes if
- # there was a subsequent update
- doc_id not in seen_ids or seen_ids.get(doc_id) < gen]
- return self.new_gen
-
- def return_docs(self, return_doc_cb):
- """Return the changed documents and their last change generation
- repeatedly invoking the callback return_doc_cb.
-
- The final step of a sync exchange.
-
- :param: return_doc_cb(doc, gen, trans_id): is a callback
- used to return the documents with their last change generation
- to the target replica.
- :return: None
- """
- changes_to_return = self.changes_to_return
- # return docs, including conflicts
- changed_doc_ids = [doc_id for doc_id, _, _ in changes_to_return]
- self._trace('before get_docs')
- docs = self._db.get_docs(
- changed_doc_ids, check_for_conflicts=False, include_deleted=True)
-
- docs_by_gen = izip(
- docs, (gen for _, gen, _ in changes_to_return),
- (trans_id for _, _, trans_id in changes_to_return))
- _outgoing_trace = [] # for tests
- for doc, gen, trans_id in docs_by_gen:
- return_doc_cb(doc, gen, trans_id)
- _outgoing_trace.append((doc.doc_id, doc.rev))
- # for tests
- self._db._last_exchange_log['return'] = {
- 'docs': _outgoing_trace,
- 'last_gen': self.new_gen}
-
-
-class LocalSyncTarget(l2db.SyncTarget):
- """Common sync target implementation logic for all local sync targets."""
-
- def __init__(self, db):
- self._db = db
- self._trace_hook = None
-
- def sync_exchange(self, docs_by_generations, source_replica_uid,
- last_known_generation, last_known_trans_id,
- return_doc_cb, ensure_callback=None):
- self._db.validate_gen_and_trans_id(
- last_known_generation, last_known_trans_id)
- sync_exch = SyncExchange(
- self._db, source_replica_uid, last_known_generation)
- if self._trace_hook:
- sync_exch._set_trace_hook(self._trace_hook)
- # 1st step: try to insert incoming docs and record progress
- for doc, doc_gen, trans_id in docs_by_generations:
- sync_exch.insert_doc_from_source(doc, doc_gen, trans_id)
- # 2nd step: find changed documents (including conflicts) to return
- new_gen = sync_exch.find_changes_to_return()
- # final step: return docs and record source replica sync point
- sync_exch.return_docs(return_doc_cb)
- return new_gen, sync_exch.new_trans_id
-
- def _set_trace_hook(self, cb):
- self._trace_hook = cb
diff --git a/common/src/leap/soledad/common/l2db/vectorclock.py b/common/src/leap/soledad/common/l2db/vectorclock.py
deleted file mode 100644
index 42bceaa8..00000000
--- a/common/src/leap/soledad/common/l2db/vectorclock.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# Copyright 2011 Canonical Ltd.
-#
-# This file is part of u1db.
-#
-# u1db is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3
-# as published by the Free Software Foundation.
-#
-# u1db is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with u1db. If not, see <http://www.gnu.org/licenses/>.
-
-"""VectorClockRev helper class."""
-
-
-class VectorClockRev(object):
- """Track vector clocks for multiple replica ids.
-
- This allows simple comparison to determine if one VectorClockRev is
- newer/older/in-conflict-with another VectorClockRev without having to
- examine history. Every replica has a strictly increasing revision. When
- creating a new revision, they include all revisions for all other replicas
- which the new revision dominates, and increment their own revision to
- something greater than the current value.
- """
-
- def __init__(self, value):
- self._values = self._expand(value)
-
- def __repr__(self):
- s = self.as_str()
- return '%s(%s)' % (self.__class__.__name__, s)
-
- def as_str(self):
- s = '|'.join(['%s:%d' % (m, r) for m, r
- in sorted(self._values.items())])
- return s
-
- def _expand(self, value):
- result = {}
- if value is None:
- return result
- for replica_info in value.split('|'):
- replica_uid, counter = replica_info.split(':')
- counter = int(counter)
- result[replica_uid] = counter
- return result
-
- def is_newer(self, other):
- """Is this VectorClockRev strictly newer than other.
- """
- if not self._values:
- return False
- if not other._values:
- return True
- this_is_newer = False
- other_expand = dict(other._values)
- for key, value in self._values.iteritems():
- if key in other_expand:
- other_value = other_expand.pop(key)
- if other_value > value:
- return False
- elif other_value < value:
- this_is_newer = True
- else:
- this_is_newer = True
- if other_expand:
- return False
- return this_is_newer
-
- def increment(self, replica_uid):
- """Increase the 'replica_uid' section of this vector clock.
-
- :return: A string representing the new vector clock value
- """
- self._values[replica_uid] = self._values.get(replica_uid, 0) + 1
-
- def maximize(self, other_vcr):
- for replica_uid, counter in other_vcr._values.iteritems():
- if replica_uid not in self._values:
- self._values[replica_uid] = counter
- else:
- this_counter = self._values[replica_uid]
- if this_counter < counter:
- self._values[replica_uid] = counter
diff --git a/common/src/leap/soledad/common/log.py b/common/src/leap/soledad/common/log.py
deleted file mode 100644
index 59a47726..00000000
--- a/common/src/leap/soledad/common/log.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# -*- coding: utf-8 -*-
-# log.py
-# Copyright (C) 2016 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-This module centralizes logging facilities and allows for different behaviours,
-as using the python logging module instead of twisted logger, and to print logs
-to stdout, mainly for development purposes.
-"""
-
-
-import os
-import sys
-import time
-
-from twisted.logger import Logger
-from twisted.logger import textFileLogObserver
-from twisted.logger import LogLevel
-from twisted.logger import InvalidLogLevelError
-from twisted.python.failure import Failure
-
-
-# What follows is a patched class to correctly log namespace and level when
-# using the default formatter and --syslog option in twistd. This seems to be a
-# known bug but it has not been reported to upstream yet.
-
-class SyslogLogger(Logger):
-
- def emit(self, level, format=None, **kwargs):
- if level not in LogLevel.iterconstants():
- self.failure(
- "Got invalid log level {invalidLevel!r} in {logger}.emit().",
- Failure(InvalidLogLevelError(level)),
- invalidLevel=level,
- logger=self,
- )
- return
-
- event = kwargs
- event.update(
- log_logger=self, log_level=level, log_namespace=self.namespace,
- log_source=self.source, log_format=format, log_time=time.time(),
- )
-
- # ---------------------------------8<---------------------------------
- # this is a workaround for the mess between twisted's legacy log system
- # and twistd's --syslog option.
- event["system"] = "%s#%s" % (self.namespace, level.name)
- # ---------------------------------8<---------------------------------
-
- if "log_trace" in event:
- event["log_trace"].append((self, self.observer))
-
- self.observer(event)
-
-
-def getLogger(*args, **kwargs):
-
- if os.environ.get('SOLEDAD_USE_PYTHON_LOGGING'):
- import logging
- return logging.getLogger(__name__)
-
- if os.environ.get('SOLEDAD_LOG_TO_STDOUT'):
- kwargs({'observer': textFileLogObserver(sys.stdout)})
-
- return SyslogLogger(*args, **kwargs)
-
-
-__all__ = ['getLogger']
diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py
deleted file mode 100644
index acebb77b..00000000
--- a/common/src/leap/soledad/common/tests/__init__.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013, 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Tests to make sure Soledad provides U1DB functionality and more.
-"""
-
-
-import os
-
-
-def load_tests():
- """
- Build a test suite that includes all tests in leap.soledad.common.tests
- but does not include tests in the u1db_tests/ subfolder. The reason for
- not including those tests are:
-
- 1. they by themselves only test u1db functionality in the u1db module
- (despite we use them as basis for testing soledad functionalities).
-
- 2. they would fail because we monkey patch u1db's remote http server
- to add soledad functionality we need.
- """
- import unittest
- import glob
- import imp
- tests_prefix = os.path.join(
- '.', 'src', 'leap', 'soledad', 'common', 'tests')
- suite = unittest.TestSuite()
- for testcase in glob.glob(os.path.join(tests_prefix, 'test_*.py')):
- modname = os.path.basename(os.path.splitext(testcase)[0])
- f, pathname, description = imp.find_module(modname, [tests_prefix])
- module = imp.load_module(modname, f, pathname, description)
- suite.addTest(unittest.TestLoader().loadTestsFromModule(module))
- return suite
diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py
deleted file mode 100644
index 2136bb8f..00000000
--- a/common/src/leap/soledad/common/tests/test_command.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# -*- coding: utf-8 -*-
-# test_command.py
-# Copyright (C) 2015 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Tests for command execution using a validator function for arguments.
-"""
-from twisted.trial import unittest
-from leap.soledad.common.command import exec_validated_cmd
-
-
-def validator(arg):
- return True if arg is 'valid' else False
-
-
-class ExecuteValidatedCommandTest(unittest.TestCase):
-
- def test_argument_validation(self):
- status, out = exec_validated_cmd("command", "invalid arg", validator)
- self.assertEquals(status, 1)
- self.assertEquals(out, "invalid argument")
- status, out = exec_validated_cmd("echo", "valid", validator)
- self.assertEquals(status, 0)
- self.assertEquals(out, "valid\n")
-
- def test_return_status_code_success(self):
- status, out = exec_validated_cmd("echo", "arg")
- self.assertEquals(status, 0)
- self.assertEquals(out, "arg\n")
-
- def test_handle_command_with_spaces(self):
- status, out = exec_validated_cmd("echo I am", "an argument")
- self.assertEquals(status, 0, out)
- self.assertEquals(out, "I am an argument\n")
-
- def test_handle_oserror_on_invalid_command(self):
- status, out = exec_validated_cmd("inexistent command with", "args")
- self.assertEquals(status, 1)
- self.assertIn("No such file or directory", out)
-
- def test_return_status_code_number_on_failure(self):
- status, out = exec_validated_cmd("ls", "user-bebacafe")
- self.assertNotEquals(status, 0)
- self.assertIn('No such file or directory\n', out)
diff --git a/common/versioneer.py b/common/versioneer.py
deleted file mode 100644
index 58339251..00000000
--- a/common/versioneer.py
+++ /dev/null
@@ -1,1774 +0,0 @@
-
-# Version: 0.16
-
-"""The Versioneer - like a rocketeer, but for versions.
-
-The Versioneer
-==============
-
-* like a rocketeer, but for versions!
-* https://github.com/warner/python-versioneer
-* Brian Warner
-* License: Public Domain
-* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy
-* [![Latest Version]
-(https://pypip.in/version/versioneer/badge.svg?style=flat)
-](https://pypi.python.org/pypi/versioneer/)
-* [![Build Status]
-(https://travis-ci.org/warner/python-versioneer.png?branch=master)
-](https://travis-ci.org/warner/python-versioneer)
-
-This is a tool for managing a recorded version number in distutils-based
-python projects. The goal is to remove the tedious and error-prone "update
-the embedded version string" step from your release process. Making a new
-release should be as easy as recording a new tag in your version-control
-system, and maybe making new tarballs.
-
-
-## Quick Install
-
-* `pip install versioneer` to somewhere to your $PATH
-* add a `[versioneer]` section to your setup.cfg (see below)
-* run `versioneer install` in your source tree, commit the results
-
-## Version Identifiers
-
-Source trees come from a variety of places:
-
-* a version-control system checkout (mostly used by developers)
-* a nightly tarball, produced by build automation
-* a snapshot tarball, produced by a web-based VCS browser, like github's
- "tarball from tag" feature
-* a release tarball, produced by "setup.py sdist", distributed through PyPI
-
-Within each source tree, the version identifier (either a string or a number,
-this tool is format-agnostic) can come from a variety of places:
-
-* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
- about recent "tags" and an absolute revision-id
-* the name of the directory into which the tarball was unpacked
-* an expanded VCS keyword ($Id$, etc)
-* a `_version.py` created by some earlier build step
-
-For released software, the version identifier is closely related to a VCS
-tag. Some projects use tag names that include more than just the version
-string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
-needs to strip the tag prefix to extract the version identifier. For
-unreleased software (between tags), the version identifier should provide
-enough information to help developers recreate the same tree, while also
-giving them an idea of roughly how old the tree is (after version 1.2, before
-version 1.3). Many VCS systems can report a description that captures this,
-for example `git describe --tags --dirty --always` reports things like
-"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
-0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
-uncommitted changes.
-
-The version identifier is used for multiple purposes:
-
-* to allow the module to self-identify its version: `myproject.__version__`
-* to choose a name and prefix for a 'setup.py sdist' tarball
-
-## Theory of Operation
-
-Versioneer works by adding a special `_version.py` file into your source
-tree, where your `__init__.py` can import it. This `_version.py` knows how to
-dynamically ask the VCS tool for version information at import time.
-
-`_version.py` also contains `$Revision$` markers, and the installation
-process marks `_version.py` to have this marker rewritten with a tag name
-during the `git archive` command. As a result, generated tarballs will
-contain enough information to get the proper version.
-
-To allow `setup.py` to compute a version too, a `versioneer.py` is added to
-the top level of your source tree, next to `setup.py` and the `setup.cfg`
-that configures it. This overrides several distutils/setuptools commands to
-compute the version when invoked, and changes `setup.py build` and `setup.py
-sdist` to replace `_version.py` with a small static file that contains just
-the generated version data.
-
-## Installation
-
-First, decide on values for the following configuration variables:
-
-* `VCS`: the version control system you use. Currently accepts "git".
-
-* `style`: the style of version string to be produced. See "Styles" below for
- details. Defaults to "pep440", which looks like
- `TAG[+DISTANCE.gSHORTHASH[.dirty]]`.
-
-* `versionfile_source`:
-
- A project-relative pathname into which the generated version strings should
- be written. This is usually a `_version.py` next to your project's main
- `__init__.py` file, so it can be imported at runtime. If your project uses
- `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
- This file should be checked in to your VCS as usual: the copy created below
- by `setup.py setup_versioneer` will include code that parses expanded VCS
- keywords in generated tarballs. The 'build' and 'sdist' commands will
- replace it with a copy that has just the calculated version string.
-
- This must be set even if your project does not have any modules (and will
- therefore never import `_version.py`), since "setup.py sdist" -based trees
- still need somewhere to record the pre-calculated version strings. Anywhere
- in the source tree should do. If there is a `__init__.py` next to your
- `_version.py`, the `setup.py setup_versioneer` command (described below)
- will append some `__version__`-setting assignments, if they aren't already
- present.
-
-* `versionfile_build`:
-
- Like `versionfile_source`, but relative to the build directory instead of
- the source directory. These will differ when your setup.py uses
- 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
- then you will probably have `versionfile_build='myproject/_version.py'` and
- `versionfile_source='src/myproject/_version.py'`.
-
- If this is set to None, then `setup.py build` will not attempt to rewrite
- any `_version.py` in the built tree. If your project does not have any
- libraries (e.g. if it only builds a script), then you should use
- `versionfile_build = None`. To actually use the computed version string,
- your `setup.py` will need to override `distutils.command.build_scripts`
- with a subclass that explicitly inserts a copy of
- `versioneer.get_version()` into your script file. See
- `test/demoapp-script-only/setup.py` for an example.
-
-* `tag_prefix`:
-
- a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
- If your tags look like 'myproject-1.2.0', then you should use
- tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
- should be an empty string, using either `tag_prefix=` or `tag_prefix=''`.
-
-* `parentdir_prefix`:
-
- a optional string, frequently the same as tag_prefix, which appears at the
- start of all unpacked tarball filenames. If your tarball unpacks into
- 'myproject-1.2.0', this should be 'myproject-'. To disable this feature,
- just omit the field from your `setup.cfg`.
-
-This tool provides one script, named `versioneer`. That script has one mode,
-"install", which writes a copy of `versioneer.py` into the current directory
-and runs `versioneer.py setup` to finish the installation.
-
-To versioneer-enable your project:
-
-* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and
- populating it with the configuration values you decided earlier (note that
- the option names are not case-sensitive):
-
- ````
- [versioneer]
- VCS = git
- style = pep440
- versionfile_source = src/myproject/_version.py
- versionfile_build = myproject/_version.py
- tag_prefix =
- parentdir_prefix = myproject-
- ````
-
-* 2: Run `versioneer install`. This will do the following:
-
- * copy `versioneer.py` into the top of your source tree
- * create `_version.py` in the right place (`versionfile_source`)
- * modify your `__init__.py` (if one exists next to `_version.py`) to define
- `__version__` (by calling a function from `_version.py`)
- * modify your `MANIFEST.in` to include both `versioneer.py` and the
- generated `_version.py` in sdist tarballs
-
- `versioneer install` will complain about any problems it finds with your
- `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all
- the problems.
-
-* 3: add a `import versioneer` to your setup.py, and add the following
- arguments to the setup() call:
-
- version=versioneer.get_version(),
- cmdclass=versioneer.get_cmdclass(),
-
-* 4: commit these changes to your VCS. To make sure you won't forget,
- `versioneer install` will mark everything it touched for addition using
- `git add`. Don't forget to add `setup.py` and `setup.cfg` too.
-
-## Post-Installation Usage
-
-Once established, all uses of your tree from a VCS checkout should get the
-current version string. All generated tarballs should include an embedded
-version string (so users who unpack them will not need a VCS tool installed).
-
-If you distribute your project through PyPI, then the release process should
-boil down to two steps:
-
-* 1: git tag 1.0
-* 2: python setup.py register sdist upload
-
-If you distribute it through github (i.e. users use github to generate
-tarballs with `git archive`), the process is:
-
-* 1: git tag 1.0
-* 2: git push; git push --tags
-
-Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at
-least one tag in its history.
-
-## Version-String Flavors
-
-Code which uses Versioneer can learn about its version string at runtime by
-importing `_version` from your main `__init__.py` file and running the
-`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
-import the top-level `versioneer.py` and run `get_versions()`.
-
-Both functions return a dictionary with different flavors of version
-information:
-
-* `['version']`: A condensed version string, rendered using the selected
- style. This is the most commonly used value for the project's version
- string. The default "pep440" style yields strings like `0.11`,
- `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
- below for alternative styles.
-
-* `['full-revisionid']`: detailed revision identifier. For Git, this is the
- full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
-
-* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
- this is only accurate if run in a VCS checkout, otherwise it is likely to
- be False or None
-
-* `['error']`: if the version string could not be computed, this will be set
- to a string describing the problem, otherwise it will be None. It may be
- useful to throw an exception in setup.py if this is set, to avoid e.g.
- creating tarballs with a version string of "unknown".
-
-Some variants are more useful than others. Including `full-revisionid` in a
-bug report should allow developers to reconstruct the exact code being tested
-(or indicate the presence of local changes that should be shared with the
-developers). `version` is suitable for display in an "about" box or a CLI
-`--version` output: it can be easily compared against release notes and lists
-of bugs fixed in various releases.
-
-The installer adds the following text to your `__init__.py` to place a basic
-version in `YOURPROJECT.__version__`:
-
- from ._version import get_versions
- __version__ = get_versions()['version']
- del get_versions
-
-## Styles
-
-The setup.cfg `style=` configuration controls how the VCS information is
-rendered into a version string.
-
-The default style, "pep440", produces a PEP440-compliant string, equal to the
-un-prefixed tag name for actual releases, and containing an additional "local
-version" section with more detail for in-between builds. For Git, this is
-TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
-tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
-that this commit is two revisions ("+2") beyond the "0.11" tag. For released
-software (exactly equal to a known tag), the identifier will only contain the
-stripped tag, e.g. "0.11".
-
-Other styles are available. See details.md in the Versioneer source tree for
-descriptions.
-
-## Debugging
-
-Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
-to return a version of "0+unknown". To investigate the problem, run `setup.py
-version`, which will run the version-lookup code in a verbose mode, and will
-display the full contents of `get_versions()` (including the `error` string,
-which may help identify what went wrong).
-
-## Updating Versioneer
-
-To upgrade your project to a new release of Versioneer, do the following:
-
-* install the new Versioneer (`pip install -U versioneer` or equivalent)
-* edit `setup.cfg`, if necessary, to include any new configuration settings
- indicated by the release notes
-* re-run `versioneer install` in your source tree, to replace
- `SRC/_version.py`
-* commit any changed files
-
-### Upgrading to 0.16
-
-Nothing special.
-
-### Upgrading to 0.15
-
-Starting with this version, Versioneer is configured with a `[versioneer]`
-section in your `setup.cfg` file. Earlier versions required the `setup.py` to
-set attributes on the `versioneer` module immediately after import. The new
-version will refuse to run (raising an exception during import) until you
-have provided the necessary `setup.cfg` section.
-
-In addition, the Versioneer package provides an executable named
-`versioneer`, and the installation process is driven by running `versioneer
-install`. In 0.14 and earlier, the executable was named
-`versioneer-installer` and was run without an argument.
-
-### Upgrading to 0.14
-
-0.14 changes the format of the version string. 0.13 and earlier used
-hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
-plus-separated "local version" section strings, with dot-separated
-components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
-format, but should be ok with the new one.
-
-### Upgrading from 0.11 to 0.12
-
-Nothing special.
-
-### Upgrading from 0.10 to 0.11
-
-You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
-`setup.py setup_versioneer`. This will enable the use of additional
-version-control systems (SVN, etc) in the future.
-
-## Future Directions
-
-This tool is designed to make it easily extended to other version-control
-systems: all VCS-specific components are in separate directories like
-src/git/ . The top-level `versioneer.py` script is assembled from these
-components by running make-versioneer.py . In the future, make-versioneer.py
-will take a VCS name as an argument, and will construct a version of
-`versioneer.py` that is specific to the given VCS. It might also take the
-configuration arguments that are currently provided manually during
-installation by editing setup.py . Alternatively, it might go the other
-direction and include code from all supported VCS systems, reducing the
-number of intermediate scripts.
-
-
-## License
-
-To make Versioneer easier to embed, all its code is dedicated to the public
-domain. The `_version.py` that it creates is also in the public domain.
-Specifically, both are released under the Creative Commons "Public Domain
-Dedication" license (CC0-1.0), as described in
-https://creativecommons.org/publicdomain/zero/1.0/ .
-
-"""
-
-from __future__ import print_function
-try:
- import configparser
-except ImportError:
- import ConfigParser as configparser
-import errno
-import json
-import os
-import re
-import subprocess
-import sys
-
-
-class VersioneerConfig:
- """Container for Versioneer configuration parameters."""
-
-
-def get_root():
- """Get the project root directory.
-
- We require that all commands are run from the project root, i.e. the
- directory that contains setup.py, setup.cfg, and versioneer.py .
- """
- root = os.path.realpath(os.path.abspath(os.getcwd()))
- setup_py = os.path.join(root, "setup.py")
- versioneer_py = os.path.join(root, "versioneer.py")
- if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
- # allow 'python path/to/setup.py COMMAND'
- root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
- setup_py = os.path.join(root, "setup.py")
- versioneer_py = os.path.join(root, "versioneer.py")
- if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
- err = ("Versioneer was unable to run the project root directory. "
- "Versioneer requires setup.py to be executed from "
- "its immediate directory (like 'python setup.py COMMAND'), "
- "or in a way that lets it use sys.argv[0] to find the root "
- "(like 'python path/to/setup.py COMMAND').")
- raise VersioneerBadRootError(err)
- try:
- # Certain runtime workflows (setup.py install/develop in a setuptools
- # tree) execute all dependencies in a single python process, so
- # "versioneer" may be imported multiple times, and python's shared
- # module-import table will cache the first one. So we can't use
- # os.path.dirname(__file__), as that will find whichever
- # versioneer.py was first imported, even in later projects.
- me = os.path.realpath(os.path.abspath(__file__))
- if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]:
- print("Warning: build in %s is using versioneer.py from %s"
- % (os.path.dirname(me), versioneer_py))
- except NameError:
- pass
- return root
-
-
-def get_config_from_root(root):
- """Read the project setup.cfg file to determine Versioneer config."""
- # This might raise EnvironmentError (if setup.cfg is missing), or
- # configparser.NoSectionError (if it lacks a [versioneer] section), or
- # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
- # the top of versioneer.py for instructions on writing your setup.cfg .
- setup_cfg = os.path.join(root, "setup.cfg")
- parser = configparser.SafeConfigParser()
- with open(setup_cfg, "r") as f:
- parser.readfp(f)
- VCS = parser.get("versioneer", "VCS") # mandatory
-
- def get(parser, name):
- if parser.has_option("versioneer", name):
- return parser.get("versioneer", name)
- return None
- cfg = VersioneerConfig()
- cfg.VCS = VCS
- cfg.style = get(parser, "style") or ""
- cfg.versionfile_source = get(parser, "versionfile_source")
- cfg.versionfile_build = get(parser, "versionfile_build")
- cfg.tag_prefix = get(parser, "tag_prefix")
- if cfg.tag_prefix in ("''", '""'):
- cfg.tag_prefix = ""
- cfg.parentdir_prefix = get(parser, "parentdir_prefix")
- cfg.verbose = get(parser, "verbose")
- return cfg
-
-
-class NotThisMethod(Exception):
- """Exception raised if a method is not valid for the current scenario."""
-
-# these dictionaries contain VCS-specific tools
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method): # decorator
- """Decorator to mark a method as the handler for a particular VCS."""
- def decorate(f):
- """Store f in HANDLERS[vcs][method]."""
- if vcs not in HANDLERS:
- HANDLERS[vcs] = {}
- HANDLERS[vcs][method] = f
- return f
- return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
- """Call the given command(s)."""
- assert isinstance(commands, list)
- p = None
- for c in commands:
- try:
- dispcmd = str([c] + args)
- # remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
- stderr=(subprocess.PIPE if hide_stderr
- else None))
- break
- except EnvironmentError:
- e = sys.exc_info()[1]
- if e.errno == errno.ENOENT:
- continue
- if verbose:
- print("unable to run %s" % dispcmd)
- print(e)
- return None
- else:
- if verbose:
- print("unable to find command, tried %s" % (commands,))
- return None
- stdout = p.communicate()[0].strip()
- if sys.version_info[0] >= 3:
- stdout = stdout.decode()
- if p.returncode != 0:
- if verbose:
- print("unable to run %s (error)" % dispcmd)
- return None
- return stdout
-LONG_VERSION_PY['git'] = '''
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.16 (https://github.com/warner/python-versioneer)
-
-"""Git implementation of _version.py."""
-
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
- """Get the keywords needed to look up the version information."""
- # these strings will be replaced by git during git-archive.
- # setup.py/versioneer.py will grep for the variable names, so they must
- # each be defined on a line of their own. _version.py will just call
- # get_keywords().
- git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
- git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
- keywords = {"refnames": git_refnames, "full": git_full}
- return keywords
-
-
-class VersioneerConfig:
- """Container for Versioneer configuration parameters."""
-
-
-def get_config():
- """Create, populate and return the VersioneerConfig() object."""
- # these strings are filled in when 'setup.py versioneer' creates
- # _version.py
- cfg = VersioneerConfig()
- cfg.VCS = "git"
- cfg.style = "%(STYLE)s"
- cfg.tag_prefix = "%(TAG_PREFIX)s"
- cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
- cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
- cfg.verbose = False
- return cfg
-
-
-class NotThisMethod(Exception):
- """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method): # decorator
- """Decorator to mark a method as the handler for a particular VCS."""
- def decorate(f):
- """Store f in HANDLERS[vcs][method]."""
- if vcs not in HANDLERS:
- HANDLERS[vcs] = {}
- HANDLERS[vcs][method] = f
- return f
- return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
- """Call the given command(s)."""
- assert isinstance(commands, list)
- p = None
- for c in commands:
- try:
- dispcmd = str([c] + args)
- # remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
- stderr=(subprocess.PIPE if hide_stderr
- else None))
- break
- except EnvironmentError:
- e = sys.exc_info()[1]
- if e.errno == errno.ENOENT:
- continue
- if verbose:
- print("unable to run %%s" %% dispcmd)
- print(e)
- return None
- else:
- if verbose:
- print("unable to find command, tried %%s" %% (commands,))
- return None
- stdout = p.communicate()[0].strip()
- if sys.version_info[0] >= 3:
- stdout = stdout.decode()
- if p.returncode != 0:
- if verbose:
- print("unable to run %%s (error)" %% dispcmd)
- return None
- return stdout
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
- """Try to determine the version from the parent directory name.
-
- Source tarballs conventionally unpack into a directory that includes
- both the project name and a version string.
- """
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%%s', but '%%s' doesn't start with "
- "prefix '%%s'" %% (root, dirname, parentdir_prefix))
- raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
- return {"version": dirname[len(parentdir_prefix):],
- "full-revisionid": None,
- "dirty": False, "error": None}
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
- """Extract version information from the given file."""
- # the code embedded in _version.py can just fetch the value of these
- # keywords. When used from setup.py, we don't want to import _version.py,
- # so we do it with a regexp instead. This function is not used from
- # _version.py.
- keywords = {}
- try:
- f = open(versionfile_abs, "r")
- for line in f.readlines():
- if line.strip().startswith("git_refnames ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["refnames"] = mo.group(1)
- if line.strip().startswith("git_full ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["full"] = mo.group(1)
- f.close()
- except EnvironmentError:
- pass
- return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
- """Get version information from git keywords."""
- if not keywords:
- raise NotThisMethod("no keywords at all, weird")
- refnames = keywords["refnames"].strip()
- if refnames.startswith("$Format"):
- if verbose:
- print("keywords are unexpanded, not using")
- raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
- refs = set([r.strip() for r in refnames.strip("()").split(",")])
- # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
- # just "foo-1.0". If we see a "tag: " prefix, prefer those.
- TAG = "tag: "
- tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
- if not tags:
- # Either we're using git < 1.8.3, or there really are no tags. We use
- # a heuristic: assume all version tags have a digit. The old git %%d
- # expansion behaves like git log --decorate=short and strips out the
- # refs/heads/ and refs/tags/ prefixes that would let us distinguish
- # between branches and tags. By ignoring refnames without digits, we
- # filter out many common branch names like "release" and
- # "stabilization", as well as "HEAD" and "master".
- tags = set([r for r in refs if re.search(r'\d', r)])
- if verbose:
- print("discarding '%%s', no digits" %% ",".join(refs-tags))
- if verbose:
- print("likely tags: %%s" %% ",".join(sorted(tags)))
- for ref in sorted(tags):
- # sorting will prefer e.g. "2.0" over "2.0rc1"
- if ref.startswith(tag_prefix):
- r = ref[len(tag_prefix):]
- if verbose:
- print("picking %%s" %% r)
- return {"version": r,
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": None
- }
- # no suitable tags, so version is "0+unknown", but full hex is still there
- if verbose:
- print("no suitable tags, using unknown + full revision id")
- return {"version": "0+unknown",
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": "no suitable tags"}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
- """Get version from 'git describe' in the root of the source tree.
-
- This only gets called if the git-archive 'subst' keywords were *not*
- expanded, and _version.py hasn't already been rewritten with a short
- version string, meaning we're inside a checked out source tree.
- """
- if not os.path.exists(os.path.join(root, "..", ".git")):
- if verbose:
- print("no .git in %%s" %% root)
- raise NotThisMethod("no .git directory")
-
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
- # if there isn't one, this yields HEX[-dirty] (no NUM)
- describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long",
- "--match", "%%s*" %% tag_prefix],
- cwd=root)
- # --long was added in git-1.5.5
- if describe_out is None:
- raise NotThisMethod("'git describe' failed")
- describe_out = describe_out.strip()
- full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if full_out is None:
- raise NotThisMethod("'git rev-parse' failed")
- full_out = full_out.strip()
-
- pieces = {}
- pieces["long"] = full_out
- pieces["short"] = full_out[:7] # maybe improved later
- pieces["error"] = None
-
- # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
- # TAG might have hyphens.
- git_describe = describe_out
-
- # look for -dirty suffix
- dirty = git_describe.endswith("-dirty")
- pieces["dirty"] = dirty
- if dirty:
- git_describe = git_describe[:git_describe.rindex("-dirty")]
-
- # now we have TAG-NUM-gHEX or HEX
-
- if "-" in git_describe:
- # TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- pieces["error"] = ("unable to parse git-describe output: '%%s'"
- %% describe_out)
- return pieces
-
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%%s' doesn't start with prefix '%%s'"
- print(fmt %% (full_tag, tag_prefix))
- pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
- %% (full_tag, tag_prefix))
- return pieces
- pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
- # distance: number of commits since tag
- pieces["distance"] = int(mo.group(2))
-
- # commit: short hex revision ID
- pieces["short"] = mo.group(3)
-
- else:
- # HEX: no tags
- pieces["closest-tag"] = None
- count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
- cwd=root)
- pieces["distance"] = int(count_out) # total number of commits
-
- return pieces
-
-
-def plus_or_dot(pieces):
- """Return a + if we don't already have one, else return a ."""
- if "+" in pieces.get("closest-tag", ""):
- return "."
- return "+"
-
-
-def render_pep440(pieces):
- """Build up version string, with post-release "local version identifier".
-
- Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
- get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
- Exceptions:
- 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += plus_or_dot(pieces)
- rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- else:
- # exception #1
- rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
- pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- return rendered
-
-
-def render_pep440_pre(pieces):
- """TAG[.post.devDISTANCE] -- No -dirty.
-
- Exceptions:
- 1: no tags. 0.post.devDISTANCE
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += ".post.dev%%d" %% pieces["distance"]
- else:
- # exception #1
- rendered = "0.post.dev%%d" %% pieces["distance"]
- return rendered
-
-
-def render_pep440_post(pieces):
- """TAG[.postDISTANCE[.dev0]+gHEX] .
-
- The ".dev0" means dirty. Note that .dev0 sorts backwards
- (a dirty tree will appear "older" than the corresponding clean one),
- but you shouldn't be releasing software with -dirty anyways.
-
- Exceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%%d" %% pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += plus_or_dot(pieces)
- rendered += "g%%s" %% pieces["short"]
- else:
- # exception #1
- rendered = "0.post%%d" %% pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += "+g%%s" %% pieces["short"]
- return rendered
-
-
-def render_pep440_old(pieces):
- """TAG[.postDISTANCE[.dev0]] .
-
- The ".dev0" means dirty.
-
- Eexceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%%d" %% pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- else:
- # exception #1
- rendered = "0.post%%d" %% pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- return rendered
-
-
-def render_git_describe(pieces):
- """TAG[-DISTANCE-gHEX][-dirty].
-
- Like 'git describe --tags --dirty --always'.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render_git_describe_long(pieces):
- """TAG-DISTANCE-gHEX[-dirty].
-
- Like 'git describe --tags --dirty --always -long'.
- The distance/hash is unconditional.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render(pieces, style):
- """Render the given version pieces into the requested style."""
- if pieces["error"]:
- return {"version": "unknown",
- "full-revisionid": pieces.get("long"),
- "dirty": None,
- "error": pieces["error"]}
-
- if not style or style == "default":
- style = "pep440" # the default
-
- if style == "pep440":
- rendered = render_pep440(pieces)
- elif style == "pep440-pre":
- rendered = render_pep440_pre(pieces)
- elif style == "pep440-post":
- rendered = render_pep440_post(pieces)
- elif style == "pep440-old":
- rendered = render_pep440_old(pieces)
- elif style == "git-describe":
- rendered = render_git_describe(pieces)
- elif style == "git-describe-long":
- rendered = render_git_describe_long(pieces)
- else:
- raise ValueError("unknown style '%%s'" %% style)
-
- return {"version": rendered, "full-revisionid": pieces["long"],
- "dirty": pieces["dirty"], "error": None}
-
-
-def get_versions():
- """Get version information or return default if unable to do so."""
- # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
- # __file__, we can work backwards from there to the root. Some
- # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
- # case we can only use expanded keywords.
-
- cfg = get_config()
- verbose = cfg.verbose
-
- try:
- return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
- verbose)
- except NotThisMethod:
- pass
-
- try:
- root = os.path.realpath(__file__)
- # versionfile_source is the relative path from the top of the source
- # tree (where the .git directory might live) to this file. Invert
- # this to find the root from __file__.
- for i in cfg.versionfile_source.split('/'):
- root = os.path.dirname(root)
- except NameError:
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to find root of source tree"}
-
- try:
- pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
- return render(pieces, cfg.style)
- except NotThisMethod:
- pass
-
- try:
- if cfg.parentdir_prefix:
- return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
- except NotThisMethod:
- pass
-
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to compute version"}
-'''
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
- """Extract version information from the given file."""
- # the code embedded in _version.py can just fetch the value of these
- # keywords. When used from setup.py, we don't want to import _version.py,
- # so we do it with a regexp instead. This function is not used from
- # _version.py.
- keywords = {}
- try:
- f = open(versionfile_abs, "r")
- for line in f.readlines():
- if line.strip().startswith("git_refnames ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["refnames"] = mo.group(1)
- if line.strip().startswith("git_full ="):
- mo = re.search(r'=\s*"(.*)"', line)
- if mo:
- keywords["full"] = mo.group(1)
- f.close()
- except EnvironmentError:
- pass
- return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
- """Get version information from git keywords."""
- if not keywords:
- raise NotThisMethod("no keywords at all, weird")
- refnames = keywords["refnames"].strip()
- if refnames.startswith("$Format"):
- if verbose:
- print("keywords are unexpanded, not using")
- raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
- refs = set([r.strip() for r in refnames.strip("()").split(",")])
- # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
- # just "foo-1.0". If we see a "tag: " prefix, prefer those.
- TAG = "tag: "
- tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
- if not tags:
- # Either we're using git < 1.8.3, or there really are no tags. We use
- # a heuristic: assume all version tags have a digit. The old git %d
- # expansion behaves like git log --decorate=short and strips out the
- # refs/heads/ and refs/tags/ prefixes that would let us distinguish
- # between branches and tags. By ignoring refnames without digits, we
- # filter out many common branch names like "release" and
- # "stabilization", as well as "HEAD" and "master".
- tags = set([r for r in refs if re.search(r'\d', r)])
- if verbose:
- print("discarding '%s', no digits" % ",".join(refs-tags))
- if verbose:
- print("likely tags: %s" % ",".join(sorted(tags)))
- for ref in sorted(tags):
- # sorting will prefer e.g. "2.0" over "2.0rc1"
- if ref.startswith(tag_prefix):
- r = ref[len(tag_prefix):]
- if verbose:
- print("picking %s" % r)
- return {"version": r,
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": None
- }
- # no suitable tags, so version is "0+unknown", but full hex is still there
- if verbose:
- print("no suitable tags, using unknown + full revision id")
- return {"version": "0+unknown",
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": "no suitable tags"}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
- """Get version from 'git describe' in the root of the source tree.
-
- This only gets called if the git-archive 'subst' keywords were *not*
- expanded, and _version.py hasn't already been rewritten with a short
- version string, meaning we're inside a checked out source tree.
- """
- if not os.path.exists(os.path.join(root, "..", ".git")):
- if verbose:
- print("no .git in %s" % root)
- raise NotThisMethod("no .git directory")
-
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
- # if there isn't one, this yields HEX[-dirty] (no NUM)
- describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long",
- "--match", "%s*" % tag_prefix],
- cwd=root)
- # --long was added in git-1.5.5
- if describe_out is None:
- raise NotThisMethod("'git describe' failed")
- describe_out = describe_out.strip()
- full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if full_out is None:
- raise NotThisMethod("'git rev-parse' failed")
- full_out = full_out.strip()
-
- pieces = {}
- pieces["long"] = full_out
- pieces["short"] = full_out[:7] # maybe improved later
- pieces["error"] = None
-
- # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
- # TAG might have hyphens.
- git_describe = describe_out
-
- # look for -dirty suffix
- dirty = git_describe.endswith("-dirty")
- pieces["dirty"] = dirty
- if dirty:
- git_describe = git_describe[:git_describe.rindex("-dirty")]
-
- # now we have TAG-NUM-gHEX or HEX
-
- if "-" in git_describe:
- # TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- pieces["error"] = ("unable to parse git-describe output: '%s'"
- % describe_out)
- return pieces
-
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%s' doesn't start with prefix '%s'"
- print(fmt % (full_tag, tag_prefix))
- pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
- % (full_tag, tag_prefix))
- return pieces
- pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
- # distance: number of commits since tag
- pieces["distance"] = int(mo.group(2))
-
- # commit: short hex revision ID
- pieces["short"] = mo.group(3)
-
- else:
- # HEX: no tags
- pieces["closest-tag"] = None
- count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
- cwd=root)
- pieces["distance"] = int(count_out) # total number of commits
-
- return pieces
-
-
-def do_vcs_install(manifest_in, versionfile_source, ipy):
- """Git-specific installation logic for Versioneer.
-
- For Git, this means creating/changing .gitattributes to mark _version.py
- for export-time keyword substitution.
- """
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- files = [manifest_in, versionfile_source]
- if ipy:
- files.append(ipy)
- try:
- me = __file__
- if me.endswith(".pyc") or me.endswith(".pyo"):
- me = os.path.splitext(me)[0] + ".py"
- versioneer_file = os.path.relpath(me)
- except NameError:
- versioneer_file = "versioneer.py"
- files.append(versioneer_file)
- present = False
- try:
- f = open(".gitattributes", "r")
- for line in f.readlines():
- if line.strip().startswith(versionfile_source):
- if "export-subst" in line.strip().split()[1:]:
- present = True
- f.close()
- except EnvironmentError:
- pass
- if not present:
- f = open(".gitattributes", "a+")
- f.write("%s export-subst\n" % versionfile_source)
- f.close()
- files.append(".gitattributes")
- run_command(GITS, ["add", "--"] + files)
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
- """Try to determine the version from the parent directory name.
-
- Source tarballs conventionally unpack into a directory that includes
- both the project name and a version string.
- """
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%s', but '%s' doesn't start with "
- "prefix '%s'" % (root, dirname, parentdir_prefix))
- raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
- return {"version": dirname[len(parentdir_prefix):],
- "full-revisionid": None,
- "dirty": False, "error": None}
-
-SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.16) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-import json
-import sys
-
-version_json = '''
-%s
-''' # END VERSION_JSON
-
-
-def get_versions():
- return json.loads(version_json)
-"""
-
-
-def versions_from_file(filename):
- """Try to determine the version from _version.py if present."""
- try:
- with open(filename) as f:
- contents = f.read()
- except EnvironmentError:
- raise NotThisMethod("unable to read _version.py")
- mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON",
- contents, re.M | re.S)
- if not mo:
- raise NotThisMethod("no version_json in _version.py")
- return json.loads(mo.group(1))
-
-
-def write_to_version_file(filename, versions):
- """Write the given version number to the given _version.py file."""
- os.unlink(filename)
- contents = json.dumps(versions, sort_keys=True,
- indent=1, separators=(",", ": "))
- with open(filename, "w") as f:
- f.write(SHORT_VERSION_PY % contents)
-
- print("set %s to '%s'" % (filename, versions["version"]))
-
-
-def plus_or_dot(pieces):
- """Return a + if we don't already have one, else return a ."""
- if "+" in pieces.get("closest-tag", ""):
- return "."
- return "+"
-
-
-def render_pep440(pieces):
- """Build up version string, with post-release "local version identifier".
-
- Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
- get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
- Exceptions:
- 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += plus_or_dot(pieces)
- rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- else:
- # exception #1
- rendered = "0+untagged.%d.g%s" % (pieces["distance"],
- pieces["short"])
- if pieces["dirty"]:
- rendered += ".dirty"
- return rendered
-
-
-def render_pep440_pre(pieces):
- """TAG[.post.devDISTANCE] -- No -dirty.
-
- Exceptions:
- 1: no tags. 0.post.devDISTANCE
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += ".post.dev%d" % pieces["distance"]
- else:
- # exception #1
- rendered = "0.post.dev%d" % pieces["distance"]
- return rendered
-
-
-def render_pep440_post(pieces):
- """TAG[.postDISTANCE[.dev0]+gHEX] .
-
- The ".dev0" means dirty. Note that .dev0 sorts backwards
- (a dirty tree will appear "older" than the corresponding clean one),
- but you shouldn't be releasing software with -dirty anyways.
-
- Exceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += plus_or_dot(pieces)
- rendered += "g%s" % pieces["short"]
- else:
- # exception #1
- rendered = "0.post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- rendered += "+g%s" % pieces["short"]
- return rendered
-
-
-def render_pep440_old(pieces):
- """TAG[.postDISTANCE[.dev0]] .
-
- The ".dev0" means dirty.
-
- Eexceptions:
- 1: no tags. 0.postDISTANCE[.dev0]
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"] or pieces["dirty"]:
- rendered += ".post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- else:
- # exception #1
- rendered = "0.post%d" % pieces["distance"]
- if pieces["dirty"]:
- rendered += ".dev0"
- return rendered
-
-
-def render_git_describe(pieces):
- """TAG[-DISTANCE-gHEX][-dirty].
-
- Like 'git describe --tags --dirty --always'.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- if pieces["distance"]:
- rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render_git_describe_long(pieces):
- """TAG-DISTANCE-gHEX[-dirty].
-
- Like 'git describe --tags --dirty --always -long'.
- The distance/hash is unconditional.
-
- Exceptions:
- 1: no tags. HEX[-dirty] (note: no 'g' prefix)
- """
- if pieces["closest-tag"]:
- rendered = pieces["closest-tag"]
- rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
- else:
- # exception #1
- rendered = pieces["short"]
- if pieces["dirty"]:
- rendered += "-dirty"
- return rendered
-
-
-def render(pieces, style):
- """Render the given version pieces into the requested style."""
- if pieces["error"]:
- return {"version": "unknown",
- "full-revisionid": pieces.get("long"),
- "dirty": None,
- "error": pieces["error"]}
-
- if not style or style == "default":
- style = "pep440" # the default
-
- if style == "pep440":
- rendered = render_pep440(pieces)
- elif style == "pep440-pre":
- rendered = render_pep440_pre(pieces)
- elif style == "pep440-post":
- rendered = render_pep440_post(pieces)
- elif style == "pep440-old":
- rendered = render_pep440_old(pieces)
- elif style == "git-describe":
- rendered = render_git_describe(pieces)
- elif style == "git-describe-long":
- rendered = render_git_describe_long(pieces)
- else:
- raise ValueError("unknown style '%s'" % style)
-
- return {"version": rendered, "full-revisionid": pieces["long"],
- "dirty": pieces["dirty"], "error": None}
-
-
-class VersioneerBadRootError(Exception):
- """The project root directory is unknown or missing key files."""
-
-
-def get_versions(verbose=False):
- """Get the project version from whatever source is available.
-
- Returns dict with two keys: 'version' and 'full'.
- """
- if "versioneer" in sys.modules:
- # see the discussion in cmdclass.py:get_cmdclass()
- del sys.modules["versioneer"]
-
- root = get_root()
- cfg = get_config_from_root(root)
-
- assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
- handlers = HANDLERS.get(cfg.VCS)
- assert handlers, "unrecognized VCS '%s'" % cfg.VCS
- verbose = verbose or cfg.verbose
- assert cfg.versionfile_source is not None, \
- "please set versioneer.versionfile_source"
- assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
-
- versionfile_abs = os.path.join(root, cfg.versionfile_source)
-
- # extract version from first of: _version.py, VCS command (e.g. 'git
- # describe'), parentdir. This is meant to work for developers using a
- # source checkout, for users of a tarball created by 'setup.py sdist',
- # and for users of a tarball/zipball created by 'git archive' or github's
- # download-from-tag feature or the equivalent in other VCSes.
-
- get_keywords_f = handlers.get("get_keywords")
- from_keywords_f = handlers.get("keywords")
- if get_keywords_f and from_keywords_f:
- try:
- keywords = get_keywords_f(versionfile_abs)
- ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
- if verbose:
- print("got version from expanded keyword %s" % ver)
- return ver
- except NotThisMethod:
- pass
-
- try:
- ver = versions_from_file(versionfile_abs)
- if verbose:
- print("got version from file %s %s" % (versionfile_abs, ver))
- return ver
- except NotThisMethod:
- pass
-
- from_vcs_f = handlers.get("pieces_from_vcs")
- if from_vcs_f:
- try:
- pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
- ver = render(pieces, cfg.style)
- if verbose:
- print("got version from VCS %s" % ver)
- return ver
- except NotThisMethod:
- pass
-
- try:
- if cfg.parentdir_prefix:
- ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
- if verbose:
- print("got version from parentdir %s" % ver)
- return ver
- except NotThisMethod:
- pass
-
- if verbose:
- print("unable to compute version")
-
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None, "error": "unable to compute version"}
-
-
-def get_version():
- """Get the short version string for this project."""
- return get_versions()["version"]
-
-
-def get_cmdclass():
- """Get the custom setuptools/distutils subclasses used by Versioneer."""
- if "versioneer" in sys.modules:
- del sys.modules["versioneer"]
- # this fixes the "python setup.py develop" case (also 'install' and
- # 'easy_install .'), in which subdependencies of the main project are
- # built (using setup.py bdist_egg) in the same python process. Assume
- # a main project A and a dependency B, which use different versions
- # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
- # sys.modules by the time B's setup.py is executed, causing B to run
- # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
- # sandbox that restores sys.modules to it's pre-build state, so the
- # parent is protected against the child's "import versioneer". By
- # removing ourselves from sys.modules here, before the child build
- # happens, we protect the child from the parent's versioneer too.
- # Also see https://github.com/warner/python-versioneer/issues/52
-
- cmds = {}
-
- # we add "version" to both distutils and setuptools
- from distutils.core import Command
-
- class cmd_version(Command):
- description = "report generated version string"
- user_options = []
- boolean_options = []
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- vers = get_versions(verbose=True)
- print("Version: %s" % vers["version"])
- print(" full-revisionid: %s" % vers.get("full-revisionid"))
- print(" dirty: %s" % vers.get("dirty"))
- if vers["error"]:
- print(" error: %s" % vers["error"])
- cmds["version"] = cmd_version
-
- # we override "build_py" in both distutils and setuptools
- #
- # most invocation pathways end up running build_py:
- # distutils/build -> build_py
- # distutils/install -> distutils/build ->..
- # setuptools/bdist_wheel -> distutils/install ->..
- # setuptools/bdist_egg -> distutils/install_lib -> build_py
- # setuptools/install -> bdist_egg ->..
- # setuptools/develop -> ?
-
- # we override different "build_py" commands for both environments
- if "setuptools" in sys.modules:
- from setuptools.command.build_py import build_py as _build_py
- else:
- from distutils.command.build_py import build_py as _build_py
-
- class cmd_build_py(_build_py):
- def run(self):
- root = get_root()
- cfg = get_config_from_root(root)
- versions = get_versions()
- _build_py.run(self)
- # now locate _version.py in the new build/ directory and replace
- # it with an updated value
- if cfg.versionfile_build:
- target_versionfile = os.path.join(self.build_lib,
- cfg.versionfile_build)
- print("UPDATING %s" % target_versionfile)
- write_to_version_file(target_versionfile, versions)
- cmds["build_py"] = cmd_build_py
-
- if "cx_Freeze" in sys.modules: # cx_freeze enabled?
- from cx_Freeze.dist import build_exe as _build_exe
-
- class cmd_build_exe(_build_exe):
- def run(self):
- root = get_root()
- cfg = get_config_from_root(root)
- versions = get_versions()
- target_versionfile = cfg.versionfile_source
- print("UPDATING %s" % target_versionfile)
- write_to_version_file(target_versionfile, versions)
-
- _build_exe.run(self)
- os.unlink(target_versionfile)
- with open(cfg.versionfile_source, "w") as f:
- LONG = LONG_VERSION_PY[cfg.VCS]
- f.write(LONG %
- {"DOLLAR": "$",
- "STYLE": cfg.style,
- "TAG_PREFIX": cfg.tag_prefix,
- "PARENTDIR_PREFIX": cfg.parentdir_prefix,
- "VERSIONFILE_SOURCE": cfg.versionfile_source,
- })
- cmds["build_exe"] = cmd_build_exe
- del cmds["build_py"]
-
- # we override different "sdist" commands for both environments
- if "setuptools" in sys.modules:
- from setuptools.command.sdist import sdist as _sdist
- else:
- from distutils.command.sdist import sdist as _sdist
-
- class cmd_sdist(_sdist):
- def run(self):
- versions = get_versions()
- self._versioneer_generated_versions = versions
- # unless we update this, the command will keep using the old
- # version
- self.distribution.metadata.version = versions["version"]
- return _sdist.run(self)
-
- def make_release_tree(self, base_dir, files):
- root = get_root()
- cfg = get_config_from_root(root)
- _sdist.make_release_tree(self, base_dir, files)
- # now locate _version.py in the new base_dir directory
- # (remembering that it may be a hardlink) and replace it with an
- # updated value
- target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
- print("UPDATING %s" % target_versionfile)
- write_to_version_file(target_versionfile,
- self._versioneer_generated_versions)
- cmds["sdist"] = cmd_sdist
-
- return cmds
-
-
-CONFIG_ERROR = """
-setup.cfg is missing the necessary Versioneer configuration. You need
-a section like:
-
- [versioneer]
- VCS = git
- style = pep440
- versionfile_source = src/myproject/_version.py
- versionfile_build = myproject/_version.py
- tag_prefix =
- parentdir_prefix = myproject-
-
-You will also need to edit your setup.py to use the results:
-
- import versioneer
- setup(version=versioneer.get_version(),
- cmdclass=versioneer.get_cmdclass(), ...)
-
-Please read the docstring in ./versioneer.py for configuration instructions,
-edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
-"""
-
-SAMPLE_CONFIG = """
-# See the docstring in versioneer.py for instructions. Note that you must
-# re-run 'versioneer.py setup' after changing this section, and commit the
-# resulting files.
-
-[versioneer]
-#VCS = git
-#style = pep440
-#versionfile_source =
-#versionfile_build =
-#tag_prefix =
-#parentdir_prefix =
-
-"""
-
-INIT_PY_SNIPPET = """
-from ._version import get_versions
-__version__ = get_versions()['version']
-del get_versions
-"""
-
-
-def do_setup():
- """Main VCS-independent setup function for installing Versioneer."""
- root = get_root()
- try:
- cfg = get_config_from_root(root)
- except (EnvironmentError, configparser.NoSectionError,
- configparser.NoOptionError) as e:
- if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
- print("Adding sample versioneer config to setup.cfg",
- file=sys.stderr)
- with open(os.path.join(root, "setup.cfg"), "a") as f:
- f.write(SAMPLE_CONFIG)
- print(CONFIG_ERROR, file=sys.stderr)
- return 1
-
- print(" creating %s" % cfg.versionfile_source)
- with open(cfg.versionfile_source, "w") as f:
- LONG = LONG_VERSION_PY[cfg.VCS]
- f.write(LONG % {"DOLLAR": "$",
- "STYLE": cfg.style,
- "TAG_PREFIX": cfg.tag_prefix,
- "PARENTDIR_PREFIX": cfg.parentdir_prefix,
- "VERSIONFILE_SOURCE": cfg.versionfile_source,
- })
-
- ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
- "__init__.py")
- if os.path.exists(ipy):
- try:
- with open(ipy, "r") as f:
- old = f.read()
- except EnvironmentError:
- old = ""
- if INIT_PY_SNIPPET not in old:
- print(" appending to %s" % ipy)
- with open(ipy, "a") as f:
- f.write(INIT_PY_SNIPPET)
- else:
- print(" %s unmodified" % ipy)
- else:
- print(" %s doesn't exist, ok" % ipy)
- ipy = None
-
- # Make sure both the top-level "versioneer.py" and versionfile_source
- # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
- # they'll be copied into source distributions. Pip won't be able to
- # install the package without this.
- manifest_in = os.path.join(root, "MANIFEST.in")
- simple_includes = set()
- try:
- with open(manifest_in, "r") as f:
- for line in f:
- if line.startswith("include "):
- for include in line.split()[1:]:
- simple_includes.add(include)
- except EnvironmentError:
- pass
- # That doesn't cover everything MANIFEST.in can do
- # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
- # it might give some false negatives. Appending redundant 'include'
- # lines is safe, though.
- if "versioneer.py" not in simple_includes:
- print(" appending 'versioneer.py' to MANIFEST.in")
- with open(manifest_in, "a") as f:
- f.write("include versioneer.py\n")
- else:
- print(" 'versioneer.py' already in MANIFEST.in")
- if cfg.versionfile_source not in simple_includes:
- print(" appending versionfile_source ('%s') to MANIFEST.in" %
- cfg.versionfile_source)
- with open(manifest_in, "a") as f:
- f.write("include %s\n" % cfg.versionfile_source)
- else:
- print(" versionfile_source already in MANIFEST.in")
-
- # Make VCS-specific changes. For git, this means creating/changing
- # .gitattributes to mark _version.py for export-time keyword
- # substitution.
- do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
- return 0
-
-
-def scan_setup_py():
- """Validate the contents of setup.py against Versioneer's expectations."""
- found = set()
- setters = False
- errors = 0
- with open("setup.py", "r") as f:
- for line in f.readlines():
- if "import versioneer" in line:
- found.add("import")
- if "versioneer.get_cmdclass()" in line:
- found.add("cmdclass")
- if "versioneer.get_version()" in line:
- found.add("get_version")
- if "versioneer.VCS" in line:
- setters = True
- if "versioneer.versionfile_source" in line:
- setters = True
- if len(found) != 3:
- print("")
- print("Your setup.py appears to be missing some important items")
- print("(but I might be wrong). Please make sure it has something")
- print("roughly like the following:")
- print("")
- print(" import versioneer")
- print(" setup( version=versioneer.get_version(),")
- print(" cmdclass=versioneer.get_cmdclass(), ...)")
- print("")
- errors += 1
- if setters:
- print("You should remove lines like 'versioneer.VCS = ' and")
- print("'versioneer.versionfile_source = ' . This configuration")
- print("now lives in setup.cfg, and should be removed from setup.py")
- print("")
- errors += 1
- return errors
-
-if __name__ == "__main__":
- cmd = sys.argv[1]
- if cmd == "setup":
- errors = do_setup()
- errors += scan_setup_py()
- if errors:
- sys.exit(1)