From cce638a8adf4e045ca5505afea4bda57753c31dd Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Mon, 11 Aug 2014 16:33:29 -0400 Subject: initial import of debian package --- docs/source/_static/default.css | 553 ++++++++++++++++++++++++++++++++++++ docs/source/_static/logo.png | Bin 0 -> 8686 bytes docs/source/_static/zeromq.ico | Bin 0 -> 1150 bytes docs/source/_templates/layout.html | 23 ++ docs/source/api/index.rst | 25 ++ docs/source/api/zmq.auth.ioloop.rst | 20 ++ docs/source/api/zmq.auth.rst | 30 ++ docs/source/api/zmq.auth.thread.rst | 24 ++ docs/source/api/zmq.devices.rst | 86 ++++++ docs/source/api/zmq.green.rst | 7 + docs/source/api/zmq.rst | 140 +++++++++ docs/source/changelog.rst | 446 +++++++++++++++++++++++++++++ docs/source/conf.py | 209 ++++++++++++++ docs/source/devices.rst | 82 ++++++ docs/source/eventloop.rst | 197 +++++++++++++ docs/source/index.rst | 95 +++++++ docs/source/logging.rst | 84 ++++++ docs/source/morethanbindings.rst | 166 +++++++++++ docs/source/pyversions.rst | 195 +++++++++++++ docs/source/serialization.rst | 90 ++++++ docs/source/ssh.rst | 80 ++++++ docs/source/unicode.rst | 188 ++++++++++++ 22 files changed, 2740 insertions(+) create mode 100644 docs/source/_static/default.css create mode 100644 docs/source/_static/logo.png create mode 100644 docs/source/_static/zeromq.ico create mode 100644 docs/source/_templates/layout.html create mode 100644 docs/source/api/index.rst create mode 100644 docs/source/api/zmq.auth.ioloop.rst create mode 100644 docs/source/api/zmq.auth.rst create mode 100644 docs/source/api/zmq.auth.thread.rst create mode 100644 docs/source/api/zmq.devices.rst create mode 100644 docs/source/api/zmq.green.rst create mode 100644 docs/source/api/zmq.rst create mode 100644 docs/source/changelog.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/devices.rst create mode 100644 docs/source/eventloop.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/logging.rst create mode 100644 docs/source/morethanbindings.rst create mode 100644 docs/source/pyversions.rst create mode 100644 docs/source/serialization.rst create mode 100644 docs/source/ssh.rst create mode 100644 docs/source/unicode.rst (limited to 'docs/source') diff --git a/docs/source/_static/default.css b/docs/source/_static/default.css new file mode 100644 index 0000000..71c9d5e --- /dev/null +++ b/docs/source/_static/default.css @@ -0,0 +1,553 @@ +/** + * Alternate Sphinx design + * Originally created by Armin Ronacher for Werkzeug, adapted by Georg Brandl. + */ + +body { + font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif; + font-size: 14px; + letter-spacing: -0.01em; + line-height: 150%; + text-align: center; + /*background-color: #AFC1C4; */ + background-color: #BFD1D4; + color: black; + padding: 0; + border: 1px solid #aaa; + + margin: 0px 80px 0px 80px; + min-width: 740px; +} + +a { + color: #CA7900; + text-decoration: none; +} + +a:hover { + color: #2491CF; +} + +pre { + font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; + font-size: 0.95em; + letter-spacing: 0.015em; + padding: 0.5em; + border: 1px solid #ccc; + background-color: #f8f8f8; +} + +td.linenos pre { + padding: 0.5em 0; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +cite, code, tt { + font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; + font-size: 0.95em; + letter-spacing: 0.01em; +} + +hr { + border: 1px solid #abc; + margin: 2em; +} + +tt { + background-color: #f2f2f2; + border-bottom: 1px solid #ddd; + color: #333; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; + border: 0; +} + +tt.descclassname { + background-color: transparent; + border: 0; +} + +tt.xref { + background-color: transparent; + font-weight: bold; + border: 0; +} + +a tt { + background-color: transparent; + font-weight: bold; + border: 0; + color: #CA7900; +} + +a tt:hover { + color: #2491CF; +} + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +.refcount { + color: #060; +} + +dt:target, +.highlight { + background-color: #fbe54e; +} + +dl.class, dl.function { + border-top: 2px solid #888; +} + +dl.method, dl.attribute { + border-top: 1px solid #aaa; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +pre { + line-height: 120%; +} + +pre a { + color: inherit; + text-decoration: underline; +} + +.first { + margin-top: 0 !important; +} + +div.document { + background-color: white; + text-align: left; + background-image: url(contents.png); + background-repeat: repeat-x; +} + +/* +div.documentwrapper { + width: 100%; +} +*/ + +div.clearer { + clear: both; +} + +div.related h3 { + display: none; +} + +div.related ul { + background-image: url(navigation.png); + height: 2em; + list-style: none; + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 0; + padding-left: 10px; +} + +div.related ul li { + margin: 0; + padding: 0; + height: 2em; + float: left; +} + +div.related ul li.right { + float: right; + margin-right: 5px; +} + +div.related ul li a { + margin: 0; + padding: 0 5px 0 5px; + line-height: 1.75em; + color: #EE9816; +} + +div.related ul li a:hover { + color: #3CA8E7; +} + +div.body { + margin: 0; + padding: 0.5em 20px 20px 20px; +} + +div.bodywrapper { + margin: 0 240px 0 0; + border-right: 1px solid #ccc; +} + +div.body a { + text-decoration: underline; +} + +div.sphinxsidebar { + margin: 0; + padding: 0.5em 15px 15px 0; + width: 210px; + float: right; + text-align: left; +/* margin-left: -100%; */ +} + +div.sphinxsidebar h4, div.sphinxsidebar h3 { + margin: 1em 0 0.5em 0; + font-size: 0.9em; + padding: 0.1em 0 0.1em 0.5em; + color: white; + border: 1px solid #86989B; + background-color: #AFC1C4; +} + +div.sphinxsidebar ul { + padding-left: 1.5em; + margin-top: 7px; + list-style: none; + padding: 0; + line-height: 130%; +} + +div.sphinxsidebar ul ul { + list-style: square; + margin-left: 20px; +} + +p { + margin: 0.8em 0 0.5em 0; +} + +p.rubric { + font-weight: bold; +} + +h1 { + margin: 0; + padding: 0.7em 0 0.3em 0; + font-size: 1.5em; + color: #11557C; +} + +h2 { + margin: 1.3em 0 0.2em 0; + font-size: 1.35em; + padding: 0; +} + +h3 { + margin: 1em 0 -0.3em 0; + font-size: 1.2em; +} + +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + color: black!important; +} + +h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor { + display: none; + margin: 0 0 0 0.3em; + padding: 0 0.2em 0 0.2em; + color: #aaa!important; +} + +h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, +h5:hover a.anchor, h6:hover a.anchor { + display: inline; +} + +h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover, +h5 a.anchor:hover, h6 a.anchor:hover { + color: #777; + background-color: #eee; +} + +table { + border-collapse: collapse; + margin: 0 -0.5em 0 -0.5em; +} + +table td, table th { + padding: 0.2em 0.5em 0.2em 0.5em; +} + +div.footer { + background-color: #E3EFF1; + color: #86989B; + padding: 3px 8px 3px 0; + clear: both; + font-size: 0.8em; + text-align: right; +} + +div.footer a { + color: #86989B; + text-decoration: underline; +} + +div.pagination { + margin-top: 2em; + padding-top: 0.5em; + border-top: 1px solid black; + text-align: center; +} + +div.sphinxsidebar ul.toc { + margin: 1em 0 1em 0; + padding: 0 0 0 0.5em; + list-style: none; +} + +div.sphinxsidebar ul.toc li { + margin: 0.5em 0 0.5em 0; + font-size: 0.9em; + line-height: 130%; +} + +div.sphinxsidebar ul.toc li p { + margin: 0; + padding: 0; +} + +div.sphinxsidebar ul.toc ul { + margin: 0.2em 0 0.2em 0; + padding: 0 0 0 1.8em; +} + +div.sphinxsidebar ul.toc ul li { + padding: 0; +} + +div.admonition, div.warning { + font-size: 0.9em; + margin: 1em 0 0 0; + border: 1px solid #86989B; + background-color: #f7f7f7; +} + +div.admonition p, div.warning p { + margin: 0.5em 1em 0.5em 1em; + padding: 0; +} + +div.admonition pre, div.warning pre { + margin: 0.4em 1em 0.4em 1em; +} + +div.admonition p.admonition-title, +div.warning p.admonition-title { + margin: 0; + padding: 0.1em 0 0.1em 0.5em; + color: white; + border-bottom: 1px solid #86989B; + font-weight: bold; + background-color: #AFC1C4; +} + +div.warning { + border: 1px solid #940000; +} + +div.warning p.admonition-title { + background-color: #CF0000; + border-bottom-color: #940000; +} + +div.admonition ul, div.admonition ol, +div.warning ul, div.warning ol { + margin: 0.1em 0.5em 0.5em 3em; + padding: 0; +} + +div.versioninfo { + margin: 1em 0 0 0; + border: 1px solid #ccc; + background-color: #DDEAF0; + padding: 8px; + line-height: 1.3em; + font-size: 0.9em; +} + + +a.headerlink { + color: #c60f0f!important; + font-size: 1em; + margin-left: 6px; + padding: 0 4px 0 4px; + text-decoration: none!important; + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + +a.headerlink:hover { + background-color: #ccc; + color: white!important; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +img.inheritance { + border: 0px +} + +form.pfform { + margin: 10px 0 20px 0; +} + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} +/* from numpy:*/ +td.field-body > blockquote { + margin-top: 0.1em; + margin-bottom: 0.5em; +} +/* spacing in see also definition lists */ +dl.last > dd { + margin-top: 1px; + margin-bottom: 5px; + margin-left: 30px; +} + +.field-list th { + color: rgb(0,102,204); + background: #eee; +} + +table.field-list th { +/* border-left: 1px solid #aaa !important;*/ + padding-left: 5px; + white-space: nowrap; +} + +table.field-list { + border-collapse: separate; + border-spacing: 10px; +} + +th.field-name { +/* border-left: 1px solid #aaa !important;*/ +padding-left: .5em; +padding-right: .5em; +text-align: right; + +} + +.rubric { + color: rgb(0,50,150); + background-color: ; + font-style: italic; +} + +.versionadded { + font-style: italic; + background: #ffe; +} \ No newline at end of file diff --git a/docs/source/_static/logo.png b/docs/source/_static/logo.png new file mode 100644 index 0000000..c3e8dab Binary files /dev/null and b/docs/source/_static/logo.png differ diff --git a/docs/source/_static/zeromq.ico b/docs/source/_static/zeromq.ico new file mode 100644 index 0000000..3f810a1 Binary files /dev/null and b/docs/source/_static/zeromq.ico differ diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html new file mode 100644 index 0000000..a3eebdb --- /dev/null +++ b/docs/source/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + + +{% block rootrellink %} +
  • home| 
    • +
  • search| 
    • +
  • API »
    • +{% endblock %} + + +{% block relbar1 %} + +
    +PyZMQ Documentation +
    +{{ super() }} +{% endblock %} + +{# put the sidebar before the body #} +{% block sidebar1 %}{{ sidebar() }}{% endblock %} +{% block sidebar2 %}{% endblock %} + diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst new file mode 100644 index 0000000..6b51f0c --- /dev/null +++ b/docs/source/api/index.rst @@ -0,0 +1,25 @@ +.. _api-index: + +################### + The PyZMQ API +################### + +:Release: |release| +:Date: |today| + +.. toctree:: + + zmq + zmq.devices + zmq.green + generated/zmq.eventloop.ioloop + generated/zmq.eventloop.zmqstream + zmq.auth + zmq.auth.thread + zmq.auth.ioloop + generated/zmq.log.handlers + generated/zmq.ssh.tunnel + generated/zmq.utils.jsonapi + generated/zmq.utils.monitor + generated/zmq.utils.z85 + diff --git a/docs/source/api/zmq.auth.ioloop.rst b/docs/source/api/zmq.auth.ioloop.rst new file mode 100644 index 0000000..efcb026 --- /dev/null +++ b/docs/source/api/zmq.auth.ioloop.rst @@ -0,0 +1,20 @@ +.. AUTO-GENERATED FILE -- DO NOT EDIT! + +auth.ioloop +=========== + +Module: :mod:`auth.ioloop` +-------------------------- +.. automodule:: zmq.auth.ioloop + +.. currentmodule:: zmq.auth.ioloop + +:class:`IOLoopAuthenticator` +---------------------------- + + +.. autoclass:: IOLoopAuthenticator + :members: + :undoc-members: + :inherited-members: + diff --git a/docs/source/api/zmq.auth.rst b/docs/source/api/zmq.auth.rst new file mode 100644 index 0000000..6ca29d7 --- /dev/null +++ b/docs/source/api/zmq.auth.rst @@ -0,0 +1,30 @@ +auth +==== + +Module: :mod:`auth` +------------------- +.. automodule:: zmq.auth + +.. currentmodule:: zmq.auth + + +:class:`Authenticator` +---------------------- + + +.. autoclass:: Authenticator + :members: + :undoc-members: + :inherited-members: + + +Functions +--------- + + +.. autofunction:: create_certificates + +.. autofunction:: load_certificate + +.. autofunction:: load_certificates + diff --git a/docs/source/api/zmq.auth.thread.rst b/docs/source/api/zmq.auth.thread.rst new file mode 100644 index 0000000..e2c0b1b --- /dev/null +++ b/docs/source/api/zmq.auth.thread.rst @@ -0,0 +1,24 @@ +.. AUTO-GENERATED FILE -- DO NOT EDIT! + +auth.thread +=========== + +Module: :mod:`auth.thread` +-------------------------- +.. automodule:: zmq.auth.thread + +.. currentmodule:: zmq.auth.thread + +Classes +------- + + +:class:`ThreadAuthenticator` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. autoclass:: ThreadAuthenticator + :members: + :undoc-members: + :inherited-members: + diff --git a/docs/source/api/zmq.devices.rst b/docs/source/api/zmq.devices.rst new file mode 100644 index 0000000..690b0b5 --- /dev/null +++ b/docs/source/api/zmq.devices.rst @@ -0,0 +1,86 @@ +devices +======= + +Functions +--------- + +.. autofunction:: zmq.device + +.. autofunction:: zmq.proxy + +Module: :mod:`zmq.devices` +-------------------------- +.. automodule:: zmq.devices + +.. currentmodule:: zmq.devices + + +Base Devices +------------ + +:class:`Device` +*************** + +.. autoclass:: Device + :members: + :exclude-members: context_factory, run, run_device + +:class:`ThreadDevice` +********************* + +.. autoclass:: ThreadDevice + :members: + +:class:`ProcessDevice` +********************** + +.. autoclass:: ProcessDevice + :members: + + +Proxy Devices +------------- + +:class:`Proxy` +******************** + +.. autoclass:: Proxy + :members: bind_mon, connect_mon, setsockopt_mon + +:class:`ThreadProxy` +******************** + +.. autoclass:: ThreadProxy + :members: + +:class:`ProcessProxy` +********************* + +.. autoclass:: ProcessProxy + :members: + + +MonitoredQueue Devices +---------------------- + +.. autofunction:: zmq.devices.monitored_queue + +:class:`MonitoredQueue` +***************************** + +.. autoclass:: MonitoredQueue + :members: + +:class:`ThreadMonitoredQueue` +***************************** + +.. autoclass:: ThreadMonitoredQueue + :members: + +:class:`ProcessMonitoredQueue` +****************************** + +.. autoclass:: ProcessMonitoredQueue + :members: + + diff --git a/docs/source/api/zmq.green.rst b/docs/source/api/zmq.green.rst new file mode 100644 index 0000000..6d0667a --- /dev/null +++ b/docs/source/api/zmq.green.rst @@ -0,0 +1,7 @@ +green +===== + +Module: :mod:`green` +-------------------- + +.. automodule:: zmq.green diff --git a/docs/source/api/zmq.rst b/docs/source/api/zmq.rst new file mode 100644 index 0000000..0a0f7fe --- /dev/null +++ b/docs/source/api/zmq.rst @@ -0,0 +1,140 @@ +zmq +=== + +.. automodule:: zmq + +.. currentmodule:: zmq + +Basic Classes +------------- + +:class:`Context` +**************** + + +.. autoclass:: Context + :members: + :undoc-members: + :inherited-members: + :exclude-members: sockopts, closed + + .. attribute:: closed + + boolean - whether the context has been terminated. + If True, you can no longer use this Context. + + +:class:`Socket` +*************** + + +.. autoclass:: Socket + :members: + :undoc-members: + :inherited-members: + :exclude-members: closed, context, getsockopt_unicode, recv_unicode, setsockopt_unicode, send_unicode + + .. attribute:: closed + + boolean - whether the socket has been closed. + If True, you can no longer use this Socket. + + + +:class:`Frame` +************** + + +.. autoclass:: Frame + :members: + :inherited-members: + + +:class:`MessageTracker` +*********************** + + +.. autoclass:: MessageTracker + :members: + :inherited-members: + + +Polling +------- + +:class:`Poller` +*************** + +.. autoclass:: Poller + :members: + :inherited-members: + + +.. autofunction:: zmq.select + + +Exceptions +---------- + +:class:`ZMQError` +***************** + +.. autoclass:: ZMQError + :members: + :inherited-members: + + +:class:`ZMQVersionError` +***************** + +.. autoclass:: ZMQVersionError + :members: + :inherited-members: + +:class:`Again` +************** + + +.. autoclass:: Again + + +:class:`ContextTerminated` +************************** + + +.. autoclass:: ContextTerminated + + +:class:`NotDone` +**************** + + +.. autoclass:: NotDone + + +:class:`ZMQBindError` +********************* + + +.. autoclass:: ZMQBindError + + + +Functions +--------- + +.. autofunction:: zmq.zmq_version + +.. autofunction:: zmq.pyzmq_version + +.. autofunction:: zmq.zmq_version_info + +.. autofunction:: zmq.pyzmq_version_info + +.. autofunction:: zmq.device + +.. autofunction:: zmq.proxy + +.. autofunction:: zmq.curve_keypair + +.. autofunction:: zmq.get_includes \ No newline at end of file diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..8af0652 --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,446 @@ +.. PyZMQ changelog summary, started by Min Ragan-Kelley, 2011 + +.. _changelog: + +================ +Changes in PyZMQ +================ + +This is a coarse summary of changes in pyzmq versions. For a real changelog, consult the +`git log `_ + +14.3.1 +====== + +Minor bugfixes to pyzmq 14.3: + +- Fixes to building bundled libzmq on OS X < 10.9 +- Fixes to import-failure warnings on Python 3.4 +- Fixes to tests +- Pull upstream fixes to zmq.ssh for ssh multiplexing + +14.3.0 +====== + +- PyZMQ no longer calls :meth:`.Socket.close` or :meth:`.Context.term` during process cleanup. + Changes to garbage collection in Python 3.4 make this impossible to do sensibly. +- :meth:`ZMQStream.close` closes its socket immediately, rather than scheduling a timeout. +- Raise the original ImportError when importing zmq fails. + Should be more informative than `no module cffi...`. + +.. warning:: + + Users of Python 3.4 should not use pyzmq < 14.3, due to changes in garbage collection. + + +14.2.0 +====== + +New Stuff +--------- + +- Raise new ZMQVersionError when a requested method is not supported by the linked libzmq. + For backward compatibility, this subclasses NotImplementedError. + + +Bugs Fixed +---------- + +- Memory leak introduced in pyzmq-14.0 in zero copy. +- OverflowError on 32 bit systems in zero copy. + + +14.1.0 +====== + +Security +-------- + +The headline features for 14.1 are adding better support for libzmq's +security features. + +- When libzmq is bundled as a Python extension (e.g. wheels, eggs), + libsodium is also bundled (excluding Windows), + ensuring that libzmq security is available to users who install from wheels +- New :mod:`zmq.auth`, implementing zeromq's ZAP authentication, + modeled on czmq zauth. + For more information, see the `examples `_. + + +Other New Stuff +--------------- + +- Add PYZMQ_BACKEND for enabling use of backends outside the pyzmq codebase. +- Add :attr:`~.Context.underlying` property and :meth:`~.Context.shadow` + method to Context and Socket, for handing off sockets and contexts. + between pyzmq and other bindings (mainly pyczmq_). +- Add TOS, ROUTER_HANDOVER, and IPC_FILTER constants from libzmq-4.1-dev. +- Add Context option support in the CFFI backend. +- Various small unicode and build fixes, as always. +- :meth:`~.Socket.send_json` and :meth:`~.Socket.recv_json` pass any extra kwargs to ``json.dumps/loads``. + + +.. _pyczmq: https://github.com/zeromq/pyczmq + + +Deprecations +------------ + +- ``Socket.socket_type`` is deprecated, in favor of ``Socket.type``, + which has been available since 2.1. + + +14.0.1 +====== + +Bugfix release + +- Update bundled libzmq to current (4.0.3). +- Fix bug in :meth:`.Context.destroy` with no open sockets. +- Threadsafety fixes in the garbage collector. +- Python 3 fixes in :mod:`zmq.ssh`. + + +14.0.0 +====== + +* Update bundled libzmq to current (4.0.1). +* Backends are now implemented in ``zmq.backend`` instead of ``zmq.core``. + This has no effect on public APIs. +* Various build improvements for Cython and CFFI backends (PyPy compiles at build time). +* Various GIL-related performance improvements - the GIL is no longer touched from a zmq IO thread. +* Adding a constant should now be a bit easier - only zmq/sugar/constant_names should need updating, + all other constant-related files should be automatically updated by ``setup.py constants``. +* add support for latest libzmq-4.0.1 + (includes ZMQ_CURVE security and socket event monitoring). + +New stuff +--------- + +- :meth:`.Socket.monitor` +- :meth:`.Socket.get_monitor_socket` +- :func:`zmq.curve_keypair` +- :mod:`zmq.utils.monitor` +- :mod:`zmq.utils.z85` + + +13.1.0 +====== + +The main new feature is improved tornado 3 compatibility. +PyZMQ ships a 'minitornado' submodule, which contains a small subset of tornado 3.0.1, +in order to get the IOLoop base class. zmq.eventloop.ioloop.IOLoop is now a simple subclass, +and if the system tornado is ≥ 3.0, then the zmq IOLoop is a proper registered subclass +of the tornado one itself, and minitornado is entirely unused. + +13.0.2 +====== + +Bugfix release! + +A few things were broken in 13.0.0, so this is a quick bugfix release. + +* **FIXED** EAGAIN was unconditionally turned into KeyboardInterrupt +* **FIXED** we used totally deprecated ctypes_configure to generate constants in CFFI backend +* **FIXED** memory leak in CFFI backend for PyPy +* **FIXED** typo prevented IPC_PATH_MAX_LEN from ever being defined +* **FIXED** various build fixes - linking with librt, Cython compatibility, etc. + +13.0.1 +====== + +defunct bugfix. We do not speak of this... + +13.0.0 +====== + +PyZMQ now officially targets libzmq-3 (3.2.2), +0MQ ≥ 2.1.4 is still supported for the indefinite future, but 3.x is recommended. +PyZMQ has detached from libzmq versioning, +and will just follow its own regular versioning scheme from now on. +PyZMQ bdists will include whatever is the latest stable libzmq release (3.2.2 for pyzmq-13.0). + +.. note:: + + set/get methods are exposed via get/setattr on all Context, Socket, and Frame classes. + This means that subclasses of these classes that require extra attributes + **must declare these attributes at the class level**. + +Experiments Removed +------------------- + +* The Threadsafe ZMQStream experiment in 2.2.0.1 was deemed inappropriate and not useful, + and has been removed. +* The :mod:`zmq.web` experiment has been removed, + to be developed as a `standalone project `_. + +New Stuff +--------- + +* Support for PyPy via CFFI backend (requires py, ctypes-configure, and cffi). +* Add support for new APIs in libzmq-3 + + - :meth:`.Socket.disconnect` + - :meth:`.Socket.unbind` + - :meth:`.Context.set` + - :meth:`.Context.get` + - :meth:`.Frame.set` + - :meth:`.Frame.get` + - :func:`zmq.proxy` + - :class:`zmq.devices.Proxy` + - Exceptions for common zmq errnos: :class:`zmq.Again`, :class:`zmq.ContextTerminated` + (subclass :class:`ZMQError`, so fully backward-compatible). + + +* Setting and getting :attr:`.Socket.hwm` sets or gets *both* SNDHWM/RCVHWM for libzmq-3. +* Implementation splits core Cython bindings from pure-Python subclasses + with sugar methods (send/recv_multipart). This should facilitate + non-Cython backends and PyPy support [spoiler: it did!]. + + +Bugs Fixed +---------- + +* Unicode fixes in log and monitored queue +* MinGW, ppc, cross-compilation, and HP-UX build fixes +* :mod:`zmq.green` should be complete - devices and tornado eventloop both work + in gevent contexts. + + +2.2.0.1 +======= + +This is a tech-preview release, to try out some new features. +It is expected to be short-lived, as there are likely to be issues to iron out, +particularly with the new pip-install support. + +Experimental New Stuff +---------------------- + +These features are marked 'experimental', which means that their APIs are not set in stone, +and may be removed or changed in incompatible ways in later releases. + + +Threadsafe ZMQStream +******************** + +With the IOLoop inherited from tornado, there is exactly one method that is threadsafe: +:meth:`.IOLoop.add_callback`. With this release, we are trying an experimental option +to pass all IOLoop calls via this method, so that ZMQStreams can be used from one thread +while the IOLoop runs in another. To try out a threadsafe stream: + +.. sourcecode:: python + + stream = ZMQStream(socket, threadsafe=True) + + +pip install pyzmq +***************** + +PyZMQ should now be pip installable, even on systems without libzmq. +In these cases, when pyzmq fails to find an appropriate libzmq to link against, +it will try to build libzmq as a Python extension. +This work is derived from `pyzmq_static `_. + +To this end, PyZMQ source distributions include the sources for libzmq (2.2.0) and libuuid (2.21), +both used under the LGPL. + + +zmq.green +********* + +The excellent `gevent_zeromq `_ socket +subclass which provides `gevent `_ compatibility has been merged as +:mod:`zmq.green`. + +.. seealso:: + + :ref:`zmq_green` + + +Bugs Fixed +---------- + +* TIMEO sockopts are properly included for libzmq-2.2.0 +* avoid garbage collection of sockets after fork (would cause ``assert (mailbox.cpp:79)``). + + +2.2.0 +===== + +Some effort has gone into refining the pyzmq API in this release to make it a model for +other language bindings. This is principally made in a few renames of objects and methods, +all of which leave the old name for backwards compatibility. + +.. note:: + + As of this release, all code outside ``zmq.core`` is BSD licensed (where + possible), to allow more permissive use of less-critical code and utilities. + +Name Changes +------------ + +* The :class:`~.Message` class has been renamed to :class:`~.Frame`, to better match other + zmq bindings. The old Message name remains for backwards-compatibility. Wherever pyzmq + docs say "Message", they should refer to a complete zmq atom of communication (one or + more Frames, connected by ZMQ_SNDMORE). Please report any remaining instances of + Message==MessagePart with an Issue (or better yet a Pull Request). + +* All ``foo_unicode`` methods are now called ``foo_string`` (``_unicode`` remains for + backwards compatibility). This is not only for cross-language consistency, but it makes + more sense in Python 3, where native strings are unicode, and the ``_unicode`` suffix + was wedded too much to Python 2. + +Other Changes and Removals +-------------------------- + +* ``prefix`` removed as an unused keyword argument from :meth:`~.Socket.send_multipart`. + +* ZMQStream :meth:`~.ZMQStream.send` default has been changed to `copy=True`, so it matches + Socket :meth:`~.Socket.send`. + +* ZMQStream :meth:`~.ZMQStream.on_err` is deprecated, because it never did anything. + +* Python 2.5 compatibility has been dropped, and some code has been cleaned up to reflect + no-longer-needed hacks. + +* Some Cython files in :mod:`zmq.core` have been split, to reduce the amount of + Cython-compiled code. Much of the body of these files were pure Python, and thus did + not benefit from the increased compile time. This change also aims to ease maintaining + feature parity in other projects, such as + `pyzmq-ctypes `_. + + +New Stuff +--------- + +* :class:`~.Context` objects can now set default options when they create a socket. These + are set and accessed as attributes to the context. Socket options that do not apply to a + socket (e.g. SUBSCRIBE on non-SUB sockets) will simply be ignored. + +* :meth:`~.ZMQStream.on_recv_stream` has been added, which adds the stream itself as a + second argument to the callback, making it easier to use a single callback on multiple + streams. + +* A :attr:`~Frame.more` boolean attribute has been added to the :class:`~.Frame` (née + Message) class, so that frames can be identified as terminal without extra queires of + :attr:`~.Socket.rcvmore`. + + +Experimental New Stuff +---------------------- + +These features are marked 'experimental', which means that their APIs are not +set in stone, and may be removed or changed in incompatible ways in later releases. + +* :mod:`zmq.web` added for load-balancing requests in a tornado webapp with zeromq. + + +2.1.11 +====== + +* remove support for LABEL prefixes. A major feature of libzmq-3.0, the LABEL + prefix, has been removed from libzmq, prior to the first stable libzmq 3.x release. + + * The prefix argument to :meth:`~.Socket.send_multipart` remains, but it continue to behave in + exactly the same way as it always has on 2.1.x, simply prepending message parts. + + * :meth:`~.Socket.recv_multipart` will always return a list, because prefixes are once + again indistinguishable from regular message parts. + +* add :meth:`.Socket.poll` method, for simple polling of events on a single socket. + +* no longer require monkeypatching tornado IOLoop. The :class:`.ioloop.ZMQPoller` class + is a poller implementation that matches tornado's expectations, and pyzmq sockets can + be used with any tornado application just by specifying the use of this poller. The + pyzmq IOLoop implementation now only trivially differs from tornado's. + + It is still recommended to use :func:`.ioloop.install`, which sets *both* the zmq and + tornado global IOLoop instances to the same object, but it is no longer necessary. + + .. warning:: + + The most important part of this change is that the ``IOLoop.READ/WRITE/ERROR`` + constants now match tornado's, rather than being mapped directly to the zmq + ``POLLIN/OUT/ERR``. So applications that used the low-level :meth:`IOLoop.add_handler` + code with ``POLLIN/OUT/ERR`` directly (used to work, but was incorrect), rather than + using the IOLoop class constants will no longer work. Fixing these to use the IOLoop + constants should be insensitive to the actual value of the constants. + +2.1.10 +====== + +* Add support for libzmq-3.0 LABEL prefixes: + + .. warning:: + + This feature has been removed from libzmq, and thus removed from future pyzmq + as well. + + * send a message with label-prefix with: + + .. sourcecode:: python + + send_multipart([b'msg', b'parts'], prefix=[b'label', b'prefix']) + + * :meth:`recv_multipart` returns a tuple of ``(prefix,msg)`` if a label prefix is detected + * ZMQStreams and devices also respect the LABEL prefix + +* add czmq-style close&term as :meth:`ctx.destroy`, so that :meth:`ctx.term` + remains threadsafe and 1:1 with libzmq. +* :meth:`Socket.close` takes optional linger option, for setting linger prior + to closing. +* add :func:`~zmq.core.version.zmq_version_info` and + :func:`~zmq.core.version.pyzmq_version_info` for getting libzmq and pyzmq versions as + tuples of numbers. This helps with the fact that version string comparison breaks down + once versions get into double-digits. +* ioloop changes merged from upstream `Tornado `_ 2.1 + +2.1.9 +===== + +* added zmq.ssh tools for tunneling socket connections, copied from IPython +* Expanded sockopt support to cover changes in libzmq-4.0 dev. +* Fixed an issue that prevented :exc:`KeyboardInterrupts` from being catchable. +* Added attribute-access for set/getsockopt. Setting/Getting attributes of :class:`Sockets` + with the names of socket options is mapped to calls of set/getsockopt. + +.. sourcecode:: python + + s.hwm = 10 + s.identity = b'whoda' + s.linger + # -1 + +* Terminating a :class:`~Context` closes the sockets it created, matching the behavior in + `czmq `_. +* :class:`ThreadDevices` use :meth:`Context.instance` to create sockets, so they can use + inproc connections to sockets in other threads. +* fixed units error on :func:`zmq.select`, where the poll timeout was 1000 times longer + than expected. +* Add missing ``DEALER/ROUTER`` socket type names (currently aliases, to be replacements for ``XREP/XREQ``). +* base libzmq dependency raised to 2.1.4 (first stable release) from 2.1.0. + + +2.1.7.1 +======= + +* bdist for 64b Windows only. This fixed a type mismatch on the ``ZMQ_FD`` sockopt + that only affected that platform. + + +2.1.7 +===== + +* Added experimental support for libzmq-3.0 API +* Add :func:`zmq.eventloop.ioloop.install` for using pyzmq's IOLoop in a tornado + application. + + +2.1.4 +===== + +* First version with binary distribution support +* Added :meth:`~Context.instance()` method for using a single Context throughout an application + without passing references around. + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..0761991 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# +# PyZMQ documentation build configuration file, created by +# sphinx-quickstart on Sat Feb 20 23:31:19 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os +import string + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0,os.path.abspath('../sphinxext')) + +# patch autodoc to work with Cython Sources +import sphinx_cython + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'ipython_console_highlighting', + 'numpydoc', + ] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'PyZMQ' +copyright = u"""2013, Brian E. Granger & Min Ragan-Kelley. +ØMQ logo © iMatix Corportation, used under the Creative Commons Attribution-Share Alike 3.0 License. +Python logo ™ of the Python Software Foundation, used by Min RK with permission from the Foundation""" + +intersphinx_mapping = {'python': ('http://docs.python.org/', None)} +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +import zmq + +# The short X.Y version. +version = zmq.__version__.split('-')[0] +# The full version, including alpha/beta/rc tags. +release = zmq.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = 'zeromq.ico' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'PyZMQdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'PyZMQ.tex', u'PyZMQ Documentation', + u'Brian E. Granger \\and Min Ragan-Kelley', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/docs/source/devices.rst b/docs/source/devices.rst new file mode 100644 index 0000000..4ef70ce --- /dev/null +++ b/docs/source/devices.rst @@ -0,0 +1,82 @@ +.. PyZMQ devices doc, by Min Ragan-Kelley, 2011 + +.. _devices: + +Devices in PyZMQ +================ + +.. seealso:: + + ØMQ Guide `Device coverage `_. + +ØMQ has a notion of Devices - simple programs that manage a send-recv pattern for +connecting two or more sockets. Being full programs, devices include a ``while(True)`` +loop and thus block execution permanently once invoked. We have provided in the +:mod:`devices` subpackage some facilities for running these devices in the background, as +well as a custom three-socket MonitoredQueue_ device. + + +BackgroundDevices +----------------- + +It seems fairly rare that in a Python program one would actually want to create a zmq +device via :func:`.device` in the main thread, since such a call would block execution +forever. The most likely model for launching devices is in background threads or +processes. We have provided classes for launching devices in a background thread with +:class:`.ThreadDevice` and via multiprocessing with :class:`.ProcessDevice`. For +threadsafety and running across processes, these methods do not take Socket objects as +arguments, but rather socket types, and then the socket creation and configuration happens +via the BackgroundDevice's :meth:`foo_in` proxy methods. For each configuration method +(bind/connect/setsockopt), there are proxy methods for calling those methods on the Socket +objects created in the background thread or process, prefixed with 'in\_' or 'out\_', +corresponding to the `in_socket` and `out_socket`:: + + from zmq.devices import ProcessDevice + + pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) + pd.bind_in('tcp://*:12345') + pd.connect_out('tcp://127.0.0.1:12543') + pd.setsockopt_in(zmq.IDENTITY, 'ROUTER') + pd.setsockopt_out(zmq.IDENTITY, 'DEALER') + pd.start() + # it will now be running in a background process + +MonitoredQueue +-------------- + +One of ØMQ's builtin devices is the ``QUEUE``. This is a symmetric two-socket device that +fully supports passing messages in either direction via any pattern. We saw a logical +extension of the ``QUEUE`` as one that behaves in the same way with respect to the in/out +sockets, but also sends every message in either direction *also* on a third `monitor` +socket. For performance reasons, this :func:`.monitored_queue` function is written in +Cython, so the loop does not involve Python, and should have the same performance as the +basic ``QUEUE`` device. + +One shortcoming of the ``QUEUE`` device is that it does not support having ``ROUTER`` +sockets as both input and output. This is because ``ROUTER`` sockets, when they receive a +message, prepend the ``IDENTITY`` of the socket that sent the message (for use in routing +the reply). The result is that the output socket will always try to route the incoming +message back to the original sender, which is presumably not the intended pattern. In +order for the queue to support a ROUTER-ROUTER connection, it must swap the first two parts +of the message in order to get the right message out the other side. + +To invoke a monitored queue is similar to invoking a regular ØMQ device:: + + from zmq.devices import monitored_queue + ins = ctx.socket(zmq.ROUTER) + outs = ctx.socket(zmq.DEALER) + mons = ctx.socket(zmq.PUB) + configure_sockets(ins,outs,mons) + monitored_queue(ins, outs, mons, in_prefix='in', out_prefix='out') + +The `in_prefix` and `out_prefix` default to 'in' and 'out' respectively, and a PUB socket +is most logical for the monitor socket, since it will never receive messages, and the +in/out prefix is well suited to the PUB/SUB topic subscription model. All messages sent on +`mons` will be multipart, the first part being the prefix corresponding to the socket that +received the message. + +Or for launching an MQ in the background, there are :class:`.ThreadMonitoredQueue` and +:class:`.ProcessMonitoredQueue`, which function just like the base +BackgroundDevice objects, but add :meth:`foo_mon` methods for configuring the monitor socket. + + diff --git a/docs/source/eventloop.rst b/docs/source/eventloop.rst new file mode 100644 index 0000000..3126b8b --- /dev/null +++ b/docs/source/eventloop.rst @@ -0,0 +1,197 @@ +.. PyZMQ eventloop doc, by Min Ragan-Kelley, 2011 + +.. _eventloop: + +==================== +Eventloops and PyZMQ +==================== + +Tornado IOLoop +============== + +Facebook's `Tornado`_ includes an eventloop for handing poll events on filedescriptors and +native sockets. We have included a small part of Tornado (specifically its +:mod:`.ioloop`), and adapted its :class:`IOStream` class into :class:`.ZMQStream` for +handling poll events on ØMQ sockets. A ZMQStream object works much like a Socket object, +but instead of calling :meth:`~.Socket.recv` directly, you register a callback with +:meth:`~.ZMQStream.on_recv`. Callbacks can also be registered for send events +with :meth:`~.ZMQStream.on_send`. + + +:func:`install()` +----------------- + +With PyZMQ's ioloop, you can use zmq sockets in any tornado application. You must first +install PyZMQ's :class:`.IOLoop`, with the :func:`.ioloop.install` function: + +.. sourcecode:: python + + from zmq.eventloop import ioloop + ioloop.install() + +This sets the global instance of :class:`tornado.ioloop.IOLoop` with the global instance of +our IOLoop class. The reason this must happen is that tornado objects avoid having to pass +the active IOLoop instance around by having a staticmethod :meth:`.IOLoop.instance`, which +always returns the active instance. If PyZMQ's IOLoop is installed after the first call to +:meth:`.IOLoop.instance()` (called in almost every tornado object constructor), then it will +raise an :exc:`AssertionError`, because the global IOLoop instance has already been +created, and proceeding would result in not all objects being associated with the right +IOLoop. + +It is possible to use PyZMQ sockets with tornado *without* calling :func:`.ioloop.install`, +but it is less convenient. First, you must instruct the tornado IOLoop to use the zmq poller: + +.. sourcecode:: python + + from zmq.eventloop.ioloop import ZMQIOLoop + + loop = ZMQIOLoop() + +Then, when you instantiate tornado and ZMQStream objects, you must pass the `io_loop` +argument to ensure that they use this loop, instead of the global instance. + +This is especially useful for writing tests, such as this: + +.. sourcecode:: python + + from tornado.testing import AsyncTestCase + from zmq.eventloop.ioloop import ZMQIOLoop + from zmq.eventloop.zmqstream import ZMQStream + + class TestZMQBridge(AsyncTestCase): + + # Use a ZMQ-compatible I/O loop so that we can use `ZMQStream`. + def get_new_ioloop(self): + return ZMQIOLoop() + +You can also manually install this IOLoop as the global tornado instance, with: + +.. sourcecode:: python + + from zmq.eventloop.ioloop import ZMQIOLoop + loop = ZMQIOLoop() + loop.install() + +but it will **NOT** be the global *pyzmq* IOLoop instance, so it must still be +passed to your ZMQStream constructors. + + +:meth:`send` +------------ + +ZMQStream objects do have :meth:`~.ZMQStream.send` and :meth:`~.ZMQStream.send_multipart` +methods, which behaves the same way as :meth:`.Socket.send`, but instead of sending right +away, the :class:`.IOLoop` will wait until socket is able to send (for instance if ``HWM`` +is met, or a ``REQ/REP`` pattern prohibits sending at a certain point). Messages sent via +send will also be passed to the callback registered with :meth:`~.ZMQStream.on_send` after +sending. + +:meth:`on_recv` +--------------- + +:meth:`.ZMQStream.on_recv` is the primary method for using a ZMQStream. It registers a +callback to fire with messages as they are received, which will *always* be multipart, +even if its length is 1. You can easily use this to build things like an echo socket: + +.. sourcecode:: python + + s = ctx.socket(zmq.REP) + s.bind('tcp://localhost:12345') + stream = ZMQStream(s) + def echo(msg): + stream.send_multipart(msg) + stream.on_recv(echo) + ioloop.IOLoop.instance().start() + +on_recv can also take a `copy` flag, just like :meth:`.Socket.recv`. If `copy=False`, then +callbacks registered with on_recv will receive tracked :class:`.Frame` objects instead of +bytes. + +.. note:: + + A callback must be registered using either :meth:`.ZMQStream.on_recv` or + :meth:`.ZMQStream.on_recv_stream` before any data will be received on the + underlying socket. This allows you to temporarily pause processing on a + socket by setting both callbacks to None. Processing can later be resumed + by restoring either callback. + + +:meth:`on_recv_stream` +---------------------- + +:meth:`.ZMQStream.on_recv_stream` is just like on_recv above, but the callback will be +passed both the message and the stream, rather than just the message. This is meant to make +it easier to use a single callback with multiple streams. + +.. sourcecode:: python + + s1 = ctx.socket(zmq.REP) + s1.bind('tcp://localhost:12345') + stream1 = ZMQStream(s1) + + s2 = ctx.socket(zmq.REP) + s2.bind('tcp://localhost:54321') + stream2 = ZMQStream(s2) + + def echo(stream, msg): + stream.send_multipart(msg) + + stream1.on_recv_stream(echo) + stream2.on_recv_stream(echo) + + ioloop.IOLoop.instance().start() + + +:meth:`flush` +------------- + +Sometimes with an eventloop, there can be multiple events ready on a single iteration of +the loop. The :meth:`~.ZMQStream.flush` method allows developers to pull messages off of +the queue to enforce some priority over the event loop ordering. flush pulls any pending +events off of the queue. You can specify to flush only recv events, only send events, or +any events, and you can specify a limit for how many events to flush in order to prevent +starvation. + +.. _Tornado: https://github.com/facebook/tornado + +.. _zmq_green: + +PyZMQ and gevent +================ + +PyZMQ ≥ 2.2.0.1 ships with a `gevent `_ compatible API as :mod:`zmq.green`. +To use it, simply: + +.. sourcecode:: python + + import zmq.green as zmq + +Then write your code as normal. + +Socket.send/recv and zmq.Poller are gevent-aware. + +In PyZMQ ≥ 2.2.0.2, green.device and green.eventloop should be gevent-friendly as well. + +.. note:: + + The green device does *not* release the GIL, unlike the true device in zmq.core. + +zmq.green.eventloop includes minimally patched IOLoop/ZMQStream in order to use the gevent-enabled Poller, +so you should be able to use the ZMQStream interface in gevent apps as well, +though using two eventloops simultaneously (tornado + gevent) is not recommended. + +.. warning:: + + There is a `known issue `_ in gevent ≤ 1.0 or libevent, + which can cause zeromq socket events to be missed. + PyZMQ works around this by adding a timeout so it will not wait forever for gevent to notice events. + The only known solution for this is to use gevent ≥ 1.0, which is currently at 1.0b3, + and does not exhibit this behavior. + +.. seealso:: + + zmq.green examples `on GitHub `_. + +:mod:`zmq.green` is simply `gevent_zeromq `_, +merged into the pyzmq project. + diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..f32c90d --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,95 @@ +.. PyZMQ documentation master file, created by + sphinx-quickstart on Sat Feb 20 23:31:19 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +PyZMQ Documentation +=================== + +:Release: |release| +:Date: |today| + + +PyZMQ is the Python bindings for ØMQ_. This +documentation currently contains notes on some important aspects of developing PyZMQ and +an overview of what the ØMQ API looks like in Python. For information on how to use +ØMQ in general, see the many examples in the excellent `ØMQ Guide`_, all of which +have a version in Python. + +PyZMQ works with Python 3 (≥ 3.2), and Python 2 (≥ 2.6), with no transformations or 2to3, +as well as PyPy (at least 2.0 beta), thanks to a new CFFI backend. + +Please don't hesitate to report pyzmq-specific issues to our tracker_ on GitHub. +General questions about ØMQ are better sent to the ØMQ `mailing list`_ or `IRC Channel`_. + +:ref:`Summary of Changes in PyZMQ ` + + +Supported LibZMQ +================ + +PyZMQ aims to support all stable ( ≥2.1.4, ≥ 3.2.2, ≥ 4.0.1 ) and active development ( ≥ 4.1.0 ) +versions of libzmq. Building the same pyzmq against various versions of libzmq is supported, +but only the functionality of the linked libzmq will be available. + +.. note:: + + libzmq 3.0-3.1 are not, and will never be supported. + There never was a stable release of either. + + +Binary distributions (eggs, MSIs, and wheels on `PyPI `__ +or `GitHub `__) of PyZMQ ship with +the stable version of libzmq at the time of release, built with default configuration. +For pyzmq-|release|, this is 4.0.3. + +Using PyZMQ +=========== + +.. toctree:: + :maxdepth: 2 + + api/index.rst + morethanbindings.rst + serialization.rst + devices.rst + eventloop.rst + logging.rst + ssh.rst + + +Notes from developing PyZMQ +=========================== + +.. toctree:: + :maxdepth: 2 + + pyversions.rst + unicode.rst + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +Links +===== + +* ØMQ_ Home +* The `ØMQ Guide`_ +* `PyZMQ Installation`_ notes on the ZeroMQ website +* PyZMQ on GitHub_ +* Issue Tracker_ + +.. _ØMQ: http://www.zeromq.org +.. _ØMQ Guide: http://zguide.zeromq.org +.. _mailing list: http://www.zeromq.org/docs:mailing-lists +.. _IRC Channel: http://www.zeromq.org/chatroom +.. _Cython: http://cython.org/ +.. _GitHub: https://www.github.com/zeromq/pyzmq +.. _ØMQ Manual: http://www.zeromq.org/intro:read-the-manual +.. _PyZMQ Installation: http://www.zeromq.org/bindings:python +.. _tracker: https://www.github.com/zeromq/pyzmq/issues + diff --git a/docs/source/logging.rst b/docs/source/logging.rst new file mode 100644 index 0000000..ea16da3 --- /dev/null +++ b/docs/source/logging.rst @@ -0,0 +1,84 @@ +.. PyZMQ logging doc, by Min Ragan-Kelley, 2011 + +.. _logging: + +Asynchronous Logging via PyZMQ +============================== + +.. seealso:: + + * The ØMQ guide `coverage `_ of PUB/SUB + messaging + * Python logging module `documentation `_ + +Python provides extensible logging facilities through its :py:mod:`logging` module. This +module allows for easily extensible logging functionality through the use of +:py:class:`~logging.Handler` objects. The most obvious case for hooking up pyzmq to +logging would be to broadcast log messages over a PUB socket, so we have provides a +:class:`.PUBHandler` class for doing just that. + +PUB/SUB and Topics +------------------ + +The ØMQ PUB/SUB pattern consists of a PUB socket broadcasting messages, and a collection +of SUB sockets that receive those messages. Each PUB message is a multipart-message, where +the first part is interpreted as a topic. SUB sockets can subscribe to topics by setting +their ``SUBSCRIBE`` sockopt, e.g.:: + + sub = ctx.socket(zmq.SUB) + sub.setsockopt(zmq.SUBSCRIBE, 'topic1') + sub.setsockopt(zmq.SUBSCRIBE, 'topic2') + +When subscribed, the SUB socket will only receive messages where the first part *starts +with* one of the topics set via ``SUBSCRIBE``. The default behavior is to exclude all +messages, and subscribing to the empty string '' will receive all messages. + +PUBHandler +---------- + +The :class:`.PUBHandler` object is created for allowing the python logging to be emitted +on a PUB socket. The main difference between a PUBHandler and a regular logging Handler is +the inclusion of topics. For the most basic logging, you can simply create a PUBHandler +with an interface or a configured PUB socket, and just let it go:: + + pub = context.socket(zmq.PUB) + pub.bind('tcp://*:12345') + handler = PUBHandler(pub) + logger = logging.getLogger() + logger.addHandler(handler) + +At this point, all messages logged with the default logger will be broadcast on the pub +socket. + +the PUBHandler does work with topics, and the handler has an attribute ``root_topic``:: + + handler.root_topic = 'myprogram' + +Python loggers also have loglevels. The base topic of messages emitted by the PUBHandler +will be of the form: ``.``, e.g. 'myprogram.INFO' or +'whatever.ERROR'. This way, subscribers can easily subscribe to subsets of the logging +messages. Log messages are always two-part, where the first part is the topic tree, and +the second part is the actual log message. + + >>> logger.info('hello there') + >>> print sub.recv_multipart() + ['myprogram.INFO', 'hello there'] + +Subtopics +********* + +You can also add to the topic tree below the loglevel on an individual message basis. +Assuming your logger is connected to a PUBHandler, you can add as many additional topics +on the front of the message, which will be added always after the loglevel. A special +delimiter defined at ``zmq.log.handlers.TOPIC_DELIM`` is scanned by the PUBHandler, so if +you pass your own subtopics prior to that symbol, they will be stripped from the message +and added to the topic tree:: + + >>> log_msg = "hello there" + >>> subtopic = "sub.topic" + >>> msg = zmq.log.handlers.TOPIC_DELIM.join([subtopic, log_msg]) + >>> logger.warn(msg) + >>> print sub.recv_multipart() + ['myprogram.WARN.sub.topic', 'hello there'] + + diff --git a/docs/source/morethanbindings.rst b/docs/source/morethanbindings.rst new file mode 100644 index 0000000..1bff8f0 --- /dev/null +++ b/docs/source/morethanbindings.rst @@ -0,0 +1,166 @@ +.. PyZMQ Bindings doc, by Min Ragan-Kelley, 2011 + +.. _bindings: + +More Than Just Bindings +======================= + +PyZMQ is ostensibly the Python bindings for `ØMQ`_, but the project, following +Python's 'batteries included' philosophy, provides more than just Python methods and +objects for calling into the ØMQ C++ library. + + + +The Core as Bindings +-------------------- + +PyZMQ is currently broken up into four subpackages. First, is the Core. :mod:`zmq.core` +contains the actual bindings for ZeroMQ, and no extended functionality beyond the very +basic. The core modules are split, such that each basic ZeroMQ object (or function, if no +object is associated) is a separate module, e.g. :mod:`zmq.core.context` contains the +:class:`.Context` object, :mod:`zmq.core.poll` contains a :class:`.Poller` object, as well +as the :func:`.select` function, etc. ZMQ constants are, for convenience, all kept +together in :mod:`zmq.core.constants`. + +There are two reasons for breaking the core into submodules: *recompilation* and +*derivative projects*. The monolithic PyZMQ became quite tedious to have to recompile +everything for a small change to a single object. With separate files, that's no longer +necessary. The second reason has to do with Cython. PyZMQ is written in Cython, a tool for +efficiently writing C-extensions for Python. By separating out our objects into individual +`pyx` files, each with their declarations in a `pxd` header, other projects can write +extensions in Cython and call directly to ZeroMQ at the C-level without the penalty of +going through our Python objects. + +Thread Safety +------------- + +In ØMQ, Contexts are threadsafe objects, but Sockets are **not**. It is safe to use a +single Context (e.g. via :meth:`zmq.Context.instance`) in your entire multithreaded +application, but you should create sockets on a per-thread basis. If you share sockets +across threads, you are likely to encounter uncatchable c-level crashes of your +application unless you use judicious application of :py:class:`threading.Lock`, but this +approach is not recommended. + +.. seealso:: + + ZeroMQ API note on threadsafety on `2.2 `_ + or `3.2 `_ + + +Socket Options as Attributes +---------------------------- + +.. versionadded:: 2.1.9 + +In 0MQ, socket options are set/retrieved with the :meth:`set/getsockopt` methods. With the +class-based approach in pyzmq, it would be logical to perform these operations with +simple attribute access, and this has been added in pyzmq 2.1.9. Simply assign to or +request a Socket attribute with the (case-insensitive) name of a sockopt, and it should +behave just as you would expect: + +.. sourcecode:: python + + s = ctx.socket(zmq.DEALER) + s.identity = b'dealer' + s.hwm = 10 + s.events + # 0 + s.fd + # 16 + + +Default Options on the Context +****************************** + +.. versionadded:: 2.1.11 + +Just like setting socket options as attributes on Sockets, you can do the same on Contexts. +This affects the default options of any *new* sockets created after the assignment. + +.. sourcecode:: python + + ctx = zmq.Context() + ctx.linger = 0 + rep = ctx.socket(zmq.REP) + req = ctx.socket(zmq.REQ) + +Socket options that do not apply to a socket (e.g. SUBSCRIBE on non-SUB sockets) will +simply be ignored. + + +Core Extensions +--------------- + +We have extended the core functionality in two ways that appear inside the :mod:`core` +bindings, and are not general ØMQ features. + +Builtin Serialization +********************* + +First, we added common serialization with the builtin :py:mod:`json` and :py:mod:`pickle` +as first-class methods to the :class:`Socket` class. A socket has the methods +:meth:`~.Socket.send_json` and :meth:`~.Socket.send_pyobj`, which correspond to sending an +object over the wire after serializing with :mod:`json` and :mod:`pickle` respectively, +and any object sent via those methods can be reconstructed with the +:meth:`~.Socket.recv_json` and :meth:`~.Socket.recv_pyobj` methods. Unicode strings are +other objects that are not unambiguously sendable over the wire, so we include +:meth:`~.Socket.send_string` and :meth:`~.Socket.recv_string` that simply send bytes +after encoding the message ('utf-8' is the default). + +.. seealso:: + + * :ref:`Further information ` on serialization in pyzmq. + + * :ref:`Our Unicode discussion ` for more information on the trials and + tribulations of working with Unicode in a C extension while supporting Python 2 and 3. + + +MessageTracker +************** + +The second extension of basic ØMQ functionality is the :class:`MessageTracker`. The +MessageTracker is an object used to track when the underlying ZeroMQ is done with a +message buffer. One of the main use cases for ØMQ in Python is the ability to perform +non-copying sends. Thanks to Python's buffer interface, many objects (including NumPy +arrays) provide the buffer interface, and are thus directly sendable. However, as with any +asynchronous non-copying messaging system like ØMQ or MPI, it can be important to know +when the message has actually been sent, so it is safe again to edit the buffer without +worry of corrupting the message. This is what the MessageTracker is for. + +The MessageTracker is a simple object, but there is a penalty to its use. Since by its +very nature, the MessageTracker must involve threadsafe communication (specifically a +builtin :py:class:`~Queue.Queue` object), instantiating a MessageTracker takes a modest +amount of time (10s of µs), so in situations instantiating many small messages, this can +actually dominate performance. As a result, tracking is optional, via the ``track`` flag, +which is optionally passed, always defaulting to ``False``, in each of the three places +where a Frame object (the pyzmq object for wrapping a segment of a message) is +instantiated: The :class:`.Frame` constructor, and non-copying sends and receives. + +A MessageTracker is very simple, and has just one method and one attribute. The property +:attr:`MessageTracker.done` will be ``True`` when the Frame(s) being tracked are no +longer in use by ØMQ, and :meth:`.MessageTracker.wait` will block, waiting for the +Frame(s) to be released. + +.. Note:: + + A Frame cannot be tracked after it has been instantiated without tracking. If a + Frame is to even have the *option* of tracking, it must be constructed with + ``track=True``. + + +Extensions +---------- + +So far, PyZMQ includes four extensions to core ØMQ that we found basic enough to be +included in PyZMQ itself: + +* :ref:`zmq.log ` : Logging handlers for hooking Python logging up to the + network +* :ref:`zmq.devices ` : Custom devices and objects for running devices in the + background +* :ref:`zmq.eventloop ` : The `Tornado`_ event loop, adapted for use + with ØMQ sockets. +* :ref:`zmq.ssh ` : Simple tools for tunneling zeromq connections via ssh. + +.. _ØMQ: http://www.zeromq.org +.. _Tornado: https://github.com/facebook/tornado diff --git a/docs/source/pyversions.rst b/docs/source/pyversions.rst new file mode 100644 index 0000000..8a509dc --- /dev/null +++ b/docs/source/pyversions.rst @@ -0,0 +1,195 @@ +.. PyZMQ Version compatibility doc, by Min Ragan-Kelley, 2010 + +.. _pyversions: + +PyZMQ, Python2.5, and Python3 +============================= + +PyZMQ is a fairly light, low-level library, so supporting as many versions +as is reasonable is our goal. Currently, we support at least Python 2.5-3.1. +Making the changes to the codebase required a few tricks, which are documented here +for future reference, either by us or by other developers looking to support several +versions of Python. + +.. Note:: + + It is far simpler to support 2.6-3.x than to include 2.5. Many of the significant + syntax changes have been backported to 2.6, so just writing new-style code would work + in many cases. I will try to note these points as they come up. + + +pyversion_compat.h +------------------ + +Many functions we use, primarily involved in converting between C-buffers and Python +objects, are not available on all supported versions of Python. In order to resolve +missing symbols, we added a header :file:`utils/pyversion_compat.h` that defines missing +symbols with macros. Some of these macros alias new names to old functions (e.g. +``PyBytes_AsString``), so that we can call new-style functions on older versions, and some +simply define the function as an empty exception raiser. The important thing is that the +symbols are defined to prevent compiler warnings and linking errors. Everywhere we use +C-API functions that may not be available in a supported version, at the top of the file +is the code: + +.. sourcecode:: guess + + cdef extern from "pyversion_compat.h": + pass + +This ensures that the symbols are defined in the Cython generated C-code. Higher level +switching logic exists in the code itself, to prevent actually calling unavailable +functions, but the symbols must still be defined. + +Bytes and Strings +----------------- + +.. Note:: + + If you are using Python >= 2.6, to prepare your PyZMQ code for Python3 you should use + the ``b'message'`` syntax to ensure all your string literal messages will still be + :class:`bytes` after you make the upgrade. + +The most cumbersome part of PyZMQ compatibility from a user's perspective is the fact +that, since ØMQ uses C-strings, and would like to do so without copying, we must use the +Py3k :class:`bytes` object, which is backported to 2.6. In order to do this in a +Python-version independent way, we added a small utility that unambiguously defines the +string types: :class:`bytes`, :class:`unicode`, :obj:`basestring`. This is important, +because :class:`str` means different things on 2.x and 3.x, and :class:`bytes` is +undefined on 2.5, and both :class:`unicode` and :obj:`basestring` are undefined on 3.x. +All typechecking in PyZMQ is done against these types: + +================= ================= ==================== +Explicit Type 2.x 3.x +================= ================= ==================== +:obj:`bytes` :obj:`str` :obj:`bytes` +:obj:`unicode` :obj:`unicode` :obj:`str` +:obj:`basestring` :obj:`basestring` :obj:`(str, bytes)` +================= ================= ==================== + +.. Note:: + + 2.5 specific + + Where we really noticed the issue of :class:`bytes` vs :obj:`strings` coming up for + users was in updating the tests to run on every version. Since the ``b'bytes + literal'`` syntax was not backported to 2.5, we must call ``"message".encode()`` for + *every* string in the test suite. + +.. seealso:: :ref:`Unicode discussion ` for more information on strings/bytes. + +``PyBytes_*`` +************* + +The standard C-API function for turning a C-string into a Python string was a set of +functions with the prefix ``PyString_*``. However, with the Unicode changes made in +Python3, this was broken into ``PyBytes_*`` for bytes objects and ``PyUnicode_*`` for +unicode objects. We changed all our ``PyString_*`` code to ``PyBytes_*``, which was +backported to 2.6. + + +.. Note:: + + 2.5 Specific: + + Since Python 2.5 doesn't support the ``PyBytes_*`` functions, we had to alias them to + the ``PyString_*`` methods in utils/pyversion_compat.h. + + .. sourcecode:: c++ + + #define PyBytes_FromStringAndSize PyString_FromStringAndSize + #define PyBytes_FromString PyString_FromString + #define PyBytes_AsString PyString_AsString + #define PyBytes_Size PyString_Size + +Buffers +------- + +The layer that is most complicated for developers, but shouldn't trouble users, is the +Python C-Buffer APIs. These are the methods for converting between Python objects and C +buffers. The reason it is complicated is that it keeps changing. + +There are two buffer interfaces for converting an object to a C-buffer, known as new-style +and old-style. Old-style buffers were introduced long ago, but the new-style is only +backported to 2.6. The old-style buffer interface is not available in 3.x. There is also +an old- and new-style interface for creating Python objects that view C-memory. The +old-style object is called a :class:`buffer`, and the new-style object is +:class:`memoryview`. Unlike the new-style buffer interface for objects, +:class:`memoryview` has only been backported to *2.7*. This means that the available +buffer-related functions are not the same in any two versions of Python 2.5, 2.6, 2.7, or +3.1. + +We have a :file:`utils/buffers.pxd` file that defines our :func:`asbuffer` and +:func:`frombuffer` functions. :file:`utils/buffers.pxd` was adapted from mpi4py_'s +:file:`asbuffer.pxi`. The :func:`frombuffer` functionality was added. These functions +internally switch based on Python version to call the appropriate C-API functions. + +.. seealso:: `Python Buffer API `_ + +.. _bufferapi: http://docs.python.org/c-api/buffer.html + + +``__str__`` +----------- + +As discussed, :class:`str` is not a platform independent type. The two places where we are +required to return native str objects are :func:`error.strerror`, and +:func:`Message.__str__`. In both of these cases, the natural return is actually a +:class:`bytes` object. In the methods, the native :class:`str` type is checked, and if the +native str is actually unicode, then we decode the bytes into unicode: + +.. sourcecode:: py + + # ... + b = natural_result() + if str is unicode: + return b.decode() + else: + return b + +Exceptions +---------- + +.. Note:: + + This section is only relevant for supporting Python 2.5 and 3.x, not for 2.6-3.x. + +The syntax for handling exceptions has `changed `_ in Python 3. The old syntax: + +.. sourcecode:: py + + try: + s.send(msg) + except zmq.ZMQError, e: + handle(e) + +is no longer valid in Python 3. Instead, the new syntax for this is: + +.. sourcecode:: py + + try: + s.send(msg) + except zmq.ZMQError as e: + handle(e) + +This new syntax is backported to Python 2.6, but is invalid on 2.5. For 2.6-3.x compatible +code, we could just use the new syntax. However, the only method we found to catch an +exception for handling on both 2.5 and 3.1 is to get the exception object inside the +exception block: + +.. sourcecode:: py + + try: + s.send(msg) + except zmq.ZMQError: + e = sys.exc_info()[1] + handle(e) + +This is certainly not as elegant as either the old or new syntax, but it's the only way we +have found to work everywhere. + +.. seealso:: PEP-3110_ + +.. _PEP-3110: http://www.python.org/dev/peps/pep-3110/ + + +.. _mpi4py: http://mpi4py.googlecode.com \ No newline at end of file diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst new file mode 100644 index 0000000..8f9095a --- /dev/null +++ b/docs/source/serialization.rst @@ -0,0 +1,90 @@ +.. PyZMQ serialization doc, by Min Ragan-Kelley, 2011 + +.. _serialization: + +Serializing messages with PyZMQ +=============================== + +When sending messages over a network, you often need to marshall your data into bytes. + + +Builtin serialization +--------------------- + +PyZMQ is primarily bindings for libzmq, but we do provide three builtin serialization +methods for convenience, to help Python developers learn libzmq. Python has two primary +packages for serializing objects: :py:mod:`json` and :py:mod:`pickle`, so we provide +simple convenience methods for sending and receiving objects serialized with these +modules. A socket has the methods :meth:`~.Socket.send_json` and +:meth:`~.Socket.send_pyobj`, which correspond to sending an object over the wire after +serializing with json and pickle respectively, and any object sent via those +methods can be reconstructed with the :meth:`~.Socket.recv_json` and +:meth:`~.Socket.recv_pyobj` methods. + + +These methods designed for convenience, not for performance, so developers who do want +to emphasize performance should use their own serialized send/recv methods. + +Using your own serialization +---------------------------- + +In general, you will want to provide your own serialization that is optimized for your +application or library availability. This may include using your own preferred +serialization ([msgpack]_, [protobuf]_), or adding compression via [zlib]_ in the standard +library, or the super fast [blosc]_ library. + +There are two simple models for implementing your own serialization: write a function +that takes the socket as an argument, or subclass Socket for use in your own apps. + +For instance, pickles can often be reduced substantially in size by compressing the data. +The following will send *compressed* pickles over the wire: + +.. sourcecode:: python + + import zlib, cPickle as pickle + + def send_zipped_pickle(socket, obj, flags=0, protocol=-1): + """pickle an object, and zip the pickle before sending it""" + p = pickle.dumps(obj, protocol) + z = zlib.compress(p) + return socket.send(z, flags=flags) + + def recv_zipped_pickle(socket, flags=0, protocol=-1): + """inverse of send_zipped_pickle""" + z = socket.recv(flags) + p = zlib.uncompress(z) + return pickle.loads(p) + +A common data structure in Python is the numpy array. PyZMQ supports sending +numpy arrays without copying any data, since they provide the Python buffer interface. +However just the buffer is not enough information to reconstruct the array on the +receiving side. Here is an example of a send/recv that allow non-copying +sends/recvs of numpy arrays including the dtype/shape data necessary for reconstructing +the array. + +.. sourcecode:: python + + import numpy + + def send_array(socket, A, flags=0, copy=True, track=False): + """send a numpy array with metadata""" + md = dict( + dtype = str(A.dtype), + shape = A.shape, + ) + socket.send_json(md, flags|zmq.SNDMORE) + return socket.send(A, flags, copy=copy, track=track) + + def recv_array(socket, flags=0, copy=True, track=False): + """recv a numpy array""" + md = socket.recv_json(flags=flags) + msg = socket.recv(flags=flags, copy=copy, track=track) + buf = buffer(msg) + A = numpy.frombuffer(buf, dtype=md['dtype']) + return A.reshape(md['shape']) + + +.. [msgpack] Message Pack serialization library http://msgpack.org +.. [protobuf] Google Protocol Buffers http://code.google.com/p/protobuf +.. [zlib] Python stdlib module for zip compression: :py:mod:`zlib` +.. [blosc] Blosc: A blocking, shuffling and loss-less (and crazy-fast) compression library http://blosc.pytables.org/trac diff --git a/docs/source/ssh.rst b/docs/source/ssh.rst new file mode 100644 index 0000000..ca2a607 --- /dev/null +++ b/docs/source/ssh.rst @@ -0,0 +1,80 @@ +.. PyZMQ ssh doc, by Min Ragan-Kelley, 2011 + +.. _ssh: + +Tunneling PyZMQ Connections with SSH +==================================== + +.. versionadded:: 2.1.9 + +You may want to connect ØMQ sockets across machines, or untrusted networks. One common way +to do this is to tunnel the connection via SSH. IPython_ introduced some tools for +tunneling ØMQ connections over ssh in simple cases. These functions have been brought into +pyzmq as :mod:`zmq.ssh` under IPython's BSD license. + +PyZMQ will use the shell ssh command via pexpect_ by default, but it also supports +using paramiko_ for tunnels, so it should work on Windows. + +.. note:: + + pexpect has no Python3 support at this time, so Python 3 users should get Thomas + Kluyver's `pexpect-u`_ fork. + +An SSH tunnel has five basic components: + +* server : the SSH server through which the tunnel will be created +* remote ip : the IP of the remote machine *as seen from the server* + (remote ip may be, but is not not generally the same machine as server). +* remote port : the port on the remote machine that you want to connect to. +* local ip : the interface on your local machine you want to use (default: 127.0.0.1) +* local port : the local port you want to forward to the remote port (default: high random) + +So once you have established the tunnel, connections to ``localip:localport`` will actually +be connections to ``remoteip:remoteport``. + +In most cases, you have a zeromq url for a remote machine, but you need to tunnel the +connection through an ssh server. This is + +So if you would use this command from the same LAN as the remote machine: + +.. sourcecode:: python + + sock.connect("tcp://10.0.1.2:5555") + +to make the same connection from another machine that is outside the network, but you have +ssh access to a machine ``server`` on the same LAN, you would simply do: + +.. sourcecode:: python + + from zmq import ssh + ssh.tunnel_connection(sock, "tcp://10.0.1.2:5555", "server") + +Note that ``"server"`` can actually be a fully specified ``"user@server:port"`` ssh url. +Since this really just launches a shell command, all your ssh configuration of usernames, +aliases, keys, etc. will be respected. If necessary, :func:`tunnel_connection` does take +arguments for specific passwords, private keys (the ssh ``-i`` option), and non-default +choice of whether to use paramiko. + +If you are on the same network as the machine, but it is only listening on localhost, you +can still connect by making the machine itself the server, and using loopback as the +remote ip: + +.. sourcecode:: python + + from zmq import ssh + ssh.tunnel_connection(sock, "tcp://127.0.0.1:5555", "10.0.1.2") + +The :func:`tunnel_connection` function is a simple utility that forwards a random +localhost port to the real destination, and connects a socket to the new local url, +rather than the remote one that wouldn't actually work. + +.. seealso:: + + A short discussion of ssh tunnels: http://www.revsys.com/writings/quicktips/ssh-tunnel.html + + +.. _IPython: http://ipython.org +.. _pexpect: http://www.noah.org/wiki/pexpect +.. _pexpect-u: http://pypi.python.org/pypi/pexpect-u +.. _paramiko: http://www.lag.net/paramiko/ + diff --git a/docs/source/unicode.rst b/docs/source/unicode.rst new file mode 100644 index 0000000..a0c7878 --- /dev/null +++ b/docs/source/unicode.rst @@ -0,0 +1,188 @@ +.. PyZMQ Unicode doc, by Min Ragan-Kelley, 2010 + +.. _unicode: + +PyZMQ and Unicode +================= + +PyZMQ is built with an eye towards an easy transition to Python 3, and part of +that is dealing with unicode strings. This is an overview of some of what we +found, and what it means for PyZMQ. + +First, Unicode in Python 2 and 3 +******************************** + +In Python < 3, a ``str`` object is really a C string with some sugar - a +specific series of bytes with some fun methods like ``endswith()`` and +``split()``. In 2.0, the ``unicode`` object was added, which handles different +methods of encoding. In Python 3, however, the meaning of ``str`` changes. A +``str`` in Python 3 is a full unicode object, with encoding and everything. If +you want a C string with some sugar, there is a new object called ``bytes``, +that behaves much like the 2.x ``str``. The idea is that for a user, a string is +a series of *characters*, not a series of bytes. For simple ascii, the two are +interchangeable, but if you consider accents and non-Latin characters, then the +character meaning of byte sequences can be ambiguous, since it depends on the +encoding scheme. They decided to avoid the ambiguity by forcing users who want +the actual bytes to specify the encoding every time they want to convert a +string to bytes. That way, users are aware of the difference between a series of +bytes and a collection of characters, and don't confuse the two, as happens in +Python 2.x. + +The problems (on both sides) come from the fact that regardless of the language +design, users are mostly going to use ``str`` objects to represent collections +of characters, and the behavior of that object is dramatically different in +certain aspects between the 2.x ``bytes`` approach and the 3.x ``unicode`` +approach. The ``unicode`` approach has the advantage of removing byte ambiguity +- it's a list of characters, not bytes. However, if you really do want the +bytes, it's very inefficient to get them. The ``bytes`` approach has the +advantage of efficiency. A ``bytes`` object really is just a char* pointer with +some methods to be used on it, so when interacting with, so interacting with C +code, etc is highly efficient and straightforward. However, understanding a +bytes object as a string with extended characters introduces ambiguity and +possibly confusion. + +To avoid ambiguity, hereafter we will refer to encoded C arrays as 'bytes' and +abstract unicode objects as 'strings'. + +Unicode Buffers +--------------- + +Since unicode objects have a wide range of representations, they are not stored +as the bytes according to their encoding, but rather in a format called UCS (an +older fixed-width Unicode format). On some platforms (OS X, Windows), the storage +is UCS-2, which is 2 bytes per character. On most \*ix systems, it is UCS-4, or +4 bytes per character. The contents of the *buffer* of a ``unicode`` object are +not encoding dependent (always UCS-2 or UCS-4), but they are *platform* +dependent. As a result of this, and the further insistence on not interpreting +``unicode`` objects as bytes without specifying encoding, ``str`` objects in +Python 3 don't even provide the buffer interface. You simply cannot get the raw +bytes of a ``unicode`` object without specifying the encoding for the bytes. In +Python 2.x, you can get to the raw buffer, but the platform dependence and the +fact that the encoding of the buffer is not the encoding of the object makes it +very confusing, so this is probably a good move. + +The efficiency problem here comes from the fact that simple ascii strings are 4x +as big in memory as they need to be (on most Linux, 2x on other platforms). +Also, to translate to/from C code that works with char*, you always have to copy +data and encode/decode the bytes. This really is horribly inefficient from a +memory standpoint. Essentially, Where memory efficiency matters to you, you +should never ever use strings; use bytes. The problem is that users will almost +always use ``str``, and in 2.x they are efficient, but in 3.x they are not. We +want to make sure that we don't help the user make this mistake, so we ensure +that zmq methods don't try to hide what strings really are. + +What This Means for PyZMQ +************************* + +PyZMQ is a wrapper for a C library, so it really should use bytes, since a +string is not a simple wrapper for ``char *`` like it used to be, but an +abstract sequence of characters. The representations of bytes in Python are +either the ``bytes`` object itself, or any object that provides the buffer +interface (aka memoryview). In Python 2.x, unicode objects do provide the buffer +interface, but as they do not in Python 3, where pyzmq requires bytes, we +specifically reject unicode objects. + +The relevant methods here are ``socket.send/recv``, ``socket.get/setsockopt``, +``socket.bind/connect``. The important consideration for send/recv and +set/getsockopt is that when you put in something, you really should get the same +object back with its partner method. We can easily coerce unicode objects to +bytes with send/setsockopt, but the problem is that the pair method of +recv/getsockopt will always be bytes, and there should be symmetry. We certainly +shouldn't try to always decode on the retrieval side, because if users just want +bytes, then we are potentially using up enormous amounts of excess memory +unnecessarily, due to copying and larger memory footprint of unicode strings. + +Still, we recognize the fact that users will quite frequently have unicode +strings that they want to send, so we have added ``socket._string()`` +wrappers. These methods simply wrap their bytes counterpart by encoding +to/decoding from bytes around them, and they all take an `encoding` keyword +argument that defaults to utf-8. Since encoding and decoding are necessary to +translate between unicode and bytes, it is impossible to perform non-copying +actions with these wrappers. + +``socket.bind/connect`` methods are different from these, in that they are +strictly setters and there is not corresponding getter method. As a result, we +feel that we can safely coerce unicode objects to bytes (always to utf-8) in +these methods. + +.. note:: + + For cross-language symmetry (including Python 3), the ``_unicode`` methods + are now ``_string``. Many languages have a notion of native strings, and + the use of ``_unicode`` was wedded too closely to the name of such objects + in Python 2. For the time being, anywhere you see ``_string``, ``_unicode`` + also works, and is the only option in pyzmq ≤ 2.1.11. + + +The Methods +----------- + +Overview of the relevant methods: + +.. py:function:: socket.bind(self, addr) + + `addr` is ``bytes`` or ``unicode``. If ``unicode``, + encoded to utf-8 ``bytes`` + +.. py:function:: socket.connect(self, addr) + + `addr` is ``bytes`` or ``unicode``. If ``unicode``, + encoded to utf-8 ``bytes`` + +.. py:function:: socket.send(self, object obj, flags=0, copy=True) + + `obj` is ``bytes`` or provides buffer interface. + + if `obj` is ``unicode``, raise ``TypeError`` + +.. py:function:: socket.recv(self, flags=0, copy=True) + + returns ``bytes`` if `copy=True` + + returns ``zmq.Message`` if `copy=False`: + + `message.buffer` is a buffer view of the ``bytes`` + + `str(message)` provides the ``bytes`` + + `unicode(message)` decodes `message.buffer` with utf-8 + +.. py:function:: socket.send_string(self, unicode s, flags=0, encoding='utf-8') + + takes a ``unicode`` string `s`, and sends the ``bytes`` + after encoding without an extra copy, via: + + `socket.send(s.encode(encoding), flags, copy=False)` + +.. py:function:: socket.recv_string(self, flags=0, encoding='utf-8') + + always returns ``unicode`` string + + there will be a ``UnicodeError`` if it cannot decode the buffer + + performs non-copying `recv`, and decodes the buffer with `encoding` + +.. py:function:: socket.setsockopt(self, opt, optval) + + only accepts ``bytes`` for `optval` (or ``int``, depending on `opt`) + + ``TypeError`` if ``unicode`` or anything else + +.. py:function:: socket.getsockopt(self, opt) + + returns ``bytes`` (or ``int``), never ``unicode`` + +.. py:function:: socket.setsockopt_string(self, opt, unicode optval, encoding='utf-8') + + accepts ``unicode`` string for `optval` + + encodes `optval` with `encoding` before passing the ``bytes`` to + `setsockopt` + +.. py:function:: socket.getsockopt_string(self, opt, encoding='utf-8') + + always returns ``unicode`` string, after decoding with `encoding` + + note that `zmq.IDENTITY` is the only `sockopt` with a string value + that can be queried with `getsockopt` + -- cgit v1.2.3