From aefc79b0343b80bb0793728d05cc20cc22974196 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 12 Nov 2017 10:24:56 -0800 Subject: [PATCH 1/3] run sphinx-quickstart --- docs/Makefile | 20 ++++++ docs/conf.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 20 ++++++ 3 files changed, 208 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.md diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..481f575 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = Magic-Wormhole +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8949501 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +# +# Magic-Wormhole documentation build configuration file, created by +# sphinx-quickstart on Sun Nov 12 10:24:09 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.md' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Magic-Wormhole' +copyright = u'2017, Brian Warner' +author = u'Brian Warner' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.10' +# The full version, including alpha/beta/rc tags. +release = u'0.10.3' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Magic-Wormholedoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Magic-Wormhole.tex', u'Magic-Wormhole Documentation', + u'Brian Warner', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'magic-wormhole', u'Magic-Wormhole Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Magic-Wormhole', u'Magic-Wormhole Documentation', + author, 'Magic-Wormhole', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..cd320b8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,20 @@ +.. Magic-Wormhole documentation master file, created by + sphinx-quickstart on Sun Nov 12 10:24:09 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Magic-Wormhole's documentation! +========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From 346cfa13e2c12f243ab5a29b8b89a7014fcf93f1 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 12 Nov 2017 15:12:36 -0800 Subject: [PATCH 2/3] enable markdown (but use index.rst), small headline tweaks --- docs/api.md | 2 +- docs/conf.py | 9 +++++++-- docs/{index.md => index.rst} | 15 +++++++++++++-- docs/introduction.md | 2 +- docs/server-protocol.md | 2 +- docs/transit.md | 14 +++++++------- 6 files changed, 30 insertions(+), 14 deletions(-) rename docs/{index.md => index.rst} (59%) diff --git a/docs/api.md b/docs/api.md index aa18ce9..f352e7d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,4 +1,4 @@ -# Magic-Wormhole +# The Magic-Wormhole API This library provides a mechanism to securely transfer small amounts of data between two computers. Both machines must be connected to the diff --git a/docs/conf.py b/docs/conf.py index 8949501..cabec8d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,11 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +from recommonmark.parser import CommonMarkParser + +source_parsers = { + ".md": CommonMarkParser, +} # -- General configuration ------------------------------------------------ @@ -38,8 +43,8 @@ templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = '.md' +source_suffix = ['.rst', '.md'] +#source_suffix = '.md' # The master toctree document. master_doc = 'index' diff --git a/docs/index.md b/docs/index.rst similarity index 59% rename from docs/index.md rename to docs/index.rst index cd320b8..5f9308b 100644 --- a/docs/index.md +++ b/docs/index.rst @@ -3,13 +3,24 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Magic-Wormhole's documentation! -========================================== +Magic-Wormhole: Get Things From One Computer To Another, Safely +=============================================================== .. toctree:: :maxdepth: 2 :caption: Contents: + introduction + attacks + tor + + api + transit + server-protocol + client-protocol + file-transfer-protocol + + journal Indices and tables diff --git a/docs/introduction.md b/docs/introduction.md index 7e2c255..a1e7f30 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,4 +1,4 @@ -# Magic-Wormhole +# Introduction The magic-wormhole (Python) distribution provides several things: an executable tool ("bin/wormhole"), an importable library (`import wormhole`), diff --git a/docs/server-protocol.md b/docs/server-protocol.md index 3afb967..345f617 100644 --- a/docs/server-protocol.md +++ b/docs/server-protocol.md @@ -220,7 +220,7 @@ any), and which ones provoke direct responses: * S->C pong {pong: int} * S->C error {error: str, orig:} -# Persistence +## Persistence The server stores all messages in a database, so it should not lose any information when it is restarted. The server will not send a direct diff --git a/docs/transit.md b/docs/transit.md index 4cbda37..2ca7872 100644 --- a/docs/transit.md +++ b/docs/transit.md @@ -1,4 +1,4 @@ -= Transit Protocol = +# Transit Protocol The Transit protocol is responsible for establishing an encrypted bidirectional record stream between two programs. It must be given a "transit @@ -19,7 +19,7 @@ The other side will attempt to connect to each of those ports, as well as listening on its own socket. After a few seconds without success, they will both connect to a relay server. -== Roles == +## Roles The Transit protocol has pre-defined "Sender" and "Receiver" roles (unlike Wormhole, which is symmetric/nobody-goes-first). Each connection must have @@ -32,7 +32,7 @@ each side.. This may be relaxed in the future, much as Wormhole was. -== Records == +## Records Transit establishes a **record-pipe**, so the two sides can send and receive whole records, rather than unframed bytes. This is a side-effect of the @@ -62,7 +62,7 @@ dropped. If a record is lost (e.g. the receiver observers records #1,#2,#4, but not #3), the connection is dropped when the unexpected sequence number is received. -== Handshake == +## Handshake The transit key is used to derive several secondary keys. Two of them are used as a "handshake", to distinguish correct Transit connections from other @@ -111,7 +111,7 @@ two handshakes), then making new connections to play back the recorded handshakes, but this level of attacker could simply drop the user's packets directly. -== Relay == +## Relay The **Transit Relay** is a host which offers TURN-like services for magic-wormhole instances. It uses a TCP-based protocol with a handshake to @@ -139,7 +139,7 @@ attempting to use the relay. If it has no viable direct hints, it will start using the relay right away. This prefers direct connections, but doesn't introduce completely unnecessary stalls. -== API == +## API First, create a Transit instance, giving it the connection information of the transit relay. The application must know whether it should use a Sender or a @@ -196,7 +196,7 @@ turns). However the blocking API does not provide a way to send records while waiting for an inbound record. This *might* work with threads, but it has not been tested. -== Twisted API == +## Twisted API The same facilities are available in the asynchronous Twisted environment. The difference is that some functions return Deferreds instead of immediate From 3f3be8c7aa170036f395502ce305582690eb8e4b Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 12 Nov 2017 15:21:56 -0800 Subject: [PATCH 3/3] move most of README into docs, add more pointers --- README.md | 274 +------------------------------------------ docs/index.rst | 5 +- docs/introduction.md | 2 +- 3 files changed, 6 insertions(+), 275 deletions(-) diff --git a/README.md b/README.md index f5cf1e9..0b325c5 100644 --- a/README.md +++ b/README.md @@ -19,274 +19,8 @@ and do not need to be memorized. * PyCon 2016 presentation: [Slides](http://www.lothar.com/~warner/MagicWormhole-PyCon2016.pdf), [Video](https://youtu.be/oFrTqQw0_3c) -## Example - -Sender: - -``` -% wormhole send README.md -Sending 7924 byte file named 'README.md' -On the other computer, please run: wormhole receive -Wormhole code is: 7-crossover-clockwork - -Sending (<-10.0.1.43:58988).. -100%|=========================| 7.92K/7.92K [00:00<00:00, 6.02MB/s] -File sent.. waiting for confirmation -Confirmation received. Transfer complete. -``` - -Receiver: - -``` -% wormhole receive -Enter receive wormhole code: 7-crossover-clockwork -Receiving file (7924 bytes) into: README.md -ok? (y/n): y -Receiving (->tcp:10.0.1.43:58986).. -100%|===========================| 7.92K/7.92K [00:00<00:00, 120KB/s] -Received file written to README.md -``` - - -## Installation - -```$ pip install magic-wormhole``` - -You either want to do this into a "user" environment (putting the -``wormhole`` executable in ``~/.local/bin/wormhole``) like this: - -``` -pip install --user magic-wormhole -``` - -or put it into a virtualenv, to avoid modifying the system python's -libraries, like this: - -``` -virtualenv venv -source venv/bin/activate -pip install magic-wormhole -``` - -You probably *don't* want to use ``sudo`` when you run ``pip``. - -### OS X - -On OS X, you may need to install `pip` and run `$ xcode-select --install` to -get GCC. - -Or with `homebrew`: - -`$ brew install magic-wormhole` - -### Linux - -On Debian 9 and Ubuntu 17.04+ with `apt`: - -```$ sudo apt install magic-wormhole``` - -On previous versions of the Debian/Ubuntu systems, or if you want to install -the latest version, you may first need: - -`$ sudo apt-get install python-pip build-essential python-dev libffi-dev libssl-dev` - -On Fedora: - -`$ sudo dnf install python-pip python-devel libffi-devel openssl-devel gcc-c++ -libtool redhat-rpm-config`. - -Note: If you get errors like `fatal error: sodium.h: No such file or -directory` on Linux, either use `SODIUM_INSTALL=bundled pip install -magic-wormhole`, or try installing the `libsodium-dev` / `libsodium-devel` -package. These work around a bug in pynacl which gets confused when the -libsodium runtime is installed (e.g. `libsodium13`) but not the development -package. - -### Windows - -On Windows, python2 may work better than python3. On older systems, `$ pip -install --upgrade pip` may be necessary to get a version that can compile all -the dependencies. Most of the dependencies are published as binary wheels, -but in case your system is unable to find these, it will have to compile -them, for which Microsoft Visual C++ 9.0 may be required. Get it from -http://aka.ms/vcpython27 . - -## Motivation - -* Moving a file to a friend's machine, when the humans can speak to each - other (directly) but the computers cannot -* Delivering a properly-random password to a new user via the phone -* Supplying an SSH public key for future login use - -Copying files onto a USB stick requires physical proximity, and is -uncomfortable for transferring long-term secrets because flash memory is hard -to erase. Copying files with ssh/scp is fine, but requires previous -arrangements and an account on the target machine, and how do you bootstrap -the account? Copying files through email first requires transcribing an email -address in the opposite direction, and is even worse for secrets, because -email is unencrypted. Copying files through encrypted email requires -bootstrapping a GPG key as well as an email address. Copying files through -Dropbox is not secure against the Dropbox server and results in a large URL -that must be transcribed. Using a URL shortener adds an extra step, reveals -the full URL to the shortening service, and leaves a short URL that can be -guessed by outsiders. - -Many common use cases start with a human-mediated communication channel, such -as IRC, IM, email, a phone call, or a face-to-face conversation. Some of -these are basically secret, or are "secret enough" to last until the code is -delivered and used. If this does not feel strong enough, users can turn on -additional verification that doesn't depend upon the secrecy of the channel. - -The notion of a "magic wormhole" comes from the image of two distant wizards -speaking the same enchanted phrase at the same time, and causing a mystical -connection to pop into existence between them. The wizards then throw books -into the wormhole and they fall out the other side. Transferring files -securely should be that easy. - -## Design - -The `wormhole` tool uses PAKE "Password-Authenticated Key Exchange", a family -of cryptographic algorithms that uses a short low-entropy password to -establish a strong high-entropy shared key. This key can then be used to -encrypt data. `wormhole` uses the SPAKE2 algorithm, due to Abdalla and -Pointcheval[1]. - -PAKE effectively trades off interaction against offline attacks. The only way -for a network attacker to learn the shared key is to perform a -man-in-the-middle attack during the initial connection attempt, and to -correctly guess the code being used by both sides. Their chance of doing this -is inversely proportional to the entropy of the wormhole code. The default is -to use a 16-bit code (use --code-length= to change this), so for each use of -the tool, an attacker gets a 1-in-65536 chance of success. As such, users can -expect to see many error messages before the attacker has a reasonable chance -of success. - -## Timing - -The program does not have any built-in timeouts, however it is expected that -both clients will be run within an hour or so of each other. This makes the -tool most useful for people who are having a real-time conversation already, -and want to graduate to a secure connection. Both clients must be left -running until the transfer has finished. - -## Relays - -The wormhole library requires a "Rendezvous Server": a simple WebSocket-based -relay that delivers messages from one client to another. This allows the -wormhole codes to omit IP addresses and port numbers. The URL of a public -server is baked into the library for use as a default, and will be freely -available until volume or abuse makes it infeasible to support. Applications -which desire more reliability can easily run their own relay and configure -their clients to use it instead. Code for the Rendezvous Server is included -in the library. - -The file-transfer commands also use a "Transit Relay", which is another -simple server that glues together two inbound TCP connections and transfers -data on each to the other. The `wormhole send` file mode shares the IP -addresses of each client with the other (inside the encrypted message), and -both clients first attempt to connect directly. If this fails, they fall back -to using the transit relay. As before, the host/port of a public server is -baked into the library, and should be sufficient to handle moderate traffic. - -The protocol includes provisions to deliver notices and error messages to -clients: if either relay must be shut down, these channels will be used to -provide information about alternatives. - -## CLI tool - -* `wormhole send [args] --text TEXT` -* `wormhole send [args] FILENAME` -* `wormhole send [args] DIRNAME` -* `wormhole receive [args]` - -Both commands accept additional arguments to influence their behavior: - -* `--code-length WORDS`: use more or fewer than 2 words for the code -* `--verify` : print (and ask user to compare) extra verification string - -## Library - -The `wormhole` module makes it possible for other applications to use these -code-protected channels. This includes Twisted support, and (in the future) -will include blocking/synchronous support too. See docs/api.md for details. - -The file-transfer tools use a second module named `wormhole.transit`, which -provides an encrypted record-pipe. It knows how to use the Transit Relay as -well as direct connections, and attempts them all in parallel. -`TransitSender` and `TransitReceiver` are distinct, although once the -connection is established, data can flow in either direction. All data is -encrypted (using nacl/libsodium "secretbox") using a key derived from the -PAKE phase. See `src/wormhole/cli/cmd_send.py` for examples. - -## Development - -* Bugs and Patches: https://github.com/warner/magic-wormhole -* Chat: #magic-wormhole on irc.freenode.net - -To set up Magic Wormhole for development, you will first need to -install [virtualenv][]. - -Once you've done that, ``git clone`` the repo, ``cd`` into the root of the -repository, and run: - -``` -virtualenv venv -source venv/bin/activate -pip install --upgrade pip setuptools -``` - -Now your virtualenv has been activated. You'll want to re-run -`source venv/bin/activate` for every new terminal session you open. - -To install Magic Wormhole and its development dependencies into your -virtualenv, run: - -``` -pip install -e .[dev] -``` - -While the virtualenv is active, running ``wormhole`` will get you the -development version. - -### Running Tests - - -Within your virtualenv, the command-line program `trial` will -run the test suite: - -``` -trial wormhole -``` - -This tests the entire `wormhole` package. If you want to run -only the tests for a specific module, or even just a specific test, -you can specify it instead via Python's standard dotted -import notation, e.g.: - -``` -trial wormhole.test.test_cli.PregeneratedCode.test_file_tor -``` - -Developers can also just clone the source tree and run `tox` to run the unit -tests on all supported (and installed) versions of python: 2.7, 3.4, 3.5, and -3.6. - -### Troubleshooting - -Every so often, you might get a traceback with the following -kind of error: - -``` -pkg_resources.DistributionNotFound: The 'magic-wormhole==0.9.1-268.g66e0d86.dirty' distribution was not found and is required by the application -``` - -If this happens, run `pip install -e .[dev]` again. - -[virtualenv]: http://python-guide-pt-br.readthedocs.io/en/latest/dev/virtualenvs/ - -### Other - -Relevant [xkcd](https://xkcd.com/949/) :-) +For complete documentation, please see https://magic-wormhole.readthedocs.io +or the docs/ subdirectory. ## License, Compatibility @@ -296,7 +30,3 @@ This library is compatible with python2.7, 3.4, 3.5, and 3.6 . It is probably compatible with py2.6, but the latest Twisted (>=15.5.0) is not. - - - -[1]: http://www.di.ens.fr/~pointche/Documents/Papers/2005_rsa.pdf "RSA 2005" diff --git a/docs/index.rst b/docs/index.rst index 5f9308b..b00ffd7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,16 +10,17 @@ Magic-Wormhole: Get Things From One Computer To Another, Safely :maxdepth: 2 :caption: Contents: - introduction - attacks + welcome tor + introduction api transit server-protocol client-protocol file-transfer-protocol + attacks journal diff --git a/docs/introduction.md b/docs/introduction.md index a1e7f30..adacc0d 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,4 +1,4 @@ -# Introduction +# Protocol/API/Library Introduction The magic-wormhole (Python) distribution provides several things: an executable tool ("bin/wormhole"), an importable library (`import wormhole`),