feat: proxy modes and proxy authentication

CHANGELOG.md

@@ -74,6 +74,14 @@ Two behaviour changes come with it:
   reported in assertion failures rather than hidden, so an assertion against a shortened
   recording cannot look complete. Requests that failed are recorded too, so a blocked
   call can be asserted on.
+- `Start Mitm Proxy` takes `mode`, so the proxy can be something other than a forward
+  proxy: `reverse:` puts it in front of one server, so a client needs no proxy settings
+  at all; `upstream:` sends everything on through another proxy, which is what a
+  corporate network needs; `transparent` and `socks5` are passed through as well. A mode
+  that cannot be understood fails the keyword with mitmproxy's own explanation, rather
+  than leaving the proxy to fail to start for an unstated reason.
+- `Start Mitm Proxy` takes `proxy_auth`, requiring clients to authenticate before the
+  proxy serves them.
 - `Get Proxy Rules` returns the loaded rules in the order they are applied.
 - Blocking rules have an alias, like every other rule, so they are removed the same way.
 - Rules survive a restart of the proxy: the registry outlives it, and only the addon

MitmLibrary/__init__.py

@@ -12,7 +12,7 @@
 applications in a more realistic and controlled environment.
 """
 
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Dict, List, Optional, Sequence, Union
 
 from mitmproxy.tools import dump
 from robot.api import logger
@@ -115,6 +115,43 @@ class MitmLibrary:
     number dropped is reported in the failure message of an assertion rather than being
     hidden. A body longer than the second cap is shortened, and the recorded request says
     so.
+    = Proxy modes =
+    By default the proxy is a normal forward proxy: a client is configured to send its
+    traffic through it. `mode` on `Start Mitm Proxy` changes that, and takes either one
+    mode or a list of them.
+
+    - `regular`: a forward proxy. The default.
+    - `reverse:http://host:port`: the proxy stands in front of one server. Clients talk
+      to the proxy directly, as if it were that server, so nothing needs to be configured
+      to use a proxy at all.
+    - `upstream:http://host:port`: a forward proxy that passes everything on to another
+      proxy. This is what a corporate network needs, where the machine running the tests
+      may not reach the internet on its own. Rules and recording still apply here, on
+      this proxy; what the proxy further up the chain makes of the traffic is its own
+      business, and it does not necessarily see it as separate requests.
+    - `transparent`: traffic is routed to the proxy by the network itself. Needs the
+      operating system to be set up for it.
+    - `socks5`: a SOCKS5 proxy rather than an HTTP one.
+
+    A mode may be followed by `@host:port` to give it its own listening address, which
+    overrides `listen_host` and `listen_port`. `Get Proxy Address` reports where the
+    proxy actually ended up listening, which is why it reads that from the proxy rather
+    than repeating the arguments back.
+
+    A specification that cannot be understood fails `Start Mitm Proxy` with mitmproxy's
+    own explanation, rather than leaving the proxy to fail to start for an unstated
+    reason.
+
+    `transparent` and `socks5` are passed through to mitmproxy but are not exercised by
+    this library's own tests, because they need the operating system or a client
+    configured for them.
+
+    == Example ==
+    | # Stand in front of a service, so a client needs no proxy settings at all
+    | Start Mitm Proxy    mode=reverse:http://127.0.0.1:5000
+
+    | # Send everything on through the network's own proxy
+    | Start Mitm Proxy    mode=upstream:http://corporate-proxy:3128
 
     = Mitm Certificates =
     To test with SSL verification or use a browser without ignoring certificates, you need to set up
@@ -188,6 +225,8 @@ def start_mitm_proxy(
         record: bool = False,
         record_limit: int = DEFAULT_LIMIT,
         record_body_limit: int = DEFAULT_BODY_LIMIT,
+        mode: Optional[Union[str, List[str]]] = None,
+        proxy_auth: Optional[str] = None,
     ) -> None:
         """
         Starts a proxy at the given host and port.
@@ -205,11 +244,19 @@ def start_mitm_proxy(
           had been called. Off by default; see the `Recording` section.
         - record_limit: How many requests to keep when recording.
         - record_body_limit: How many bytes of each body to keep when recording.
+        - mode: How the proxy handles the traffic it receives. A single mode or a list of
+          them. Defaults to a normal forward proxy. See the `Proxy modes` section.
+        - proxy_auth: Require clients to authenticate before the proxy serves them.
+          `username:password` for one account, `any` to accept any combination, or
+          `@path/to/htpasswd` for an Apache htpasswd file.
 
-        Fails if the proxy cannot be started, for example when the port is already in use.
+        Fails if the proxy cannot be started, for example when the port is already in use,
+        or if a mode cannot be understood.
 
         Example:
         | Start Mitm Proxy    192.168.1.100    8888    /path/to/certificates    True
+        | Start Mitm Proxy    mode=reverse:http://127.0.0.1:5000
+        | Start Mitm Proxy    proxy_auth=tester:secret
 
         See the 'Mitm Certificates' section in the documentation for more information.
         """
@@ -223,6 +270,8 @@ def start_mitm_proxy(
                 certificates_directory,
                 ssl_insecure,
                 self._build_addons,
+                mode,
+                proxy_auth,
             )
         except Exception:
             # The controller has already discarded the master it could not start. The

MitmLibrary/proxy_controller.py

@@ -13,9 +13,10 @@
 import logging
 import time
 from concurrent.futures import Future, TimeoutError as FutureTimeoutError
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from mitmproxy import options
+from mitmproxy.proxy import mode_specs
 from mitmproxy.tools import dump
 from robot.api import logger
 
@@ -33,14 +34,30 @@ class StartupErrorCollector(logging.Handler):
 
     mitmproxy reports bind failures through the logging module rather than by raising,
     so this is the only way to tell the user *why* the proxy did not start.
+
+    It listens on the root logger, which means it also hears errors that have nothing to
+    do with this proxy - a proxy stopped moments ago still logs from its own teardown,
+    and so does the rest of the test run. Only records from mitmproxy are kept, and only
+    those that report a failure to listen are treated as a reason to give up. Everything
+    else is remembered for the failure message but does not, by itself, fail a startup
+    that would otherwise have succeeded.
     """
 
+    #: What mitmproxy says when it cannot bind, which is the failure worth acting on.
+    BIND_FAILURE = "failed to listen"
+
     def __init__(self) -> None:
         super().__init__(level=logging.ERROR)
         self.messages: List[str] = []
+        self.bind_failures: List[str] = []
 
     def emit(self, record: logging.LogRecord) -> None:
-        self.messages.append(record.getMessage())
+        if not record.name.startswith("mitmproxy"):
+            return
+        message = record.getMessage()
+        self.messages.append(message)
+        if self.BIND_FAILURE in message:
+            self.bind_failures.append(message)
 
 
 class ProxyController:
@@ -68,6 +85,8 @@ def start(
         certificates_directory: Optional[str],
         ssl_insecure: bool,
         addon_factory: AddonFactory,
+        mode: Optional[Union[str, Sequence[str]]] = None,
+        proxy_auth: Optional[str] = None,
     ) -> None:
         """Starts the proxy and waits until it is actually listening.
 
@@ -84,6 +103,8 @@ def start(
         }
         if certificates_directory is not None:
             option_kwargs["confdir"] = certificates_directory
+        if mode is not None:
+            option_kwargs["mode"] = self._parse_modes(mode)
         opts = options.Options(**option_kwargs)
         # Bind the master to the loop it will actually run on. Without this it binds
         # to whatever loop happens to be running on the calling thread, which is not
@@ -95,6 +116,11 @@ def start(
             with_dumper=False,
         )
         self.master = master
+        if proxy_auth is not None:
+            # proxyauth belongs to the addon of the same name, which registers it when
+            # the master loads its addons, so it does not exist yet when the options are
+            # built above.
+            master.options.update(proxyauth=proxy_auth)
         self._disable_errorcheck(master)
         for addon in addon_factory(master):
             master.addons.add(addon)
@@ -110,6 +136,27 @@ def start(
         finally:
             logging.getLogger().removeHandler(collector)
 
+    @staticmethod
+    def _parse_modes(mode: Union[str, Sequence[str]]) -> List[str]:
+        """Checks the mode specifications and returns them as mitmproxy wants them.
+
+        Parsing here means an unusable specification fails the keyword that gave it,
+        with mitmproxy's own explanation of what is wrong. Left to the proxy it would
+        surface as a startup timeout with nothing useful attached, because mitmproxy
+        logs the problem rather than raising it.
+        """
+        modes = [mode] if isinstance(mode, str) else list(mode)
+        for spec in modes:
+            try:
+                mode_specs.ProxyMode.parse(spec)
+            except ValueError as error:
+                raise ValueError(
+                    f"'{spec}' is not a usable proxy mode: {error}. Modes look like "
+                    f"'regular', 'reverse:http://host:port', 'upstream:http://host:port', "
+                    f"'transparent' or 'socks5', optionally followed by '@host:port'."
+                ) from error
+        return modes
+
     @staticmethod
     def _disable_errorcheck(master: dump.DumpMaster) -> None:
         """Removes mitmproxy's errorcheck addon, which is wrong for a library.
@@ -156,11 +203,13 @@ def _fail_on_startup_error(
                 )
             if self.listen_addresses(proxy_master):
                 return
-            if collector.messages:
+            if collector.bind_failures:
                 break
             time.sleep(STARTUP_POLL_INTERVAL)
 
-        reported = "; ".join(collector.messages) or "no error reported"
+        reported = "; ".join(
+            collector.bind_failures or collector.messages
+        ) or "no error reported"
         self.discard(wait=False)
         raise RuntimeError(
             f"Could not start the proxy on {listen_host}:{listen_port}: {reported}"
@@ -228,9 +277,29 @@ def discard(self, wait: bool = True) -> None:
                 )
             except Exception as error:  # pylint: disable=broad-exception-caught
                 logger.info(f"The proxy stopped with an error: {error}")
+        self._uninstall_log_handler()
         self.master = None
         self.future = None
 
+    def _uninstall_log_handler(self) -> None:
+        """Removes the root logger handler mitmproxy installed for this master.
+
+        mitmproxy attaches a handler to the root logger that forwards every log record
+        to the master's event loop. It never removes it, so once the proxy has stopped
+        and its loop is closed, any later log record - from anywhere in the test run -
+        raises "Event loop is closed" inside logging. Starting several proxies in one run
+        leaves one such handler behind each time.
+        """
+        if self.master is None:
+            return
+        handler = getattr(self.master, "_legacy_log_events", None)
+        if handler is None:  # pragma: no cover - present in every version we support
+            return
+        try:
+            handler.uninstall()
+        except Exception as error:  # pylint: disable=broad-exception-caught
+            logger.info(f"Could not remove the mitmproxy log handler: {error}")
+
     def _close_servers(self) -> None:
         """Closes the listening sockets held by the proxyserver addon.
 

README.md

@@ -111,6 +111,24 @@ Be aware that `0.0.0.0` exposes an intercepting proxy on every network interface
 who can reach the machine can route their traffic through it.
 
 
+### Proxy modes
+
+By default the proxy is a forward proxy: a client is configured to send traffic through
+it. `mode` changes that:
+
+```robotframework
+# Stand in front of a service, so a client needs no proxy settings at all
+Start Mitm Proxy    mode=reverse:http://127.0.0.1:5000
+
+# Send everything on through the network's own proxy
+Start Mitm Proxy    mode=upstream:http://corporate-proxy:3128
+```
+
+`transparent` and `socks5` are passed through to mitmproxy too. A mode that cannot be
+understood fails `Start Mitm Proxy` rather than leaving the proxy to fail to start for an
+unstated reason. `proxy_auth` requires clients to authenticate before the proxy serves
+them.
+
 ### Why use Mitm?
 Mitm allows manipulation on single browser instance, by using a proxy. It does not
 require you to set up stubs or mocks that might influence the entire application at

tests/test_proxy_integration.py

@@ -100,6 +100,40 @@ def test_port_zero_reports_the_port_the_system_picked(self):
             with self.assertRaises(OSError):
                 sock.bind(("127.0.0.1", address.port))
 
+    def test_an_unrelated_error_during_startup_does_not_fail_the_keyword(self):
+        """A proxy stopped moments ago still logs from its own teardown, and the rest of
+        the test run logs too. Those must not be read as this proxy failing to bind: the
+        collector listens on the root logger, so it hears all of them.
+        """
+        noisy = threading.Thread(target=self._log_errors_briefly)
+        noisy.start()
+        self.addCleanup(noisy.join, 5)
+        self.library.start_mitm_proxy(listen_port=self.port)
+        self.assertEqual(
+            self.library.controller.listen_addresses(), [("127.0.0.1", self.port)]
+        )
+
+    @staticmethod
+    def _log_errors_briefly():
+        """Logs the kind of noise a stopping proxy leaves behind."""
+        deadline = time.monotonic() + 1
+        mitm_logger = logging.getLogger("mitmproxy.addons.something")
+        while time.monotonic() < deadline:
+            mitm_logger.error("Addon error: Event loop is closed")
+            logging.getLogger("asyncio").error("Task was destroyed but it is pending!")
+            time.sleep(0.02)
+
+    def test_a_real_bind_failure_is_still_reported(self):
+        """The noise filter must not swallow the failure it exists to report."""
+        blocker = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        blocker.bind(("127.0.0.1", self.port))
+        blocker.listen(1)
+        self.addCleanup(blocker.close)
+        with self.assertRaises(RuntimeError) as context:
+            self.library.start_mitm_proxy(listen_port=self.port)
+        self.assertIn("failed to listen", str(context.exception))
+
     def test_an_unrelated_logged_error_does_not_kill_the_proxy(self):
         """mitmproxy's errorcheck addon exits the process when anything logged an error
         while a master starts. It watches the root logger, so the error can come from a
@@ -168,6 +202,46 @@ def _request_through_proxy(proxy_port, url):
             opener.open(url, timeout=5).close()
         except Exception:  # noqa: BLE001 - the answer does not matter, only the record
             pass
+    def test_stopping_removes_the_mitmproxy_log_handler(self):
+        """mitmproxy leaves a root logger handler behind that outlives its own loop.
+
+        Every record logged afterwards is forwarded to a closed event loop, which raises
+        inside logging, and a run that starts several proxies accumulates one handler
+        per proxy. Nothing in the library logs enough to notice; a long suite does.
+        """
+        from mitmproxy import log as mitmproxy_log
+
+        def installed():
+            return [
+                handler
+                for handler in logging.getLogger().handlers
+                if isinstance(handler, mitmproxy_log.MitmLogHandler)
+            ]
+
+        before = len(installed())
+        self.library.start_mitm_proxy(listen_port=self.port)
+        self.assertGreater(len(installed()), before)
+        self.library.stop_mitm_proxy()
+        self.assertEqual(len(installed()), before)
+
+        # Logging after the proxy is gone must not raise into the logging machinery.
+        logging.getLogger("some.other.component").warning("after the proxy stopped")
+
+    def test_starting_several_proxies_does_not_pile_up_log_handlers(self):
+        from mitmproxy import log as mitmproxy_log
+
+        def installed():
+            return [
+                handler
+                for handler in logging.getLogger().handlers
+                if isinstance(handler, mitmproxy_log.MitmLogHandler)
+            ]
+
+        before = len(installed())
+        for _ in range(3):
+            self.library.start_mitm_proxy(listen_port=free_port())
+            self.library.stop_mitm_proxy()
+        self.assertEqual(len(installed()), before)
 
 
 if __name__ == "__main__":