Keep-alive connection handling, refactor state machine parser

Makefile

@@ -101,7 +101,7 @@ deploy-config:
 
 
 # -----------------------------
-# Deploy index page # TODO use install_www from assets module
+# Deploy index page
 # -----------------------------
 .PHONY: deploy-www
 deploy-www:

README.md

@@ -11,6 +11,7 @@ A lightweight HTTP server library for MicroPython designed for constrained embed
 - Finite-state-machine parser with linear sliding buffer
 - Robust byte-stream handling
 - Query parameter parsing with percent encoding support
+- Persistent connections (set by **connection: keep-alive**)
 - TLS support
 
 # Installation

assets/www/examples.html

@@ -92,7 +92,7 @@ <h2>Simple Server Application</h2>
         if value_format == "%":
             free = round(100 * free / (free + mem_alloc()), 2)
 
-    return "text/plain", (f"Free memory [{value_format}]: {free}\n")
+    return "text/plain", f"Free memory [{value_format}]: {free}\n"
 
 async def run_server():
     server = HttpServer()

docs/dimensioning/http_dimensioning.md

@@ -28,19 +28,19 @@ with no active network traffic.
 
 | id | http_mem_cap | http_multipart | socket_max_con | tls | footprint_bytes |
 | --- | --- | --- | --- | --- | --- |
-| base | 0.05 | False | 1 | False | 38448 |
-| low_mem_cap_001 | 0.0127 | False | 1 | False | 38448 |
-| low_mem_cap_002 | 0.0253 | False | 2 | False | 39664 |
-| low_mem_cap_003 | 0.0505 | False | 4 | False | 42096 |
-| high_mem_cap_001 | 0.0568 | False | 1 | False | 45616 |
-| high_mem_cap_002 | 0.114 | False | 2 | False | 54000 |
-| high_mem_cap_003 | 0.228 | False | 4 | False | 70768 |
-| multipart_001 | 0.0127 | True | 1 | False | 40704 |
-| multipart_002 | 0.0253 | True | 2 | False | 41920 |
-| multipart_003 | 0.0505 | True | 4 | False | 44352 |
-| tls_001 | 0.0127 | False | 1 | True | 41120 |
-| tls_002 | 0.0253 | False | 2 | True | 42336 |
-| tls_003 | 0.0505 | False | 4 | True | 44768 |
+| base | 0.05 | False | 1 | False | 38512 |
+| low_mem_cap_001 | 0.0127 | False | 1 | False | 38512 |
+| low_mem_cap_002 | 0.0253 | False | 2 | False | 39728 |
+| low_mem_cap_003 | 0.0505 | False | 4 | False | 42112 |
+| high_mem_cap_001 | 0.0568 | False | 1 | False | 45680 |
+| high_mem_cap_002 | 0.114 | False | 2 | False | 54064 |
+| high_mem_cap_003 | 0.228 | False | 4 | False | 70832 |
+| multipart_001 | 0.0127 | True | 1 | False | 40608 |
+| multipart_002 | 0.0253 | True | 2 | False | 41824 |
+| multipart_003 | 0.0505 | True | 4 | False | 44256 |
+| tls_001 | 0.0127 | False | 1 | True | 41184 |
+| tls_002 | 0.0253 | False | 2 | True | 42400 |
+| tls_003 | 0.0505 | False | 4 | True | 44832 |
 
 ## Heap usage under network traffic
 ![image info](./img/esp32_c3/base.png)
@@ -60,9 +60,9 @@ with no active network traffic.
 | high_mem_cap_001 | 0.00111 | False | 1 | False | 45520 |
 | high_mem_cap_002 | 0.00222 | False | 2 | False | 53904 |
 | high_mem_cap_003 | 0.00443 | False | 4 | False | 70672 |
-| multipart_001 | 0.000247 | True | 1 | False | 40656 |
-| multipart_002 | 0.000493 | True | 2 | False | 41872 |
-| multipart_003 | 0.000985 | True | 4 | False | 44304 |
+| multipart_001 | 0.000247 | True | 1 | False | 40528 |
+| multipart_002 | 0.000493 | True | 2 | False | 41744 |
+| multipart_003 | 0.000985 | True | 4 | False | 44176 |
 | tls_001 | 0.000247 | False | 1 | True | 40736 |
 | tls_002 | 0.000493 | False | 2 | True | 41952 |
 | tls_003 | 0.000985 | False | 4 | True | 44384 |

example/demo_app/app.py

@@ -20,7 +20,7 @@ def app(http_ctx, payload):
         if value_format == "%":
             free = round(100 * free / (free + mem_alloc()), 2)
 
-    return "text/plain", (f"Free memory [{value_format}]: {free}\n")
+    return "text/plain", f"Free memory [{value_format}]: {free}\n"
 
 
 async def main():

src/pyrobusta/bindings/http_connection.py

@@ -7,7 +7,7 @@
 
 from ..stream.buffer import BufferFullError
 from ..transport.connection import BaseConnection
-from ..protocol.http import HttpEngine, ServerBusyError, HeaderParsingError
+from ..protocol.http import HttpEngine
 from ..utils import logging
 
 
@@ -19,7 +19,6 @@ class HttpConnection(BaseConnection):
 
     MTU_SIZE = 1460
     STATE_MACHINE_SLEEP_MS = 2
-    RESP_HANDLER_SLEEP_MS = 2
     RECV_TIMEOUT_SECONDS = 10
 
     __slots__ = ("_engine", "_prev_state", "_recv_buf", "_send_buf")
@@ -36,9 +35,12 @@ async def run(self):
         Handle socket connection with HTTP state machine parser.
         """
         self._prev_state = None
-        while self._engine.state is not None:
+        while not self._engine.is_terminated():
             await self._run_state_machine()
             await sleep_ms(self.STATE_MACHINE_SLEEP_MS)
+            if self._engine.is_terminated() and self._engine.do_keep_alive():
+                self._engine.reset()
+                self._prev_state = None
 
     async def _flush_response(self):
         data = self._send_buf.peek()
@@ -49,90 +51,69 @@ async def _flush_response(self):
     async def _read_to_buf(self):
         buf_free = self._recv_buf.capacity - self._recv_buf.size()
         if not buf_free:
-            self._engine.on_buffer_full(self._send_buf)
-            await self._flush_response()
-            return 0
-        try:
-            request = await self.read(
-                read_bytes=buf_free,
-                decoding=None,
-                timeout_seconds=self.RECV_TIMEOUT_SECONDS,
-            )
-        except asyncio.TimeoutError:
-            self._engine.on_timeout(self._send_buf)
-            await self._flush_response()
-            return 0
-        except Exception as e:  # pylint: disable=W0718
-            self._engine.on_failure(
-                self._send_buf, b"Read error: " + str(e).encode("ascii")
-            )
-            await self._flush_response()
-            return 0
+            raise BufferFullError()
+        request = await self.read(
+            read_bytes=buf_free,
+            decoding=None,
+            timeout_seconds=self.RECV_TIMEOUT_SECONDS,
+        )
         self._recv_buf.write(request)
         logging.debug(__name__ + f"._read_to_buf: [{request}]")
         return len(request)
 
     async def _run_state_machine(self):
-        if self._prev_state == self._engine.state or self._prev_state is None:
-            num_read = await self._read_to_buf()
-            if not num_read:
-                # Reject incomplete request
-                self._engine.on_client_error(
-                    self._send_buf, self._engine.BAD_REQUEST_ERROR
-                )
-                await self._flush_response()
-                return
-        try:
-            resp_handler = None
-            while self._engine.state is not None:
-                self._prev_state = self._engine.state
-                resp_handler = self._engine.state(self._recv_buf, self._send_buf)
-                if not self._send_buf.size():
-                    break
-                await self._flush_response()
-                await sleep_ms(self.STATE_MACHINE_SLEEP_MS)
-        except BufferFullError:
-            self._engine.on_failure(self._send_buf, b"Buffer full")
-            await self._flush_response()
-            return
-        except ServerBusyError:
-            self._engine.on_unavailable(self._send_buf)
-            await self._flush_response()
-            return
-        except HeaderParsingError:
-            self._engine.on_client_error(self._send_buf, self._engine.HEADER_ERROR)
-            await self._flush_response()
-            return
-        except Exception as e:  # pylint: disable=W0718
-            logging.warning(__name__ + f"._run_state_machine: {e}")
-            self._engine.on_failure(self._send_buf, str(e).encode("ascii"))
+        # [1] read request
+        if self._prev_state == self._engine.state or (
+            self._prev_state is None and not self._recv_buf.size()
+        ):
+            try:
+                num_read = await self._read_to_buf()
+                if not num_read:
+                    self._engine.abort(400)
+                    self._engine.set_response_body(b"Incomplete request")
+            except BufferFullError:
+                self._engine.abort(413)
+            except asyncio.TimeoutError:
+                self._engine.abort(408)
+            except Exception as e:  # pylint: disable=W0718
+                self._engine.abort(500)
+                self._engine.set_response_body(b"Read error: " + str(e).encode("ascii"))
+
+        # [2] process request by state machine
+        for _ in self._engine.run(self._recv_buf):
+            if self._prev_state == self._engine.state:
+                # No state transition occurred, read more data
+                break
+            self._prev_state = self._engine.state
+            await sleep_ms(self.STATE_MACHINE_SLEEP_MS)
+
+        # [3] write response
+        if self._engine.is_started() and self._engine.is_terminated():
+            self._engine.write_response_head(self._send_buf)
             await self._flush_response()
-            return
-        if self._engine.state is None and resp_handler is not None:
-            await self._response_handler(resp_handler)
+            if self._engine.resp_handler is not None:
+                await self._response_handler(self._engine.resp_handler)
 
     async def _response_handler(self, resp_handler):
         if "closure" == type(resp_handler).__name__:
             for is_finished in resp_handler(self._send_buf):
                 await self._flush_response()
                 if is_finished:
                     break
-                await sleep_ms(self.RESP_HANDLER_SLEEP_MS)
+                await sleep_ms(self.STATE_MACHINE_SLEEP_MS)
         elif type(resp_handler).__name__ in ("FileIO", "BytesIO"):
-            with resp_handler as rh:
+            try:
                 while True:
                     view = self._send_buf.writable_view()
-                    num_read = rh.readinto(view)
+                    num_read = resp_handler.readinto(view)
                     if not num_read:
                         break
                     self._send_buf.commit(num_read)
                     await self._flush_response()
-                    await sleep_ms(self.RESP_HANDLER_SLEEP_MS)
+                    await sleep_ms(self.STATE_MACHINE_SLEEP_MS)
+            finally:
+                resp_handler.close()
         else:
-            self._engine.on_failure(
-                self._send_buf,
-                f"Invalid response handler {type(resp_handler).__name__}".encode(
-                    "ascii"
-                ),
+            raise RuntimeError(
+                f"Invalid response handler {type(resp_handler).__name__}"
             )
-            await self._flush_response()

src/pyrobusta/connectivity/wifi.py

@@ -1,5 +1,5 @@
 """
-Helpers for setting up Wi-Fi in station mode
+Helpers for setting up Wi-Fi in station mode.
 """
 
 from time import sleep
@@ -12,7 +12,7 @@
 
 def initialize():
     """
-    Initialize WLAN interface in station mode
+    Initialize WLAN station interface.
     """
     ssid = get_config(CONF_WIFI_SSID)
     password = get_config(CONF_WIFI_PASSWORD)
@@ -44,7 +44,7 @@ def initialize():
 
 def get_address():
     """
-    Get the address of the WLAN interface
+    Get the IP address of the WLAN station.
     """
     sta_if = WLAN(STA_IF)
     if sta_if.isconnected():