Connection handling improvements, measurements for dimensioning

Makefile

@@ -25,7 +25,7 @@ MPY_TARGETS = $(patsubst $(SRC_DIR)/%.py,$(BUILD_DIR)/%.mpy,$(NON_INIT_PY))
 INIT_TARGETS = $(patsubst $(SRC_DIR)/%.py,$(BUILD_DIR)/%.py,$(filter %__init__.py,$(PY_FILES)))
 
 .PHONY: all
-all: clean toolchain static-checkers unit-test build test-unix deploy publish
+all: clean toolchain static-checkers unit-test build test-unix deploy deploy-config tls-cert deploy-cert deploy-example
 
 # ================================================
 # Build
@@ -179,22 +179,23 @@ run-unix: stage-example
 # -----------------------------
 .PHONY: deploy-example
 deploy-example:
-	@echo "Uploading boot.py"
+	@echo "Uploading boot.py, app.py"
 	@mpremote $(DEVICE) soft-reset
 	mpremote $(DEVICE) cp $(EXAMPLE_DIR)/boot.py :boot.py
 	mpremote $(DEVICE) cp $(EXAMPLE_DIR)/app.py :app.py
 
 	@echo "Uploading pyrobusta.env"
 	@if [ -f pyrobusta.env ]; then mpremote $(DEVICE) cp pyrobusta.env :pyrobusta.env; fi
 	@mpremote $(DEVICE) reset
+	@echo "\e[32m$(EXAMPLE_DIR) example is successfully deployed, \n"\
+	"run 'make DEVICE=$(DEVICE) run-device' to restart the device and check the output.\e[0m"
 
 # -----------------------------
-# Run example directly
+# Connect to device through REPL
 # -----------------------------
 .PHONY: run-device
 run-device:
-	@mpremote $(DEVICE) soft-reset
-	mpremote $(DEVICE) run $(EXAMPLE_DIR)/app.py
+	@mpremote $(DEVICE) reset repl
 
 
 # ================================================

README.md

@@ -60,8 +60,8 @@ make build              # Cross-compile, create build artifacts
 make deploy             # Upload build artifacts to device using mpremote
 make tls-cert           # Optional: generate self-signed certificate for the device
 make deploy-cert        # Optional: upload generated certificate to the device
-make deploy-example     # Upload example application using mpremote
-make run-device         # Run application on the device using mpremote run
+make deploy-example     # Deploy the selected example app using mpremote
+make run-device         # Optional: Reset the device and connect through REPL
 ```
 ```deploy-example``` and ```run-device``` uses the DEVICE argument
 set to ```u0``` (/dev/ttyUSB0) by default, passed to mpremote.

assets/www/examples.html

@@ -103,14 +103,14 @@ <h2>Simple Server Application</h2>
         logging.warning(f"loop stopped: {e}")
         asyncio.get_event_loop().close()
     </textarea>
-    <textarea readonly="true" rows="13">
+    <textarea readonly="true" rows="15">
 # /boot.py
 
 # This file is executed on every boot (including wake-boot from deepsleep)
 import machine
 from os import listdir
 
-from pyrobusta.con import wifi
+from pyrobusta.connectivity import wifi
 
 connected = wifi.initialize()
 if connected and not machine.reset_cause() == machine.SOFT_RESET:

dist/pyrobusta/assets/www/examples.html

@@ -108,7 +108,7 @@ <h2>Simple Server Application</h2>
 import machine
 from os import listdir
 
-from pyrobusta.con import wifi
+from pyrobusta.connectivity import wifi
 
 connected = wifi.initialize()
 if connected and not machine.reset_cause() == machine.SOFT_RESET:

docs/dimensioning/http_dimensioning.md

@@ -0,0 +1,71 @@
+Use the following measurement data to guide configuration choices when dimensioning the\
+HTTP server for specific constraints such as memory footprint, request throughput, and\
+feature enablement (e.g., TLS, multipart handling, file serving).
+
+The tables below are derived from controlled benchmarks. Each measurement varies a subset\
+of parameters relative to a defined baseline configuration.
+
+```.env
+# Base configuration
+socket_max_con=1
+http_mem_cap=0.05
+http_multipart=False
+http_serve_files=True
+tls=False
+http_port=8080
+https_port=4443
+log_level=info
+http_served_paths=/lib/pyrobusta /www
+```
+
+# ESP32-C3 "SuperMini" (ESP32-C3FH4)
+The ESP32-C3 provides approximately 162KB of usable heap. It is recommended to limit the maximum\
+number of socket connections to 2 (socket_max_con).
+
+## Idle heap usage
+The table below reports heap consumption after module imports, measured under idle conditions\
+with no active network traffic.
+
+| id | http_mem_cap | http_multipart | socket_max_con | tls | footprint_bytes |
+| --- | --- | --- | --- | --- | --- |
+| base | 0.05 | False | 1 | False | 38448 |
+| low_mem_cap_001 | 0.0127 | False | 1 | False | 38448 |
+| low_mem_cap_002 | 0.0253 | False | 2 | False | 39664 |
+| low_mem_cap_003 | 0.0505 | False | 4 | False | 42096 |
+| high_mem_cap_001 | 0.0568 | False | 1 | False | 45616 |
+| high_mem_cap_002 | 0.114 | False | 2 | False | 54000 |
+| high_mem_cap_003 | 0.228 | False | 4 | False | 70768 |
+| multipart_001 | 0.0127 | True | 1 | False | 40704 |
+| multipart_002 | 0.0253 | True | 2 | False | 41920 |
+| multipart_003 | 0.0505 | True | 4 | False | 44352 |
+| tls_001 | 0.0127 | False | 1 | True | 41120 |
+| tls_002 | 0.0253 | False | 2 | True | 42336 |
+| tls_003 | 0.0505 | False | 4 | True | 44768 |
+
+## Heap usage under network traffic
+![image info](./img/esp32_c3/base.png)
+
+
+# ESP32-S3 (8MB PSRAM)
+The table below reports heap consumption after module imports, measured under idle conditions\
+with no active network traffic.
+
+## Idle heap usage
+| id | http_mem_cap | http_multipart | socket_max_con | tls | footprint_bytes |
+| --- | --- | --- | --- | --- | --- |
+| base | 0.05 | False | 1 | False | 45520 |
+| low_mem_cap_001 | 0.000247 | False | 1 | False | 38352 |
+| low_mem_cap_002 | 0.000493 | False | 2 | False | 39568 |
+| low_mem_cap_003 | 0.000985 | False | 4 | False | 42000 |
+| high_mem_cap_001 | 0.00111 | False | 1 | False | 45520 |
+| high_mem_cap_002 | 0.00222 | False | 2 | False | 53904 |
+| high_mem_cap_003 | 0.00443 | False | 4 | False | 70672 |
+| multipart_001 | 0.000247 | True | 1 | False | 40656 |
+| multipart_002 | 0.000493 | True | 2 | False | 41872 |
+| multipart_003 | 0.000985 | True | 4 | False | 44304 |
+| tls_001 | 0.000247 | False | 1 | True | 40736 |
+| tls_002 | 0.000493 | False | 2 | True | 41952 |
+| tls_003 | 0.000985 | False | 4 | True | 44384 |
+
+## Heap usage under network traffic
+![image info](./img/esp32_s3/base.png)

example/demo_app/app.py

@@ -3,7 +3,6 @@
 
 from pyrobusta.server import http_server
 from pyrobusta.protocol.http import HttpEngine
-from pyrobusta.utils import logging
 
 
 @HttpEngine.route("/app", "GET")
@@ -24,10 +23,8 @@ def app(http_ctx, payload):
     return "text/plain", (f"Free memory [{value_format}]: {free}\n")
 
 
-def main():
-    http_server.main()
-    try:
-        asyncio.get_event_loop().run_forever()
-    except Exception as e:
-        logging.warning(f"loop stopped: {e}")
-        asyncio.get_event_loop().close()
+async def main():
+    server = http_server.HttpServer()
+    asyncio.create_task(server.start_socket_server())
+    while True:
+        await asyncio.sleep(1)

example/demo_app/boot.py

@@ -1,12 +1,13 @@
 # This file is executed on every boot (including wake-boot from deepsleep)
+import asyncio
 import machine
 from os import listdir
 
-from pyrobusta.con import wifi
+from pyrobusta.connectivity import wifi
 
 connected = wifi.initialize()
 if connected and not machine.reset_cause() == machine.SOFT_RESET:
     if "app.py" in listdir():
         import app
 
-        app.main()
+        asyncio.run(app.main())

example/mem_usage/app.py

@@ -3,7 +3,6 @@
 
 import pyrobusta.server.http_server as http_server
 from pyrobusta.protocol.http import HttpEngine
-from pyrobusta.utils import logging
 
 
 @HttpEngine.route("/mem-usage", "GET")
@@ -23,11 +22,11 @@ def mem_usage(http_ctx, _):
         selector = http_ctx.get_url_encoded_query_param(http_ctx.query, "key", "")
         if selector == "free":
             if value_format == "%":
-                free = round(100 * free / (used + free),2)
+                free = round(100 * free / (used + free), 2)
             return "text/plain", f"Free   [{value_format}]: {free}\n"
         if selector == "used":
             if value_format == "%":
-                used = round(100 * used / (used + free),2)
+                used = round(100 * used / (used + free), 2)
             return "text/plain", f"Used   [{value_format}]: {used}\n"
         if selector == "total":
             return "text/plain", f"Total  [bytes]: {used + free}\n"
@@ -43,10 +42,8 @@ def mem_usage(http_ctx, _):
     )
 
 
-http_server.main()
-
-try:
-    asyncio.get_event_loop().run_forever()
-except Exception as e:
-    logging.warning(f"[asyncio] loop stopped: {e}")
-    asyncio.get_event_loop().close()
+async def main():
+    server = http_server.HttpServer()
+    asyncio.create_task(server.start_socket_server())
+    while True:
+        await asyncio.sleep(1)

example/mem_usage/boot.py

@@ -1,12 +1,13 @@
 # This file is executed on every boot (including wake-boot from deepsleep)
+import asyncio
 import machine
 from os import listdir
 
-from pyrobusta.con import wifi
+from pyrobusta.connectivity import wifi
 
 connected = wifi.initialize()
 if connected and not machine.reset_cause() == machine.SOFT_RESET:
     if "app.py" in listdir():
         import app
 
-        app.main()
+        asyncio.run(app.main())

example/mip_repo/app.py

@@ -1,5 +1,4 @@
 import asyncio
-import os
 
 import pyrobusta.server.http_server as http_server
 from pyrobusta.protocol.http import HttpEngine
@@ -42,10 +41,8 @@ def self_serve_mip_package(http_ctx, _):
     return "application/json", package_files
 
 
-http_server.main()
-
-try:
-    asyncio.get_event_loop().run_forever()
-except Exception as e:
-    logging.warning(f"[asyncio] loop stopped: {e}")
-    asyncio.get_event_loop().close()
+async def main():
+    server = http_server.HttpServer()
+    asyncio.create_task(server.start_socket_server())
+    while True:
+        await asyncio.sleep(1)

example/mip_repo/boot.py

@@ -1,12 +1,13 @@
 # This file is executed on every boot (including wake-boot from deepsleep)
+import asyncio
 import machine
 from os import listdir
 
-from pyrobusta.con import wifi
+from pyrobusta.connectivity import wifi
 
 connected = wifi.initialize()
 if connected and not machine.reset_cause() == machine.SOFT_RESET:
     if "app.py" in listdir():
         import app
 
-        app.main()
+        asyncio.run(app.main())

requirements.txt

@@ -1,3 +1,5 @@
 mpremote==1.27.0
 pylint
 black
+requests
+matplotlib

scripts/clean_device.py

@@ -1,18 +1,28 @@
-import uos
+from os import listdir, remove, rmdir
 
 
-def delete_all(path):
-    try:
-        for name in uos.listdir(path):
+def delete_path(path):
+    for name in listdir(path):
+        if path == "/":
+            full = "/" + name
+        else:
             full = path + "/" + name
+
+        try:
+            remove(full)
+        except OSError:
+            delete_path(full)
             try:
-                uos.remove(full)
+                rmdir(full)
             except OSError:
-                delete_all(full)
-                uos.rmdir(full)
-        uos.rmdir(path)
-    except OSError:
-        pass
+                pass
 
 
-delete_all("/pyrobusta")
+delete_path("/lib/pyrobusta")
+delete_path("/www")
+
+for f in ("/app.py", "/boot.py", "/pyrobusta.env", "/cert.der", "/key.der"):
+    try:
+        remove(f)
+    except OSError:
+        pass