fix: drop dashboard page path from server URL (Froyo widget)

.github/e2e/gateway.py

@@ -144,6 +144,8 @@ def __init__(self, name):
         self.chats = []
         # True while a test wants Froyo's dashboard read to fail (the widget keeps its list).
         self.froyo_down = False
+        # True while a test wants the dashboard to answer 200 with no readable list.
+        self.froyo_bad = False
 
     def now_ts(self):
         # Strictly increasing timestamps, so two notes pushed back to back never tie.
@@ -186,13 +188,20 @@ def froyo_items(many=False):
 
 
 def froyo_dashboard(items):
-    """Like GET /api/members/{slug}/dashboard: the page's values sit in a JSON data island."""
-    island = json.dumps({"fields": {"open_items": items, "items_as_of": "now"}, "agentic": ["open_items"]})
+    """Like GET /api/members/{slug}/dashboard for Froyo's page v4: the values sit in a JSON data
+    island. open_items_live (the sync job's list, an object) is current; the agentic open_items
+    lags behind and carries one stale row, which the widget must never show."""
+    stale = items + [{"n": 99, "title": "E2E stale row", "due": None, "kind": "deadline", "flag": None}]
+    island = json.dumps({
+        "fields": {"open_items": stale, "open_items_live": {"items": items, "as_of": "now"}, "items_as_of": "now"},
+        "agentic": ["open_items", "items_as_of"],
+    })
     island = island.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
     html = ('<!doctype html><html><head><title>Froyo</title></head><body>'
             '<script type="application/json" id="kirocrew-dashboard-data">' + island + '</script>'
             '<div id="open"></div></body></html>')
-    return json.dumps({"state": "live", "template": "e2e", "rendered_html": html})
+    return json.dumps({"instance_version": 4, "dashboard": "e2e", "state": "live", "scripts": True,
+                       "fields": ["items_as_of", "open_items", "open_items_live"], "rendered_html": html})
 
 
 def make_handler(state):
@@ -300,11 +309,13 @@ def do_GET(self):
                 return self.send(200, "ok")
             if p == "/test/froyo":
                 # mode=reset (the made-up list), many (8 items, more than fit), empty,
-                # or down (reads fail, the list is kept; any other mode brings it back up).
+                # down (503) or bad (200, no list): reads fail and the list is kept; any other
+                # mode brings it back up.
                 mode = q.get("mode")
                 with state.lock:
                     state.froyo_down = mode == "down"
-                    if mode != "down":
+                    state.froyo_bad = mode == "bad"
+                    if mode not in ("down", "bad"):
                         state.froyo = [] if mode == "empty" else froyo_items(many=mode == "many")
                         state.chats = []
                 return self.send(200, "ok")
@@ -317,9 +328,11 @@ def do_GET(self):
                 if q.get("member") != FROYO_MEMBER:
                     return self.send(400, json.dumps({"code": "missing_member"}), "application/json")
                 with state.lock:
-                    items, down = list(state.froyo), state.froyo_down
+                    items, down, bad = list(state.froyo), state.froyo_down, state.froyo_bad
                 if down:
                     return self.send(503, json.dumps({"error": "down"}), "application/json")
+                if bad:
+                    return self.send(200, json.dumps({"state": "empty"}), "application/json")
                 return self.send(200, froyo_dashboard(items), "application/json")
             if p == "/api/update/check":
                 if not self.signed_in():

.github/e2e/run.sh

@@ -1103,10 +1103,22 @@ PY
   has_text "E2E test parcel" || fail "a failed read hid the rows"
   ! has_text "Updated" || fail "a failed read still says Updated"
   check "a failed refresh keeps the rows and says 'as of HH:mm'"
+  has_text "· network" || fail "the header does not say why the read failed (503 -> network)"
+  check "the header says why: 'as of HH:mm · network'"
+
+  # A 200 with no readable list is a format problem, and says so.
+  curl -s "$GW/test/froyo?mode=bad" >/dev/null
+  tap_now "Refresh"
+  wait_text "· format" 30
+  has_text "E2E test parcel" || fail "a format failure hid the rows"
+  check "a page with no list says 'as of HH:mm · format' and keeps the rows"
   curl -s "$GW/test/froyo?mode=reset" >/dev/null
   tap_now "Refresh"
   wait_text "5 open · Updated" 30
+  ! has_text "· network" || fail "the reason stayed after a good read"
   check "the next good read says Updated again"
+  ! has_text "E2E stale row" || fail "the lagging open_items field was shown instead of open_items_live"
+  check "the live list (open_items_live) wins over the lagging open_items"
   # Ends on the home screen for the screenshot.
 }
 

README.md

@@ -96,10 +96,10 @@ A home-screen widget, **Froyo · Upcoming**, lists the open items of the crewmat
 - Due within 24 hours is orange. Overdue is red and says **late**.
 - Tap a row to open Froyo's chat in the app.
 - Tap **Done** on a row. The app asks first. Say yes and the row goes, with **Undo** for 5 seconds. Then the app sends `Done: <title>` to Froyo's chat, and Froyo marks it done. Undo after that sends `Undo done: <title>`. With app lock on, Done asks you to unlock first.
-- It refreshes about every 30 minutes, when you open the app, after a Done, and when you tap **⟳** in the header (it shows **Updating…** while it reads). The header says **Updated HH:mm** after a good read. Nothing open shows **Nothing coming up**. When the server cannot be reached, the last list stays, with **as of HH:mm**.
+- It refreshes about every 30 minutes, when you open the app, after a Done, and when you tap **⟳** in the header (it shows **Updating…** while it reads). The header says **Updated HH:mm** after a good read. Nothing open shows **Nothing coming up**. When a read fails, the last list stays, with **as of HH:mm** and why: **sign-in** (open the app to sign in), **network**, **timeout**, **server** or **format** (the page held no list the widget can read).
 - Resize it from 4×1 to 4×4. A taller widget shows more rows at once; the list shows whole rows and scrolls to the rest. Rows are never squeezed or overlapped.
 
-How it works: the app reads Froyo's thread with `POST /api/members/froyo-personal-butler/thread` and its open items from `GET /api/members/froyo-personal-butler/dashboard` (the `open_items` field of Froyo's dashboard), with the app's own login cookie. Done is a chat message sent with `POST /api/chat`, the same call the dashboard's chat box makes.
+How it works: the app reads Froyo's thread with `POST /api/members/froyo-personal-butler/thread` and its open items from `GET /api/members/froyo-personal-butler/dashboard` (Froyo's `open_items_live` field, kept current by its sync job, else `open_items`), with the app's own login cookie. Done is a chat message sent with `POST /api/chat`, the same call the dashboard's chat box makes.
 
 The widget shows item titles on the home screen, also while app lock is on.
 

app/src/main/java/io/github/rayvader/kirocrew/AlertsClient.kt

@@ -97,14 +97,21 @@ class AlertsClient(private val ctx: Context) {
      * the dashboard's Origin, which the gateway's CSRF check asks for, and [json] as its body.
      * Throws IOException on network errors.
      */
-    fun call(profile: Profile, method: String, path: String, json: String? = null): Response {
-        var res = callOnce(profile, method, path, json)
-        if (res.authRequired && refresh(profile)) res = callOnce(profile, method, path, json)
+    fun call(
+        profile: Profile,
+        method: String,
+        path: String,
+        json: String? = null,
+        readTimeoutMs: Int = READ_TIMEOUT_MS,
+    ): Response {
+        var res = callOnce(profile, method, path, json, readTimeoutMs)
+        if (res.authRequired && refresh(profile)) res = callOnce(profile, method, path, json, readTimeoutMs)
         return res
     }
 
-    private fun callOnce(profile: Profile, method: String, path: String, json: String?): Response {
+    private fun callOnce(profile: Profile, method: String, path: String, json: String?, readTimeoutMs: Int): Response {
         val conn = open(profile, method, path)
+        conn.readTimeout = readTimeoutMs
         try {
             conn.setRequestProperty("Accept", "application/json")
             if (method != "GET") {

app/src/main/java/io/github/rayvader/kirocrew/FroyoWidget.kt

@@ -27,11 +27,14 @@ import androidx.work.Worker
 import androidx.work.WorkerParameters
 import androidx.work.workDataOf
 import io.github.rayvader.kirocrew.core.DueState
+import io.github.rayvader.kirocrew.core.FailReason
 import io.github.rayvader.kirocrew.core.Profile
 import io.github.rayvader.kirocrew.core.SyncNote
 import io.github.rayvader.kirocrew.core.Upcoming
 import io.github.rayvader.kirocrew.core.UpcomingItem
+import android.util.Log
 import java.io.IOException
+import java.net.SocketTimeoutException
 import java.time.LocalDateTime
 import java.time.ZoneId
 import java.util.Locale
@@ -70,6 +73,7 @@ object FroyoWidgets {
     private const val SP_ITEMS_AT = "froyo_items_at"
     private const val SP_ERROR = "froyo_error"
     private const val SP_REFRESHING_AT = "froyo_refreshing_at"
+    private const val SP_REASON = "froyo_fail_reason"
 
     /** The header's refresh button. Only this app can send it: the button's intent is explicit. */
     const val ACTION_REFRESH = "io.github.rayvader.kirocrew.FROYO_REFRESH"
@@ -133,33 +137,69 @@ object FroyoWidgets {
 
     /** Reads Froyo's thread and open items from the active server. Off the main thread. */
     fun fetch(ctx: Context): Error {
-        val profile = profile(ctx) ?: return saveError(ctx, Error.NO_SERVER)
+        val profile = profile(ctx) ?: return saveError(ctx, Error.NO_SERVER, null, "no server")
         val client = AlertsClient(ctx)
         return try {
-            val thread = thread(ctx, client, profile) ?: return saveError(ctx, lastAuth ?: Error.FAILED)
-            val res = client.call(profile, "GET", Upcoming.dashboardPath(thread.member))
-            if (res.authRequired) return saveError(ctx, Error.SIGNED_OUT)
-            if (res.code != 200) return saveError(ctx, Error.FAILED)
-            val items = Upcoming.extractItems(res.body) ?: return saveError(ctx, Error.FAILED)
+            var thread = thread(ctx, client, profile)
+                ?: return failThread(ctx)
+            var res = client.call(profile, "GET", Upcoming.dashboardPath(thread.member), readTimeoutMs = DASHBOARD_READ_TIMEOUT_MS)
+            if (!res.authRequired && !Upcoming.isHtml(res.contentType) && Upcoming.threadMayBeStale(res.code)) {
+                // The remembered member or slot may be gone: ask for the thread again, once.
+                forgetThread(ctx)
+                thread = thread(ctx, client, profile) ?: return failThread(ctx)
+                res = client.call(profile, "GET", Upcoming.dashboardPath(thread.member), readTimeoutMs = DASHBOARD_READ_TIMEOUT_MS)
+            }
+            val items = if (res.code == 200 && !res.authRequired && !Upcoming.isHtml(res.contentType)) Upcoming.extractItems(res.body) else null
+            val reason = Upcoming.failReason(res.code, res.authRequired, items != null, res.contentType)
+            if (reason != null || items == null) {
+                val error = if (reason == FailReason.SIGN_IN) Error.SIGNED_OUT else Error.FAILED
+                return saveError(ctx, error, reason ?: FailReason.FORMAT, "HTTP ${res.code}")
+            }
             sp(ctx).edit {
                 putString(SP_ITEMS, items)
                 putLong(SP_ITEMS_AT, System.currentTimeMillis())
                 putString(SP_ERROR, Error.NONE.name)
+                remove(SP_REASON)
             }
             Error.NONE
+        } catch (e: SocketTimeoutException) {
+            saveError(ctx, Error.FAILED, FailReason.TIMEOUT, e.javaClass.simpleName)
         } catch (e: IOException) {
-            saveError(ctx, Error.FAILED)
+            saveError(ctx, Error.FAILED, FailReason.NETWORK, e.javaClass.simpleName)
         } catch (e: RuntimeException) {
             // CookieManager throws while the WebView package is being updated.
-            saveError(ctx, Error.FAILED)
+            saveError(ctx, Error.FAILED, FailReason.NETWORK, e.javaClass.simpleName)
+        }
+    }
+
+    private fun failThread(ctx: Context): Error {
+        val auth = lastAuth
+        return if (auth == Error.SIGNED_OUT) {
+            saveError(ctx, Error.SIGNED_OUT, FailReason.SIGN_IN, "thread: sign-in")
+        } else {
+            saveError(ctx, Error.FAILED, lastThreadReason ?: FailReason.SERVER, "thread")
+        }
+    }
+
+    private fun forgetThread(ctx: Context) {
+        sp(ctx).edit {
+            remove(SP_SLOT)
+            remove(SP_MEMBER)
+            remove(SP_PROFILE)
         }
     }
 
+    /** The dashboard composes Froyo's whole page per read, which can be slow over a far link. */
+    private const val DASHBOARD_READ_TIMEOUT_MS = 45_000
+
+    @Volatile private var lastThreadReason: FailReason? = null
+
     @Volatile private var lastAuth: Error? = null
 
     /** Froyo's pinned chat, asked once per server and then remembered. */
     fun thread(ctx: Context, client: AlertsClient, profile: Profile): io.github.rayvader.kirocrew.core.FroyoThread? {
         lastAuth = null
+        lastThreadReason = null
         val sp = sp(ctx)
         val slot = sp.getString(SP_SLOT, null)
         val member = sp.getString(SP_MEMBER, null)
@@ -168,8 +208,16 @@ object FroyoWidgets {
         }
         val res = client.call(profile, "POST", Upcoming.THREAD_PATH)
         if (res.authRequired) lastAuth = Error.SIGNED_OUT
-        if (res.code != 200) return null
-        val thread = Upcoming.parseThread(res.body) ?: return null
+        if (res.code != 200 || Upcoming.isHtml(res.contentType)) {
+            lastThreadReason = Upcoming.failReason(res.code, res.authRequired, false, res.contentType)
+            if (Upcoming.isHtml(res.contentType)) lastAuth = null
+            return null
+        }
+        val thread = Upcoming.parseThread(res.body)
+        if (thread == null) {
+            lastThreadReason = FailReason.FORMAT
+            return null
+        }
         sp.edit {
             putString(SP_SLOT, thread.slotKey)
             putString(SP_MEMBER, thread.member)
@@ -188,15 +236,26 @@ object FroyoWidgets {
             remove(SP_MEMBER)
             remove(SP_PROFILE)
             putString(SP_ERROR, Error.SIGNED_OUT.name)
+            putString(SP_REASON, FailReason.SIGN_IN.name)
         }
         render(ctx)
     }
 
-    private fun saveError(ctx: Context, error: Error): Error {
-        sp(ctx).edit { putString(SP_ERROR, error.name) }
+    private fun saveError(ctx: Context, error: Error, reason: FailReason? = null, detail: String = ""): Error {
+        // In logcat too, so a stuck widget can be traced: adb logcat -s FroyoWidget
+        Log.w(TAG, "read failed: ${error.name} ${reason?.word ?: "-"} $detail")
+        sp(ctx).edit {
+            putString(SP_ERROR, error.name)
+            if (reason != null) putString(SP_REASON, reason.name) else remove(SP_REASON)
+        }
         return error
     }
 
+    private const val TAG = "FroyoWidget"
+
+    private fun savedReason(ctx: Context): FailReason? =
+        sp(ctx).getString(SP_REASON, null)?.let { runCatching { FailReason.valueOf(it) }.getOrNull() }
+
     fun hide(ctx: Context, title: String) {
         val hidden = Upcoming.decodeHidden(sp(ctx).getString(SP_HIDDEN, null)).toMutableMap()
         hidden[title] = System.currentTimeMillis()
@@ -260,10 +319,11 @@ object FroyoWidgets {
         // line, so no row is given up for it.
         val open = ctx.getString(R.string.froyo_open, items.size)
         val refreshing = Upcoming.isRefreshing(sp(ctx).getLong(SP_REFRESHING_AT, 0L), System.currentTimeMillis())
-        val line = Upcoming.syncLine(refreshing, error != Error.NONE, loaded, at, ZoneId.systemDefault())
+        val line = Upcoming.syncLine(refreshing, error != Error.NONE, loaded, at, ZoneId.systemDefault(), savedReason(ctx))
         val count = when (line.note) {
             SyncNote.UPDATED -> ctx.getString(R.string.froyo_updated, open, line.time)
-            SyncNote.AS_OF -> ctx.getString(R.string.froyo_as_of, open, line.time)
+            SyncNote.AS_OF -> line.reason?.let { ctx.getString(R.string.froyo_as_of_why, open, line.time, it.word) }
+                ?: ctx.getString(R.string.froyo_as_of, open, line.time)
             SyncNote.UPDATING -> ctx.getString(R.string.froyo_updating_count, open)
             SyncNote.NONE -> open
         }
@@ -278,7 +338,8 @@ object FroyoWidgets {
             loaded -> null
             error == Error.NO_SERVER -> ctx.getString(R.string.froyo_no_server)
             error == Error.SIGNED_OUT -> ctx.getString(R.string.froyo_signed_out)
-            error == Error.FAILED -> ctx.getString(R.string.froyo_failed)
+            error == Error.FAILED -> savedReason(ctx)?.let { ctx.getString(R.string.froyo_failed_why, it.word) }
+                ?: ctx.getString(R.string.froyo_failed)
             else -> ctx.getString(R.string.froyo_loading)
         }
         v.setViewVisibility(R.id.froyo_message, if (message != null) View.VISIBLE else View.GONE)

app/src/main/java/io/github/rayvader/kirocrew/core/Profile.kt

@@ -21,7 +21,10 @@ object ProfileCodec {
             val id = dec(parts[0]) ?: return@mapNotNull null
             val name = dec(parts[1]) ?: return@mapNotNull null
             val url = dec(parts[2]) ?: return@mapNotNull null
-            if (id.isBlank() || url.isBlank()) null else Profile(id, name.ifBlank { url }, url)
+            if (id.isBlank() || url.isBlank()) return@mapNotNull null
+            // An older save may hold a page path ("…/chat/x"); API calls need the server base.
+            val base = UrlTools.serverBase(url)
+            Profile(id, name.ifBlank { base }, base)
         }
     }