diff --git a/AGENTS.md b/AGENTS.md index 98c1af5..a91cd7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,19 @@ RAILS_ENV=test bin/rails app:db:migrate # one-time, after adding a migration After this, `schema.rb` is updated and subsequent runs of `db:schema:load` will pick up the new tables. Commit `schema.rb` alongside the migration file. + +### Troubleshooting `ActiveRecord::DatabaseAlreadyExists` + +If `bin/rails test` (or `db:test:prepare`) fails with +`ActiveRecord::DatabaseAlreadyExists` in `SQLiteDatabaseTasks#purge`, there +are stray sqlite files in the **engine-root** `storage/`. The dummy's +databases live in `test/dummy/storage/`; `purge` drops the file relative to +the app root but then `create` checks `File.exist?` relative to the current +directory, so a leftover `storage/*.sqlite3` in the engine root makes it +think the database already exists. This happens after running a db task from +the wrong place. Clear the strays and reset: + +```bash +rm -f storage/*.sqlite3* +RAILS_ENV=test bin/rails db:drop db:create db:schema:load +``` diff --git a/app/assets/stylesheets/upright/incidents.css b/app/assets/stylesheets/upright/incidents.css new file mode 100644 index 0000000..f54d53a --- /dev/null +++ b/app/assets/stylesheets/upright/incidents.css @@ -0,0 +1,598 @@ +@layer components { + :root { + --impact-minor: oklch(78% 0.18 95); + --impact-major: oklch(70% 0.18 50); + --impact-critical: oklch(60% 0.21 28); + --impact-maintenance: oklch(64% 0.14 250); + } + + .incident-editor { + --accent: var(--color-ink-dark); + margin: 0 auto; + max-width: 42rem; + padding: calc(var(--block-space) * 2) var(--main-padding) calc(var(--block-space) * 4); + } + + .incident-editor--minor { --accent: var(--impact-minor); } + .incident-editor--major { --accent: var(--impact-major); } + .incident-editor--critical { --accent: var(--impact-critical); } + .incident-editor--maintenance { --accent: var(--impact-maintenance); } + + .incident-editor__back { + color: var(--color-ink-medium); + font-family: var(--font-mono); + font-size: var(--text-x-small); + letter-spacing: 0.04em; + text-transform: uppercase; + } + + .incident-form { + --accent: var(--color-ink-dark); + background: var(--color-canvas); + border: var(--border); + border-radius: 0.75rem; + box-shadow: var(--shadow); + margin-top: var(--block-space); + overflow: hidden; + position: relative; + } + + .incident-form:has(#impact_minor:checked) { --accent: var(--impact-minor); } + .incident-form:has(#impact_major:checked) { --accent: var(--impact-major); } + .incident-form:has(#impact_critical:checked) { --accent: var(--impact-critical); } + .incident-form:has(#kind_maintenance:checked), + .incident-form--maintenance { --accent: var(--impact-maintenance); } + + .incident-form__spine { + background: var(--accent); + inset: 0 auto 0 0; + position: absolute; + transition: background 200ms ease-out; + width: 4px; + } + + .incident-form__body { + display: flex; + flex-direction: column; + gap: calc(var(--block-space) * 1.5); + padding: calc(var(--block-space) * 2); + padding-left: calc(var(--block-space) * 2 + 4px); + } + + .incident-form__head { + display: flex; + flex-direction: column; + gap: 0.35rem; + } + + .incident-form__eyebrow { + align-items: center; + color: var(--accent); + display: flex; + font-family: var(--font-mono); + font-size: var(--text-x-small); + gap: 0.5rem; + letter-spacing: 0.12em; + margin: 0; + text-transform: uppercase; + transition: color 200ms ease-out; + } + + .incident-form__eyebrow::before { + background: var(--accent); + border-radius: 50%; + content: ""; + height: 0.5rem; + transition: background 200ms ease-out; + width: 0.5rem; + } + + .incident-form__title { + color: var(--color-ink); + font-size: var(--text-large); + font-weight: 600; + margin: 0; + } + + .incident-errors { + background: color-mix(in oklab, var(--impact-critical) 12%, var(--color-canvas)); + border: 1px solid color-mix(in oklab, var(--impact-critical) 40%, var(--color-canvas)); + border-radius: 0.5rem; + color: var(--color-ink); + font-size: var(--text-small); + list-style: none; + margin: 0; + padding: 0.75rem 1rem; + } + + .field { + display: flex; + flex-direction: column; + gap: 0.4rem; + } + + .field__label { + color: var(--color-ink-dark); + font-family: var(--font-mono); + font-size: var(--text-x-small); + font-weight: 500; + letter-spacing: 0.08em; + text-transform: uppercase; + } + + .field__hint { + color: var(--color-ink-medium); + font-size: var(--text-small); + } + + .field__input { + background: var(--color-canvas); + border: 1px solid var(--color-ink-light); + border-radius: 0.5rem; + color: var(--color-ink); + color-scheme: light dark; + font: inherit; + padding: 0.6em 0.75em; + width: 100%; + } + + .field__input:hover { border-color: var(--color-ink-dark); } + + .field__input:focus-visible { + border-color: var(--accent); + outline: var(--focus-ring-size) solid var(--accent); + outline-offset: 1px; + } + + .field__input::-webkit-calendar-picker-indicator { + cursor: pointer; + opacity: 0.55; + } + + .field__input::-webkit-calendar-picker-indicator:hover { + opacity: 1; + } + + textarea.field__input { + min-height: 6rem; + resize: vertical; + } + + .incident-form:has(#kind_maintenance:checked) .field--incident, + .incident-form--maintenance .field--incident { display: none; } + + .incident-form:has(#kind_incident:checked) .field--maintenance, + .incident-form--incident .field--maintenance { display: none; } + + .segmented { + background: var(--color-ink-lightest); + border: var(--border); + border-radius: 0.6rem; + display: grid; + gap: 2px; + grid-auto-flow: column; + grid-auto-columns: 1fr; + padding: 3px; + } + + .segmented input { + position: absolute; + opacity: 0; + pointer-events: none; + } + + .segmented label { + border-radius: 0.45rem; + color: var(--color-ink-dark); + cursor: pointer; + font-size: var(--text-small); + font-weight: 500; + padding: 0.5rem 0.75rem; + text-align: center; + transition: background 120ms ease-out, color 120ms ease-out; + } + + .segmented label:hover { color: var(--color-ink); } + + .segmented input:checked + label { + background: var(--color-canvas); + box-shadow: var(--shadow); + color: var(--color-ink); + } + + .segmented input:focus-visible + label { + outline: var(--focus-ring-size) solid var(--accent); + outline-offset: 1px; + } + + .segmented input#kind_maintenance:checked + label { color: var(--impact-maintenance); } + .segmented input#kind_incident:checked + label { color: var(--color-ink); } + + .impact-picker { + display: grid; + gap: 0.5rem; + grid-template-columns: repeat(3, 1fr); + } + + .impact { + align-items: center; + background: var(--color-canvas); + border: 1px solid var(--color-ink-light); + border-radius: 0.5rem; + cursor: pointer; + display: flex; + gap: 0.5rem; + padding: 0.6rem 0.75rem; + transition: border-color 120ms ease-out, background 120ms ease-out; + } + + .impact__dot { + border-radius: 50%; + box-shadow: inset 0 0 0 2px var(--color-canvas), 0 0 0 1px currentColor; + height: 0.7rem; + width: 0.7rem; + } + + .impact--minor { color: var(--impact-minor); } + .impact--major { color: var(--impact-major); } + .impact--critical { color: var(--impact-critical); } + + .impact__name { + color: var(--color-ink-dark); + font-size: var(--text-small); + font-weight: 500; + } + + .impact input { position: absolute; opacity: 0; pointer-events: none; } + + .impact:hover { border-color: currentColor; } + + .impact:has(:checked) { + background: color-mix(in oklab, currentColor 10%, var(--color-canvas)); + border-color: currentColor; + } + + .impact:has(:checked) .impact__dot { background: currentColor; } + + .impact:has(:checked) .impact__name { color: var(--color-ink); } + + .impact:has(:focus-visible) { + outline: var(--focus-ring-size) solid currentColor; + outline-offset: 1px; + } + + .chips { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + } + + .chip { + align-items: center; + background: var(--color-canvas); + border: 1px solid var(--color-ink-light); + border-radius: 2rem; + color: var(--color-ink-dark); + cursor: pointer; + display: inline-flex; + font-size: var(--text-small); + gap: 0.4rem; + padding: 0.4rem 0.85rem; + transition: border-color 120ms ease-out, background 120ms ease-out, color 120ms ease-out; + } + + .chip input { position: absolute; opacity: 0; pointer-events: none; } + + .chip::before { + border: 1.5px solid var(--color-ink-light); + border-radius: 50%; + content: ""; + height: 0.7rem; + width: 0.7rem; + } + + .chip:hover { border-color: var(--color-ink-dark); } + + .chip:has(:checked) { + background: var(--color-selected); + border-color: var(--accent); + color: var(--color-ink); + } + + .chip:has(:checked)::before { + background: var(--accent); + border-color: var(--accent); + box-shadow: inset 0 0 0 2px var(--color-selected); + } + + .chip:has(:focus-visible) { + outline: var(--focus-ring-size) solid var(--accent); + outline-offset: 1px; + } + + .incident-form__actions { + align-items: center; + border-top: var(--border); + display: flex; + gap: 1rem; + justify-content: flex-end; + margin-top: 0.5rem; + padding-top: calc(var(--block-space) * 1.25); + } + + .incident-form__cancel { + color: var(--color-ink-medium); + font-size: var(--text-small); + } + + .lifecycle { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; + list-style: none; + margin: 0 0 calc(var(--block-space) * 1.5); + padding: 0; + } + + .lifecycle__step { + align-items: center; + color: var(--color-ink-medium); + display: flex; + font-family: var(--font-mono); + font-size: var(--text-x-small); + gap: 0.4rem; + letter-spacing: 0.06em; + text-transform: uppercase; + } + + .lifecycle__step::after { + color: var(--color-ink-light); + content: "→"; + margin-left: 0.4rem; + } + + .lifecycle__step:last-child::after { content: ""; } + + .lifecycle__step--done { color: var(--color-ink-dark); } + + .lifecycle__step--current { + color: var(--accent); + font-weight: 600; + } + + .incident-section { + margin-top: calc(var(--block-space) * 2.5); + } + + .incident-section__title { + color: var(--color-ink-dark); + font-family: var(--font-mono); + font-size: var(--text-x-small); + letter-spacing: 0.1em; + margin: 0 0 var(--block-space); + text-transform: uppercase; + } + + .composer { + background: var(--color-canvas); + border: var(--border); + border-radius: 0.75rem; + display: flex; + flex-direction: column; + gap: var(--block-space); + padding: calc(var(--block-space) * 1.5); + } + + .timeline { + display: flex; + flex-direction: column; + gap: 0; + list-style: none; + margin: 0; + padding: 0; + } + + .timeline__item { + border-left: 2px solid var(--color-ink-lighter); + padding: 0 0 calc(var(--block-space) * 1.25) calc(var(--block-space) * 1.25); + position: relative; + } + + .timeline__item:last-child { border-left-color: transparent; padding-bottom: 0; } + + .timeline__item::before { + background: var(--color-canvas); + border: 2px solid var(--accent); + border-radius: 50%; + content: ""; + height: 0.7rem; + left: calc(-0.35rem - 1px); + position: absolute; + top: 0.15rem; + width: 0.7rem; + } + + .timeline__status { + color: var(--color-ink); + font-family: var(--font-mono); + font-size: var(--text-x-small); + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + } + + .timeline__time { + color: var(--color-ink-medium); + font-family: var(--font-mono); + font-size: var(--text-x-small); + margin-left: 0.5rem; + } + + .timeline__body { + color: var(--color-ink-dark); + font-size: var(--text-small); + margin: 0.3rem 0 0; + } + + .incidents-index { + margin: 0 auto; + max-width: 48rem; + padding: calc(var(--block-space) * 2) var(--main-padding) calc(var(--block-space) * 4); + } + + .incidents-header { + align-items: baseline; + display: flex; + flex-wrap: wrap; + gap: 1rem; + justify-content: space-between; + margin-bottom: calc(var(--block-space) * 2); + } + + .incidents-header__title { + font-size: var(--text-x-large); + font-weight: 600; + letter-spacing: -0.02em; + margin: 0; + } + + .incidents-actions { + display: flex; + gap: 0.5rem; + } + + .btn--ghost { + background: transparent; + border: var(--border); + color: var(--color-ink-dark); + } + + .btn--ghost:hover { + background: var(--color-ink-lightest); + color: var(--color-ink); + } + + .incidents-empty { + border: 1px dashed var(--color-ink-light); + border-radius: 0.75rem; + padding: calc(var(--block-space) * 3); + text-align: center; + } + + .incidents-empty__headline { + color: var(--color-ink); + font-size: var(--text-large); + font-weight: 600; + margin: 0 0 0.4rem; + } + + .incidents-empty__body { + color: var(--color-ink-medium); + font-size: var(--text-small); + margin: 0 auto; + max-width: 28rem; + } + + .incident-list { + display: flex; + flex-direction: column; + gap: 0.5rem; + } + + .incident-row { + --accent: var(--color-ink-dark); + background: var(--color-canvas); + border: var(--border); + border-radius: 0.6rem; + display: flex; + overflow: hidden; + position: relative; + text-decoration: none; + transition: border-color 120ms ease-out, background 120ms ease-out; + } + + .incident-row--minor { --accent: var(--impact-minor); } + .incident-row--major { --accent: var(--impact-major); } + .incident-row--critical { --accent: var(--impact-critical); } + .incident-row--maintenance { --accent: var(--impact-maintenance); } + + .incident-row:hover { + background: var(--color-ink-lightest); + border-color: var(--color-ink-dark); + } + + .incident-row:focus-visible { + outline: var(--focus-ring-size) solid var(--accent); + outline-offset: 1px; + } + + .incident-row__spine { + background: var(--accent); + flex-shrink: 0; + width: 4px; + } + + .incident-row__main { + display: flex; + flex-direction: column; + gap: 0.3rem; + min-width: 0; + padding: 0.85rem 1rem; + width: 100%; + } + + .incident-row__top { + align-items: center; + display: flex; + gap: 0.75rem; + } + + .incident-row__kind { + color: var(--accent); + font-family: var(--font-mono); + font-size: var(--text-x-small); + letter-spacing: 0.1em; + text-transform: uppercase; + } + + .incident-row__status { + background: var(--color-ink-lightest); + border-radius: 2rem; + color: var(--color-ink-dark); + font-size: var(--text-x-small); + margin-left: auto; + padding: 0.15rem 0.6rem; + } + + .incident-row__title { + color: var(--color-ink); + font-size: var(--text-normal); + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .incident-row__meta { + color: var(--color-ink-medium); + font-size: var(--text-small); + margin: 0; + } + + .incident-row__meta time { + font-family: var(--font-mono); + font-size: var(--text-x-small); + } + + @media (prefers-reduced-motion: reduce) { + .incident-form__spine, + .incident-form__eyebrow, + .incident-form__eyebrow::before, + .segmented label, + .impact, + .chip, + .incident-row { + transition: none; + } + } +} diff --git a/app/assets/stylesheets/upright/public_status.css b/app/assets/stylesheets/upright/public_status.css index be868bd..ce71915 100644 --- a/app/assets/stylesheets/upright/public_status.css +++ b/app/assets/stylesheets/upright/public_status.css @@ -4,6 +4,7 @@ --status-degraded: oklch(78% 0.18 95); --status-partial: oklch(70% 0.18 50); --status-major: oklch(60% 0.21 28); + --status-maintenance: oklch(64% 0.14 250); --status-none: oklch(var(--lch-ink-lightest)); } @@ -40,6 +41,7 @@ .status-banner--degraded { background: var(--status-degraded); color: oklch(20% 0 0); } .status-banner--partial_outage { background: var(--status-partial); } .status-banner--major_outage { background: var(--status-major); } + .status-banner--maintenance { background: var(--status-maintenance); } .status-banner__dot { background: currentColor; @@ -116,6 +118,7 @@ .service__status--degraded { color: var(--status-degraded); } .service__status--partial_outage { color: var(--status-partial); } .service__status--major_outage { color: var(--status-major); } + .service__status--maintenance { color: var(--status-maintenance); } .service__description { color: var(--color-ink-medium); @@ -233,4 +236,188 @@ .uptime__tooltip-dot--partial_outage { background: var(--status-partial); } .uptime__tooltip-dot--major_outage { background: var(--status-major); } .uptime__tooltip-dot--none { background: var(--status-none); } + + .events, + .upcoming { + margin: calc(var(--block-space) * -2) 0 calc(var(--block-space) * 3); + } + + .event, + .upcoming__item { + border: var(--border); + border-left: 3px solid var(--status-none); + border-radius: 0.5rem; + padding: 0.75rem 1rem; + } + + .event + .event, + .upcoming__item + .upcoming__item { + margin-top: 0.5rem; + } + + .event--minor { border-left-color: var(--status-degraded); } + .event--major { border-left-color: var(--status-partial); } + .event--critical { border-left-color: var(--status-major); } + .event--maintenance { border-left-color: var(--status-maintenance); } + + .upcoming__title { + font-size: var(--text-large); + font-weight: 600; + margin: 0 0 0.75rem; + } + + .event__head { + align-items: baseline; + display: flex; + gap: 1rem; + justify-content: space-between; + } + + .event__title { font-weight: 600; } + + .event__status, + .upcoming__when { + color: var(--color-ink-medium); + flex-shrink: 0; + font-size: var(--text-small); + text-transform: uppercase; + letter-spacing: 0.05em; + } + + .event__meta, + .event__services { + color: var(--color-ink-medium); + font-size: var(--text-small); + margin: 0.35rem 0 0; + } + + .event__update { + margin: 0.5rem 0 0; + } + + .event__time { + color: var(--color-ink-medium); + display: block; + font-size: var(--text-x-small); + margin-top: 0.15rem; + } + + .event, + .upcoming__item { + color: inherit; + display: block; + text-decoration: none; + transition: border-color 120ms ease-out; + } + + .event:hover, + .upcoming__item:hover { + border-color: var(--color-ink-medium); + } + + .event:focus-visible, + .upcoming__item:focus-visible { + outline: 2px solid var(--color-ink); + outline-offset: 2px; + } + + .incident-detail { + --accent: var(--status-none); + margin: 0 auto; + max-width: 44rem; + padding: calc(var(--block-space) * 3) var(--main-padding); + } + + .incident-detail--minor { --accent: var(--status-degraded); } + .incident-detail--major { --accent: var(--status-partial); } + .incident-detail--critical { --accent: var(--status-major); } + .incident-detail--maintenance { --accent: var(--status-maintenance); } + + .incident-detail__back { + color: var(--color-ink-medium); + font-size: var(--text-small); + } + + .incident-detail__head { + border-bottom: var(--border); + margin: var(--block-space) 0 calc(var(--block-space) * 2); + padding-bottom: calc(var(--block-space) * 1.5); + } + + .incident-detail__eyebrow { + color: var(--accent); + font-size: var(--text-small); + font-weight: 600; + letter-spacing: 0.04em; + margin: 0 0 0.4rem; + text-transform: uppercase; + } + + .incident-detail__title { + font-size: var(--text-x-large); + font-weight: 600; + letter-spacing: -0.02em; + margin: 0 0 0.75rem; + } + + .incident-detail__meta { + align-items: center; + color: var(--color-ink-medium); + display: flex; + flex-wrap: wrap; + font-size: var(--text-small); + gap: 0.4rem 0.9rem; + } + + .incident-detail__status { + background: color-mix(in oklab, var(--accent) 16%, var(--color-canvas)); + border-radius: 2rem; + color: var(--color-ink); + font-weight: 500; + padding: 0.15rem 0.7rem; + } + + .incident-detail__timeline { + list-style: none; + margin: 0; + padding: 0; + } + + .incident-detail__update { + border-left: 2px solid var(--color-ink-lighter); + padding: 0 0 calc(var(--block-space) * 1.75) calc(var(--block-space) * 1.5); + position: relative; + } + + .incident-detail__update:last-child { + border-left-color: transparent; + padding-bottom: 0; + } + + .incident-detail__update::before { + background: var(--color-canvas); + border: 2px solid var(--accent); + border-radius: 50%; + content: ""; + height: 0.75rem; + left: calc(-0.375rem - 1px); + position: absolute; + top: 0.2rem; + width: 0.75rem; + } + + .incident-detail__update-status { + font-weight: 600; + } + + .incident-detail__update-time { + color: var(--color-ink-medium); + font-size: var(--text-x-small); + margin-left: 0.5rem; + } + + .incident-detail__update-body { + color: var(--color-ink-dark); + margin: 0.4rem 0 0; + } } diff --git a/app/controllers/upright/incidents/updates_controller.rb b/app/controllers/upright/incidents/updates_controller.rb new file mode 100644 index 0000000..412bcd6 --- /dev/null +++ b/app/controllers/upright/incidents/updates_controller.rb @@ -0,0 +1,14 @@ +class Upright::Incidents::UpdatesController < Upright::ApplicationController + def create + @incident = Upright::Incident.find(params[:incident_id]) + @incident.record_update(status: update_params[:status], body: update_params[:body]) + redirect_to edit_incident_path(@incident), notice: "Update posted." + rescue ActiveRecord::RecordInvalid + redirect_to edit_incident_path(@incident), alert: "Couldn't post update — check the status and message." + end + + private + def update_params + params.expect(incident_update: [ :status, :body ]) + end +end diff --git a/app/controllers/upright/incidents_controller.rb b/app/controllers/upright/incidents_controller.rb new file mode 100644 index 0000000..c11bd64 --- /dev/null +++ b/app/controllers/upright/incidents_controller.rb @@ -0,0 +1,54 @@ +class Upright::IncidentsController < Upright::ApplicationController + before_action :set_incident, only: %i[ edit update destroy ] + + def index + @current = Upright::Incident.active.order(starts_at: :desc) + @upcoming = Upright::Maintenance.upcoming.order(:starts_at) + @past = Upright::Incident.past.limit(50) + end + + def new + @incident = incident_class.new(starts_at: Time.current) + @incident.impact ||= "minor" unless @incident.maintenance? + end + + def create + @incident = incident_class.new(incident_params) + + if @incident.save + redirect_to incidents_path, notice: "#{@incident.model_name.human} created." + else + render :new, status: :unprocessable_entity + end + end + + def edit + end + + def update + if @incident.update(incident_params) + redirect_to incidents_path, notice: "Saved." + else + render :edit, status: :unprocessable_entity + end + end + + def destroy + @incident.destroy + redirect_to incidents_path, notice: "Deleted." + end + + private + def set_incident + @incident = Upright::Incident.find(params[:id]) + end + + def incident_class + maintenance = ActiveModel::Type::Boolean.new.cast(params[:maintenance]) + maintenance ? Upright::Maintenance : Upright::Incident + end + + def incident_params + params.expect(incident: [ :title, :impact, :starts_at, :ends_at, :body, service_codes: [] ]) + end +end diff --git a/app/controllers/upright/public/incidents_controller.rb b/app/controllers/upright/public/incidents_controller.rb new file mode 100644 index 0000000..c4f8dcd --- /dev/null +++ b/app/controllers/upright/public/incidents_controller.rb @@ -0,0 +1,6 @@ +class Upright::Public::IncidentsController < Upright::Public::BaseController + def show + @incident = Upright::Incident.find(params[:id]) + expires_in 15.seconds, public: true + end +end diff --git a/app/controllers/upright/public/services_controller.rb b/app/controllers/upright/public/services_controller.rb index ee16a85..3d7f2b2 100644 --- a/app/controllers/upright/public/services_controller.rb +++ b/app/controllers/upright/public/services_controller.rb @@ -1,6 +1,9 @@ class Upright::Public::ServicesController < Upright::Public::BaseController def index @services = Upright::Service.public_facing + @active_incidents = Upright::Incident.reactive.active.order(starts_at: :desc) + @active_maintenances = Upright::Maintenance.active.order(:starts_at) + @upcoming_maintenances = Upright::Maintenance.upcoming.order(:starts_at) expires_in 15.seconds, public: true end end diff --git a/app/helpers/upright/application_helper.rb b/app/helpers/upright/application_helper.rb index ac1810e..0b96aac 100644 --- a/app/helpers/upright/application_helper.rb +++ b/app/helpers/upright/application_helper.rb @@ -7,6 +7,10 @@ def site_name(site) "#{country_flag(site.country)} #{site.city}" end + def page_title_tag(app_name = "Upright") + tag.title [ @page_title, app_name ].compact.join(" · ") + end + def upright_stylesheet_link_tag(**options) Upright::Engine.root.join("app/assets/stylesheets/upright").glob("*.css") .map { |f| "upright/#{f.basename('.css')}" }.sort diff --git a/app/helpers/upright/public/services_helper.rb b/app/helpers/upright/public/services_helper.rb index 3fc8279..a07957c 100644 --- a/app/helpers/upright/public/services_helper.rb +++ b/app/helpers/upright/public/services_helper.rb @@ -3,18 +3,32 @@ module Upright::Public::ServicesHelper operational: "All Systems Operational", degraded: "Some Systems Degraded", partial_outage: "Partial Outage", - major_outage: "Major Outage" + major_outage: "Major Outage", + maintenance: "Service Under Maintenance" } def overall_status_label(status) OVERALL_STATUS_LABELS.fetch(status) end + def maintenance_window_description(maintenance) + start, finish = maintenance.starts_at, maintenance.ends_at + same_day = finish && start.to_date == finish.to_date + + if same_day + "#{start.to_fs(:month_day_at)}–#{finish.to_fs(:clock_zone)}" + elsif finish + "#{start.to_fs(:month_day_at)} – #{finish.to_fs(:month_day_at_zone)}" + else + start.to_fs(:month_day_at_zone) + end + end + def status_label(status) status.to_s.humanize end - def outage_duration_phrase(started_at:) + def outage_duration_description(started_at:) if started_at "for #{distance_of_time_in_words(started_at, Time.current)}" else diff --git a/app/jobs/upright/maintenance_advance_job.rb b/app/jobs/upright/maintenance_advance_job.rb new file mode 100644 index 0000000..441ed36 --- /dev/null +++ b/app/jobs/upright/maintenance_advance_job.rb @@ -0,0 +1,8 @@ +class Upright::MaintenanceAdvanceJob < Upright::ApplicationJob + queue_as :default + + def perform + Upright::Maintenance.where(resolved_at: nil).find_each(&:auto_advance_status) + Upright::Maintenance.export_service_metrics + end +end diff --git a/app/models/concerns/upright/incidents/lifecycle.rb b/app/models/concerns/upright/incidents/lifecycle.rb new file mode 100644 index 0000000..1b22d21 --- /dev/null +++ b/app/models/concerns/upright/incidents/lifecycle.rb @@ -0,0 +1,28 @@ +module Upright::Incidents::Lifecycle + extend ActiveSupport::Concern + + included do + scope :active, -> { where(resolved_at: nil).where(starts_at: ..Time.current) } + scope :upcoming, -> { where(resolved_at: nil).where(starts_at: Time.current..) } + scope :past, -> { where.not(resolved_at: nil).order(starts_at: :desc) } + + scope :reactive, -> { where.not(type: "Upright::Maintenance").or(where(type: nil)) } + + scope :for_service, ->(code) { + joins(:affected_services).where(upright_incident_affected_services: { service_code: code }) + } + end + + def active? = resolved_at.nil? && starts_at <= Time.current + def upcoming? = resolved_at.nil? && starts_at > Time.current + def past? = resolved_at.present? + + def record_update(status:, body:) + transaction do + updates.create!(status: status, body: body) + self.status = status + self.resolved_at = Time.current if resolved_at.nil? && self.class::TERMINAL_STATUSES.include?(status) + save! + end + end +end diff --git a/app/models/concerns/upright/probeable.rb b/app/models/concerns/upright/probeable.rb index 246a735..b4cd75f 100644 --- a/app/models/concerns/upright/probeable.rb +++ b/app/models/concerns/upright/probeable.rb @@ -68,6 +68,10 @@ def probe_service try(:service) end + def probe_key + [ probe_name, probe_type, probe_target ] + end + def probe_alert_severity severity = try(:alert_severity)&.to_sym ALERT_SEVERITIES.include?(severity) ? severity : :high diff --git a/app/models/concerns/upright/services/maintenance_status.rb b/app/models/concerns/upright/services/maintenance_status.rb new file mode 100644 index 0000000..bcf2d06 --- /dev/null +++ b/app/models/concerns/upright/services/maintenance_status.rb @@ -0,0 +1,19 @@ +module Upright::Services::MaintenanceStatus + extend ActiveSupport::Concern + + def active_maintenance + Upright::Maintenance.active.for_service(code).first + end + + def maintenance_active? + active_maintenance.present? + end + + def upcoming_maintenances + Upright::Maintenance.upcoming.for_service(code).order(:starts_at) + end + + def active_incidents + Upright::Incident.reactive.active.for_service(code) + end +end diff --git a/app/models/upright/incident.rb b/app/models/upright/incident.rb new file mode 100644 index 0000000..84b6dcb --- /dev/null +++ b/app/models/upright/incident.rb @@ -0,0 +1,57 @@ +class Upright::Incident < Upright::PersistentRecord + include Upright::Incidents::Lifecycle + + attr_accessor :body + + enum :status, %w[ investigating monitoring resolved + scheduled in_progress completed ].index_with(&:itself) + enum :impact, %w[ minor major critical maintenance ].index_with(&:itself), prefix: true + + STATUSES = %w[ investigating monitoring resolved ] + TERMINAL_STATUSES = %w[ resolved ] + IMPACTS = %w[ minor major critical ] + + IMPACT_STATUS = { "minor" => :degraded, "major" => :partial_outage, "critical" => :major_outage } + + def self.active_statuses + reactive.active.map { |incident| IMPACT_STATUS.fetch(incident.impact) } + end + + has_many :updates, -> { order(created_at: :desc) }, class_name: "Upright::IncidentUpdate", inverse_of: :incident, dependent: :destroy + has_many :affected_services, class_name: "Upright::IncidentAffectedService", inverse_of: :incident, dependent: :destroy + + validates :title, :starts_at, presence: true + validates :status, inclusion: { in: ->(incident) { incident.class::STATUSES } } + validates :impact, inclusion: { in: ->(incident) { incident.class::IMPACTS } } + + before_validation :set_default_status, on: :create + after_create :record_initial_update + + def maintenance? = false + + def service_codes = affected_services.map(&:service_code) + + def service_codes=(codes) + self.affected_services = Array(codes).reject(&:blank?).uniq.map do |code| + affected_services.find { |s| s.service_code == code } || + affected_services.build(service_code: code) + end + end + + def services + service_codes.filter_map { |code| Upright::Service.find_by(code: code) } + end + + private + def set_default_status + self.status ||= self.class::STATUSES.first + end + + def record_initial_update + updates.create!(status: status, body: body.presence || default_initial_body) + end + + def default_initial_body + maintenance? ? "Maintenance scheduled." : "We are investigating." + end +end diff --git a/app/models/upright/incident_affected_service.rb b/app/models/upright/incident_affected_service.rb new file mode 100644 index 0000000..3150e9f --- /dev/null +++ b/app/models/upright/incident_affected_service.rb @@ -0,0 +1,9 @@ +class Upright::IncidentAffectedService < Upright::PersistentRecord + belongs_to :incident, class_name: "Upright::Incident", inverse_of: :affected_services + + validates :service_code, presence: true, + uniqueness: { scope: :incident_id }, + inclusion: { in: ->(_) { Upright::Service.all.map(&:code) } } + + def service = Upright::Service.find_by(code: service_code) +end diff --git a/app/models/upright/incident_update.rb b/app/models/upright/incident_update.rb new file mode 100644 index 0000000..f7ea096 --- /dev/null +++ b/app/models/upright/incident_update.rb @@ -0,0 +1,5 @@ +class Upright::IncidentUpdate < Upright::PersistentRecord + belongs_to :incident, class_name: "Upright::Incident", inverse_of: :updates + + validates :status, :body, presence: true +end diff --git a/app/models/upright/maintenance.rb b/app/models/upright/maintenance.rb new file mode 100644 index 0000000..e84e95c --- /dev/null +++ b/app/models/upright/maintenance.rb @@ -0,0 +1,36 @@ +class Upright::Maintenance < Upright::Incident + STATUSES = %w[ scheduled in_progress completed ] + TERMINAL_STATUSES = %w[ completed ] + IMPACTS = %w[ maintenance ] + + SUPPRESSION_LEAD = 1.minute + + scope :suppressing, -> { where(resolved_at: nil).where(starts_at: ..SUPPRESSION_LEAD.from_now) } + + validates :ends_at, presence: true + validate :ends_after_start + + before_validation :set_maintenance_impact + + def maintenance? = true + + def self.export_service_metrics + Upright::Service.all.each do |service| + Yabeda.upright_service_under_maintenance.set({ probe_service: service.code }, suppressing.for_service(service.code).exists? ? 1 : 0) + end + end + + def auto_advance_status(now: Time.current) + record_update(status: "in_progress", body: "Maintenance is underway.") if scheduled? && now >= starts_at + record_update(status: "completed", body: "Maintenance is complete.") if in_progress? && now >= ends_at + end + + private + def set_maintenance_impact + self.impact = "maintenance" + end + + def ends_after_start + errors.add(:ends_at, "must be after the start") if ends_at.present? && starts_at.present? && ends_at <= starts_at + end +end diff --git a/app/models/upright/probes/status.rb b/app/models/upright/probes/status.rb index cda57e7..de4060e 100644 --- a/app/models/upright/probes/status.rb +++ b/app/models/upright/probes/status.rb @@ -23,8 +23,7 @@ def label_selector(probe_type) end def build_probes(results) - # Group results by probe identity (name + type + probe_target) - grouped = results.group_by { |r| [ r[:metric][:name], r[:metric][:type], r[:metric][:probe_target] ] } + grouped = results.group_by { |r| Probe.key_for(**r[:metric]) } grouped.map do |(_name, _type, _target), series| site_statuses = series.map do |s| diff --git a/app/models/upright/probes/status/probe.rb b/app/models/upright/probes/status/probe.rb index cbefc53..e9354d1 100644 --- a/app/models/upright/probes/status/probe.rb +++ b/app/models/upright/probes/status/probe.rb @@ -10,6 +10,14 @@ def initialize(name:, type:, probe_target:, site_statuses:) @site_statuses = site_statuses end + def self.key_for(name:, type:, probe_target:, **) + [ name, type, probe_target ] + end + + def key + self.class.key_for(name:, type:, probe_target:) + end + def status_for_site(code) site_statuses.find { |s| s.site_code == code.to_s } end diff --git a/app/models/upright/rollups/probe_rollup.rb b/app/models/upright/rollups/probe_rollup.rb index 582eb25..ee59c8e 100644 --- a/app/models/upright/rollups/probe_rollup.rb +++ b/app/models/upright/rollups/probe_rollup.rb @@ -13,13 +13,22 @@ class Upright::Rollups::ProbeRollup < Upright::PersistentRecord def self.rollup_day(day) fetch_uptime_for(day).each do |probe_uptime| - find_or_create_by(probe_name: probe_uptime.fetch(:probe_name), period_start: day.beginning_of_day) do |rollup| + find_or_create_by( + probe_name: probe_uptime.fetch(:probe_name), + probe_type: probe_uptime[:probe_type], + probe_target: probe_uptime[:probe_target], + period_start: day.beginning_of_day + ) do |rollup| rollup.probe_service = probe_uptime[:probe_service] rollup.uptime_fraction = probe_uptime.fetch(:uptime_fraction) end end end + def probe_key + [ probe_name, probe_type, probe_target ] + end + def self.fetch_uptime_for(day) query_time = [ day.end_of_day, Time.current ].min @@ -28,6 +37,8 @@ def self.fetch_uptime_for(day) Array(response[:result]).map do |series| { probe_name: series.dig(:metric, :name), + probe_type: series.dig(:metric, :type), + probe_target: series.dig(:metric, :probe_target), probe_service: series.dig(:metric, :probe_service).presence, uptime_fraction: series.dig(:value, 1).to_f } diff --git a/app/models/upright/service.rb b/app/models/upright/service.rb index 6703f86..df5a98f 100644 --- a/app/models/upright/service.rb +++ b/app/models/upright/service.rb @@ -1,5 +1,6 @@ class Upright::Service < FrozenRecord::Base include Upright::Services::LiveStatus + include Upright::Services::MaintenanceStatus def self.file_path Upright.configuration.services_path.to_s @@ -8,7 +9,9 @@ def self.file_path scope :public_facing, -> { where(public: true) } def self.overall_status - Upright::Status::PRIORITY.find { |status| all.any? { |service| service.live_status == status } } || :operational + probe_statuses = all.reject(&:maintenance_active?).map(&:live_status) + worst = Upright::Status.worst(probe_statuses + Upright::Incident.active_statuses) + worst == :operational && Upright::Maintenance.active.any? ? :maintenance : worst end def self.by_history(past: 90.days) @@ -17,6 +20,7 @@ def self.by_history(past: 90.days) def self.degraded all.filter_map do |service| + next if service.maintenance_active? status = service.live_status unless status == :operational { service: service, status: status, started_at: service.current_outage_started_at } diff --git a/app/models/upright/status.rb b/app/models/upright/status.rb index e1947ed..f54b396 100644 --- a/app/models/upright/status.rb +++ b/app/models/upright/status.rb @@ -4,6 +4,10 @@ module Upright::Status # nil fraction means we have no measurement for the period — distinct from # :operational, so we return nil rather than guessing. + def self.worst(statuses) + PRIORITY.find { |status| statuses.include?(status) } || :operational + end + def self.for(uptime_fraction) if uptime_fraction case uptime_fraction diff --git a/app/views/layouts/upright/_header.html.erb b/app/views/layouts/upright/_header.html.erb index 2b91837..0ede6cf 100644 --- a/app/views/layouts/upright/_header.html.erb +++ b/app/views/layouts/upright/_header.html.erb @@ -13,6 +13,7 @@ <%= link_to "Probes", upright.site_root_url(subdomain: current_or_default_site.code), class: "header__link" %> <%= link_to "Uptime", upright.dashboards_uptime_url(subdomain: Upright.configuration.global_subdomain), class: "header__link" %> <%= link_to "Status", upright.dashboards_probe_status_url(subdomain: Upright.configuration.global_subdomain), class: "header__link" %> + <%= link_to "Incidents", upright.incidents_url(subdomain: Upright.configuration.global_subdomain), class: "header__link" %> <%= link_to "Jobs", upright.jobs_url(subdomain: current_or_default_site.code), class: "header__link" %> diff --git a/app/views/layouts/upright/application.html.erb b/app/views/layouts/upright/application.html.erb index 01a0880..1e45d92 100644 --- a/app/views/layouts/upright/application.html.erb +++ b/app/views/layouts/upright/application.html.erb @@ -1,7 +1,7 @@ - Upright + <%= page_title_tag %> diff --git a/app/views/layouts/upright/public.html.erb b/app/views/layouts/upright/public.html.erb index 330f020..edc4bbe 100644 --- a/app/views/layouts/upright/public.html.erb +++ b/app/views/layouts/upright/public.html.erb @@ -1,7 +1,7 @@ - Status + <%= page_title_tag "Status" %> <%= csrf_meta_tags %> <%= csp_meta_tag %> diff --git a/app/views/upright/incidents/_form.html.erb b/app/views/upright/incidents/_form.html.erb new file mode 100644 index 0000000..96122d4 --- /dev/null +++ b/app/views/upright/incidents/_form.html.erb @@ -0,0 +1,93 @@ +<% url = @incident.new_record? ? incidents_path : incident_path(@incident) %> +<%= form_with model: @incident, scope: :incident, url: url, + class: class_names("incident-form", + "incident-form--maintenance" => @incident.persisted? && @incident.maintenance?, + "incident-form--incident" => @incident.persisted? && !@incident.maintenance?) do |f| %> + + +
+
+

<%= @incident.new_record? ? "New event" : (@incident.maintenance? ? "Maintenance" : "Incident") %>

+ <% unless @incident.new_record? %> +

<%= @incident.title %>

+ <% end %> +
+ + <% if @incident.errors.any? %> + + <% end %> + + <% if @incident.new_record? %> +
+ Type +
+ > + + > + +
+
+ <% end %> + +
+ <%= f.label :title, "Title", class: "field__label" %> + <%= f.text_field :title, class: "field__input", + placeholder: "Short summary customers will see", autofocus: @incident.new_record? %> +
+ +
+ Impact +
+ <% Upright::Incident::IMPACTS.each do |level| %> + + <% end %> +
+
+ +
+ <%= f.label :starts_at, "Starts", class: "field__label" %> + <%= f.datetime_field :starts_at, class: "field__input" %> +
+ +
+ <%= f.label :ends_at, "Ends", class: "field__label" %> + <%= f.datetime_field :ends_at, class: "field__input" %> + Affected services read as “Under maintenance” for this window and stay out of the outage alarm. +
+ +
+ Affected services +
+ + <% Upright::Service.all.each do |service| %> + + <% end %> +
+
+ + <% if @incident.new_record? %> +
+ <%= label_tag "incident_body", "First update", class: "field__label" %> + <%= text_area_tag "incident[body]", nil, id: "incident_body", class: "field__input", + placeholder: "What’s happening, and what you’re doing about it." %> +
+ <% end %> + +
+ <%= link_to "Cancel", incidents_path, class: "incident-form__cancel" %> + <%= f.submit(@incident.new_record? ? "Create" : "Save changes", class: "btn") %> +
+
+<% end %> diff --git a/app/views/upright/incidents/_row.html.erb b/app/views/upright/incidents/_row.html.erb new file mode 100644 index 0000000..36b0e28 --- /dev/null +++ b/app/views/upright/incidents/_row.html.erb @@ -0,0 +1,15 @@ +<%= link_to edit_incident_path(incident), + class: "incident-row incident-row--#{incident.maintenance? ? "maintenance" : incident.impact}" do %> + +
+
+ <%= incident.maintenance? ? "Maintenance" : "Incident" %> + <%= incident.status.humanize %> +
+ <%= incident.title %> +

+ <% if incident.services.any? %><%= incident.services.map(&:name).to_sentence %> · <% end %> + +

+
+<% end %> diff --git a/app/views/upright/incidents/edit.html.erb b/app/views/upright/incidents/edit.html.erb new file mode 100644 index 0000000..e7f81c6 --- /dev/null +++ b/app/views/upright/incidents/edit.html.erb @@ -0,0 +1,55 @@ +<% @page_title = @incident.title %> + +
"> + <%= link_to "← All incidents", incidents_path, class: "incident-editor__back" %> + + <%= render "form" %> + +
+

Lifecycle

+
    + <% statuses = @incident.class::STATUSES %> + <% current = statuses.index(@incident.status) %> + <% statuses.each_with_index do |status, index| %> +
  1. <%= "lifecycle__step--current" if index == current %>"><%= status.humanize %>
  2. + <% end %> +
+
+ +
+

Post an update

+ <%= form_with url: incident_updates_path(@incident), scope: :incident_update, class: "composer" do |f| %> +
+ <%= f.label :status, "Status", class: "field__label" %> + <%= f.select :status, @incident.class::STATUSES.map { |status| [ status.humanize, status ] }, + { selected: @incident.status }, class: "input--select" %> +
+
+ <%= f.label :body, "Message", class: "field__label" %> + <%= f.text_area :body, class: "field__input", placeholder: "What changed." %> +
+
+ <%= f.submit "Post update", class: "btn" %> +
+ <% end %> +
+ +
+

Timeline

+
    + <% @incident.updates.each do |update| %> +
  1. + <%= update.status.humanize %> + +

    <%= update.body %>

    +
  2. + <% end %> +
+
+ +
+ <%= button_to "Delete #{@incident.maintenance? ? "maintenance" : "incident"}", + incident_path(@incident), method: :delete, class: "btn", + form: { data: { turbo_confirm: "Delete this #{@incident.maintenance? ? "maintenance" : "incident"}?" } } %> +
+
diff --git a/app/views/upright/incidents/index.html.erb b/app/views/upright/incidents/index.html.erb new file mode 100644 index 0000000..6c31a6d --- /dev/null +++ b/app/views/upright/incidents/index.html.erb @@ -0,0 +1,53 @@ +<% @page_title = "Incidents and maintenance" %> + +
+
+

Incidents & maintenance

+
+ <%= link_to "New incident", new_incident_path, class: "btn" %> + <%= link_to "New maintenance", new_incident_path(maintenance: true), class: "btn btn--ghost" %> +
+
+ + <% if @current.none? && @upcoming.none? && @past.none? %> +
+

All clear

+

Nothing to report. Incidents and maintenance you declare here appear on the public status page.

+
+ <% end %> + + <% if @current.any? %> +
+

Active

+
+ <% @current.each do |incident| %> + <%= render "row", incident: incident, + when_label: incident.maintenance? ? maintenance_window_description(incident) : "Started #{incident.starts_at.to_fs(:month_day_at_zone)}" %> + <% end %> +
+
+ <% end %> + + <% if @upcoming.any? %> +
+

Upcoming maintenance

+
+ <% @upcoming.each do |incident| %> + <%= render "row", incident: incident, when_label: maintenance_window_description(incident) %> + <% end %> +
+
+ <% end %> + + <% if @past.any? %> +
+

Past

+
+ <% @past.each do |incident| %> + <%= render "row", incident: incident, + when_label: "Resolved #{incident.resolved_at&.to_fs(:month_day_at_zone)}" %> + <% end %> +
+
+ <% end %> +
diff --git a/app/views/upright/incidents/new.html.erb b/app/views/upright/incidents/new.html.erb new file mode 100644 index 0000000..9c876a1 --- /dev/null +++ b/app/views/upright/incidents/new.html.erb @@ -0,0 +1,6 @@ +<% @page_title = "New #{@incident.maintenance? ? "maintenance" : "incident"}" %> + +
+ <%= link_to "← All incidents", incidents_path, class: "incident-editor__back" %> + <%= render "form" %> +
diff --git a/app/views/upright/public/incidents/show.html.erb b/app/views/upright/public/incidents/show.html.erb new file mode 100644 index 0000000..0cb02be --- /dev/null +++ b/app/views/upright/public/incidents/show.html.erb @@ -0,0 +1,29 @@ +<% @page_title = @incident.title %> + +
"> + <%= link_to "← Status", "/", class: "incident-detail__back" %> + +
+

<%= @incident.maintenance? ? "Scheduled maintenance" : "Incident" %>

+

<%= @incident.title %>

+
+ <%= status_label(@incident.status) %> + <% if @incident.maintenance? %> + <%= maintenance_window_description(@incident) %> + <% end %> + <% if @incident.services.any? %> + <%= @incident.services.map(&:name).to_sentence %> + <% end %> +
+
+ +
    + <% @incident.updates.each do |update| %> +
  1. + <%= status_label(update.status) %> + +

    <%= update.body %>

    +
  2. + <% end %> +
+
diff --git a/app/views/upright/public/services/_active_events.html.erb b/app/views/upright/public/services/_active_events.html.erb new file mode 100644 index 0000000..024eeb7 --- /dev/null +++ b/app/views/upright/public/services/_active_events.html.erb @@ -0,0 +1,28 @@ +<% events = incidents.to_a + maintenances.to_a %> +<% if events.any? %> +
+ <% events.each do |event| %> + <%= link_to public_incident_path(event), class: "event event--#{event.maintenance? ? "maintenance" : event.impact}" do %> +
+ <%= event.title %> + <%= status_label(event.status) %> +
+ + <% if event.maintenance? %> +

<%= maintenance_window_description(event) %>

+ <% end %> + + <% if event.services.any? %> +

<%= event.services.map(&:name).to_sentence %>

+ <% end %> + + <% if update = event.updates.first %> +

+ <%= update.body %> + +

+ <% end %> + <% end %> + <% end %> +
+<% end %> diff --git a/app/views/upright/public/services/_degraded_list.html.erb b/app/views/upright/public/services/_degraded_list.html.erb index cb870c4..70e065f 100644 --- a/app/views/upright/public/services/_degraded_list.html.erb +++ b/app/views/upright/public/services/_degraded_list.html.erb @@ -6,7 +6,7 @@ <%= item[:service].name %> <%= status_label(item[:status]).downcase %> - <%= outage_duration_phrase(started_at: item[:started_at]) %> + <%= outage_duration_description(started_at: item[:started_at]) %> <% end %> diff --git a/app/views/upright/public/services/_service.html.erb b/app/views/upright/public/services/_service.html.erb index 15b3511..f46e1e0 100644 --- a/app/views/upright/public/services/_service.html.erb +++ b/app/views/upright/public/services/_service.html.erb @@ -3,9 +3,13 @@
<%= service.name %> - - <%= status_label(today_status) %> - + <% if service.maintenance_active? %> + Under maintenance + <% else %> + + <%= status_label(today_status) %> + + <% end %>
<% if description = service.try(:description).presence %> diff --git a/app/views/upright/public/services/_upcoming_maintenance.html.erb b/app/views/upright/public/services/_upcoming_maintenance.html.erb new file mode 100644 index 0000000..34c3860 --- /dev/null +++ b/app/views/upright/public/services/_upcoming_maintenance.html.erb @@ -0,0 +1,21 @@ +<% if maintenances.any? %> +
+

Scheduled maintenance

+ <% maintenances.each do |maintenance| %> + <%= link_to public_incident_path(maintenance), class: "upcoming__item" do %> +
+ <%= maintenance.title %> + <%= maintenance_window_description(maintenance) %> +
+ + <% if maintenance.services.any? %> +

<%= maintenance.services.map(&:name).to_sentence %>

+ <% end %> + + <% if update = maintenance.updates.first %> +

<%= update.body %>

+ <% end %> + <% end %> + <% end %> +
+<% end %> diff --git a/app/views/upright/public/services/index.html.erb b/app/views/upright/public/services/index.html.erb index 3340faa..92af3db 100644 --- a/app/views/upright/public/services/index.html.erb +++ b/app/views/upright/public/services/index.html.erb @@ -4,6 +4,9 @@ <%= render "overall_banner", status: @services.overall_status %> <%= render "degraded_list", degraded: @services.degraded %> + <%= render "active_events", incidents: @active_incidents, maintenances: @active_maintenances %> + <%= render "upcoming_maintenance", maintenances: @upcoming_maintenances %> + <% @services.by_history.each do |service, history| %> <%= render "service", service: service, history: history %> <% end %> diff --git a/app/views/upright/public/services/index.rss.builder b/app/views/upright/public/services/index.rss.builder index 418a627..d7c963a 100644 --- a/app/views/upright/public/services/index.rss.builder +++ b/app/views/upright/public/services/index.rss.builder @@ -9,10 +9,21 @@ xml.rss(version: "2.0") do @services.degraded.each do |issue| xml.item do xml.title "#{issue[:service].name} — #{status_label(issue[:status])}" - xml.description "#{issue[:service].name} is currently #{status_label(issue[:status]).downcase} #{outage_duration_phrase(started_at: issue[:started_at])}." + xml.description "#{issue[:service].name} is currently #{status_label(issue[:status]).downcase} #{outage_duration_description(started_at: issue[:started_at])}." xml.pubDate issue[:started_at].rfc822 if issue[:started_at] xml.guid feed_item_guid(issue), isPermaLink: "false" end end + + (@active_incidents.to_a + @active_maintenances.to_a).each do |event| + update = event.updates.first + xml.item do + xml.title "#{event.title} — #{status_label(event.status)}" + xml.description update&.body.to_s + xml.link public_incident_url(event) + xml.pubDate (update&.created_at || event.starts_at).rfc822 + xml.guid "incident-#{event.id}-#{update&.id}", isPermaLink: "false" + end + end end end diff --git a/config/initializers/date_formats.rb b/config/initializers/date_formats.rb index 818da7e..19aa3f0 100644 --- a/config/initializers/date_formats.rb +++ b/config/initializers/date_formats.rb @@ -1 +1,5 @@ Date::DATE_FORMATS[:month_day] ||= "%b %-d" + +Time::DATE_FORMATS[:month_day_at] ||= "%b %-d, %H:%M" +Time::DATE_FORMATS[:month_day_at_zone] ||= "%b %-d, %H:%M %Z" +Time::DATE_FORMATS[:clock_zone] ||= "%H:%M %Z" diff --git a/config/routes.rb b/config/routes.rb index c0aee5f..a08d7aa 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -17,6 +17,10 @@ resource :probe_status, only: :show end + resources :incidents do + resources :updates, only: :create, controller: "incidents/updates" + end + scope :framed do resource :prometheus, only: :show, controller: :prometheus_proxy resource :alertmanager, only: :show, controller: :alertmanager_proxy @@ -44,6 +48,7 @@ scope module: :public, as: :public do root "services#index", as: :services_root get "feed", to: "services#index", as: :services_feed, defaults: { format: :rss } + resources :incidents, only: :show end end diff --git a/db/persistent_migrate/20260702000001_create_upright_incidents.rb b/db/persistent_migrate/20260702000001_create_upright_incidents.rb new file mode 100644 index 0000000..1f363fe --- /dev/null +++ b/db/persistent_migrate/20260702000001_create_upright_incidents.rb @@ -0,0 +1,34 @@ +class CreateUprightIncidents < ActiveRecord::Migration[8.0] + def change + create_table :upright_incidents do |t| + t.string :type + t.string :title, null: false + t.string :status, null: false + t.string :impact, null: false + t.datetime :starts_at, null: false + t.datetime :ends_at + t.datetime :resolved_at + + t.timestamps + end + + add_index :upright_incidents, [ :resolved_at, :starts_at ] + add_index :upright_incidents, [ :type, :starts_at ] + + create_table :upright_incident_updates do |t| + t.references :incident, null: false, foreign_key: { to_table: :upright_incidents } + t.string :status, null: false + t.text :body + + t.datetime :created_at, null: false + end + + create_table :upright_incident_affected_services do |t| + t.references :incident, null: false, foreign_key: { to_table: :upright_incidents } + t.string :service_code, null: false + end + + add_index :upright_incident_affected_services, [ :incident_id, :service_code ], unique: true + add_index :upright_incident_affected_services, :service_code + end +end diff --git a/db/persistent_migrate/20260707000001_add_probe_identity_to_upright_rollups.rb b/db/persistent_migrate/20260707000001_add_probe_identity_to_upright_rollups.rb new file mode 100644 index 0000000..6cb1db8 --- /dev/null +++ b/db/persistent_migrate/20260707000001_add_probe_identity_to_upright_rollups.rb @@ -0,0 +1,11 @@ +class AddProbeIdentityToUprightRollups < ActiveRecord::Migration[8.0] + def change + add_column :upright_rollups_probe_rollups, :probe_type, :string + add_column :upright_rollups_probe_rollups, :probe_target, :string + + remove_index :upright_rollups_probe_rollups, column: [ :probe_name, :period_start ], unique: true + add_index :upright_rollups_probe_rollups, + [ :probe_name, :probe_type, :probe_target, :period_start ], + unique: true, name: "idx_probe_rollups_identity_period" + end +end diff --git a/lib/upright/metrics.rb b/lib/upright/metrics.rb index 1d18373..b39f2a9 100644 --- a/lib/upright/metrics.rb +++ b/lib/upright/metrics.rb @@ -55,6 +55,11 @@ def define_metrics comment: "HTTP response status code", aggregation: :max, tags: %i[name probe_target probe_service] + + gauge :service_under_maintenance, + comment: "Whether a service is under active maintenance (1 = yes, 0 = no)", + aggregation: :most_recent, + tags: %i[probe_service] end end end diff --git a/test/dummy/db/persistent_schema.rb b/test/dummy/db/persistent_schema.rb index c415e6c..69a0cc9 100644 --- a/test/dummy/db/persistent_schema.rb +++ b/test/dummy/db/persistent_schema.rb @@ -10,16 +10,51 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_05_12_000001) do +ActiveRecord::Schema[8.1].define(version: 2026_07_07_000001) do + create_table "upright_incident_affected_services", force: :cascade do |t| + t.integer "incident_id", null: false + t.string "service_code", null: false + t.index ["incident_id", "service_code"], name: "idx_on_incident_id_service_code_188b04aae6", unique: true + t.index ["incident_id"], name: "index_upright_incident_affected_services_on_incident_id" + t.index ["service_code"], name: "index_upright_incident_affected_services_on_service_code" + end + + create_table "upright_incident_updates", force: :cascade do |t| + t.text "body" + t.datetime "created_at", null: false + t.integer "incident_id", null: false + t.string "status", null: false + t.index ["incident_id"], name: "index_upright_incident_updates_on_incident_id" + end + + create_table "upright_incidents", force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "ends_at" + t.string "impact", null: false + t.datetime "resolved_at" + t.datetime "starts_at", null: false + t.string "status", null: false + t.string "title", null: false + t.string "type" + t.datetime "updated_at", null: false + t.index ["resolved_at", "starts_at"], name: "index_upright_incidents_on_resolved_at_and_starts_at" + t.index ["type", "starts_at"], name: "index_upright_incidents_on_type_and_starts_at" + end + create_table "upright_rollups_probe_rollups", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "period_start", null: false t.string "probe_name", null: false t.string "probe_service" + t.string "probe_target" + t.string "probe_type" t.integer "status", default: 0, null: false t.datetime "updated_at", null: false t.float "uptime_fraction", null: false - t.index ["probe_name", "period_start"], name: "idx_on_probe_name_period_start_3a6029f64e", unique: true + t.index ["probe_name", "probe_type", "probe_target", "period_start"], name: "idx_probe_rollups_identity_period", unique: true t.index ["probe_service", "period_start"], name: "idx_on_probe_service_period_start_c65e2bccc5" end + + add_foreign_key "upright_incident_affected_services", "upright_incidents", column: "incident_id" + add_foreign_key "upright_incident_updates", "upright_incidents", column: "incident_id" end diff --git a/test/fixtures/upright_incident_affected_services.yml b/test/fixtures/upright_incident_affected_services.yml new file mode 100644 index 0000000..114953a --- /dev/null +++ b/test/fixtures/upright_incident_affected_services.yml @@ -0,0 +1,6 @@ +_fixture: + model_class: Upright::IncidentAffectedService + +reactive_with_service_app: + incident: reactive_with_service + service_code: example_app diff --git a/test/fixtures/upright_incident_updates.yml b/test/fixtures/upright_incident_updates.yml new file mode 100644 index 0000000..c65ccb5 --- /dev/null +++ b/test/fixtures/upright_incident_updates.yml @@ -0,0 +1,7 @@ +_fixture: + model_class: Upright::IncidentUpdate + +reactive_resolved_initial: + incident: reactive_resolved + status: investigating + body: "We are investigating." diff --git a/test/fixtures/upright_incidents.yml b/test/fixtures/upright_incidents.yml new file mode 100644 index 0000000..ae793a0 --- /dev/null +++ b/test/fixtures/upright_incidents.yml @@ -0,0 +1,58 @@ +_fixture: + model_class: Upright::Incident + +upcoming: + type: Upright::Maintenance + title: "Scheduled upgrade" + impact: maintenance + status: scheduled + starts_at: 2026-06-15 13:00:00 + ends_at: 2026-06-15 14:00:00 + +in_progress: + type: Upright::Maintenance + title: "Rolling restart" + impact: maintenance + status: in_progress + starts_at: 2026-06-15 11:00:00 + ends_at: 2026-06-15 13:00:00 + +started_scheduled: + type: Upright::Maintenance + title: "Window open, not yet advanced" + impact: maintenance + status: scheduled + starts_at: 2026-06-15 11:00:00 + ends_at: 2026-06-15 13:00:00 + +elapsed_scheduled: + type: Upright::Maintenance + title: "Window fully elapsed" + impact: maintenance + status: scheduled + starts_at: 2026-06-15 10:00:00 + ends_at: 2026-06-15 11:00:00 + +reactive_resolved: + type: Upright::Incident + title: "Resolved outage" + impact: minor + status: resolved + starts_at: 2026-06-15 10:00:00 + resolved_at: 2026-06-15 11:00:00 + +reactive_other: + type: Upright::Incident + title: "Another resolved outage" + impact: major + status: resolved + starts_at: 2026-06-15 09:00:00 + resolved_at: 2026-06-15 10:00:00 + +reactive_with_service: + type: Upright::Incident + title: "Service-scoped resolved incident" + impact: minor + status: resolved + starts_at: 2026-06-15 10:00:00 + resolved_at: 2026-06-15 11:00:00 diff --git a/test/fixtures/upright_rollups_probe_rollups.yml b/test/fixtures/upright_rollups_probe_rollups.yml index 919aed2..b61cba2 100644 --- a/test/fixtures/upright_rollups_probe_rollups.yml +++ b/test/fixtures/upright_rollups_probe_rollups.yml @@ -3,6 +3,8 @@ _fixture: example_web_may_05: probe_name: Web + probe_type: http + probe_target: https://example.com probe_service: example_app period_start: 2026-05-05 00:00:00 uptime_fraction: 0.85 @@ -10,6 +12,8 @@ example_web_may_05: example_api_may_05: probe_name: API + probe_type: http + probe_target: https://example.com/api probe_service: example_app period_start: 2026-05-05 00:00:00 uptime_fraction: 1.0 @@ -17,6 +21,8 @@ example_api_may_05: example_web_may_11: probe_name: Web + probe_type: http + probe_target: https://example.com probe_service: example_app period_start: 2026-05-11 00:00:00 uptime_fraction: 0.95 @@ -24,6 +30,8 @@ example_web_may_11: example_web_may_12: probe_name: Web + probe_type: http + probe_target: https://example.com probe_service: example_app period_start: 2026-05-12 00:00:00 uptime_fraction: 1.0 @@ -31,6 +39,8 @@ example_web_may_12: example_api_may_12: probe_name: API + probe_type: http + probe_target: https://example.com/api probe_service: example_app period_start: 2026-05-12 00:00:00 uptime_fraction: 0.8 diff --git a/test/models/upright/incident_test.rb b/test/models/upright/incident_test.rb new file mode 100644 index 0000000..e77086c --- /dev/null +++ b/test/models/upright/incident_test.rb @@ -0,0 +1,85 @@ +require "test_helper" + +class Upright::IncidentTest < ActiveSupport::TestCase + setup { travel_to Time.utc(2026, 6, 15, 12) } + + test "a reactive incident is valid and is not a maintenance" do + incident = upright_incidents(:reactive_resolved) + + assert incident.valid? + assert_not incident.maintenance? + end + + test "rejects a status outside the reactive lifecycle" do + incident = upright_incidents(:reactive_resolved) + incident.status = "scheduled" + + assert_not incident.valid? + assert incident.errors[:status].any? + end + + test "record_update appends an update, moves status, and stamps resolved_at on a terminal status" do + incident = activate(upright_incidents(:reactive_resolved)) + + incident.record_update(status: "monitoring", body: "Watching recovery.") + assert_equal "monitoring", incident.reload.status + assert_nil incident.resolved_at + assert_equal 2, incident.updates.count + + incident.record_update(status: "resolved", body: "All clear.") + assert_equal "resolved", incident.reload.status + assert_not_nil incident.resolved_at + end + + test "creating an incident seeds an initial update from the default status" do + incident = Upright::Incident.create!(title: "x", impact: "minor", starts_at: Time.current, body: "Looking into it.") + + assert_equal "investigating", incident.status + assert_equal 1, incident.updates.count + assert_equal "Looking into it.", incident.updates.first.body + end + + test "active_statuses maps active reactive incident impact to a page status" do + activate upright_incidents(:reactive_resolved), impact: "major" + + assert_equal [ :partial_outage ], Upright::Incident.active_statuses + end + + test "service_codes= assigns affected services and rejects unknown codes" do + incident = upright_incidents(:reactive_resolved) + + incident.service_codes = [ "example_app" ] + assert incident.save + assert_equal [ "example_app" ], incident.reload.service_codes + + incident.service_codes = [ "does_not_exist" ] + assert_not incident.valid? + end + + test "active, upcoming, and past scopes key off timestamps and resolved_at" do + active = activate(upright_incidents(:reactive_other)) + resolved = upright_incidents(:reactive_resolved) + + assert_includes Upright::Incident.active, active + assert_not_includes Upright::Incident.active, resolved + assert_includes Upright::Incident.past, resolved + end + + test "reactive scope excludes the maintenance subclass" do + assert_includes Upright::Incident.reactive, upright_incidents(:reactive_resolved) + assert_not_includes Upright::Incident.reactive, upright_incidents(:in_progress) + end + + test "for_service filters by affected service code" do + incident = upright_incidents(:reactive_with_service) + + assert_includes Upright::Incident.for_service("example_app"), incident + assert_not_includes Upright::Incident.for_service("internal_tools"), incident + end + + private + def activate(incident, **overrides) + incident.update!({ status: "investigating", starts_at: 1.hour.ago, resolved_at: nil }.merge(overrides)) + incident + end +end diff --git a/test/models/upright/maintenance_test.rb b/test/models/upright/maintenance_test.rb new file mode 100644 index 0000000..1f19b3a --- /dev/null +++ b/test/models/upright/maintenance_test.rb @@ -0,0 +1,55 @@ +require "test_helper" + +class Upright::MaintenanceTest < ActiveSupport::TestCase + setup { travel_to Time.utc(2026, 6, 15, 12) } + + test "is an Incident via STI and forces the maintenance impact" do + maintenance = Upright::Maintenance.create!(title: "Upgrade", status: "scheduled", impact: "major", + starts_at: 1.hour.from_now, ends_at: 2.hours.from_now) + + assert maintenance.maintenance? + assert_kind_of Upright::Incident, maintenance + assert_equal "maintenance", maintenance.impact + assert_equal "Upright::Maintenance", maintenance.type + end + + test "requires an end that is after the start" do + maintenance = upright_incidents(:upcoming) + maintenance.ends_at = maintenance.starts_at - 1.hour + + assert_not maintenance.valid? + assert maintenance.errors[:ends_at].any? + end + + test "rejects a reactive-incident status" do + maintenance = upright_incidents(:upcoming) + maintenance.status = "investigating" + + assert_not maintenance.valid? + end + + test "auto_advance_status starts an in-window maintenance without completing it" do + maintenance = upright_incidents(:started_scheduled) + + maintenance.auto_advance_status + + assert_equal "in_progress", maintenance.reload.status + assert_nil maintenance.resolved_at + end + + test "auto_advance_status catches up a fully-elapsed window through to completed" do + maintenance = upright_incidents(:elapsed_scheduled) + + maintenance.auto_advance_status + + assert_equal "completed", maintenance.reload.status + assert_not_nil maintenance.resolved_at + end + + test "upcoming and active scopes" do + assert_includes Upright::Maintenance.upcoming, upright_incidents(:upcoming) + assert_not_includes Upright::Maintenance.upcoming, upright_incidents(:in_progress) + assert_includes Upright::Maintenance.active, upright_incidents(:in_progress) + assert_not_includes Upright::Maintenance.active, upright_incidents(:upcoming) + end +end diff --git a/test/models/upright/rollups/probe_rollup_test.rb b/test/models/upright/rollups/probe_rollup_test.rb index 74dd4b6..27c044a 100644 --- a/test/models/upright/rollups/probe_rollup_test.rb +++ b/test/models/upright/rollups/probe_rollup_test.rb @@ -20,8 +20,8 @@ class Upright::Rollups::ProbeRollupTest < ActiveSupport::TestCase test "rollup_day creates a rollup per probe uptime with derived status" do day = Date.new(2026, 5, 1) probe_uptimes = [ - { probe_name: "Web", probe_service: "example_app", uptime_fraction: 1.0 }, - { probe_name: "API", probe_service: "example_app", uptime_fraction: 0.85 } + { probe_name: "Web", probe_type: "http", probe_target: "https://example.com", probe_service: "example_app", uptime_fraction: 1.0 }, + { probe_name: "API", probe_type: "http", probe_target: "https://example.com/api", probe_service: "example_app", uptime_fraction: 0.85 } ] Upright::Rollups::ProbeRollup.stubs(:fetch_uptime_for).with(day).returns(probe_uptimes) @@ -37,11 +37,32 @@ class Upright::Rollups::ProbeRollupTest < ActiveSupport::TestCase assert_equal "partial_outage", api.status end + test "rollup_day keeps probes that share a name but differ by type as distinct rows" do + day = Date.new(2026, 5, 1) + probe_uptimes = [ + { probe_name: "BC3", probe_type: "traceroute", probe_target: "3.basecamp.com", probe_service: nil, uptime_fraction: 1.0 }, + { probe_name: "BC3", probe_type: "http", probe_target: "https://app.basecamp.com/up", probe_service: "bc5", uptime_fraction: 0.9 } + ] + + Upright::Rollups::ProbeRollup.stubs(:fetch_uptime_for).with(day).returns(probe_uptimes) + Upright::Rollups::ProbeRollup.rollup_day(day) + + rollups = Upright::Rollups::ProbeRollup.where(probe_name: "BC3", period_start: day.beginning_of_day) + assert_equal 2, rollups.count + + http = rollups.find_by(probe_type: "http") + assert_equal "bc5", http.probe_service + assert_equal 0.9, http.uptime_fraction + + traceroute = rollups.find_by(probe_type: "traceroute") + assert_nil traceroute.probe_service + end + test "rollup_day leaves existing rollups unchanged" do existing = upright_rollups_probe_rollups(:example_web_may_11) Upright::Rollups::ProbeRollup.stubs(:fetch_uptime_for).with(existing.period_start.to_date).returns([ - { probe_name: existing.probe_name, probe_service: existing.probe_service, uptime_fraction: 1.0 } + { probe_name: existing.probe_name, probe_type: existing.probe_type, probe_target: existing.probe_target, probe_service: existing.probe_service, uptime_fraction: 1.0 } ]) Upright::Rollups::ProbeRollup.rollup_day(existing.period_start.to_date) diff --git a/test/models/upright/service_maintenance_test.rb b/test/models/upright/service_maintenance_test.rb new file mode 100644 index 0000000..d862aa7 --- /dev/null +++ b/test/models/upright/service_maintenance_test.rb @@ -0,0 +1,86 @@ +require "test_helper" + +class Upright::ServiceMaintenanceTest < ActiveSupport::TestCase + setup { travel_to Time.utc(2026, 6, 15, 12) } + + test "maintenance_active? is true while a covering window is in progress" do + maintain "example_app" + + assert Upright::Service.find_by(code: "example_app").maintenance_active? + assert_not Upright::Service.find_by(code: "internal_tools").maintenance_active? + end + + test "degraded suppresses a service that is under maintenance" do + maintain "example_app" + stub_live_status :major_outage + Upright::Service.any_instance.stubs(:current_outage_started_at).returns(nil) + + codes = Upright::Service.degraded.map { |item| item[:service].code } + + assert_not_includes codes, "example_app" + assert_includes codes, "internal_tools" + end + + test "overall_status reads maintenance when the only down services are under maintenance" do + maintain "example_app" + maintain "internal_tools", using: :started_scheduled + stub_live_status :major_outage + + assert_equal :maintenance, Upright::Service.overall_status + end + + test "a real outage on another service still outranks maintenance" do + maintain "example_app" + stub_live_status :major_outage + + assert_equal :major_outage, Upright::Service.overall_status + end + + test "an active incident raises the overall status even when probes are green" do + raise_incident impact: "critical" + stub_live_status :operational + + assert_equal :major_outage, Upright::Service.overall_status + end + + test "overall status takes the worse of probe status and incident impact" do + raise_incident impact: "minor" + stub_live_status :major_outage + + assert_equal :major_outage, Upright::Service.overall_status + end + + test "export_service_metrics reports 1 for maintained services and 0 otherwise" do + maintain "example_app" + + Upright::Maintenance.export_service_metrics + + assert_equal 1, yabeda_gauge_value(:service_under_maintenance, probe_service: "example_app") + assert_equal 0, yabeda_gauge_value(:service_under_maintenance, probe_service: "internal_tools") + end + + test "export_service_metrics arms suppression before a window opens but not too early" do + upright_incidents(:upcoming).update!(starts_at: 30.seconds.from_now, service_codes: [ "example_app" ]) + + Upright::Maintenance.export_service_metrics + assert_equal 1, yabeda_gauge_value(:service_under_maintenance, probe_service: "example_app") + + upright_incidents(:upcoming).update!(starts_at: 5.minutes.from_now) + Upright::Maintenance.export_service_metrics + assert_equal 0, yabeda_gauge_value(:service_under_maintenance, probe_service: "example_app") + end + + private + def maintain(code, using: :in_progress) + upright_incidents(using).update!(service_codes: [ code ]) + end + + def raise_incident(impact:, code: "example_app") + upright_incidents(:reactive_resolved).update!(impact: impact, status: "investigating", + starts_at: 1.hour.ago, resolved_at: nil, service_codes: [ code ]) + end + + def stub_live_status(status) + Upright::Service.any_instance.stubs(:live_status).returns(status) + end +end diff --git a/test/models/upright/status_test.rb b/test/models/upright/status_test.rb index fd7d53b..9be0f4d 100644 --- a/test/models/upright/status_test.rb +++ b/test/models/upright/status_test.rb @@ -20,4 +20,13 @@ class Upright::StatusTest < ActiveSupport::TestCase test "for returns major_outage when below half" do assert_equal :major_outage, Upright::Status.for(0.4) end + + test "worst returns the highest-priority status present" do + assert_equal :major_outage, Upright::Status.worst([ :operational, :major_outage, :degraded ]) + assert_equal :degraded, Upright::Status.worst([ :operational, :degraded ]) + end + + test "worst returns operational for an empty list" do + assert_equal :operational, Upright::Status.worst([]) + end end