diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..ac1fa5469 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,248 @@ +# CLAUDE.md + +Maddy Mail Server fork with REST API for user/mailbox management. See [architecture.md](./architecture.md) for detailed system design. + +## Quick Reference + +### Build +```bash +./build.sh build # Build binaries +./build.sh --tags 'libdns_route53' build # With Route53 support +docker build -t maddy . # Docker build +``` + +### Test +```bash +go test ./... # Run all tests +go test ./internal/rest/... # Test REST API only +``` + +### Run +```bash +# Required for REST API +export ENABLE_API=true +export ADMIN_EMAIL=admin@example.com +export ADMIN_PASSWORD=secure-password + +# Database config +export DB_HOST=localhost +export DB_PORT=5432 +export DB_NAME=maddy +export DB_USER=maddy +export DB_PASSWORD=password +export MAILBOXES_DB_NAME=mailboxes + +./build/maddy run +``` + +## Fork Context + +**Fork point:** `cbeadf169c8dfc3a824c9b83ab605fd792a300b7` + +**Changes made:** +1. REST API for user/mailbox management (Echo framework) +2. User/mailbox lifecycle decoupling +3. Route53/Cloudflare DNS provider support for ACME + +**Key files added/modified:** +- `api.go` - REST API routes and initialization +- `users.go` - User CRUD handlers +- `imapAccounts.go` - Mailbox handlers +- `util.go` - DB access helpers +- `internal/rest/` - REST API utilities and models + +## Development Patterns + +### Adding REST Endpoints + +1. Define route in `api.go`: +```go +func NewV1(e *echo.Echo) { + // ... + v1.GET("/new-endpoint", newHandler) +} +``` + +2. Create handler in appropriate file (`users.go`, `imapAccounts.go`, or new file): +```go +func newHandler(c echo.Context) error { + // Use global vars: userDb, imapDb, mailboxes + return c.JSON(http.StatusOK, result) +} +``` + +3. Add DTOs to `internal/rest/model/` if needed: +```go +type NewDto struct { + Field string `json:"field" validate:"required"` +} +``` + +### Request Binding and Validation + +```go +func handler(c echo.Context) error { + r := model.SomeDto{} + if err := c.Bind(&r); err != nil { + return err // Returns 400 with validation errors + } + // Validation runs automatically via CustomBinder + // ... +} +``` + +### Error Handling + +Return errors directly - Echo handles HTTP status codes: +```go +if err != nil { + return err // Returns 500 Internal Server Error +} +return c.NoContent(http.StatusOK) +``` + +For specific status codes: +```go +return c.NoContent(http.StatusNotFound) +return echo.NewHTTPError(http.StatusBadRequest, "message") +``` + +### Database Access + +Global variables initialized in `api.go`: +- `userDb` (`module.PlainUserDB`) - User credentials operations +- `imapDb` (`module.ManageableStorage`) - Mailbox operations +- `mailboxes` (`Mailboxes`) - Folder names from config + +```go +// User operations +userDb.ListUsers() +userDb.DeleteUser(username) +userDb.SetUserPassword(username, password) + +// Mailbox operations +imapDb.CreateIMAPAcct(username) +imapDb.DeleteIMAPAcct(username) +imapDb.GetIMAPAcct(username) +``` + +## Code Organization + +### Where to add new features + +| Feature Type | Location | +|-------------|----------| +| New REST endpoint | `api.go` (route), `*.go` (handler) | +| New DTO/model | `internal/rest/model/` | +| New middleware | `internal/rest/util/middleware/` | +| Server config | `internal/rest/util/server/` | +| Module interface changes | `framework/module/` | + +### Important module names (in config) + +The REST API looks for these specific module names: +- `local_authdb` - Password authentication module +- `local_mailboxes` - IMAP storage module + +### Database Schema (Mailboxes DB) + +The imapsql module manages three key tables: + +| Table | Purpose | Key Columns | +|-------|---------|-------------| +| `users` | IMAP accounts | `id`, `username`, `msgsizelimit`, `inboxid` | +| `mboxes` | Mailbox folders | `id`, `uid` (FK→users), `name`, `specialuse`, `msgscount` | +| `msgs` | Message metadata | `mboxid` (FK→mboxes), `msgid`, `bodylen`, `extbodykey`, `seen` | + +**Useful for future features:** +- `msgs.bodylen` - Calculate storage usage/quotas (not yet implemented) +- `mboxes.msgscount` - Message count per folder +- `msgs.extbodykey` - Links to S3 blob storage + +See [architecture.md](./architecture.md#database-tables) for full schema details. + +## Testing + +### Manual API Testing + +```bash +# Health check +curl http://localhost:8080/ + +# Create user (with auth) +curl -X POST http://localhost:8080/v1/users \ + -u admin@example.com:password \ + -H "Content-Type: application/json" \ + -d '{"username":"user@example.com","password":"pass","createMailboxes":true}' + +# List users +curl http://localhost:8080/v1/users \ + -u admin@example.com:password + +# Delete user +curl -X DELETE http://localhost:8080/v1/users/user@example.com \ + -u admin@example.com:password +``` + +### Integration Tests + +Place tests alongside handlers or in `internal/rest/` directory. + +## Configuration + +### REST API Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `ENABLE_API` | Yes | - | Set to `true` to enable | +| `ADMIN_EMAIL` | Yes* | - | Admin username for Basic Auth | +| `ADMIN_PASSWORD` | Yes* | - | Admin password for Basic Auth | +| `REQUESTS_PER_SECOND` | No | 3 | Rate limit per IP | +| `CORS_ALLOW_ORIGINS` | No | `*` | Comma-separated origins | +| `CONTENT_SECURITY_POLICY` | No | - | CSP header value | + +*Required when `ENABLE_API=true` + +### Database Environment Variables + +| Variable | Description | +|----------|-------------| +| `DB_HOST` | PostgreSQL host | +| `DB_PORT` | PostgreSQL port (usually 5432) | +| `DB_NAME` | Credentials database | +| `DB_USER` | Database user | +| `DB_PASSWORD` | Database password | +| `MAILBOXES_DB_NAME` | Mailbox metadata database | + +## Common Tasks + +### Add a new user property + +1. Update DTO in `internal/rest/model/user.go` +2. Update handler in `users.go` +3. May require upstream module interface changes + +### Change mailbox folder names + +Update `maddy.conf`: +``` +storage.imapsql local_mailboxes { + sent_mailbox "Sent Messages" + trash_mailbox "Deleted Items" + # ... +} +``` + +### Add new authentication provider + +This requires upstream module changes in `internal/auth/` and implementing `module.PlainUserDB` interface. + +### Debug REST API + +```bash +# Enable debug mode (set in api.go) +# Check server logs for request/response info + +# Test rate limiting +for i in {1..10}; do curl http://localhost:8080/; done +``` diff --git a/architecture.md b/architecture.md new file mode 100644 index 000000000..21366907b --- /dev/null +++ b/architecture.md @@ -0,0 +1,537 @@ +# Maddy Mail Server - Architecture + +This document describes the architecture of this Maddy Mail Server fork, focusing on the additions made after forking from upstream (commit `cbeadf1`). + +## Project Overview + +Maddy is a composable, all-in-one email server written in Go that replaces Postfix, Dovecot, OpenDKIM, OpenSPF, and OpenDMARC with a single daemon. It supports SMTP (inbound/outbound), IMAP, and email security protocols (DKIM, SPF, DMARC). + +**Fork Purpose:** Add REST API for user and mailbox management, enabling programmatic provisioning without CLI access. + +## High-Level Architecture + +``` + +------------------+ + | REST API | + | (Port 8080) | + | [FORK ADDITION]| + +--------+---------+ + | + +------------------------------------+------------------------------------+ + | | | + v v v ++-------+--------+ +--------+--------+ +--------+--------+ +| SMTP Inbound | | Submission | | IMAP | +| (Port 25) | | (Port 587) | | (Port 143) | ++-------+--------+ +--------+--------+ +--------+--------+ + | | | + +------------------------------------+------------------------------------+ + | + +--------v--------+ + | Message | + | Pipeline | + +--------+--------+ + | + +------------------------+------------------------+ + | | | + +--------v--------+ +--------v--------+ +--------v--------+ + | Credentials | | Mailboxes | | Message Blobs | + | (PostgreSQL) | | (PostgreSQL) | | (S3) | + +-----------------+ +-----------------+ +-----------------+ +``` + +## Upstream Components (Brief) + +### Protocol Endpoints + +| Endpoint | Port | Purpose | +|----------|------|---------| +| SMTP | 25 | Inbound mail reception (MX) | +| Submission | 587 | Authenticated outbound mail | +| IMAP | 143 | Mailbox access for clients | + +Key files: +- `internal/endpoint/smtp/smtp.go` - SMTP server +- `internal/endpoint/imap/imap.go` - IMAP server + +### Message Pipeline + +Routes messages through checks, modifiers, and delivery targets based on configuration rules. + +Key file: `internal/msgpipeline/msgpipeline.go` + +### Authentication System + +Pluggable authentication providers: +- `pass_table` - Database-backed password storage (used by this fork) +- `ldap`, `pam`, `shadow` - Alternative providers + +Key interface: `module.PlainUserDB` in `framework/module/auth.go` + +### Storage Layer + +SQL-based IMAP storage with pluggable blob backends. + +Key file: `internal/storage/imapsql/imapsql.go` + +--- + +## Fork Additions (Detailed) + +### 1. REST API Architecture + +The REST API enables programmatic user and mailbox management. + +``` ++------------------+ +------------------+ +------------------+ +| HTTP Client | --> | Echo Server | --> | Middleware | +| | | (Port 8080) | | Stack | ++------------------+ +------------------+ +--------+---------+ + | + +-------------------------------------+ + | + v ++-------------------+-------------------+-------------------+ +| | | | +v v v v +Logger Recovery CORS Rate Limiter + (3 req/sec) +| | | | ++-------------------+-------------------+-------------------+ + | + v ++-------------------+-------------------+-------------------+ +| | | | +v v v v +Security CSRF Request ID Gzip +Headers (optional) Compression +| | | | ++-------------------+-------------------+-------------------+ + | + v + +--------+--------+ + | Basic Auth | + | (Admin Only) | + +--------+--------+ + | + v + +--------+--------+ + | Handlers | + | (users.go, | + | imapAccounts.go)| + +-----------------+ +``` + +#### API Endpoints + +| Method | Path | Description | Auth | +|--------|------|-------------|------| +| GET | `/` | Health check | No | +| GET | `/version` | Server version | No | +| POST | `/v1/users` | Create user | Yes | +| GET | `/v1/users` | List users (optional `?domain=` filter) | Yes | +| GET | `/v1/users/:id` | Get user | Yes | +| POST | `/v1/users/:id/password` | Update password | Yes | +| DELETE | `/v1/users/:id` | Delete user (optional `?delete_mailbox=true`) | Yes | +| POST | `/v1/users/:id/mailboxes` | Create mailbox | Yes | +| DELETE | `/v1/users/:id/mailboxes` | Delete mailbox | Yes | + +#### Request/Response Models + +```go +// Create user with optional mailbox provisioning +type CreateUserDto struct { + Username string `json:"username" validate:"email"` + Password string `json:"password,omitempty"` + CreateMailboxes bool `json:"createMailboxes"` +} + +// User representation +type User struct { + Username string `json:"username" validate:"email"` + Password string `json:"password,omitempty"` +} + +// Password update +type Password struct { + Password string `json:"password,omitempty" validate:"required"` +} +``` + +#### Key Files + +| File | Purpose | +|------|---------| +| `api.go` | Route registration, server startup, DB initialization | +| `users.go` | User CRUD handlers and business logic | +| `imapAccounts.go` | Mailbox create/delete handlers | +| `util.go` | DB access helpers, mailbox config extraction | +| `internal/rest/model/user.go` | Request/response DTOs | +| `internal/rest/util/server/server.go` | Echo server setup with middleware | +| `internal/rest/util/server/secure.go` | Security headers, CORS configuration | +| `internal/rest/util/middleware/basic_auth/` | Admin authentication | + +#### Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `ENABLE_API` | Set to `true` to enable REST API | Yes | +| `ADMIN_EMAIL` | Admin username for Basic Auth | Yes (if API enabled) | +| `ADMIN_PASSWORD` | Admin password for Basic Auth | Yes (if API enabled) | +| `REQUESTS_PER_SECOND` | Rate limit (default: 3) | No | +| `CORS_ALLOW_ORIGINS` | Comma-separated allowed origins | No | +| `CONTENT_SECURITY_POLICY` | CSP header value | No | +| `FQDN` | Domain for CSRF cookie | No | +| `USE_COOKIES` | Enable CSRF (default: false) | No | + +### 2. User/Mailbox Decoupling + +Users and mailboxes have independent lifecycles, allowing flexible provisioning workflows. + +``` +WORKFLOW A: Create user without mailbox ++-------------------+ +| POST /v1/users | +| createMailboxes: | +| false | ++--------+----------+ + | + v ++--------+----------+ +| User created in | +| credentials DB | ++-------------------+ + + +WORKFLOW B: Create user with mailbox ++-------------------+ +| POST /v1/users | +| createMailboxes: | +| true | ++--------+----------+ + | + v ++--------+----------+ +-------------------+ +| User created in | --> | Mailbox created | +| credentials DB | | with SPECIAL-USE | ++-------------------+ | folders | + +-------------------+ + + +WORKFLOW C: Provision mailbox later ++-------------------+ +-------------------+ +| POST /v1/users/ | --> | Mailbox created | +| :id/mailboxes | | Sent, Trash, | ++-------------------+ | Junk, Drafts, | + | Archive | + +-------------------+ + + +WORKFLOW D: Delete user, keep mailbox ++-------------------+ +| DELETE /v1/users/ | +| :id | ++--------+----------+ + | + v ++--------+----------+ +| User removed from | +| credentials DB | +| (mailbox remains) | ++-------------------+ +``` + +#### Mailbox Creation Details + +When a mailbox is created, these IMAP SPECIAL-USE folders are provisioned: + +| Folder | SPECIAL-USE Attribute | +|--------|----------------------| +| Sent | `\Sent` | +| Trash | `\Trash` | +| Spam/Junk | `\Junk` | +| Drafts | `\Drafts` | +| Archive | `\Archive` | + +Folder names are read from the `local_mailboxes` configuration block. + +### 3. DNS Provider Support + +Added build support for AWS Route53 and Cloudflare DNS providers for ACME certificate automation. + +**Dockerfile change:** +```dockerfile +RUN ./build.sh --builddir /tmp/maddy-build \ + --destdir /tmp/maddy-install \ + --tags 'libdns_route53 libdns_cloudflare' \ + build install +``` + +These providers enable automatic DNS-01 challenges for Let's Encrypt certificates. + +--- + +## Storage Architecture + +``` ++------------------+ +------------------+ +------------------+ +| REST API / | | maddy.conf | | Environment | +| Handlers | | | | Variables | ++--------+---------+ +--------+---------+ +--------+---------+ + | | | + +------------------------+------------------------+ + | + v + +-------------+-------------+ + | storage.imapsql | + | (local_mailboxes) | + +-------------+-------------+ + | + +-------------------+-------------------+ + | | + v v ++-------------+-------------+ +-------------+-------------+ +| PostgreSQL Database | | S3 Blob Storage | +| | | | +| - Message metadata | | - Message bodies | +| - Mailbox structure | | - Attachments | +| - Flags, UIDs | | | +| | | | +| DSN: host={DB_HOST} | | endpoint: S3 URL | +| port={DB_PORT} | | bucket: bucket name | +| dbname={MAILBOXES_ | | access_key/secret_key | +| DB_NAME} | | | ++---------------------------+ +---------------------------+ + ++---------------------------+ +| Credentials Database | +| (table.sql_query) | +| | +| - username (PK) | +| - password (bcrypt) | +| | +| DSN: host={DB_HOST} | +| port={DB_PORT} | +| dbname={DB_NAME} | ++---------------------------+ +``` + +### Database Tables + +**Credentials Database (managed by `table.sql_query`):** +```sql +CREATE TABLE credentials ( + username varchar NOT NULL PRIMARY KEY, + password varchar NOT NULL -- bcrypt hash +); +``` + +**Mailboxes Database (managed by `go-imap-sql`):** + +The imapsql module creates and manages three key tables: + +**1. users** - IMAP account records +``` ++---------------+--------------+------+-----------------------------------+ +| Column | Type | Null | Description | ++---------------+--------------+------+-----------------------------------+ +| id | int8 | NO | Primary key (auto-increment) | +| username | varchar(255) | NO | Email address (unique) | +| msgsizelimit | int4 | YES | Per-user message size limit | +| inboxid | int8 | YES | Reference to INBOX mailbox | ++---------------+--------------+------+-----------------------------------+ +``` + +**2. mboxes** - Mailbox/folder records +``` ++---------------+--------------+------+-----------------------------------+ +| Column | Type | Null | Description | ++---------------+--------------+------+-----------------------------------+ +| id | int8 | NO | Primary key (auto-increment) | +| uid | int4 | NO | FK to users.id | +| name | varchar(255) | NO | Folder name (INBOX, Sent, etc.) | +| sub | int4 | NO | Subscribed flag (default: 1) | +| mark | int4 | NO | Internal marker | +| msgsizelimit | int4 | YES | Per-mailbox size limit | +| uidnext | int4 | NO | Next UID to assign | +| uidvalidity | int8 | NO | IMAP UIDVALIDITY value | +| specialuse | varchar(255) | YES | SPECIAL-USE attribute (\Sent etc) | +| msgscount | int4 | NO | Message count in mailbox | ++---------------+--------------+------+-----------------------------------+ +``` + +**3. msgs** - Message metadata +``` ++---------------+--------------+------+-----------------------------------+ +| Column | Type | Null | Description | ++---------------+--------------+------+-----------------------------------+ +| mboxid | int8 | NO | FK to mboxes.id | +| msgid | int8 | NO | Message UID within mailbox | +| date | int8 | NO | Message date (Unix timestamp) | +| bodylen | int4 | NO | Message body size in bytes | +| mark | int4 | NO | Internal marker (default: 0) | +| bodystructure | bytea | NO | IMAP BODYSTRUCTURE (serialized) | +| cachedheader | bytea | NO | Cached headers (serialized) | +| extbodykey | varchar(255) | YES | FK to extkeys.id (S3 blob key) | +| seen | int4 | NO | \Seen flag (default: 0) | +| compressalgo | varchar(255) | YES | Compression algorithm used | +| recent | int4 | NO | \Recent flag (default: 1) | ++---------------+--------------+------+-----------------------------------+ +``` + +**Usage Notes:** +- `msgs.bodylen` can be used to calculate storage usage and quotas (not yet implemented in Maddy) +- `msgs.extbodykey` links to blob storage (S3) for actual message content +- `mboxes.msgscount` tracks message count per folder +- `mboxes.specialuse` stores IMAP SPECIAL-USE attributes (`\Sent`, `\Trash`, `\Junk`, `\Drafts`, `\Archive`) + +--- + +## Directory Structure + +### Fork-Added Directories + +``` +/ +├── api.go # REST API initialization +├── users.go # User endpoint handlers +├── imapAccounts.go # Mailbox endpoint handlers +├── util.go # DB helpers, config extraction +│ +└── internal/rest/ + ├── model/ + │ └── user.go # Request/response DTOs + │ + └── util/ + ├── middleware/ + │ └── basic_auth/ + │ └── basic_auth.go # Admin auth validator + │ + └── server/ + ├── server.go # Echo server, middleware stack + ├── secure.go # Security headers, CORS + ├── ratelimiter.go # Rate limiting + └── binding.go # Custom validation +``` + +### Key Upstream Directories + +``` +/ +├── cmd/maddy/ # Main executable +├── framework/ +│ ├── module/ # Module interfaces +│ └── config/ # Configuration parsing +├── internal/ +│ ├── endpoint/smtp/ # SMTP server +│ ├── endpoint/imap/ # IMAP server +│ ├── storage/imapsql/ # SQL storage backend +│ ├── auth/pass_table/ # Password authentication +│ └── msgpipeline/ # Message routing +└── maddy.go # Server entry point +``` + +--- + +## Configuration + +### Enabling REST API + +Set environment variables before starting: + +```bash +export ENABLE_API=true +export ADMIN_EMAIL=admin@example.com +export ADMIN_PASSWORD=secure-password +``` + +### Database Environment Variables + +| Variable | Description | +|----------|-------------| +| `DB_HOST` | PostgreSQL host | +| `DB_PORT` | PostgreSQL port | +| `DB_NAME` | Credentials database name | +| `DB_USER` | Database user | +| `DB_PASSWORD` | Database password | +| `MAILBOXES_DB_NAME` | Mailboxes database name | + +--- + +## Reference Configuration + +Below is an annotated example configuration used in production: + +``` +# Base variables +$(hostname) = example.org +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) + +# Credentials stored in PostgreSQL +table.sql_query credentials { + driver postgres + dsn "host={env:DB_HOST} port={env:DB_PORT} dbname={env:DB_NAME} user={env:DB_USER} password={env:DB_PASSWORD} sslmode=disable" + init "CREATE TABLE IF NOT EXISTS credentials (...)" + lookup "SELECT password FROM credentials WHERE username = $1" + list "SELECT username FROM credentials" + add "INSERT INTO credentials (username, password) VALUES ($1, $2)" + del "DELETE from credentials where username=$1" + set "UPDATE credentials set password=$2 where username=$1" +} + +# Password authentication using the credentials table +auth.pass_table local_authdb { + table &credentials +} + +# IMAP storage with PostgreSQL metadata and S3 blob storage +storage.imapsql local_mailboxes { + driver postgres + dsn "host={env:DB_HOST} port={env:DB_PORT} dbname={env:MAILBOXES_DB_NAME} ..." + + msg_store s3 { + creds access_key + endpoint "s3-endpoint" + access_key "key" + secret_key "secret" + bucket "bucket-name" + } + + # Mailbox names used by REST API + sent_mailbox Sent + trash_mailbox Trash + junk_mailbox Spam + drafts_mailbox Drafts + archive_mailbox Archive +} + +# SMTP inbound (port 25) +smtp tcp://0.0.0.0:25 { + # ... routing rules +} + +# Submission (port 587) +submission tcp://0.0.0.0:587 { + auth &local_authdb + # ... routing rules with DKIM signing +} + +# IMAP (port 143) +imap tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} + +# Outbound relay +target.smtp outbound_delivery_relay { + targets tcp://relay-host:25 + auth plain "user" "password" +} +``` + +**Key Points:** +- `local_authdb` and `local_mailboxes` are the module names the REST API looks for +- The REST API reads mailbox folder names from the `storage.imapsql` config +- S3 blob storage enables scalable message storage +- Relay delivery is used for outbound mail through an external SMTP server