diff --git a/config.toml b/config.toml index 7854e90..304a9f2 100644 --- a/config.toml +++ b/config.toml @@ -115,8 +115,18 @@ pages = [ "generators/hardware-requirements/index.md", "generators/install/index.md", "generators/register/index.md", + "generators/config-reference/index.md", "generators/faq/index.md", ] +subsections = [ + { label = "Deployment examples", pages = [ + "generators/deployment-examples/validator-local-geth/index.md", + "generators/deployment-examples/managed-node-gcp/index.md", + "generators/deployment-examples/nginx-tls-snapshots/index.md", + "generators/deployment-examples/high-throughput-tuning/index.md", + "generators/deployment-examples/archival-vs-pruned/index.md", + ] }, +] [[extra.sidebar]] label = "Hosts" @@ -127,8 +137,18 @@ pages = [ "hosts/register/index.md", "hosts/quickstart/index.md", "hosts/examples/index.md", + "hosts/config-reference/index.md", "hosts/faq/index.md", ] +subsections = [ + { label = "Deployment examples", pages = [ + "hosts/deployment-examples/event-filter-allowlist/index.md", + "hosts/deployment-examples/snapshot-bootstrap/index.md", + "hosts/deployment-examples/prod-vm-nginx-tls/index.md", + "hosts/deployment-examples/gcp-local-ssd-raid0/index.md", + "hosts/deployment-examples/watchtower-auto-deploy/index.md", + ] }, +] [[extra.sidebar]] label = "Views" diff --git a/content/generators/config-reference/index.md b/content/generators/config-reference/index.md new file mode 100644 index 0000000..f8eb22e --- /dev/null +++ b/content/generators/config-reference/index.md @@ -0,0 +1,188 @@ ++++ +title = "Config reference" +description = "Reference for every Generator client config field: YAML key, default, env var override, and whether the field is actually read." ++++ + +The Generator client loads configuration from a YAML file (default `config/config.yaml`) and then applies environment variable overrides on top. Fields left unset in both the YAML and env vars receive code defaults from `applyDefaults` in `config/config.go` and `SetDefaults` in `pkg/pruner/config.go` and `pkg/snapshot/snapshot.go`. + +Env vars override YAML values. Where the shipped `config.yaml` sets a different value than the code default, both are noted below. + +## chain + +Identifies which EVM chain to index. Collection names are derived as `{name}__{network}__Block`, etc. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `name` | string | `Ethereum` | no | `CHAIN_NAME` | Chain name. Also supports `Arbitrum`, `Optimism`, `Avalanche`, or any EVM chain. | +| `network` | string | `Mainnet` | no | `CHAIN_NETWORK` | Network name, for example `Mainnet` or `Testnet`. | + +Ethereum is the only officially supported chain today. The shipped `config.yaml` lists Arbitrum, Optimism, and Avalanche as supported, and any EVM-compatible chain can likely be indexed by setting `chain.name` and `chain.network` to the correct values and pointing `geth.node_url` at a compatible RPC endpoint. The codebase is being refactored from EVM-only to a `Chain` interface with chain-specific Fetcher and Converter components, which will formalize multi-chain support. See [Chain abstraction](/reference/components/generator-client#chain-abstraction-in-progress) for the current state. + +## defradb + +Embedded database configuration. DefraDB handles storage, P2P replication, content addressing, and query serving. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `url` | string | empty | yes if `embedded` is false | `DEFRADB_URL` | DefraDB API URL. When `embedded` is true and `url` is empty, DefraDB runs on a random port. `DEFRADB_HOST` with optional `DEFRADB_PORT` also constructs this URL. | +| `keyring_secret` | string | empty | no | `DEFRADB_KEYRING_SECRET` | Encryption secret for the DefraDB keyring. Must stay the same across restarts or the existing identity becomes unreadable. | +| `embedded` | bool | false | no | (none) | When true, runs an embedded DefraDB instance. When false, requires a non-empty `url` pointing to an external DefraDB. Shipped `config.yaml` sets true. | + +### defradb p2p + +P2P networking configuration. The Generator pushes data out over libp2p and rejects all incoming replication. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | `DEFRADB_P2P_ENABLED` | Enable P2P networking. Shipped `config.yaml` sets true. | +| `accept_incoming` | bool | false | no | `DEFRADB_P2P_ACCEPT_INCOMING` | Accept incoming P2P documents. false means reject all. The Generator is a source of truth and should keep this false. | +| `bootstrap_peers` | string array | empty | no | (none) | P2P bootstrap peer multiaddrs. Peers are also discovered through `EntityRegistered` events from ShinzoHub. | +| `listen_addr` | string | empty | no | `DEFRADB_P2P_LISTEN_ADDR` | Multiaddr to listen on for P2P connections. Shipped `config.yaml` sets `/ip4/0.0.0.0/tcp/9171`. | +| `max_retries` | int | 0 | no | (none) | Connection attempts before marking a peer as failed. Shipped `config.yaml` sets 5. | +| `retry_base_delay_ms` | int | 0 | no | (none) | Base delay in milliseconds for exponential backoff (1s, 2s, 4s, 8s, 16s). Shipped `config.yaml` sets 1000. | +| `reconnect_interval_ms` | int | 0 | no | (none) | Interval in milliseconds to check for disconnected peers. Shipped `config.yaml` sets 60000. | +| `enable_auto_reconnect` | bool | false | no | (none) | Automatically reconnect to failed or disconnected peers. Shipped `config.yaml` sets true. | + +### defradb store + +Badger storage engine configuration. All cache and compaction fields map directly to Badger options. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `path` | string | empty | no | `DEFRADB_STORE_PATH` | Path to the DefraDB data directory. Shipped `config.yaml` sets `./.defra`. | +| `block_cache_mb` | int64 | 0 | no | `DEFRADB_BLOCK_CACHE_MB` | Badger block cache size in MB. Shipped `config.yaml` sets 512. | +| `memtable_mb` | int64 | 0 | no | `DEFRADB_MEMTABLE_MB` | Badger memtable size in MB. Shipped `config.yaml` sets 64. | +| `index_cache_mb` | int64 | 0 | no | `DEFRADB_INDEX_CACHE_MB` | Badger index cache size in MB. Shipped `config.yaml` sets 256. | +| `num_compactors` | int | 0 | no | `DEFRADB_NUM_COMPACTORS` | Number of Badger compaction workers. Shipped `config.yaml` sets 4. | +| `num_level_zero_tables` | int | 0 | no | `DEFRADB_NUM_LEVEL_ZERO_TABLES` | L0 tables before compaction starts. Shipped `config.yaml` comments this out. | +| `num_level_zero_tables_stall` | int | 0 | no | `DEFRADB_NUM_LEVEL_ZERO_TABLES_STALL` | L0 tables that trigger write stalls. Shipped `config.yaml` comments this out. | +| `value_log_file_size_mb` | int64 | 0 | no | (none) | Size of each value log file in MB. Smaller files mean more effective GC. Shipped `config.yaml` sets 128. | + +## geth + +Connection details for the Ethereum execution node. The Generator does not run a node. It reads from one you provide. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `node_url` | string | empty | yes | `GETH_RPC_URL` | Geth JSON-RPC endpoint URL. Used for full block fetches, gap fills, and historical ranges. | +| `ws_url` | string | empty | yes | `GETH_WS_URL` | Geth WebSocket endpoint URL. Used for real-time new block header subscriptions. | +| `api_key` | string | empty | no | `GETH_API_KEY` | API key for node authentication. Leave empty for same-VPC nodes without auth. | +| `api_key_type` | string | empty | no | `GETH_API_KEY_TYPE` | Header name for the API key. `x-goog-api-key` for GCP Blockchain Node Engine, `x-api-key` for most others. | + +## indexer + +Controls how the Generator fetches and processes blocks. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `start_height` | int | 0 | no | `INDEXER_START_HEIGHT` | Block number to start indexing from on first run with no existing data. 0 means auto-detect from chain tip. Must be 0 or higher. | +| `concurrent_blocks` | int | 8 | no | `INDEXER_CONCURRENT_BLOCKS` | Number of blocks to process concurrently. Shipped `config.yaml` sets 1. | +| `receipt_workers` | int | 16 | no | `INDEXER_RECEIPT_WORKERS` | Concurrent receipt fetchers per block. Shipped `config.yaml` sets 8. | +| `max_docs_per_txn` | int | 1000 | no | `INDEXER_MAX_DOCS_PER_TXN` | Document threshold for single-transaction block creation. Shipped `config.yaml` sets 100. | +| `max_tx_docs_per_batch` | int | 0 | no | `INDEXER_MAX_TX_DOCS` | Per-batch document size for transactions. 0 means use `max_docs_per_txn`. Shipped `config.yaml` sets 100. | +| `max_log_docs_per_batch` | int | 0 | no | `INDEXER_MAX_LOG_DOCS` | Per-batch document size for logs. 0 means use `max_docs_per_txn`. Shipped `config.yaml` sets 125. | +| `max_ale_docs_per_batch` | int | 0 | no | `INDEXER_MAX_ALE_DOCS` | Per-batch document size for access list entries. 0 means use `max_docs_per_txn`. Shipped `config.yaml` sets 500. | +| `blocks_per_minute` | int | 0 | no | `INDEXER_BLOCKS_PER_MINUTE` | Block indexing rate limit. 0 means no limit. Shipped `config.yaml` sets 60. | +| `health_server_port` | int | 8080 | no | `INDEXER_HEALTH_SERVER_PORT` | Health server port. Set to -1 to disable. | +| `open_browser_on_start` | bool | false | no | (none) | Auto-open the health page in a browser on startup. | +| `start_buffer` | int | 100 | no | `INDEXER_START_BUFFER` | Start this many blocks before chain tip when skipping ahead. | +| `schema_auth_mode` | string | `token` | no | `SCHEMA_AUTH_MODE` | Auth mode for the `/api/v1/schema` endpoints. One of `none`, `token`, or `mtls`. Empty defaults to `token`. `mtls` is not yet implemented. Shipped `config.yaml` sets `${SCHEMA_AUTH_MODE}` (env-substituted). | +| `schema_api_keys` | string array | empty | yes if `schema_auth_mode` is `token` | `SCHEMA_API_KEYS` | Bearer tokens accepted for `/api/v1/schema` endpoints when mode is `token`. Comma-separated. Not settable in `config.yaml` (the field uses `yaml:"-"`); provide via `SCHEMA_API_KEYS` or every schema request returns 503. | + +## pruner + +Removes old data to keep storage bounded. Defaults are applied by `SetDefaults` in `pkg/pruner/config.go` when fields are unset. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | `PRUNER_ENABLED` | Enable automatic pruning. Shipped `config.yaml` sets true. | +| `max_blocks` | int64 | 10000 | no | `PRUNER_MAX_BLOCKS` | Number of blocks to retain. Shipped `config.yaml` sets 1000. | +| `docs_per_block` | int | 1000 | no | (none) | Average docs per block, used with `max_blocks` to compute the max document count. Approximately 1057 on Ethereum mainnet. | +| `prune_threshold` | int64 | 0 | no | `PRUNER_PRUNE_THRESHOLD` | Deprecated. Kept for backward compatibility but unused by the pruner. Shipped `config.yaml` sets 1. | +| `interval_seconds` | int | 60 | no | `PRUNER_INTERVAL_SECONDS` | How often in seconds to check and prune. Shipped `config.yaml` sets 30. | +| `prune_history` | bool | false | no | (none) | Walk DAG chains to delete historical block versions. Two to three times slower. | + +## snapshot + +Bundles multiple blocks into a single signed file for faster initial sync. Defaults are applied by `SetDefaults` in `pkg/snapshot/snapshot.go` when fields are unset. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | `SNAPSHOT_ENABLED` | Enable automatic snapshots before pruning. Shipped `config.yaml` sets true. | +| `dir` | string | `./snapshots` | no | `SNAPSHOT_DIR` | Directory to store snapshot files. Shipped `config.yaml` sets `./.defra/snapshots`. | +| `blocks_per_file` | int64 | 1000 | no | `SNAPSHOT_BLOCKS_PER_FILE` | Blocks per snapshot file. Shipped `config.yaml` sets 100. | +| `interval_seconds` | int | 60 | no | `SNAPSHOT_INTERVAL_SECONDS` | How often in seconds to check and create snapshots. Shipped `config.yaml` sets 30. | + +## logger + +Logging configuration. The logger is zap-based. The log level is controlled by `development`, not by a `level` field. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `development` | bool | false | no | `LOGGER_DEBUG` | Enable development mode logging. When true, shows debug and test-level output. When false, shows info level and above. Shipped `config.yaml` sets true. | + +The shipped `config.yaml` also sets `logger.level: "info"`. This field is not in the `LoggerConfig` struct and has no effect. The log level is determined entirely by `development`. + +## Environment variables + +The table below lists every environment variable the Generator client reads, the YAML key it overrides, and the source file that reads it. + +| Env var | Overrides | Read in | +| --- | --- | --- | +| `CHAIN_NAME` | `chain.name` | `config/config.go` | +| `CHAIN_NETWORK` | `chain.network` | `config/config.go` | +| `GETH_RPC_URL` | `geth.node_url` | `config/config.go` | +| `GETH_WS_URL` | `geth.ws_url` | `config/config.go` | +| `GETH_API_KEY` | `geth.api_key` | `config/config.go` | +| `GETH_API_KEY_TYPE` | `geth.api_key_type` | `config/config.go` | +| `INDEXER_START_HEIGHT` | `indexer.start_height` | `config/config.go` | +| `INDEXER_CONCURRENT_BLOCKS` | `indexer.concurrent_blocks` | `config/config.go` | +| `INDEXER_RECEIPT_WORKERS` | `indexer.receipt_workers` | `config/config.go` | +| `INDEXER_MAX_DOCS_PER_TXN` | `indexer.max_docs_per_txn` | `config/config.go` | +| `INDEXER_MAX_TX_DOCS` | `indexer.max_tx_docs_per_batch` | `config/config.go` | +| `INDEXER_MAX_LOG_DOCS` | `indexer.max_log_docs_per_batch` | `config/config.go` | +| `INDEXER_MAX_ALE_DOCS` | `indexer.max_ale_docs_per_batch` | `config/config.go` | +| `INDEXER_BLOCKS_PER_MINUTE` | `indexer.blocks_per_minute` | `config/config.go` | +| `INDEXER_HEALTH_SERVER_PORT` | `indexer.health_server_port` | `config/config.go` | +| `INDEXER_START_BUFFER` | `indexer.start_buffer` | `config/config.go` | +| `SCHEMA_AUTH_MODE` | `indexer.schema_auth_mode` | `config/config.go` | +| `SCHEMA_API_KEYS` | `indexer.schema_api_keys` | `config/config.go` | +| `DEFRADB_URL` | `defradb.url` | `config/config.go` | +| `DEFRADB_HOST` | `defradb.url` | `config/config.go` (combined with `DEFRADB_PORT`) | +| `DEFRADB_PORT` | `defradb.url` | `config/config.go` (combined with `DEFRADB_HOST`) | +| `DEFRADB_KEYRING_SECRET` | `defradb.keyring_secret` | `pkg/defradb/defra.go` | +| `DEFRADB_P2P_ENABLED` | `defradb.p2p.enabled` | `config/config.go` | +| `DEFRADB_P2P_LISTEN_ADDR` | `defradb.p2p.listen_addr` | `config/config.go` | +| `DEFRADB_P2P_ACCEPT_INCOMING` | `defradb.p2p.accept_incoming` | `config/config.go` | +| `DEFRADB_STORE_PATH` | `defradb.store.path` | `config/config.go` | +| `DEFRADB_BLOCK_CACHE_MB` | `defradb.store.block_cache_mb` | `config/config.go` | +| `DEFRADB_MEMTABLE_MB` | `defradb.store.memtable_mb` | `config/config.go` | +| `DEFRADB_INDEX_CACHE_MB` | `defradb.store.index_cache_mb` | `config/config.go` | +| `DEFRADB_NUM_COMPACTORS` | `defradb.store.num_compactors` | `config/config.go` | +| `DEFRADB_NUM_LEVEL_ZERO_TABLES` | `defradb.store.num_level_zero_tables` | `config/config.go` | +| `DEFRADB_NUM_LEVEL_ZERO_TABLES_STALL` | `defradb.store.num_level_zero_tables_stall` | `config/config.go` | +| `PRUNER_ENABLED` | `pruner.enabled` | `config/config.go` | +| `PRUNER_MAX_BLOCKS` | `pruner.max_blocks` | `config/config.go` | +| `PRUNER_PRUNE_THRESHOLD` | `pruner.prune_threshold` | `config/config.go` | +| `PRUNER_INTERVAL_SECONDS` | `pruner.interval_seconds` | `config/config.go` | +| `SNAPSHOT_ENABLED` | `snapshot.enabled` | `config/config.go` | +| `SNAPSHOT_DIR` | `snapshot.dir` | `config/config.go` | +| `SNAPSHOT_BLOCKS_PER_FILE` | `snapshot.blocks_per_file` | `config/config.go` | +| `SNAPSHOT_INTERVAL_SECONDS` | `snapshot.interval_seconds` | `config/config.go` | +| `LOGGER_DEBUG` | `logger.development` | `config/config.go` | + +### Env vars that are not read + +The following env vars appear in some deployment artifacts but are not read by the Generator client code. They have no effect on runtime behavior. + +| Env var | Where it appears | Notes | +| --- | --- | --- | +| `LOG_LEVEL` | `docker-compose-prod.yml`, `indexer-prod-setup.sh` | Not read. Log level is controlled by `logger.development` and `LOGGER_DEBUG`. | +| `LOG_SOURCE` | `docker-compose-prod.yml`, `indexer-prod-setup.sh` | Not read. | +| `LOG_STACKTRACE` | `docker-compose-prod.yml`, `indexer-prod-setup.sh` | Not read. | +| `DEFRADB_PLAYGROUND` | Install page `docker run` example | Not read. The playground is controlled by build tags, not env vars. | +| `GOMEMLIMIT` | `docker-compose-prod.yml` | Not read by the client. This is a Go runtime soft memory limit, honored by the Go runtime itself. Set it to control garbage collection behavior under memory pressure. | + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/generators/deployment-examples/_index.md b/content/generators/deployment-examples/_index.md new file mode 100644 index 0000000..4a4aaf5 --- /dev/null +++ b/content/generators/deployment-examples/_index.md @@ -0,0 +1,4 @@ ++++ +render = false +page_template = "page.html" ++++ diff --git a/content/generators/deployment-examples/archival-vs-pruned/index.md b/content/generators/deployment-examples/archival-vs-pruned/index.md new file mode 100644 index 0000000..3fb8c9e --- /dev/null +++ b/content/generators/deployment-examples/archival-vs-pruned/index.md @@ -0,0 +1,221 @@ ++++ +title = "Archival versus pruned" +description = "Decide whether your Generator keeps all historical blocks or prunes old ones. A pruned Generator uses less storage and can still serve snapshots; an archival Generator keeps everything but grows without bound." +[extra] +mermaid = true ++++ + +When to use this: you need to decide whether your Generator keeps all historical data or prunes old blocks. A pruned Generator uses less storage but can still serve snapshots for Host bootstrap. An archival Generator keeps everything but grows without bound. + +These scenarios use Ethereum Mainnet and Geth. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing `chain.name` and pointing the RPC URLs at a compatible node. See the [chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart TB + subgraph Pruned["Pruned Generator"] + direction LR + GenP["Generator
pruner.enabled: true
max_blocks: 1000"] + SnapP["Snapshot files
blocks_per_file: 1000"] + GenP -- "writes snapshots
before pruning" --> SnapP + end + + subgraph Archival["Archival Generator"] + GenA["Generator
pruner.enabled: false
snapshot.enabled: false"] + end + + Hosts["Hosts"] + + GenP -- "P2P (libp2p)" --> Hosts + GenA -- "P2P (libp2p)" --> Hosts + Hosts -- "HTTPS snapshot
bootstrap" --> SnapP +{% end %} + +The pruned Generator writes snapshot files before deleting old blocks. Hosts can download those snapshots for historical bootstrap, then receive live blocks over P2P. The archival Generator never deletes anything. It uses more disk space but does not need snapshots because it can serve any block range directly over P2P. + +## Pruned Generator config + +This config keeps 1000 blocks in the database and produces snapshot files before pruning. The `pruner` and `snapshot` values are drawn from the shipped `config.yaml`, with `docs_per_block` filled in from the code default in `SetDefaults` (it is not present in the shipped file): + +```yaml +chain: + name: "Ethereum" + network: "Mainnet" + +defradb: + url: "http://localhost:9181" + keyring_secret: "" + embedded: true + p2p: + enabled: true + accept_incoming: false + bootstrap_peers: [] + listen_addr: "/ip4/0.0.0.0/tcp/9171" + store: + path: "./.defra" + block_cache_mb: 512 + memtable_mb: 64 + index_cache_mb: 256 + num_compactors: 4 + value_log_file_size_mb: 128 + +geth: + node_url: "${GETH_RPC_URL}" + ws_url: "${GETH_WS_URL}" + api_key: "${GETH_API_KEY}" + api_key_type: "${GETH_API_KEY_TYPE}" + +indexer: + start_height: 0 + concurrent_blocks: 1 + receipt_workers: 8 + max_docs_per_txn: 100 + blocks_per_minute: 60 + health_server_port: 8080 + open_browser_on_start: false + start_buffer: 100 + +logger: + development: true + +pruner: + enabled: true + max_blocks: 1000 + docs_per_block: 1000 + interval_seconds: 30 + prune_history: false + +snapshot: + enabled: true + dir: "./.defra/snapshots" + blocks_per_file: 100 + interval_seconds: 30 +``` + +### Pruned key values + +- `pruner.enabled: true`: Turn on automatic pruning. See [pruner config](/generators/config-reference#pruner). +- `pruner.max_blocks: 1000`: Keep the last 1000 blocks in the database. At roughly 12 seconds per Ethereum block, that is about 3.3 hours of history. See [pruner config](/generators/config-reference#pruner). +- `pruner.docs_per_block: 1000`: Average docs per block. The pruner triggers at `max_blocks` times `docs_per_block` documents. See [pruner config](/generators/config-reference#pruner). +- `pruner.interval_seconds: 30`: Check for pruning every 30 seconds. See [pruner config](/generators/config-reference#pruner). +- `pruner.prune_history: false`: Do not walk DAG chains to delete historical block versions. Setting this to true is two to three times slower. See [pruner config](/generators/config-reference#pruner). +- `snapshot.enabled: true`: Produce snapshot files before pruning. Hosts can download these for historical bootstrap. See [snapshot config](/generators/config-reference#snapshot). +- `snapshot.blocks_per_file: 100`: Bundle 100 blocks per snapshot file. The shipped `config.yaml` sets 100, while the code default in `SetDefaults` is 1000. See [snapshot config](/generators/config-reference#snapshot). + +## Archival Generator config + +This config disables both pruning and snapshots. The database grows without bound. Mount this file at `/app/config.yaml`: + +```yaml +chain: + name: "Ethereum" + network: "Mainnet" + +defradb: + url: "http://localhost:9181" + keyring_secret: "" + embedded: true + p2p: + enabled: true + accept_incoming: false + bootstrap_peers: [] + listen_addr: "/ip4/0.0.0.0/tcp/9171" + store: + path: "./.defra" + block_cache_mb: 512 + memtable_mb: 64 + index_cache_mb: 256 + num_compactors: 4 + value_log_file_size_mb: 128 + +geth: + node_url: "${GETH_RPC_URL}" + ws_url: "${GETH_WS_URL}" + api_key: "${GETH_API_KEY}" + api_key_type: "${GETH_API_KEY_TYPE}" + +indexer: + start_height: 0 + concurrent_blocks: 1 + receipt_workers: 8 + max_docs_per_txn: 100 + blocks_per_minute: 60 + health_server_port: 8080 + open_browser_on_start: false + start_buffer: 100 + +logger: + development: true + +pruner: + enabled: false + +snapshot: + enabled: false +``` + +### Archival key values + +- `pruner.enabled: false`: No pruning. The database retains every block, transaction, log, and signature from `start_height` onward. See [pruner config](/generators/config-reference#pruner). +- `snapshot.enabled: false`: No snapshots. An archival Generator does not need snapshots because it never deletes blocks. Hosts can replicate the full history over P2P. See [snapshot config](/generators/config-reference#snapshot). + +## Storage comparison + +The pruned Generator with `max_blocks: 1000` and `docs_per_block: 1000` holds roughly 1 million documents at steady state. On Ethereum mainnet, each block produces approximately 150 documents on average (blocks with many transactions produce more, empty blocks produce fewer). The actual document count varies, but 1000 blocks typically uses 2 to 5 GB of Badger storage. + +The archival Generator grows by roughly 150 documents per block, or about 650,000 documents per day. At Ethereum's 12-second block time, a year of mainnet history is approximately 2.6 million blocks, producing hundreds of GB of storage. Plan disk capacity accordingly. + +## Compose file + +Either config can be mounted with the same compose file. Change the config file path as needed: + +```yaml +networks: + shinzo-net: + driver: bridge + +services: + shinzo-generator: + container_name: shinzo-generator + platform: linux/amd64 + image: ghcr.io/shinzonetwork/shinzo-generator-client:standard + user: "1001:1001" + restart: unless-stopped + networks: + - shinzo-net + mem_limit: 16g + mem_reservation: 13g + ports: + - "9171:9171" + volumes: + - ~/shinzo-data/defradb:/app/.defra + - ./config.yaml:/app/config.yaml:ro + environment: + - GETH_RPC_URL=http://localhost:8545 + - GETH_WS_URL=ws://localhost:8546 + - DEFRADB_KEYRING_SECRET=pingpong + - GOMEMLIMIT=14GiB + logging: + options: + max-size: "50m" + max-file: "3" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s +``` + +## Gotchas + +- `pruner.prune_threshold` appears in the shipped `config.yaml` with value 1. It is deprecated and unused by the pruner. It has been omitted from both configs above. See [pruner config](/generators/config-reference#pruner). +- `snapshot.blocks_per_file` defaults to 1000 in the code (`SetDefaults`), but the shipped `config.yaml` sets 100. If you omit the field entirely, you get 1000. If you mount the shipped config, you get 100. +- When switching from pruned to archival, old pruned data is gone. The Generator will not re-fetch blocks that were already deleted. You need to restart with `start_height` set to a block you want to begin archival from, or accept that history before the switch is incomplete. +- When switching from archival to pruned, the pruner will start deleting old blocks on the next interval. Snapshot files are created before pruning begins, but if the snapshot directory does not have enough disk space, snapshots may fail and blocks will be pruned without a snapshot backup. +- The pruner uses `DefaultCollectionConfig` which prunes `Block`, `BlockSignature`, `AccessListEntry`, `Log`, and `Transaction` collections. It does not prune `SnapshotSignature` documents, which accumulate over time even on a pruned Generator. +- For an archival Generator, consider increasing `block_cache_mb` and `index_cache_mb` to keep more of the larger database in memory. See the [high-throughput tuning scenario](../high-throughput-tuning/) for cache sizing guidance. + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/generators/deployment-examples/high-throughput-tuning/index.md b/content/generators/deployment-examples/high-throughput-tuning/index.md new file mode 100644 index 0000000..1bdccf0 --- /dev/null +++ b/content/generators/deployment-examples/high-throughput-tuning/index.md @@ -0,0 +1,172 @@ ++++ +title = "High-throughput and catch-up tuning" +description = "Tune the Generator for high throughput and fast catch-up from a historical start height by raising block and receipt concurrency, Badger cache sizes, and the Go runtime memory limit." +[extra] +mermaid = true ++++ + +When to use this: your Generator needs to catch up from a historical start height quickly, or you want to maximize block processing throughput on a machine with available CPU and memory. + +These scenarios use Ethereum Mainnet and Geth. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing `chain.name` and pointing the RPC URLs at a compatible node. See the [chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph VM["Generator VM"] + direction TB + Gen["Generator client
concurrent_blocks: 8
receipt_workers: 32
GOMEMLIMIT: 14GiB"] + DB[("Badger store
block_cache: 1024MB
memtable: 128MB
index_cache: 512MB")] + Gen --> DB + end + + Geth["Geth node"] + Hosts["Hosts"] + + Geth -- "HTTP + WS" --> Gen + Gen -- "P2P (libp2p)" --> Hosts +{% end %} + +The Generator fetches blocks from Geth and processes them with a configurable number of concurrent workers. Receipt fetching happens in parallel per block. Badger cache sizes control how much data stays in memory before hitting disk. The `GOMEMLIMIT` env var tells the Go runtime when to trigger garbage collection, preventing OOM kills under load. + +## Prerequisites + +- Docker installed on the VM. +- A Geth node endpoint with enough capacity to handle parallel RPC requests. A local node or a high-throughput managed provider works best. +- At least 16 GB of RAM on the VM. The tuning values below assume this. + +## Config file + +This `config.yaml` tunes the Generator for high throughput. The values are drawn from the `indexer` and `defradb.store` sections of the shipped `config.yaml`, adjusted upward from defaults. Mount this file into the container at `/app/config.yaml`: + +```yaml +chain: + name: "Ethereum" + network: "Mainnet" + +defradb: + url: "http://localhost:9181" + keyring_secret: "" + embedded: true + p2p: + enabled: true + accept_incoming: false + bootstrap_peers: [] + listen_addr: "/ip4/0.0.0.0/tcp/9171" + store: + path: "./.defra" + block_cache_mb: 1024 + memtable_mb: 128 + index_cache_mb: 512 + num_compactors: 8 + value_log_file_size_mb: 128 + +geth: + node_url: "${GETH_RPC_URL}" + ws_url: "${GETH_WS_URL}" + api_key: "${GETH_API_KEY}" + api_key_type: "${GETH_API_KEY_TYPE}" + +indexer: + start_height: 0 + concurrent_blocks: 8 + receipt_workers: 32 + max_docs_per_txn: 100 + blocks_per_minute: 0 + health_server_port: 8080 + open_browser_on_start: false + start_buffer: 100 + +logger: + development: true + +pruner: + enabled: true + max_blocks: 1000 + interval_seconds: 30 + prune_history: false + +snapshot: + enabled: true + dir: "./.defra/snapshots" + blocks_per_file: 100 + interval_seconds: 30 +``` + +## Compose file + +Mount the config file and set `GOMEMLIMIT` via the environment. The Geth connection details come from env vars, which override the `${GETH_*}` placeholders in the config file: + +```yaml +networks: + shinzo-net: + driver: bridge + +services: + shinzo-generator: + container_name: shinzo-generator + platform: linux/amd64 + image: ghcr.io/shinzonetwork/shinzo-generator-client:standard + user: "1001:1001" + restart: unless-stopped + networks: + - shinzo-net + mem_limit: 16g + mem_reservation: 13g + ports: + - "9171:9171" + volumes: + - ~/shinzo-data/defradb:/app/.defra + - ./config.yaml:/app/config.yaml:ro + environment: + - GETH_RPC_URL=http://localhost:8545 + - GETH_WS_URL=ws://localhost:8546 + - DEFRADB_KEYRING_SECRET=pingpong + - GOMEMLIMIT=14GiB + logging: + options: + max-size: "50m" + max-file: "3" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s +``` + +### What each tuning value does + +- `concurrent_blocks: 8`: Process 8 blocks at the same time instead of the shipped default of 1. This is the code default from `applyDefaults`. Increase it if your Geth node can handle parallel requests. See [indexer config](/generators/config-reference#indexer). +- `receipt_workers: 32`: Fetch 32 receipts concurrently per block, up from the shipped 8. Receipts are the bottleneck for blocks with many transactions. See [indexer config](/generators/config-reference#indexer). +- `blocks_per_minute: 0`: Disable the rate limit. The shipped `config.yaml` sets 60, which caps indexing speed. Set to 0 for maximum throughput during catch-up. See [indexer config](/generators/config-reference#indexer). +- `block_cache_mb: 1024`: Double the shipped 512. More cache means fewer disk reads for recently written blocks. See [defradb store config](/generators/config-reference#defradb-store). +- `memtable_mb: 128`: Double the shipped 64. Larger memtables reduce the frequency of flushes to disk. See [defradb store config](/generators/config-reference#defradb-store). +- `index_cache_mb: 512`: Double the shipped 256. More index cache speeds up point lookups during pruning and snapshot creation. See [defradb store config](/generators/config-reference#defradb-store). +- `num_compactors: 8`: Double the shipped 4. More compaction workers prevent L0 table buildup during high write rates. See [defradb store config](/generators/config-reference#defradb-store). +- `GOMEMLIMIT=14GiB`: Go runtime soft memory limit, set below the 16g container limit. The runtime uses this to decide when to trigger GC, which prevents OOM kills. See [env vars](/generators/config-reference#environment-variables). + +## Start the Generator + +```shell +docker compose -f docker-compose.yml up -d +``` + +Monitor the catch-up rate: + +```shell +curl -s http://localhost:8080/metrics | jq '.blocks_processed, .blocks_per_minute' +``` + +## Gotchas + +- The shipped `config.yaml` sets `concurrent_blocks: 1`, but the code default in `applyDefaults` is 8. If you mount no config file and set no env var, you get the code default of 8. If the container image includes the shipped `config.yaml`, you get 1 unless you override it. +- High `concurrent_blocks` and `receipt_workers` values generate parallel RPC requests against your Geth node. A local Geth node can usually handle this. A shared or rate-limited managed provider may throttle or reject connections. Monitor your Geth node's request queue and RPC error rate. +- `blocks_per_minute: 0` removes the rate limit entirely. This is useful for catch-up but means the Generator will process blocks as fast as Geth can serve them. If Geth is also serving other consumers, this can starve them of RPC capacity. +- `GOMEMLIMIT` is a Go runtime soft memory limit, not a Generator client config var. It is honored by the Go runtime's garbage collector. Set it below the container `mem_limit` to leave headroom for non-Go memory allocations. If you set it too high, the container can be OOM-killed by the kernel. +- Increasing Badger cache sizes raises memory usage. The values above (`block_cache_mb: 1024`, `memtable_mb: 128`, `index_cache_mb: 512`) add up to roughly 1.6 GB of cache alone. Make sure the total stays within the `GOMEMLIMIT` and container memory limit. +- `num_compactors: 8` uses more CPU. On a machine with fewer than 8 cores, this can cause CPU contention with block processing. Match it to your available cores. + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/generators/deployment-examples/managed-node-gcp/index.md b/content/generators/deployment-examples/managed-node-gcp/index.md new file mode 100644 index 0000000..493529a --- /dev/null +++ b/content/generators/deployment-examples/managed-node-gcp/index.md @@ -0,0 +1,134 @@ ++++ +title = "Managed node on GCP Blockchain Node Engine" +description = "Run the Generator client against GCP Blockchain Node Engine or another managed node provider that requires an API key header." +[extra] +mermaid = true ++++ + +When to use this: you want to run the Generator client against GCP Blockchain Node Engine or another managed node provider, authenticating with an API key header. + +These scenarios use Ethereum Mainnet and Geth. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing `chain.name` and pointing the RPC URLs at a compatible node. See the [chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph VM["Generator VM"] + Gen["Generator client
:9171 P2P
:8080 Health/Metrics"] + end + + GCP["GCP Blockchain Node Engine
json-rpc...blockchainnodeengine.com"] + Hosts["Hosts"] + + GCP -- "HTTPS
x-goog-api-key header" --> Gen + Gen -- "P2P (libp2p)" --> Hosts +{% end %} + +The Generator runs on its own VM and connects to GCP Blockchain Node Engine over HTTPS. GCP BNE requires an API key in the `x-goog-api-key` header. The Generator sends this header on every JSON-RPC and WebSocket request. Data flows out to Hosts over P2P. + +## Prerequisites + +- Docker installed on the VM. +- A GCP Blockchain Node Engine instance provisioned. See the [GCP documentation](https://cloud.google.com/blockchain-node-engine) for setup. +- Your GCP API key for the BNE instance. +- Port 9171 open for P2P traffic from Hosts. Port 8080 open for health and metrics if you want to monitor externally. + +## Compose file + +This compose file is drawn from `docker-compose-prod.yml` in the `shinzo-generator-client` repo. Replace `` with your GCP API key: + +```yaml +networks: + shinzo-net: + driver: bridge + +services: + shinzo-generator: + container_name: shinzo-generator + platform: linux/amd64 + image: ghcr.io/shinzonetwork/shinzo-generator-client:standard + user: "1001:1001" + restart: unless-stopped + networks: + - shinzo-net + mem_limit: 16g + mem_reservation: 13g + ports: + - "9171:9171" + volumes: + - ~/shinzo-data/defradb:/app/.defra + environment: + - GETH_RPC_URL=https://json-rpc.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com + - GETH_WS_URL=wss://ws.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com + - GETH_API_KEY= + - GETH_API_KEY_TYPE=x-goog-api-key + - INDEXER_START_HEIGHT=0 + - DEFRADB_KEYRING_SECRET=pingpong + - GOMEMLIMIT=14GiB + - SNAPSHOT_ENABLED=false + - SCHEMA_AUTH_MODE=none + logging: + options: + max-size: "50m" + max-file: "3" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s +``` + +### What each value means + +- `GETH_RPC_URL` and `GETH_WS_URL`: Your GCP BNE endpoints. Replace the example URLs with your own BNE instance URLs. See [geth config](/generators/config-reference#geth). +- `GETH_API_KEY=`: Your GCP API key. Replace this with the actual key from your GCP console. See [geth config](/generators/config-reference#geth). +- `GETH_API_KEY_TYPE=x-goog-api-key`: The header name GCP BNE expects for authentication. See [geth config](/generators/config-reference#geth). +- `INDEXER_START_HEIGHT=0`: Start indexing from the chain tip. See [indexer config](/generators/config-reference#indexer). +- `DEFRADB_KEYRING_SECRET=pingpong`: Encryption secret for the DefraDB keyring. Change this to your own secret and keep it consistent across restarts. See [defradb config](/generators/config-reference#defradb). +- `GOMEMLIMIT=14GiB`: Go runtime soft memory limit. Set below the container `mem_limit` to leave headroom for non-Go memory. See [env vars](/generators/config-reference#environment-variables). +- `SNAPSHOT_ENABLED=false`: Disable snapshots. Enable if you want the Generator to produce snapshot files for Host bootstrap. See [snapshot config](/generators/config-reference#snapshot). +- `SCHEMA_AUTH_MODE=none`: Disable authentication on the `/api/v1/schema` endpoints. The code default is `token`, which requires keys via `SCHEMA_API_KEYS` or every schema request returns 503. The repo's `docker-compose-prod.yml` sets `none`. See [indexer config](/generators/config-reference#indexer). + +## Env file + +You can also use an `.env` file instead of inline environment variables. This mirrors the vars in the compose above: + +```shell +GETH_RPC_URL=https://json-rpc.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com +GETH_WS_URL=wss://ws.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com +GETH_API_KEY=YOUR_API_KEY +GETH_API_KEY_TYPE=x-goog-api-key +INDEXER_START_HEIGHT=0 +DEFRADB_KEYRING_SECRET=YOUR_DEFRADB_KEYRING_SECRET +SCHEMA_AUTH_MODE=none +``` + +To use it with `docker run`, mount the `.env` file with `--env-file .env`. + +## Start the Generator + +```shell +docker compose -f docker-compose.yml up -d +``` + +Verify health. The health server listens on port 8080 inside the container but is not published to the host in this compose, so use `docker compose exec`: + +```shell +docker compose exec shinzo-generator curl -f http://localhost:8080/health +``` + +## Registration + +Once the Generator is running, register it with the Shinzo Network. See [Registration](/generators/register/). + +## Gotchas + +- The image tag `ghcr.io/shinzonetwork/shinzo-generator-client:standard` in this compose matches `docker-compose-prod.yml`. The repo's `indexer-prod-setup.sh` pins a versioned tag (`ghcr.io/shinzonetwork/shinzo-generator-client:v0.6.5.1-ethereum-mainnet`), and the [install page](/generators/install/) uses `ghcr.io/shinzonetwork/shinzo-generator-client:ethereum-mainnet-latest`. Docker pull/run tag strategy is being consolidated in issues #326 and #327; align with whatever those land on rather than mixing tags across deployments. +- `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` appear in the original `docker-compose-prod.yml` but are not read by the Generator client, so they are omitted here. `SCHEMA_AUTH_MODE=none` is kept because the Generator client does read it. Log level is controlled by `LOGGER_DEBUG`. See the [env vars table](/generators/config-reference#environment-variables) for details. +- The `GETH_WS_URL` in this compose file uses `wss://` (secure WebSocket) because GCP BNE requires TLS on all connections. If you switch to a non-TLS WebSocket provider, use `ws://` instead. +- `GOMEMLIMIT` is not a Generator client config var. It is a Go runtime soft memory limit, honored by the Go runtime itself. Set it below the container memory limit to control garbage collection under pressure. + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/generators/deployment-examples/nginx-tls-snapshots/index.md b/content/generators/deployment-examples/nginx-tls-snapshots/index.md new file mode 100644 index 0000000..22734ee --- /dev/null +++ b/content/generators/deployment-examples/nginx-tls-snapshots/index.md @@ -0,0 +1,246 @@ ++++ +title = "Nginx with TLS serving snapshots" +description = "Run a production Generator behind nginx with TLS termination and serve signed snapshot files to Hosts for fast historical bootstrap." +[extra] +mermaid = true ++++ + +When to use this: you want to run a production Generator behind Nginx with TLS, serving snapshot files to Host clients for fast historical bootstraping. + +These scenarios use Ethereum Mainnet and Geth. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing `chain.name` and pointing the RPC URLs at a compatible node. See the [chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph VM["Prod VM"] + direction LR + Nginx["Nginx
:443 TLS"] + Gen["Generator client
:8080 Health/Metrics/Snapshots
:9171 P2P"] + Nginx -- "proxy :8080" --> Gen + end + + Hosts["Hosts"] + + Gen -- "P2P (libp2p)
:9171" --> Hosts + Hosts -- "HTTPS
/snapshots" --> Nginx +{% end %} + +Nginx terminates TLS on port 443 and proxies health, metrics, and snapshot requests to the Generator on port 8080. Hosts connect to the Generator over P2P on port 9171 for live block data. Hosts can also pull snapshot files over HTTPS through Nginx for historical bootstrap. + +## Prerequisites + +- Docker installed on the VM. +- TLS certificate and key files at `~/ssl/nginx.crt` and `~/ssl/nginx.key`. You can use Let's Encrypt or your own CA. +- A Geth node endpoint (self-hosted or managed provider). +- Ports 443 and 9171 open externally. + +## Compose file + +This compose file is drawn from `docker-compose-prod.yml` in the `shinzo-generator-client` repo. Two things are changed for this scenario: `SNAPSHOT_ENABLED` is flipped to `true` so the Generator produces snapshot files Hosts can download through Nginx, and the Nginx port mapping is `443:443` (instead of the repo's `8080:8080`) so Nginx terminates TLS on 443 as described below. Replace `` with your Geth API key: + +```yaml +networks: + shinzo-net: + driver: bridge + +services: + shinzo-generator: + container_name: shinzo-generator + platform: linux/amd64 + image: ghcr.io/shinzonetwork/shinzo-generator-client:standard + user: "1001:1001" + restart: unless-stopped + networks: + - shinzo-net + mem_limit: 16g + mem_reservation: 13g + ports: + - "9171:9171" + volumes: + - ~/shinzo-data/defradb:/app/.defra + environment: + - GETH_RPC_URL=https://json-rpc.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com + - GETH_WS_URL=wss://ws.aouzyll7wj7e9xe4g7t82w88c.blockchainnodeengine.com + - GETH_API_KEY= + - GETH_API_KEY_TYPE=x-goog-api-key + - INDEXER_START_HEIGHT=0 + - DEFRADB_KEYRING_SECRET=pingpong + - GOMEMLIMIT=14GiB + - SNAPSHOT_ENABLED=true + - SCHEMA_AUTH_MODE=none + logging: + options: + max-size: "50m" + max-file: "3" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + nginx: + image: nginx:alpine + ports: + - "443:443" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ~/ssl/nginx.crt:/etc/nginx/ssl/nginx.crt:ro + - ~/ssl/nginx.key:/etc/nginx/ssl/nginx.key:ro + depends_on: + - shinzo-generator + networks: + - shinzo-net + restart: unless-stopped +``` + +## Nginx configuration + +This Nginx config is drawn from `nginx.conf` (and the copy generated by `indexer-prod-setup.sh`) in the `shinzo-generator-client` repo. It proxies health, registration, registration-app, metrics, snapshot, and schema endpoints, and returns 404 for unmatched routes. The CORS headers allow requests from `shinzo.network` origins. + +The repo's `nginx.conf` listens on `8080` as plaintext HTTP (the repo compose maps `8080:8080`) and does not terminate TLS — the mounted certificate files are not referenced. To make this scenario actually terminate TLS as the title implies, the server block below changes `listen 8080;` to `listen 443 ssl;` and adds the `ssl_certificate` / `ssl_certificate_key` directives pointing at the cert and key files mounted from `~/ssl/`. Everything else matches the repo file verbatim: + +```nginx +events { worker_connections 1024; } + +http { + map $http_origin $cors_origin { + default ""; + "~^https://[^/]+\.shinzo\.network$" $http_origin; + } + + server { + listen 443 ssl; + server_name _; + + ssl_certificate /etc/nginx/ssl/nginx.crt; + ssl_certificate_key /etc/nginx/ssl/nginx.key; + + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + + location = /health { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/health; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /registration { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/registration; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /registration-app { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/registration-app; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /metrics { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/metrics; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /snapshots { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/snapshots; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location ~ ^/snapshots/(.+)$ { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/snapshots/$1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + client_max_body_size 0; + } + + # Schema endpoint + location = /api/v1/schema { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/api/v1/schema; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location ~ ^/api/v1/schema/(.+)$ { + if ($request_method = OPTIONS) { return 204; } + proxy_pass http://shinzo-generator:8080/api/v1/schema/$1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Default 404 for unmatched routes + location / { + return 404; + } + } +} +``` + +### What the key values mean + +- `SNAPSHOT_ENABLED=true`: The Generator produces signed snapshot files before pruning. The original `docker-compose-prod.yml` sets this to `false`. This scenario enables it so Hosts can bootstrap from snapshots. See [snapshot config](/generators/config-reference#snapshot). +- `SCHEMA_AUTH_MODE=none`: Disables authentication on the `/api/v1/schema` endpoints that Nginx proxies. The code default is `token`, which requires keys via `SCHEMA_API_KEYS` or every schema request returns 503. The repo's `docker-compose-prod.yml` sets `none`. See [indexer config](/generators/config-reference#indexer). +- `GETH_API_KEY_TYPE=x-goog-api-key`: Header name for GCP BNE authentication. See [geth config](/generators/config-reference#geth). +- `GOMEMLIMIT=14GiB`: Go runtime soft memory limit, set below the 16g container limit. See [env vars](/generators/config-reference#environment-variables). +- The Nginx `proxy_buffering off` and extended timeouts on `/snapshots/(.+)$` prevent Nginx from buffering large snapshot downloads. `client_max_body_size 0` removes the request body size limit. + +## Start the Generator + +Place both files in the same directory and start: + +```shell +docker compose -f docker-compose.yml up -d +``` + +Verify the Generator is healthy through Nginx: + +```shell +curl https://your-domain/health +``` + +List available snapshots: + +```shell +curl https://your-domain/snapshots +``` + +## Registration + +Once the Generator is running, register it with the Shinzo Network. See [Registration](/generators/register/). + +## Gotchas + +- The original `docker-compose-prod.yml` sets `SNAPSHOT_ENABLED=false`. This scenario changes it to `true` because the purpose is to serve snapshots to Hosts. If you keep it `false`, no snapshot files are produced and the `/snapshots` endpoint returns nothing. +- The Nginx config terminates TLS on port 443 (`listen 443 ssl;` with `ssl_certificate` / `ssl_certificate_key` pointing at `/etc/nginx/ssl/nginx.crt` and `/etc/nginx/ssl/nginx.key`), and the compose maps host `443` to container `443`. You must place your certificate and key at `~/ssl/nginx.crt` and `~/ssl/nginx.key` or Nginx will fail to start. The repo's `nginx.conf` listens on `8080` as plaintext and maps `8080:8080` — it does not terminate TLS — so the `listen … ssl` and `ssl_certificate*` directives above are the only additions this scenario makes beyond the repo file. +- The image tag `ghcr.io/shinzonetwork/shinzo-generator-client:standard` in this compose matches `docker-compose-prod.yml`. The repo's `indexer-prod-setup.sh` pins a versioned tag (`ghcr.io/shinzonetwork/shinzo-generator-client:v0.6.5.1-ethereum-mainnet`), and the [install page](/generators/install/) uses `ghcr.io/shinzonetwork/shinzo-generator-client:ethereum-mainnet-latest`. Docker pull/run tag strategy is being consolidated in issues #326 and #327; align with whatever those land on rather than mixing tags across deployments. +- `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` appear in the original `docker-compose-prod.yml` but are not read by the Generator client, so they are omitted here. `SCHEMA_AUTH_MODE=none` is kept because the Generator client does read it (it controls auth on the `/api/v1/schema` endpoints Nginx proxies). See the [env vars table](/generators/config-reference#environment-variables) for details. +- The snapshot directory defaults to `./snapshots` inside the container. Snapshots are written to the DefraDB data directory at `~/shinzo-data/defradb/snapshots` on the host because of the volume mount. Hosts download them through Nginx, not directly from the filesystem. +- The `proxy_read_timeout` and `proxy_send_timeout` are set to 300 seconds for snapshot downloads. Large snapshot files can take time to transfer. If Hosts time out downloading, increase these values. + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/generators/deployment-examples/validator-local-geth/index.md b/content/generators/deployment-examples/validator-local-geth/index.md new file mode 100644 index 0000000..483c3e5 --- /dev/null +++ b/content/generators/deployment-examples/validator-local-geth/index.md @@ -0,0 +1,102 @@ ++++ +title = "Validator with local Geth" +description = "Run the Generator client beside your own Geth node on the same machine or VPC, with no API key and minimal latency." +[extra] +mermaid = true ++++ + +When to use this: you run a validator and want to run the Generator client beside your own Geth node on the same machine or VPC, with no API key and minimal latency. + +These scenarios use Ethereum Mainnet and Geth. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing `chain.name` and pointing the RPC URLs at a compatible node. See the [chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph VM["Validator machine"] + direction LR + Geth["Geth node
:8545 HTTP
:8546 WS"] + Gen["Generator client
:9171 P2P
:8080 Health/Metrics"] + Geth -- "HTTP + WS
(localhost)" --> Gen + end + + Hosts["Hosts"] + Gen -- "P2P (libp2p)" --> Hosts +{% end %} + +The Generator and Geth share a single machine. The Generator connects to Geth over localhost on the standard JSON-RPC and WebSocket ports. No API key is needed because Geth has no authentication on a private network. The Generator pushes data out to Hosts over P2P on port 9171. + +## Prerequisites + +- Docker installed on the validator machine. +- Geth running and synced, with HTTP and WebSocket enabled on ports 8545 and 8546. See the [Geth documentation](https://geth.ethereum.org/docs/fundamentals/security) for configuration. +- Ports 9171, 9181, and 8080 available on the validator machine. See [exposed ports](/generators/install#exposed-ports) for details. + +## Run the Generator + +Start the Generator client with a direct localhost connection to Geth. No API key or API key type is needed: + +```shell +docker run -d \ + --name shinzo-generator \ + --restart unless-stopped \ + -e GETH_RPC_URL=http://localhost:8545 \ + -e GETH_WS_URL=ws://localhost:8546 \ + -e INDEXER_START_HEIGHT=0 \ + -e DEFRADB_KEYRING_SECRET=testnet-secret \ + -e DEFRADB_P2P_ENABLED=true \ + -e DEFRADB_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/9171 \ + -e LOGGER_DEBUG=false \ + -p 9181:9181 \ + -p 9171:9171 \ + -p 8080:8080 \ + ghcr.io/shinzonetwork/shinzo-generator-client:ethereum-mainnet-latest +``` + +### What each value means + +- `GETH_RPC_URL=http://localhost:8545`: Geth JSON-RPC on localhost. See [geth config](/generators/config-reference#geth). +- `GETH_WS_URL=ws://localhost:8546`: Geth WebSocket on localhost. See [geth config](/generators/config-reference#geth). +- `INDEXER_START_HEIGHT=0`: Start indexing from the chain tip. See [indexer config](/generators/config-reference#indexer). +- `DEFRADB_KEYRING_SECRET=testnet-secret`: Encryption secret for the DefraDB keyring. Change this to your own secret and keep it consistent across restarts. See [defradb config](/generators/config-reference#defradb). +- `DEFRADB_P2P_ENABLED=true`: Enable P2P networking so the Generator can push data to Hosts. See [defradb p2p config](/generators/config-reference#defradb-p2p). +- `DEFRADB_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/9171`: Listen on all interfaces so Hosts outside the machine can connect. See [defradb p2p config](/generators/config-reference#defradb-p2p). +- `LOGGER_DEBUG=false`: Production log level. Set to `true` for debug-level output during setup. See [logger config](/generators/config-reference#logger). + +## Verify the connection + +Check that the Generator is healthy and processing blocks: + +```shell +curl http://localhost:8080/health +``` + +Expected response: + +```json +{ "status": "healthy", "defradb_connected": true } +``` + +Check processing metrics: + +```shell +curl -s http://localhost:8080/metrics | jq +``` + +If `blocks_processed` is climbing, the Generator is receiving blocks from Geth and writing them to DefraDB. + +## Registration + +Once the Generator is running, register it with the Shinzo Network. See [Registration](/generators/register/). + +## Gotchas + +- `DEFRADB_KEYRING_SECRET` must stay the same across restarts. If it changes, the Generator cannot load its existing DefraDB identity and will fail to start. The value `testnet-secret` is from the install page. Replace it with your own secret for any real deployment. +- The install page includes `DEFRADB_PLAYGROUND=true` in its `docker run` example. This env var is not read by the Generator client. The playground is controlled by build tags, not environment variables. See the [env vars table](/generators/config-reference#environment-variables) for which vars are effective. +- If Geth is on a different machine in the same VPC, replace `localhost` with the internal IP or hostname of the Geth machine. Still no API key needed as long as the traffic stays within the private network. +- If the Generator falls back to HTTP-only mode, the WebSocket connection to Geth failed. Check that `GETH_WS_URL` is correct and that Geth has WebSocket enabled with `--ws --ws.port 8546`. +- The default `config.yaml` in the container image enables pruning with `max_blocks: 1000` and snapshots with `blocks_per_file: 100`. For an archival Generator, see the [archival versus pruned scenario](../archival-vs-pruned/). + +## Need help + +{{ need_help(client="Generator", repo_name="shinzo-generator-client", repo="https://github.com/shinzonetwork/shinzo-generator-client/issues") }} diff --git a/content/guides/configuring-event-filters-on-a-shinzo-host/index.md b/content/guides/configuring-event-filters-on-a-shinzo-host/index.md index c8725ce..d5d25fc 100644 --- a/content/guides/configuring-event-filters-on-a-shinzo-host/index.md +++ b/content/guides/configuring-event-filters-on-a-shinzo-host/index.md @@ -123,9 +123,9 @@ Or use an [online keccak256 tool](https://emn178.github.io/online-tools/keccak_2 p2p: enabled: true bootstrap_peers: - - '/ip4/34.63.13.57/tcp/9171/p2p/12D3KooWBT2F45LH7Gy6EadTE3sm7PKofyJ3RXcCnKufdu4L5c4M' - - '/ip4/35.208.241.78/tcp/9171/p2p/12D3KooWDYXkjdncFL3X1SaaYBpFi4XfWskbXv4y5gYdTvmGm3bo' - - '/ip4/35.209.45.53/tcp/9171/p2p/12D3KooWNLCXZEVZoM6NwU1i7zAZDscEZHhynsF74M5hP99sptM9' + - '/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' + - '/ip4/34.57.239.57/tcp/9171/p2p/12D3KooWBAgCEJHYqzuCFEXzjsw2CnV9JqvqMgTKYDww58aCxwW5' + - '/ip4/34.134.119.63/tcp/9171/p2p/12D3KooWQQTuSQaz4HfuvnJHakkQy3PhWbKBBbS3RkmBw4ZsFkyT' ``` These are the addresses of the peers your host connects to for data. Check the [Shinzo Validators list](https://registration.shinzo.network/validators) for updated peer lists; these change when the network is updated. diff --git a/content/hosts/config-reference/index.md b/content/hosts/config-reference/index.md new file mode 100644 index 0000000..37bab0d --- /dev/null +++ b/content/hosts/config-reference/index.md @@ -0,0 +1,233 @@ ++++ +title = "Config reference" +description = "Every Host client config field with its YAML key, default, env var override, and notes on which keys are silently ignored." ++++ + +The Host client loads configuration from a YAML file (default `config/config.yaml`). Environment variable overrides are applied in `config/config.go` for `START_HEIGHT`, `BOOTSTRAP_PEERS`, `DEFRA_URL`, `INDEXER_SCHEMA_ENDPOINT`, and `INDEXER_SCHEMA_ENDPOINT_AUTH_TOKEN`. A sixth env var, `DEFRA_KEYRING_SECRET`, is read separately in `pkg/defradb/defra.go`. + +The Host has no `applyDefaults` function. Unset numeric fields stay at Go zero values (0, false, empty string) unless the shipped `config.yaml` sets them. The pruner is the one exception: `cfg.Pruner.SetDefaults()` is called in `pkg/host/host.go` when pruning is enabled, filling in defaults for `max_blocks`, `docs_per_block`, and `interval_seconds`. Several P2P fields also receive defaults inside `pkg/defradb/network_handler.go` and `pkg/defradb/defra.go` at startup. + +Some fields in the shipped `config.yaml` are not in the `config.go` struct and are silently ignored. These are listed in the no-op keys section below. + +## defradb + +Embedded database configuration. DefraDB handles storage, P2P replication, content addressing, CRDT merging, and query serving. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `url` | string | `http://localhost:9181` | no | `DEFRA_URL` | DefraDB API URL. Used for the playground address and health endpoint. `DEFRA_URL` overrides this value at runtime (read in `config/config.go`), so deployment artifacts can pick a different bind address such as `0.0.0.0:9181` without editing the YAML. Shipped `config.yaml` sets `localhost:9181`. | +| `keyring_secret` | string | from `DEFRA_KEYRING_SECRET` | no | `DEFRA_KEYRING_SECRET` | Encryption secret for the DefraDB keyring. Read in `pkg/defradb/defra.go`, not in `config.go`. Shipped `config.yaml` sets `pingpong`. | + +The Host uses the env var name `DEFRA_KEYRING_SECRET` (with `DEFRA_` prefix). The Generator client uses `DEFRADB_KEYRING_SECRET` (with `DEFRADB_` prefix). The two clients use different env var names for the same concept. + +### defradb p2p + +P2P networking configuration. The Host receives data from Generator clients over libp2p. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | true | no | (none) | Enable P2P networking. Shipped `config.yaml` sets true. | +| `bootstrap_peers` | string array | empty | no | `BOOTSTRAP_PEERS` | P2P bootstrap peer multiaddrs. Env var is comma-separated. Shipped `config.yaml` lists three indexer peers. | +| `listen_addr` | string | `/ip4/127.0.0.1/tcp/9171` | no | (none) | Multiaddr to listen on for P2P connections. Applied as a fallback in `StartDefraInstance` when empty. Shipped `config.yaml` sets `/ip4/0.0.0.0/tcp/9171`. | +| `max_retries` | int | 5 | no | (none) | Connection attempts before marking a peer as failed. Default applied in `network_handler.go`. | +| `retry_base_delay_ms` | int | 1000 | no | (none) | Base delay in milliseconds for exponential backoff. Default applied in `network_handler.go`. | +| `reconnect_interval_ms` | int | 60000 | no | (none) | Interval in milliseconds to check for disconnected peers. Default applied in `network_handler.go`. | +| `enable_auto_reconnect` | bool | true | no | (none) | Automatically reconnect to failed or disconnected peers. | +| `peer_discovery_timeout_ms` | int | 10000 | no | (none) | Timeout in milliseconds for auto-discovering peer IDs. When 0, falls back to 10 seconds via `DefaultPeerDiscoveryTimeout` in `peer_discovery.go`. | + +### defradb store + +Badger storage engine configuration. All cache and compaction fields map directly to Badger options. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `path` | string | `.defra` | no | (none) | Path to the DefraDB data directory. Shipped `config.yaml` sets `./.defra`. | +| `block_cache_mb` | int64 | 0 | no | (none) | Badger block cache size in MB. Shipped `config.yaml` sets 512. | +| `memtable_mb` | int64 | 0 | no | (none) | Badger memtable size in MB. Shipped `config.yaml` sets 64. | +| `index_cache_mb` | int64 | 0 | no | (none) | Badger index cache size in MB. Shipped `config.yaml` sets 256. | +| `num_compactors` | int | 0 | no | (none) | Number of Badger compaction workers. Shipped `config.yaml` sets 4. | +| `num_level_zero_tables` | int | 0 | no | (none) | L0 tables before compaction starts. Shipped `config.yaml` comments this out. | +| `num_level_zero_tables_stall` | int | 0 | no | (none) | L0 tables that trigger write stalls. Shipped `config.yaml` comments this out. | +| `value_log_file_size_mb` | int64 | 0 | no | (none) | Size of each value log file in MB. Smaller files mean faster GC. Shipped `config.yaml` sets 128. | + +## shinzo + +Shinzo-specific host configuration: attestation processing, view management, document queueing, and event filtering. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `minimum_attestations` | int | 1 | no | (none) | Loaded from YAML but not consumed in the host startup path. Shipped `config.yaml` sets 1. | +| `hub_base_url` | string | empty | no | (none) | ShinzoHub CometBFT RPC URL. Used to construct the RPC, WebSocket, and LCD endpoints. Shipped `config.yaml` sets `testnet.shinzo.network:26657`. | +| `start_height` | uint64 | 0 | no | `START_HEIGHT` | Block number to start from. 0 means auto-detect from chain tip. | +| `cache_queue_size` | int | 50000 | no | (none) | Job queue size for document processing. Shipped `config.yaml` sets 50000. | +| `batch_writer_count` | int | 8 | no | (none) | Number of batch writers for attestation processing. Shipped `config.yaml` sets 8. | +| `batch_size` | int | 1000 | no | (none) | Max attestations per batch. Shipped `config.yaml` sets 500. | +| `batch_flush_interval` | int | 100 | no | (none) | Flush interval in milliseconds for attestation batches. Shipped `config.yaml` sets 100. | +| `max_concurrent_verifications` | int | 50 | no | (none) | Max concurrent signature verifications. Shipped `config.yaml` does not set this, so the code default of 50 applies. | +| `use_block_signatures` | bool | false | no | (none) | Use block signatures for attestation verification. Shipped `config.yaml` sets true. | +| `doc_worker_count` | int | 16 | no | (none) | Number of document processing workers. Shipped `config.yaml` sets 32. | +| `doc_queue_size` | int | 5000 | no | (none) | Queue size for document event notifications. Shipped `config.yaml` sets 50000. | + +### shinzo event filter + +Controls which P2P documents the Host stores. Blocks and BlockSignatures always pass through regardless of filter settings. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | (none) | Master switch for filtering. false means accept all documents. Shipped `config.yaml` sets false. | +| `mode` | string | `allowlist` | no | (none) | `allowlist` means only accept documents matching a group. `blocklist` means accept everything except documents matching a group. | +| `cascade_filters` | bool | false | no | (none) | When true, a contract filter with `types: ["transaction"]` also applies to logs and access-list entries from the same address. Shipped `config.yaml` sets true. | +| `block_range` | object | (none) | no | (none) | Optional block number range gate. Omit to accept all blocks. | +| `groups` | object array | empty | no | (none) | Named filter groups combined with OR logic. In allowlist mode a document must match at least one enabled group. | + +### shinzo event filter block range + +Restricts processing to a range of block numbers. All three conditions must be true for snapshot bootstrap to run: `enabled`, a non-empty `indexer_url`, and at least one `historical_ranges` entry. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `min_block` | uint64 | 0 | no | (none) | Minimum block number, inclusive. | +| `max_block` | uint64 | 0 | no | (none) | Maximum block number, inclusive. 0 means no upper limit. | + +### shinzo event filter groups + +Each group bundles related contract and topic filters. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `name` | string | empty | no | (none) | Human-readable label for logging. | +| `enabled` | bool | false | no | (none) | Toggle this group on or off without removing it. | +| `contracts` | object array | empty | no | (none) | Contract address filters. | +| `topics` | object array | empty | no | (none) | Event topic filters. | + +### shinzo event filter groups contracts + +Matches documents by on-chain address. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `address` | string | empty | no | (none) | Contract address, 0x-prefixed. | +| `name` | string | empty | no | (none) | Human-readable name for logging. | +| `types` | string array | empty | no | (none) | Collection types to apply to: `transaction`, `log`, or `accessListEntry`. Multiple types can be combined. | + +### shinzo event filter groups topics + +Matches log events by topic values. `topic0` is the event signature hash. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `topic0` | string | empty | no | (none) | Event signature hash. Required for topic matching. | +| `topic1` | string | empty | no | (none) | Optional indexed parameter 1. Omit for wildcard. | +| `topic2` | string | empty | no | (none) | Optional indexed parameter 2. Omit for wildcard. | +| `topic3` | string | empty | no | (none) | Optional indexed parameter 3. Omit for wildcard. | +| `name` | string | empty | no | (none) | Human-readable name, for example `Swap` or `Transfer`. | + +## host + +Host-level configuration: lens registry path, health server, and snapshot bootstrap. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `lens_registry_path` | string | `./.defra/lens` | no | (none) | Path to store lens WASM files. | +| `health_server_port` | int | 8080 | no | (none) | Health server port. When 0, falls back to 8080. | +| `open_browser_on_start` | bool | false | no | (none) | Auto-open the metrics page in a browser on startup. | +| `snapshot` | object | (none) | no | (none) | Snapshot bootstrap configuration. See below. | + +### host snapshot + +Downloads historical data from an indexer on first startup for fast initial sync. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | (none) | Enable snapshot bootstrap on startup. Shipped `config.yaml` sets false. | +| `indexer_url` | string | empty | no | (none) | HTTP base URL of the indexer serving snapshots. Shipped `config.yaml` sets `http://35.206.105.60:8080`. | +| `historical_ranges` | object array | empty | no | (none) | Block ranges to download during bootstrap. | + +#### host snapshot historical ranges + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `start` | int64 | 0 | no | (none) | Start block number, inclusive. | +| `end` | int64 | 0 | no | (none) | End block number, inclusive. | + +## schema + +Dynamic schema fetching from an indexer. The Host fetches the indexer's schema on startup so its collection definitions match the Generator's. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `indexer_schema_endpoint` | string | `/api/v1/schema` | no | `INDEXER_SCHEMA_ENDPOINT` | HTTP endpoint path on the indexer for fetching the schema. Defaults to `DefaultIndexerSchemaEndpoint` in `config/config.go` when empty. Shipped `config.yaml` sets `/api/v1/schema`. | +| `http_client_timeout_secs` | int | 30 | no | (none) | HTTP client timeout in seconds for schema fetches. Must be non-negative and cannot exceed 300 (`MaxSchemaHTTPClientTimeout`); 0 defaults to 30 (`DefaultSchemaHTTPClientTimeout`). Shipped `config.yaml` sets 30. | +| `auth_token` | string | empty | yes if the indexer uses `SCHEMA_AUTH_MODE=token` | `INDEXER_SCHEMA_ENDPOINT_AUTH_TOKEN` | Bearer token used to authenticate schema fetches against indexers with `SCHEMA_AUTH_MODE=token` (the Generator's default). Not settable in `config.yaml` (the field uses `yaml:"-"`); provide via `INDEXER_SCHEMA_ENDPOINT_AUTH_TOKEN` or schema fetches receive 401/503 and fall back to the embedded schema. | + +## pruner + +Removes old data to keep storage bounded. Defaults are applied by `SetDefaults` in `pkg/pruner/config.go`, called from `pkg/host/host.go` when `enabled` is true. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | bool | false | no | (none) | Enable automatic pruning. Shipped `config.yaml` sets true. | +| `max_blocks` | int64 | 10000 | no | (none) | Number of blocks to retain. Shipped `config.yaml` sets 2000. | +| `docs_per_block` | int | 1000 | no | (none) | Average docs per block, used with `max_blocks` to compute the max document count. Shipped `config.yaml` sets 1000. | +| `prune_threshold` | int64 | 0 | no | (none) | Deprecated. Kept for backward compatibility but unused by the pruner. | +| `interval_seconds` | int | 60 | no | (none) | How often in seconds to check and prune. Shipped `config.yaml` sets 30. | +| `prune_history` | bool | false | no | (none) | Walk DAG chains to delete historical block versions. Two to three times slower. | + +## logger + +Logging configuration. The logger is zap-based. + +| Key | Type | Default | Required | Env var | Description | +| --- | --- | --- | --- | --- | --- | +| `development` | bool | false | no | (none) | Enable development mode logging. Shipped `config.yaml` sets true. The prod setup script sets false. | + +The shipped `config.yaml` also sets `logger.level: "info"` and the prod setup script sets `logger.level: "error"`. This field is not in the `LoggerConfig` struct and has no effect. The log level is determined by `development`. + +## No-op keys + +The following keys appear in the shipped `config.yaml` but are not in the `config.go` struct. They are parsed by the YAML loader and then silently discarded. Tuning them has no effect on runtime behavior. + +| Key | Shipped value | Notes | +| --- | --- | --- | +| `shinzo.wait_for_gaps` | true | Not in `ShinzoConfig` struct. | +| `shinzo.max_gap_size` | 1000 | Not in `ShinzoConfig` struct. | +| `shinzo.batch_processing_enabled` | true | Not in `ShinzoConfig` struct. | +| `shinzo.batch_max_views_per_job` | 50 | Not in `ShinzoConfig` struct. | +| `shinzo.batch_query_cache_size` | 1000 | Not in `ShinzoConfig` struct. | +| `shinzo.p2p_enabled` | not set | In the struct but not consumed anywhere in non-test code. | +| `shinzo.view_inactivity_timeout` | `24h` | In the struct but not consumed anywhere in non-test code. | +| `shinzo.view_cleanup_interval` | `1h` | In the struct but not consumed anywhere in non-test code. | +| `shinzo.view_worker_count` | 20 | In the struct but not consumed anywhere in non-test code. | +| `shinzo.view_queue_size` | 5000 | In the struct but not consumed anywhere in non-test code. | +| `shinzo.minimum_attestations` | 1 | In the struct and loaded from YAML but not read from the config in the host startup path. | +| `logger.level` | `info` or `error` | Not in `LoggerConfig` struct. | + +## Environment variables + +The Host client reads five env vars in `config/config.go` and one in `pkg/defradb/defra.go`. + +| Env var | Overrides | Read in | +| --- | --- | --- | +| `START_HEIGHT` | `shinzo.start_height` | `config/config.go` | +| `BOOTSTRAP_PEERS` | `defradb.p2p.bootstrap_peers` | `config/config.go` (comma-separated) | +| `DEFRA_URL` | `defradb.url` | `config/config.go` | +| `INDEXER_SCHEMA_ENDPOINT` | `schema.indexer_schema_endpoint` | `config/config.go` | +| `INDEXER_SCHEMA_ENDPOINT_AUTH_TOKEN` | `schema.auth_token` | `config/config.go` | +| `DEFRA_KEYRING_SECRET` | `defradb.keyring_secret` | `pkg/defradb/defra.go` | + +Note the keyring env var prefix: the Host uses `DEFRA_KEYRING_SECRET` (with `DEFRA_`), while the Generator uses `DEFRADB_KEYRING_SECRET` (with `DEFRADB_`). + +### Env vars that are not read + +The following env vars appear in some deployment artifacts but are not read by the Host client code. They have no effect on runtime behavior. + +| Env var | Where it appears | Notes | +| --- | --- | --- | +| `LOG_LEVEL` | `docker-compose-prod.yml`, `host-prod-setup.sh`, GCP startup scripts | Not read. Log level is controlled by `logger.development`. | +| `LOG_SOURCE` | `docker-compose-prod.yml`, `host-prod-setup.sh`, GCP startup scripts | Not read. | +| `LOG_STACKTRACE` | `docker-compose-prod.yml`, `host-prod-setup.sh`, GCP startup scripts | Not read. | +| `GOMEMLIMIT` | `docker-compose-prod.yml` | Not read by the client. This is a Go runtime soft memory limit, honored by the Go runtime itself. Set it to control garbage collection behavior under memory pressure. | + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/deployment-examples/_index.md b/content/hosts/deployment-examples/_index.md new file mode 100644 index 0000000..4a4aaf5 --- /dev/null +++ b/content/hosts/deployment-examples/_index.md @@ -0,0 +1,4 @@ ++++ +render = false +page_template = "page.html" ++++ diff --git a/content/hosts/deployment-examples/event-filter-allowlist/index.md b/content/hosts/deployment-examples/event-filter-allowlist/index.md new file mode 100644 index 0000000..9763fe6 --- /dev/null +++ b/content/hosts/deployment-examples/event-filter-allowlist/index.md @@ -0,0 +1,189 @@ ++++ +title = "Event filter with allowlist and blocklist" +description = "Configure a Host to store only specific contract events using the event filter in allowlist mode, with a blocklist variant and an optional block range gate." +[extra] +mermaid = true ++++ + +When to use this: you want your Host to store only specific contract events instead of every document from every Generator. This cuts storage and speeds up view processing. + +These scenarios use Ethereum Mainnet data. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing the contract addresses and topic hashes to match the target chain. See the [Generator chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + Gen1["Generator A"] + Gen2["Generator B"] + + subgraph HOST["Host"] + direction TB + Filter["Event filter
allowlist mode"] + Store[("DefraDB
filtered documents only")] + Filter --> Store + end + + Gen1 -- "P2P" --> HOST + Gen2 -- "P2P" --> HOST +{% end %} + +Generators push all block, transaction, and log data to the Host over P2P. The event filter inspects each document before it is written to DefraDB. In allowlist mode, only documents matching an enabled group are stored. Blocks and BlockSignatures always pass through regardless of filter settings. + +## Config file + +This config enables the event filter in allowlist mode with two groups: Uniswap V3 swaps and stablecoin transfers. It is drawn from the `event_filter` section of the shipped `host-client/config/config.yaml`: + +```yaml +defradb: + url: "localhost:9181" + keyring_secret: "pingpong" + p2p: + enabled: true + bootstrap_peers: + - '/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' + - '/ip4/34.57.239.57/tcp/9171/p2p/12D3KooWBAgCEJHYqzuCFEXzjsw2CnV9JqvqMgTKYDww58aCxwW5' + - '/ip4/34.134.119.63/tcp/9171/p2p/12D3KooWQQTuSQaz4HfuvnJHakkQy3PhWbKBBbS3RkmBw4ZsFkyT' + listen_addr: "/ip4/0.0.0.0/tcp/9171" + max_retries: 5 + retry_base_delay_ms: 1000 + reconnect_interval_ms: 60000 + enable_auto_reconnect: true + store: + path: "./.defra" + block_cache_mb: 512 + memtable_mb: 64 + index_cache_mb: 256 + num_compactors: 4 + value_log_file_size_mb: 128 + +shinzo: + start_height: 0 + hub_base_url: testnet.shinzo.network:26657 + cache_queue_size: 50000 + batch_writer_count: 8 + batch_size: 500 + batch_flush_interval: 100 + use_block_signatures: true + doc_worker_count: 32 + doc_queue_size: 50000 + event_filter: + enabled: true + mode: "allowlist" + cascade_filters: true + groups: + - name: "uniswap-v3" + enabled: true + contracts: + - name: "Uniswap V3 Router 2" + address: "0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45" + types: ["transaction", "log"] + topics: + - name: "Swap" + topic0: "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" + - name: "stablecoins" + enabled: true + contracts: + - name: "USDT" + address: "0xdAC17F958D2ee523a2206206994597C13D831ec7" + types: ["log"] + - name: "USDC" + address: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + types: ["log"] + topics: + - name: "Transfer" + topic0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + +pruner: + enabled: true + max_blocks: 2000 + docs_per_block: 1000 + interval_seconds: 30 + prune_history: false + +logger: + development: true + +host: + lens_registry_path: "./.defra/lens" + health_server_port: 8080 + open_browser_on_start: false + snapshot: + enabled: false +``` + +### What each filter value does + +- `event_filter.enabled: true`: Turn on content-based filtering. Without this, the Host stores every document it receives. See [shinzo event filter](/hosts/config-reference#shinzo-event-filter). +- `event_filter.mode: "allowlist"`: Only accept documents matching an enabled group. Set to `blocklist` to invert: accept everything except documents matching a group. See [shinzo event filter](/hosts/config-reference#shinzo-event-filter). +- `event_filter.cascade_filters: true`: A contract filter with `types: ["transaction"]` also applies to logs and access-list entries from the same address. Set to false for strict per-type matching only. See [shinzo event filter](/hosts/config-reference#shinzo-event-filter). +- `groups[].name`: Human-readable label for logging. See [shinzo event filter groups](/hosts/config-reference#shinzo-event-filter-groups). +- `groups[].enabled`: Toggle a group on or off without removing it. See [shinzo event filter groups](/hosts/config-reference#shinzo-event-filter-groups). +- `contracts[].address`: The on-chain contract address to match. See [shinzo event filter groups contracts](/hosts/config-reference#shinzo-event-filter-groups-contracts). +- `contracts[].types`: Which collection types the address filter applies to: `transaction`, `log`, or `accessListEntry`. Multiple types can be combined. See [shinzo event filter groups contracts](/hosts/config-reference#shinzo-event-filter-groups-contracts). +- `topics[].topic0`: The keccak256 hash of the event signature. Required for topic matching. See [shinzo event filter groups topics](/hosts/config-reference#shinzo-event-filter-groups-topics). + +## Using blocklist mode + +To accept everything except Uniswap V3 swaps and stablecoin transfers, change `mode` to `blocklist` and keep the same groups: + +```yaml + event_filter: + enabled: true + mode: "blocklist" + cascade_filters: true + groups: + - name: "uniswap-v3" + enabled: true + contracts: + - name: "Uniswap V3 Router 2" + address: "0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45" + types: ["transaction", "log"] + topics: + - name: "Swap" + topic0: "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" +``` + +In blocklist mode, a document is rejected if it matches any enabled group. Everything else is stored. + +## Optional block range gate + +Add a `block_range` to restrict filtering to a specific block range. Omit it to accept all blocks: + +```yaml + event_filter: + enabled: true + mode: "allowlist" + cascade_filters: true + block_range: + min_block: 20000000 + max_block: 0 + groups: + - name: "uniswap-v3" + enabled: true + contracts: + - name: "Uniswap V3 Router 2" + address: "0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45" + types: ["transaction", "log"] + topics: + - name: "Swap" + topic0: "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" +``` + +`max_block: 0` means no upper limit. See [shinzo event filter block range](/hosts/config-reference#shinzo-event-filter-block-range). + +## Tutorial companion + +For a step-by-step walkthrough of configuring a single USDT Transfer filter, see the [Configuring Event Filters guide](/guides/configuring-event-filters-on-a-shinzo-host/). This scenario shows the fuller multi-group topology from the shipped config. + +## Gotchas + +- The shipped `config.yaml` sets `event_filter.enabled: false`. This scenario changes it to `true`. If you keep it false, no filtering happens and the Host stores everything. +- Blocks and BlockSignatures always pass through the filter regardless of mode or group settings. You cannot filter out block headers or signatures. +- `cascade_filters: true` means a `transaction` type filter on a contract address also filters logs and access-list entries from that address. If you want strict per-type matching, set it to false. +- The shipped `config.yaml` includes several `shinzo.*` keys that are not in the `config.go` struct and are silently ignored: `wait_for_gaps`, `max_gap_size`, `batch_processing_enabled`, `batch_max_views_per_job`, `batch_query_cache_size`. They have been omitted from this config. See [no-op keys](/hosts/config-reference#no-op-keys) for the full list. +- The `topic0` for ERC-20 Transfer is the same across all tokens because it is derived from the function signature `Transfer(address,address,uint256)`, not from the contract address. To filter a different event, compute its keccak256 hash from the canonical signature. +- The bootstrap peer IDs in this config are the three indexer peers from the shipped `config.yaml` and may be stale. Check the [Shinzo Validators list](https://registration.shinzo.network/validators) for current peers. + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/deployment-examples/gcp-local-ssd-raid0/index.md b/content/hosts/deployment-examples/gcp-local-ssd-raid0/index.md new file mode 100644 index 0000000..3086ace --- /dev/null +++ b/content/hosts/deployment-examples/gcp-local-ssd-raid0/index.md @@ -0,0 +1,163 @@ ++++ +title = "GCP VM with local SSD RAID-0" +description = "Run a Host on a GCP VM with local SSDs striped into a RAID-0 array for high IOPS, using the bundled startup script that detects, formats, and mounts the disks." +[extra] +mermaid = true ++++ + +When to use this: you need high IOPS for DefraDB storage and want to use GCP local SSDs in a RAID-0 array. Local SSDs provide much higher throughput than standard persistent disks but are ephemeral. + +These scenarios use Ethereum Mainnet data. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing the contract addresses and topic hashes to match the target chain. See the [Generator chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph GCP["GCP VM"] + direction TB + subgraph RAID["RAID-0 array"] + direction LR + SSD1["Local SSD 1"] + SSD2["Local SSD 2"] + SSDN["Local SSD N"] + end + MNT["/mnt/localssd
ext4, noatime, discard"] + Host["Host container
--network host
:9181 :9171 :8080"] + RAID --> MNT --> Host + end + + Gens["Generators"] + Gens -- "P2P (libp2p)" --> Host +{% end %} + +The GCP startup script detects all local SSDs attached to the VM, creates a RAID-0 array across them, formats it as ext4, and mounts it at `/mnt/localssd`. The Host container runs with `--network host` and stores its DefraDB data on the RAID-0 mount. This gives the Host the combined IOPS and throughput of all attached local SSDs. + +## Prerequisites + +- A GCP VM with one or more local SSDs attached. See the [GCP local SSD documentation](https://cloud.google.com/compute/docs/disks/local-ssd) for how to create a VM with local SSDs. +- The `gcp-startup-host-local-ssd.sh` script from the `shinzo-host-client` repo, set as the VM startup script or run manually. +- A `config.yaml` file in the working directory where the script runs. + +## Startup script + +This script is drawn from `scripts/gcp-startup-host-local-ssd.sh` in the `shinzo-host-client` repo (two leading comment lines omitted for brevity). It installs Docker, detects local SSDs by their `nvme_card` model, creates a RAID-0 array if there are two or more, formats and mounts the array, then pulls and starts the Host container: + +```shell +#!/bin/bash + +set -euxo pipefail + +apt-get update +apt-get install -y docker.io mdadm + +DEVICES=() +for dev in /dev/nvme*n*; do + [[ "$dev" =~ p[0-9]+$ ]] && continue + CTRL=$(echo "$dev" | sed 's|/dev/\(nvme[0-9]*\)n.*|\1|') + MODEL=$(cat /sys/class/nvme/$CTRL/model 2>/dev/null | xargs) + echo "Checking $dev - controller: $CTRL, model: '$MODEL'" + if [[ "$MODEL" == "nvme_card" ]]; then + DEVICES+=("$dev") + fi +done + +echo "Found ${#DEVICES[@]} Local SSD(s): ${DEVICES[*]}" + +if [ "${#DEVICES[@]}" -eq 0 ]; then + echo "ERROR: No Local SSDs detected! Check that Local SSDs are attached to this VM." + exit 1 +fi + +if [ "${#DEVICES[@]}" -ge 2 ]; then + echo "Creating RAID-0 over ${#DEVICES[@]} Local SSDs" + RAID_DEV=/dev/md0 + if [ ! -e "$RAID_DEV" ]; then + mdadm --create $RAID_DEV \ + --level=0 \ + --raid-devices=${#DEVICES[@]} \ + "${DEVICES[@]}" + fi + TARGET_DEV=$RAID_DEV +else + echo "Only one Local SSD found, using it directly" + TARGET_DEV=${DEVICES[0]} +fi + +if ! blkid $TARGET_DEV; then + mkfs.ext4 -F $TARGET_DEV +fi + +MNT=/mnt/localssd +mkdir -p $MNT +mountpoint -q $MNT || mount -o noatime,discard $TARGET_DEV $MNT +chmod 777 $MNT +mkdir -p \ + $MNT/defradb \ + $MNT/logs +chown -R 1003:1006 $MNT/defradb + +docker pull ghcr.io/shinzonetwork/shinzo-host-client:v0.5.1 +docker rm -f shinzo-host || true +docker run -d \ + --name shinzo-host \ + --restart unless-stopped \ + --network host \ + -u 1003:1006 \ + -v $MNT/defradb:/app/.defra \ + -v $(pwd)/config.yaml:/app/config.yaml:ro \ + -e DEFRA_URL=0.0.0.0:9181 \ + -e START_HEIGHT=${START_HEIGHT:-} \ + -e BOOTSTRAP_PEERS=${BOOTSTRAP_PEERS:-} \ + -e LOG_LEVEL=error \ + -e LOG_SOURCE=false \ + -e LOG_STACKTRACE=false \ + --health-cmd="wget --no-verbose --tries=1 --spider http://localhost:8080/metrics || exit 1" \ + --health-interval=30s \ + --health-timeout=10s \ + --health-retries=3 \ + --health-start-period=40s \ + --log-opt max-size=50m \ + --log-opt max-file=3 \ + ghcr.io/shinzonetwork/shinzo-host-client:v0.5.1 +``` + +### What each part does + +- `apt-get install -y docker.io mdadm`: Installs Docker and `mdadm` for RAID management. +- The `for dev in /dev/nvme*n*` loop: Detects all NVMe devices and filters for GCP local SSDs by checking the `nvme_card` model string. GCP local SSDs can appear as multiple namespaces on one controller (`nvme0n1`, `nvme0n2`) or as separate controllers (`nvme0n1`, `nvme1n1`). +- `mdadm --create`: Creates a RAID-0 array across all detected local SSDs. If only one SSD is found, it is used directly without RAID. +- `mount -o noatime,discard`: Mounts the filesystem with `noatime` (no access time updates) and `discard` (TRIM support for SSD wear leveling). +- `chown -R 1003:1006 $MNT/defradb`: Sets ownership so the Host container (UID 1003, GID 1006) can read and write. +- `--network host`: The Host container uses the host network directly. No port mappings are needed because all ports are exposed on the VM. +- `-v $MNT/defradb:/app/.defra`: Mounts the RAID-0 filesystem as the DefraDB data directory. See [defradb store](/hosts/config-reference#defradb-store). +- `START_HEIGHT=${START_HEIGHT:-}` and `BOOTSTRAP_PEERS=${BOOTSTRAP_PEERS:-}`: These env vars are passed through from the VM environment if set. See [environment variables](/hosts/config-reference#environment-variables). + +## Running the script + +Set the script as the GCP VM startup script, or run it manually after SSH-ing into the VM: + +```shell +chmod +x gcp-startup-host-local-ssd.sh +sudo ./gcp-startup-host-local-ssd.sh +``` + +Set `START_HEIGHT` and `BOOTSTRAP_PEERS` as environment variables before running if you want to override the config file values: + +```shell +export START_HEIGHT=20000000 +export BOOTSTRAP_PEERS='/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' +sudo -E ./gcp-startup-host-local-ssd.sh +``` + +## Gotchas + +- GCP local SSDs are ephemeral. If the VM stops or restarts, all data on the local SSDs is lost. Use this setup only for data that can be re-synced from Generators over P2P, or pair it with the [snapshot bootstrap scenario](../snapshot-bootstrap/) to re-import historical data on restart. +- The script pins the image to `ghcr.io/shinzonetwork/shinzo-host-client:v0.5.1`. The [Watchtower scenario](../watchtower-auto-deploy/) uses `:latest`, and the [prod VM scenario](../prod-vm-nginx-tls/) uses `:v0.6.5-ethereum-mainnet`. Pick one tag and be consistent. The `v0.5.1` pin protects against unexpected updates but means you need to manually update the script to get new releases. +- `DEFRA_URL` in the `docker run` command overrides `defradb.url` and is read by the Host client (`config/config.go`). It is kept here because it is in the original script, where it binds the DefraDB API to `0.0.0.0:9181` instead of the loopback-only YAML default. `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` are also kept from the original script but are not read by the Host client and have no effect. See [env vars that are not read](/hosts/config-reference#env-vars-that-are-not-read). +- The script uses `--network host`, so no Docker port mappings are specified. All Host ports (9181, 9171, 8080) are directly exposed on the VM. If you need nginx as a reverse proxy, run it as a separate container or install it on the host. +- The `nvme_card` model check is specific to GCP local SSDs. If you are running on a different cloud provider, the NVMe model string will differ. Adjust the `MODEL` check accordingly. +- The `chmod 777 $MNT` sets world-writable permissions on the mount point. This is permissive but matches the original script. If you want tighter permissions, adjust the chmod and ensure the Host container's UID (1003) can still access the directory. + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/deployment-examples/prod-vm-nginx-tls/index.md b/content/hosts/deployment-examples/prod-vm-nginx-tls/index.md new file mode 100644 index 0000000..0b6e953 --- /dev/null +++ b/content/hosts/deployment-examples/prod-vm-nginx-tls/index.md @@ -0,0 +1,369 @@ ++++ +title = "Prod VM with nginx and TLS" +description = "Run a production Host on a VM behind nginx with TLS termination and persistent volumes for DefraDB data, keys, and lens files." +[extra] +mermaid = true ++++ + +When to use this: you want to run a production Host on a VM with nginx as a reverse proxy, TLS termination, and persistent volumes for DefraDB data, keys, and lens files. + +These scenarios use Ethereum Mainnet data. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing the contract addresses and topic hashes to match the target chain. See the [Generator chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph VM["Prod VM"] + direction TB + Nginx["nginx
:443 TLS / :8080"] + Host["Host container
:9181 GraphQL API
:9171 P2P
:8080 Health/Metrics"] + Nginx -- "proxy" --> Host + end + + Gens["Generators"] + Clients["API clients"] + + Gens -- "P2P (libp2p)
:9171" --> Host + Clients -- "HTTPS
/api/v0/graphql
/metrics" --> Nginx +{% end %} + +nginx terminates incoming HTTPS traffic and proxies GraphQL and metrics requests to the Host container. P2P traffic from Generators goes directly to the Host on port 9171, bypassing nginx. Persistent volumes on the VM keep DefraDB data, keys, and lens files across container restarts. + +## Prerequisites + +- Docker installed on the VM. +- A `config.yaml` file with your Host configuration. +- TLS certificate and key files at `~/ssl/nginx.crt` and `~/ssl/nginx.key` for nginx TLS termination. +- Ports 8080, 80, 443, and 9171 open externally. + +## Config file + +This config is drawn from `host-prod-setup.sh` in the `shinzo-host-client` repo. It sets the DefraDB URL, keyring secret, P2P bootstrap peers, and the ShinzoHub endpoint. The event filter is disabled, accepting all documents: + +```yaml +defradb: + url: "localhost:9181" + keyring_secret: "pingpong" + p2p: + enabled: true + bootstrap_peers: + - '/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' + - '/ip4/34.57.239.57/tcp/9171/p2p/12D3KooWBAgCEJHYqzuCFEXzjsw2CnV9JqvqMgTKYDww58aCxwW5' + - '/ip4/34.134.119.63/tcp/9171/p2p/12D3KooWQQTuSQaz4HfuvnJHakkQy3PhWbKBBbS3RkmBw4ZsFkyT' + listen_addr: "/ip4/0.0.0.0/tcp/9171" + max_retries: 5 + retry_base_delay_ms: 1000 + reconnect_interval_ms: 60000 + enable_auto_reconnect: true + store: + path: "./.defra" + block_cache_mb: 512 + memtable_mb: 64 + index_cache_mb: 256 + num_compactors: 4 + value_log_file_size_mb: 128 + +shinzo: + minimum_attestations: 1 + start_height: 0 + hub_base_url: testnet.shinzo.network:26657 + cache_queue_size: 50000 + batch_writer_count: 8 + batch_size: 500 + batch_flush_interval: 100 + use_block_signatures: true + doc_worker_count: 32 + doc_queue_size: 50000 + event_filter: + enabled: false + mode: "allowlist" + cascade_filters: true + groups: + - name: "uniswap-v3" + enabled: false + contracts: + - name: "Uniswap V3 Router 2" + address: "0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45" + types: ["transaction", "log"] + topics: + - name: "Swap" + topic0: "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" + - name: "stablecoins" + enabled: false + contracts: + - name: "USDT" + address: "0xdAC17F958D2ee523a2206206994597C13D831ec7" + types: ["log"] + - name: "USDC" + address: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + types: ["log"] + topics: + - name: "Transfer" + topic0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + +pruner: + enabled: true + max_blocks: 2000 + docs_per_block: 1000 + interval_seconds: 30 + prune_history: false + +logger: + development: false + +host: + lens_registry_path: "./.defra/lens" + health_server_port: 8080 + open_browser_on_start: false + snapshot: + enabled: false + indexer_url: "http://35.206.105.60:8080" + historical_ranges: + - start: 24528700 + end: 24528999 +``` + +## Compose file + +This compose file is drawn from `host-prod-setup.sh`. It runs the Host container and an nginx container on a shared Docker network. The Host container exposes the DefraDB API on host port 443, the GraphQL Playground on 444, and P2P on 9171. nginx proxies health, registration, metrics, and GraphQL on ports 8080 and 80, and terminates TLS on 443 via the mounted certificate: + +```yaml +networks: + shinzo-net: + driver: bridge + +services: + shinzo-host: + image: ghcr.io/shinzonetwork/shinzo-host-client:v0.6.5-ethereum-mainnet + user: "1001:1001" + mem_limit: 16g + mem_reservation: 13g + restart: unless-stopped + container_name: shinzo-host + networks: + - shinzo-net + ports: + - "443:9181" + - "444:9182" + - "9171:9171" + volumes: + - ~/data/defradb:/app/.defra + - ~/data/keys:/app/.defra/keys + - ~/data/lens:/app/.defra/lens + - ~/config.yaml:/app/config.yaml:ro + environment: + - DEFRA_URL=0.0.0.0:9181 + - GOMEMLIMIT=14GiB + - LOG_LEVEL=error + - LOG_SOURCE=false + - LOG_STACKTRACE=false + - ALLOWED_ORIGINS=https://*.shinzo.network + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/metrics"] + interval: 15s + timeout: 30s + retries: 10 + start_period: 120s + + nginx: + image: nginx:alpine + ports: + - "8080:8080" + - "80:80" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ~/ssl/nginx.crt:/etc/nginx/ssl/nginx.crt:ro + - ~/ssl/nginx.key:/etc/nginx/ssl/nginx.key:ro + depends_on: + - shinzo-host + networks: + - shinzo-net + restart: unless-stopped +``` + +## nginx configuration + +This nginx config is drawn from `host-prod-setup.sh`. It listens on 80, 443 (with TLS), and 8080, and proxies health, registration, metrics, and GraphQL requests to the Host container via a `$backend` variable. CORS headers are restricted to `https://explorer.shinzo.network`. The `resolver 127.0.0.11` directive lets nginx resolve the `shinzo-host` service name on the Docker network: + +```nginx +events { worker_connections 1024; } + +http { + resolver 127.0.0.11 valid=10s; + + map $http_origin $cors_origin { + default ""; + "https://explorer.shinzo.network" $http_origin; + } + + server { + listen 80; + listen 443 ssl; + listen 8080; + server_name api.shinzo.network; + + ssl_certificate /etc/nginx/ssl/nginx.crt; + ssl_certificate_key /etc/nginx/ssl/nginx.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + set $backend "shinzo-host"; + + location = /metrics { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:8080/metrics; + proxy_set_header Host $host; + } + + location = /health { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:8080/health; + proxy_set_header Host $host; + } + + location = /registration { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:8080/registration; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /registration-app { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:8080/registration-app; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /api/v0/graphql { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:9181/api/v0/graphql; + proxy_set_header Host $host; + } + + location / { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Access-Control-Max-Age' 3600 always; + add_header 'Vary' 'Origin' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Accept, Origin' always; + add_header 'Vary' 'Origin' always; + proxy_pass http://$backend:9181; + proxy_set_header Host $host; + } + } +} +``` + +### What the key values mean + +- `user: "1001:1001"`: The container runs as UID 1001, GID 1001. Update this to match your user and group ID on the VM (the repo script notes this). The data directories must be owned by this UID and GID. +- `DEFRA_URL=0.0.0.0:9181`: Overrides `defradb.url` so the DefraDB API binds to all interfaces instead of the loopback-only YAML default. Read in `config/config.go`. See [environment variables](/hosts/config-reference#environment-variables). +- `GOMEMLIMIT=14GiB`: Go runtime soft memory limit, set below the 16g container limit. See [env vars](/hosts/config-reference#environment-variables). +- `ALLOWED_ORIGINS=https://*.shinzo.network`: CORS origins the Host accepts. +- `~/data/defradb:/app/.defra`: Persistent volume for DefraDB data. See [defradb store](/hosts/config-reference#defradb-store). +- `~/data/keys:/app/.defra/keys`: Persistent volume for DefraDB keyring keys. See [defradb](/hosts/config-reference#defradb). +- `~/data/lens:/app/.defra/lens`: Persistent volume for lens WASM files. See [host](/hosts/config-reference#host). +- `~/config.yaml:/app/config.yaml:ro`: Mount the config file read-only. +- The `443:9181` port mapping exposes the DefraDB GraphQL API on host port 443. The `444:9182` mapping exposes the GraphQL Playground on port 444. The playground is built with the `hostplayground` build tag, not controlled by env vars. +- nginx terminates TLS on 443 using `/etc/nginx/ssl/nginx.crt` and `/etc/nginx/ssl/nginx.key` (mounted from `~/ssl/`), and also listens on 80 and 8080. + +## Start the Host + +Place all three files (`config.yaml`, `docker-compose.yml`, `nginx.conf`) in the same directory, then start: + +```shell +docker compose -f docker-compose.yml up -d +``` + +Verify the Host is healthy through nginx: + +```shell +curl http://localhost:8080/health +``` + +Query GraphQL over TLS on 443: + +```shell +curl -s -X POST https://localhost/api/v0/graphql \ + -H "Content-Type: application/json" \ + -d '{"query": "{ Ethereum__Mainnet__Block(limit: 1) { number hash } }"}' +``` + +## Gotchas + +- The `user: "1001:1001"` in the compose file must match the ownership of the data directories. If the directories are owned by root, the Host container will fail to write. Fix ownership with `chown -R 1001:1001 ~/data/defradb ~/data/keys ~/data/lens`. +- `DEFRA_URL` overrides `defradb.url` and is read by the Host client (`config/config.go`); it binds the DefraDB API to `0.0.0.0:9181`. `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` appear in the original `host-prod-setup.sh` compose but are not read by the Host client and have no effect. See [env vars that are not read](/hosts/config-reference#env-vars-that-are-not-read). +- The `DEFRA_KEYRING_SECRET` env var uses the `DEFRA_` prefix. The Generator client uses `DEFRADB_KEYRING_SECRET` with the `DEFRADB_` prefix. If you are running both clients on the same VM, do not confuse the two env var names. See [environment variables](/hosts/config-reference#environment-variables). +- The shipped `config.yaml` includes several `shinzo.*` keys that are not in the `config.go` struct and are silently ignored: `wait_for_gaps`, `max_gap_size`, `batch_processing_enabled`, `batch_max_views_per_job`, `batch_query_cache_size`. They have been omitted from the config above. See [no-op keys](/hosts/config-reference#no-op-keys). +- The `logger.level` field in the shipped config is not in the `LoggerConfig` struct and has no effect. It has been omitted. See [logger](/hosts/config-reference#logger). +- The bootstrap peer IDs in this config are the three indexer peers from `host-prod-setup.sh` (identical to the shipped `config.yaml`) and may be stale. Check the [Shinzo Validators list](https://registration.shinzo.network/validators) for current peers. +- The nginx CORS config restricts origins to `https://explorer.shinzo.network`. If you need to allow other origins, add them to the `map` block in the nginx config. +- The Host image tag in this compose file is `ghcr.io/shinzonetwork/shinzo-host-client:v0.6.5-ethereum-mainnet`, matching `host-prod-setup.sh`. The [GCP local SSD scenario](../gcp-local-ssd-raid0/) pins `:v0.5.1`, and the [Watchtower setup](../watchtower-auto-deploy/) uses `:latest`. Pick one tag and be consistent across your deployment. + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/deployment-examples/snapshot-bootstrap/index.md b/content/hosts/deployment-examples/snapshot-bootstrap/index.md new file mode 100644 index 0000000..25df3af --- /dev/null +++ b/content/hosts/deployment-examples/snapshot-bootstrap/index.md @@ -0,0 +1,148 @@ ++++ +title = "Snapshot bootstrap from an indexer" +description = "Bootstrap a Host with historical data on first startup by downloading signed snapshot files from an indexer over HTTPS, then receive live blocks over P2P." +[extra] +mermaid = true ++++ + +When to use this: you want your Host to sync historical data quickly on first startup by downloading signed snapshot files from an indexer, instead of waiting for P2P replication to catch up from the chain tip. + +These scenarios use Ethereum Mainnet data. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing the contract addresses and topic hashes to match the target chain. See the [Generator chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + subgraph Startup["First startup"] + direction TB + S1["1. Download snapshots
from indexer over HTTPS"] + S2["2. Import snapshots
into DefraDB"] + S3["3. Start P2P
for live blocks"] + S1 --> S2 --> S3 + end + + Indexer["Indexer
HTTP :8080
/snapshots endpoint"] + Gens["Generators
(live P2P feed)"] + Host["Host"] + + Indexer -- "HTTPS
snapshot files" --> Host + Gens -- "P2P (libp2p)" --> Host +{% end %} + +On first startup, the Host downloads signed snapshot files from an indexer over HTTPS and imports them into DefraDB. After the snapshot import completes, the Host connects to Generators over P2P and starts receiving live blocks from the chain tip. This is faster than waiting for P2P replication to fill in historical data block by block. + +## Prerequisites + +- Docker installed on the VM. +- An indexer serving snapshots over HTTP. The indexer must have `SNAPSHOT_ENABLED=true` and be reachable over HTTPS or HTTP. See the [nginx with TLS scenario](/generators/deployment-examples/nginx-tls-snapshots/) for how to set up an indexer that serves snapshots. +- The block range you want to bootstrap. The indexer must have snapshot files covering that range. + +## Config file + +This config enables snapshot bootstrap from the indexer at `http://35.206.105.60:8080` for blocks 24528700 through 24528999. It is drawn from the `host.snapshot` section of the shipped `host-client/config/config.yaml`: + +```yaml +defradb: + url: "localhost:9181" + keyring_secret: "pingpong" + p2p: + enabled: true + bootstrap_peers: + - '/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' + - '/ip4/34.57.239.57/tcp/9171/p2p/12D3KooWBAgCEJHYqzuCFEXzjsw2CnV9JqvqMgTKYDww58aCxwW5' + - '/ip4/34.134.119.63/tcp/9171/p2p/12D3KooWQQTuSQaz4HfuvnJHakkQy3PhWbKBBbS3RkmBw4ZsFkyT' + listen_addr: "/ip4/0.0.0.0/tcp/9171" + max_retries: 5 + retry_base_delay_ms: 1000 + reconnect_interval_ms: 60000 + enable_auto_reconnect: true + store: + path: "./.defra" + block_cache_mb: 512 + memtable_mb: 64 + index_cache_mb: 256 + num_compactors: 4 + value_log_file_size_mb: 128 + +shinzo: + start_height: 0 + hub_base_url: testnet.shinzo.network:26657 + cache_queue_size: 50000 + batch_writer_count: 8 + batch_size: 500 + batch_flush_interval: 100 + use_block_signatures: true + doc_worker_count: 32 + doc_queue_size: 50000 + +pruner: + enabled: true + max_blocks: 2000 + docs_per_block: 1000 + interval_seconds: 30 + prune_history: false + +logger: + development: true + +host: + lens_registry_path: "./.defra/lens" + health_server_port: 8080 + open_browser_on_start: false + snapshot: + enabled: true + indexer_url: "http://35.206.105.60:8080" + historical_ranges: + - start: 24528700 + end: 24528999 +``` + +### What each snapshot value does + +- `host.snapshot.enabled: true`: Run snapshot bootstrap on startup. The shipped `config.yaml` sets this to false. See [host snapshot](/hosts/config-reference#host-snapshot). +- `host.snapshot.indexer_url`: HTTP base URL of the indexer serving snapshots. Replace `http://35.206.105.60:8080` with your indexer URL. See [host snapshot](/hosts/config-reference#host-snapshot). +- `host.snapshot.historical_ranges`: Block ranges to download during bootstrap. Each range is inclusive. See [host snapshot](/hosts/config-reference#host-snapshot). + +## How bootstrap works + +All three conditions must be true for snapshot bootstrap to run: `snapshot.enabled` is true, `indexer_url` is non-empty, and `historical_ranges` has at least one entry. If any condition is false, bootstrap is skipped and the Host goes straight to P2P replication. + +The Host downloads snapshot files from `{indexer_url}/snapshots/{filename}` for each block range. Each snapshot file contains signed blocks bundled by the Generator. The Host imports them into DefraDB, then starts P2P replication for live blocks. + +## Running with Docker + +Mount the config file into the container: + +```shell +docker run -d \ + --name shinzo-host \ + --restart unless-stopped \ + -p 9181:9181 \ + -p 9171:9171 \ + -p 8080:8080 \ + -v $(pwd)/data/defradb:/app/.defra \ + -v $(pwd)/config.yaml:/app/config.yaml:ro \ + -e DEFRA_KEYRING_SECRET=pingpong \ + ghcr.io/shinzonetwork/shinzo-host-client:latest +``` + +Check the startup logs for snapshot import progress: + +```shell +docker logs -f shinzo-host +``` + +You should see log lines indicating snapshot downloads and imports, followed by P2P network startup. + +## Gotchas + +- Snapshot bootstrap only runs on first startup when DefraDB has no existing data. If the Host already has data for the requested block range, bootstrap is skipped. +- The `indexer_url` in the shipped `config.yaml` is `http://35.206.105.60:8080`. This is a development indexer. Replace it with your own indexer URL or a production indexer that has snapshots enabled. +- The indexer must have `SNAPSHOT_ENABLED=true` on the Generator side. If the indexer is not producing snapshot files, the `/snapshots` endpoint will return nothing and bootstrap will fail. See the [nginx with TLS scenario](/generators/deployment-examples/nginx-tls-snapshots/) for setting up an indexer that serves snapshots. +- The `DEFRA_URL` env var overrides `defradb.url` at runtime and is read by the Host client (`config/config.go`). The `docker run` above does not set it, so the DefraDB URL comes from `defradb.url` in the YAML config. See [environment variables](/hosts/config-reference#environment-variables). +- The `DEFRA_KEYRING_SECRET` env var uses the `DEFRA_` prefix. The Generator client uses `DEFRADB_KEYRING_SECRET` with the `DEFRADB_` prefix. The two clients use different env var names for the same concept. See [environment variables](/hosts/config-reference#environment-variables). +- `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` env vars appear in some deployment scripts but are not read by the Host client. They have been omitted from the `docker run` above. See [env vars that are not read](/hosts/config-reference#env-vars-that-are-not-read). + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/deployment-examples/watchtower-auto-deploy/index.md b/content/hosts/deployment-examples/watchtower-auto-deploy/index.md new file mode 100644 index 0000000..e3f50bd --- /dev/null +++ b/content/hosts/deployment-examples/watchtower-auto-deploy/index.md @@ -0,0 +1,171 @@ ++++ +title = "Watchtower auto-deploy" +description = "Auto-deploy Host updates on push to main: GitHub Actions builds and pushes the image to GHCR, and Watchtower on the VM pulls the new image and restarts the container." +[extra] +mermaid = true ++++ + +When to use this: you want pushes to `main` to automatically deploy to your Host VM. GitHub Actions builds and pushes the image to GHCR, then Watchtower on the VM detects the new image and restarts the container. + +These scenarios use Ethereum Mainnet data. Ethereum is the only officially supported chain today, but any EVM-compatible chain should work by changing the contract addresses and topic hashes to match the target chain. See the [Generator chain config](/generators/config-reference#chain) for details. + +## Topology + +{% mermaid() %} +flowchart LR + Dev["Push to main"] + CI["GitHub Actions
test + build"] + GHCR["GHCR
:latest + :sha-"] + + subgraph VM["Host VM"] + direction TB + WT["Watchtower
polls every 5 min"] + Host["Host container
auto-restarts"] + WT -- "pull + restart" --> Host + end + + Dev --> CI --> GHCR + GHCR -- "pull" --> WT +{% end %} + +A developer pushes to `main`. GitHub Actions runs tests, builds the Docker image, and pushes it to GitHub Container Registry with two tags: `:latest` and `:sha-`. Watchtower runs on the VM and polls GHCR every 5 minutes. When it detects a new `:latest` image, it stops the old Host container, pulls the new image, and starts a new container with the same configuration. + +## Prerequisites + +- Docker installed on the VM. +- A GitHub Personal Access Token (PAT) with `read:packages` scope for GHCR authentication. +- A `config.yaml` file with your Host configuration on the VM. +- The Host GitHub repo set up with GitHub Actions and the `DEFRA_KEYRING_SECRET` secret for tests. + +## VM setup + +### Authenticate to GHCR + +Log in to GHCR once on the VM. The credentials are saved to `~/.docker/config.json` so Watchtower can use them: + +```shell +echo "YOUR_GITHUB_PAT" | docker login ghcr.io -u YOUR_USERNAME --password-stdin +``` + +Replace `YOUR_GITHUB_PAT` with your GitHub Personal Access Token and `YOUR_USERNAME` with your GitHub username. + +### Start Watchtower + +Watchtower watches for containers with the `com.centurylinklabs.watchtower.enable=true` label and pulls new images for them. It polls every 300 seconds (5 minutes) and cleans up old images: + +```shell +docker run -d \ + --name watchtower \ + --restart unless-stopped \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ~/.docker/config.json:/config.json:ro \ + -e WATCHTOWER_POLL_INTERVAL=300 \ + -e WATCHTOWER_CLEANUP=true \ + -e WATCHTOWER_LABEL_ENABLE=true \ + containrrr/watchtower +``` + +### Start the Host container + +The Host container needs the Watchtower label so Watchtower knows to update it. This `docker run` is drawn from `.github/DEPLOYMENT.md` in the `shinzo-host-client` repo: + +```shell +docker run -d \ + --label com.centurylinklabs.watchtower.enable=true \ + --name shinzo-host \ + --restart unless-stopped \ + -p 9181:9181 \ + -p 9182:9182 \ + -p 9171:9171 \ + -v $(pwd)/data/defradb:/app/.defra \ + -v $(pwd)/data/lens:/app/.lens \ + -v $(pwd)/config.yaml:/app/config.yaml:ro \ + -e DEFRA_URL=0.0.0.0:9181 \ + -e LOG_LEVEL=error \ + -e LOG_SOURCE=false \ + -e LOG_STACKTRACE=false \ + ghcr.io/shinzonetwork/shinzo-host-client:latest +``` + +The `--label com.centurylinklabs.watchtower.enable=true` tells Watchtower to manage this container. Without it, Watchtower ignores the container. + +## GitHub Actions workflow + +The GitHub Actions workflow triggers on push to `main`. It runs tests, builds the Docker image, and pushes it to GHCR with two tags. The required GitHub secret is `DEFRA_KEYRING_SECRET`, used for tests. + +### Image tags + +| Tag | Purpose | +| --- | --- | +| `ghcr.io/shinzonetwork/shinzo-host-client:latest` | Most recent main build. Watchtower watches this tag. | +| `ghcr.io/shinzonetwork/shinzo-host-client:sha-<7chars>` | Specific commit, used for rollback. | + +### Deploy process + +1. Run tests. +1. Build Docker image with SHA tag. +1. Push to GHCR with `:latest` and `:sha-` tags. +1. Watchtower detects the new `:latest` image within 5 minutes. +1. Watchtower stops the old container, pulls the new image, and starts a new container with the same configuration. + +## Troubleshooting + +### Check Watchtower logs + +```shell +docker logs watchtower --tail 50 +``` + +### Check if Watchtower can pull images + +```shell +docker pull ghcr.io/shinzonetwork/shinzo-host-client:latest +``` + +If this fails, check your GHCR credentials in `~/.docker/config.json`. + +### Force immediate update + +```shell +docker exec watchtower /watchtower --run-once +``` + +### Manual rollback + +Stop the current container and start with a specific SHA tag: + +```shell +docker stop shinzo-host && docker rm shinzo-host + +docker run -d \ + --label com.centurylinklabs.watchtower.enable=true \ + --name shinzo-host \ + --restart unless-stopped \ + -p 9181:9181 \ + -p 9182:9182 \ + -p 9171:9171 \ + -v $(pwd)/data/defradb:/app/.defra \ + -v $(pwd)/data/lens:/app/.lens \ + -v $(pwd)/config.yaml:/app/config.yaml:ro \ + -e DEFRA_URL=0.0.0.0:9181 \ + -e LOG_LEVEL=error \ + -e LOG_SOURCE=false \ + -e LOG_STACKTRACE=false \ + ghcr.io/shinzonetwork/shinzo-host-client:sha-abc1234 +``` + +Replace `sha-abc1234` with the actual commit SHA tag you want to roll back to. + +## Gotchas + +- The container must have the `com.centurylinklabs.watchtower.enable=true` label or Watchtower will ignore it. Verify with `docker inspect shinzo-host | grep watchtower`. +- `DEFRA_URL` in the `docker run` commands overrides `defradb.url` and is read by the Host client (`config/config.go`), binding the DefraDB API to `0.0.0.0:9181`. `LOG_LEVEL`, `LOG_SOURCE`, and `LOG_STACKTRACE` are also kept from the original DEPLOYMENT.md but are not read by the Host client and have no effect. See [env vars that are not read](/hosts/config-reference#env-vars-that-are-not-read). +- The Host image tag `:latest` is used here. The [prod VM scenario](../prod-vm-nginx-tls/) uses `:v0.6.5-ethereum-mainnet`, and the [GCP local SSD scenario](../gcp-local-ssd-raid0/) pins `:v0.5.1`. If you use Watchtower with `:latest`, you get automatic updates. If you pin a specific tag, Watchtower will not detect new images. +- Watchtower preserves the container's configuration (ports, volumes, env vars, labels) across restarts. It only replaces the image. If you need to change the container configuration, stop and remove it manually, then start a new container with the updated configuration. +- The `DEFRA_KEYRING_SECRET` GitHub secret is used for tests in the CI pipeline, not for runtime. The runtime keyring secret comes from your `config.yaml` on the VM. See [defradb config](/hosts/config-reference#defradb). +- There is a 5-minute delay between the image push and the container restart, controlled by `WATCHTOWER_POLL_INTERVAL=300`. To reduce this, lower the interval. To increase it, raise the interval. Lower intervals mean more frequent GHCR API calls. +- Local SSDs are ephemeral on GCP. If you use Watchtower with the [GCP local SSD scenario](../gcp-local-ssd-raid0/), a VM restart will lose all DefraDB data. Watchtower will restart the container with the new image, but the data directory will be empty. Pair with the [snapshot bootstrap scenario](../snapshot-bootstrap/) to re-import data on restart. + +## Need help + +{{ need_help(client="Host", repo_name="shinzo-host-client", repo="https://github.com/shinzonetwork/shinzo-host-client/issues") }} diff --git a/content/hosts/quickstart/index.md b/content/hosts/quickstart/index.md index 46cf114..60a26ff 100644 --- a/content/hosts/quickstart/index.md +++ b/content/hosts/quickstart/index.md @@ -175,7 +175,7 @@ defradb: p2p: enabled: true bootstrap_peers: - - '/ip4/34.63.13.57/tcp/9171/p2p/12D3KooWMYhYNBo4zAi9j7TpyGQJBSvbwSSNkgsMrLs6vHUnFUzY' + - '/ip4/35.254.135.221/tcp/9171/p2p/12D3KooWDUdHSCXBM5Wb7te6ZdWMgqddw7tJ7npWSzXK5tQgBsbT' listen_addr: "/ip4/0.0.0.0/tcp/9171" enable_auto_reconnect: true store: