feat(datastore): PostgreSQL dual-database support

charts/fleet/templates/deployment.yaml

@@ -188,7 +188,11 @@ spec:
                 name: {{ .Values.fleet.license.secretName }}
           {{- end }}
           ## END FLEET SECTION
-          ## BEGIN MYSQL SECTION
+          ## BEGIN DATABASE SECTION
+          {{- if .Values.database.driver }}
+          - name: FLEET_MYSQL_DRIVER
+            value: "{{ .Values.database.driver }}"
+          {{- end }}
           - name: FLEET_MYSQL_ADDRESS
             value: "{{ .Values.database.address }}"
           - name: FLEET_MYSQL_DATABASE
@@ -224,7 +228,7 @@ spec:
           - name: FLEET_MYSQL_TLS_SERVER_NAME
             value: "{{ .Values.database.tls.serverName }}"
           {{- end }}
-          ## END MYSQL SECTION
+          ## END DATABASE SECTION
           ## BEGIN MYSQL READ REPLICA SECTION
           {{- if .Values.database_read_replica }}
           {{- if .Values.database_read_replica.address }}

charts/fleet/values.yaml

@@ -221,9 +221,13 @@ osquery:
       resultTopic: ""
 
 ## Section: database
-# All of the connection settings for MySQL
+# All of the connection settings for the primary database.
+# Supports MySQL (default) and PostgreSQL (experimental).
 database:
-  # Name of the Secret resource containing MySQL password and TLS secrets
+  # Driver selects the database backend: "mysql" (default) or "postgres".
+  # PostgreSQL support is experimental for self-hosted deployments.
+  driver: mysql
+  # Name of the Secret resource containing database password and TLS secrets
   secretName: mysql
   address: 127.0.0.1:3306
   database: fleet
@@ -323,5 +327,11 @@ environments:
 mysql:
   enabled: false
 
+# PostgreSQL subchart (experimental). Enable when database.driver is "postgres"
+# and you want Helm to provision a PG instance. For production, use a managed
+# PostgreSQL service (e.g., AWS RDS, GCP Cloud SQL) and set database.address.
+postgresql:
+  enabled: false
+
 redis:
   enabled: false

docker-compose.yml

@@ -81,6 +81,38 @@ services:
       - /var/lib/mysql:rw,noexec,nosuid
       - /tmpfs
 
+  # PostgreSQL test instance for POSTGRES_TEST=1 integration tests.
+  # Usage: docker compose up -d postgres_test
+  # Then:  POSTGRES_TEST=1 go test ./server/datastore/mysql/...
+  postgres_test:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: fleet
+      POSTGRES_PASSWORD: insecure
+      POSTGRES_DB: fleet
+    ports:
+      - "${FLEET_POSTGRES_TEST_PORT:-5434}:5432"
+    tmpfs:
+      - /var/lib/postgresql/data:rw,noexec,nosuid
+    command: [
+        "postgres",
+        "-c", "fsync=off",
+        "-c", "full_page_writes=off",
+        "-c", "synchronous_commit=off",
+    ]
+
+  # PostgreSQL development instance.
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: fleet
+      POSTGRES_PASSWORD: insecure
+      POSTGRES_DB: fleet
+    ports:
+      - "5433:5432"
+    volumes:
+      - postgres-persistent-volume:/var/lib/postgresql/data
+
   # Unauthenticated SMTP server.
   mailhog:
     image: mailhog/mailhog:latest
@@ -170,4 +202,5 @@ services:
 
 volumes:
   mysql-persistent-volume:
+  postgres-persistent-volume:
   data-s3:

go.mod

@@ -286,6 +286,10 @@ require (
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.8.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect

go.sum

@@ -543,6 +543,14 @@ github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
+github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=

server/config/config.go

@@ -39,6 +39,9 @@ const (
 
 // MysqlConfig defines configs related to MySQL
 type MysqlConfig struct {
+	// Driver selects the database driver. Only "mysql" is valid in Phase 1.
+	// Future values: "postgres" (Phase 4+).
+	Driver           string `yaml:"driver"`
 	Protocol         string `yaml:"protocol"`
 	Address          string `yaml:"address"`
 	Username         string `yaml:"username"`
@@ -1128,6 +1131,8 @@ func (t *TLS) ToTLSConfig() (*tls.Config, error) {
 // filled into the FleetConfig struct
 func (man Manager) addConfigs() {
 	addMysqlConfig := func(prefix, defaultAddr, usageSuffix string) {
+		man.addConfigString(prefix+".driver", "",
+			"Database driver: mysql (default) or postgres"+usageSuffix)
 		man.addConfigString(prefix+".protocol", "tcp",
 			"MySQL server communication protocol (tcp,unix,...)"+usageSuffix)
 		man.addConfigString(prefix+".address", defaultAddr,
@@ -1637,6 +1642,7 @@ func (man Manager) LoadConfig() FleetConfig {
 
 	loadMysqlConfig := func(prefix string) MysqlConfig {
 		return MysqlConfig{
+			Driver:           man.getConfigString(prefix + ".driver"),
 			Protocol:         man.getConfigString(prefix + ".protocol"),
 			Address:          man.getConfigString(prefix + ".address"),
 			Username:         man.getConfigString(prefix + ".username"),

server/datastore/mysql/activities.go

@@ -60,7 +60,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 				'script_name', COALESCE(ses.name, scr.name, ''),
 				'script_execution_id', ua.execution_id,
 				'batch_execution_id', bahr.batch_execution_id,
-				'async', NOT ua.payload->'$.sync_request',
+				'async', COALESCE(ua.payload->>'$.sync_request', '0') != '1',
 				'policy_id', sua.policy_id,
 				'policy_name', p.name
 			) as details,
@@ -104,7 +104,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 				'software_package', COALESCE(si.filename, ua.payload->>'$.installer_filename', ''),
 				'install_uuid', ua.execution_id,
 				'status', 'pending_install',
-				'self_service', ua.payload->'$.self_service' IS TRUE,
+				'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1',
 				'source', COALESCE(st.source, ua.payload->>'$.source'),
 				'policy_id', siua.policy_id,
 				'policy_name', p.name
@@ -146,7 +146,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 				'software_title', COALESCE(st.name, ua.payload->>'$.software_title_name', ''),
 				'script_execution_id', ua.execution_id,
 				'status', 'pending_uninstall',
-				'self_service', COALESCE(ua.payload->'$.self_service', FALSE) IS TRUE,
+				'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1',
 				'source', COALESCE(st.source, ua.payload->>'$.source'),
 				'policy_id', siua.policy_id,
 				'policy_name', p.name
@@ -188,7 +188,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 				'software_title', COALESCE(st.name, ''),
 				'app_store_id', vaua.adam_id,
 				'command_uuid', ua.execution_id,
-				'self_service', ua.payload->'$.self_service' IS TRUE,
+				'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1',
 				'status', 'pending_install',
 				'host_platform', h.platform
 			) AS details,
@@ -228,7 +228,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint
 				'host_display_name', COALESCE(hdn.display_name, ''),
 				'software_title', COALESCE(st.name, ''),
 				'command_uuid', ua.execution_id,
-				'self_service', ua.payload->'$.self_service' IS TRUE,
+				'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1',
 				'status', 'pending_install'
 			) AS details,
 			IF(ua.activated_at IS NULL, 0, 1) as topmost,
@@ -809,10 +809,10 @@ func (ds *Datastore) GetHostUpcomingActivityMeta(ctx context.Context, hostID uin
 		ua.activated_at,
 		ua.activity_type,
 		CASE
-			WHEN hma.lock_ref = :execution_id THEN :lock_action
-			WHEN hma.unlock_ref = :execution_id THEN :unlock_action
-			WHEN hma.wipe_ref = :execution_id THEN :wipe_action
-			ELSE :none_action
+			WHEN hma.lock_ref = :execution_id THEN CAST(:lock_action AS UNSIGNED)
+			WHEN hma.unlock_ref = :execution_id THEN CAST(:unlock_action AS UNSIGNED)
+			WHEN hma.wipe_ref = :execution_id THEN CAST(:wipe_action AS UNSIGNED)
+			ELSE CAST(:none_action AS UNSIGNED)
 		END AS well_known_action
 	FROM
 		upcoming_activities ua
@@ -1070,9 +1070,9 @@ SELECT
 	sua.script_id,
 	sua.policy_id,
 	ua.user_id,
-	COALESCE(ua.payload->'$.sync_request', 0),
+	COALESCE(ua.payload->>'$.sync_request', '0') = '1',
 	sua.setup_experience_script_id,
-	COALESCE(ua.payload->'$.is_internal', 0)
+	COALESCE(ua.payload->>'$.is_internal', '0') = '1'
 FROM
 	upcoming_activities ua
 	INNER JOIN script_upcoming_activities sua
@@ -1108,7 +1108,7 @@ SELECT
 	ua.host_id,
 	siua.software_installer_id,
 	ua.user_id,
-	COALESCE(ua.payload->'$.self_service', 0),
+	COALESCE(ua.payload->>'$.self_service', '0') = '1',
 	siua.policy_id,
 	COALESCE(si.filename, ua.payload->>'$.installer_filename', '[deleted installer]'),
 	COALESCE(si.version, ua.payload->>'$.version', 'unknown'),
@@ -1120,7 +1120,7 @@ SELECT
 	-- the number of prior tries. +1 makes this the next attempt in sequence:
 	-- first install = 1, first retry = 2, second retry = 3, etc.
 	CASE
-		WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->'$.with_retries', 0) = 1 THEN (
+		WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->>'$.with_retries', '0') = '1' THEN (
 			SELECT COUNT(*) + 1
 			FROM host_software_installs hsi2
 			WHERE hsi2.host_id = ua.host_id
@@ -1255,7 +1255,7 @@ SELECT
 	ua.execution_id,
 	ua.user_id,
 	ua.payload->>'$.associated_event_id',
-	COALESCE(ua.payload->'$.self_service', 0),
+	COALESCE(ua.payload->>'$.self_service', '0') = '1',
 	vaua.policy_id
 FROM
 	upcoming_activities ua
@@ -1291,7 +1291,7 @@ SELECT
 	ua.execution_id,
 	ua.user_id,
 	iha.platform,
-	COALESCE(ua.payload->'$.self_service', 0)
+	COALESCE(ua.payload->>'$.self_service', '0') = '1'
 FROM
 	upcoming_activities ua
 	INNER JOIN in_house_app_upcoming_activities ihua

server/datastore/mysql/activities_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestActivity(t *testing.T) {
-	ds := CreateMySQLDS(t)
+	ds := CreateDS(t)
 
 	cases := []struct {
 		name string

server/datastore/mysql/aggregated_stats.go

@@ -129,8 +129,7 @@ func (ds *Datastore) CalculateAggregatedPerfStatsPercentiles(ctx context.Context
 		`
 		INSERT INTO aggregated_stats(id, type, global_stats, json_value)
 		VALUES (?, ?, 0, ?)
-		ON DUPLICATE KEY UPDATE json_value=VALUES(json_value)
-		`,
+		`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value=VALUES(json_value)`),
 		queryID, aggregate, statsJson,
 	)
 	if err != nil {

server/datastore/mysql/aggregated_stats_test.go

@@ -46,7 +46,7 @@ func slowStats(t *testing.T, ds *Datastore, id uint, percentile int, column stri
 }
 
 func TestAggregatedStats(t *testing.T) {
-	ds := CreateMySQLDS(t)
+	ds := CreateDS(t)
 
 	var args []interface{}