From 95cc062d97c36757f487c67af05006a8ef824cea Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 15:54:13 -0400 Subject: [PATCH 1/6] feat(datastore): add PostgreSQL rebind driver and platform support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the pgx-rebind SQL driver that wraps pgx/v5 to automatically translate MySQL-dialect SQL to PostgreSQL at query time. Handles 30+ transformation categories including placeholder conversion, function rewrites, boolean/integer fixes, JSON operators, upsert syntax, and more. New files: - server/platform/postgres/rebind_driver.go — SQL rewrite layer (1,185 lines) - server/platform/postgres/errors.go — PG SQLSTATE error classification - server/platform/postgres/common.go — shared PG utilities Config/infrastructure: - server/config/config.go — --mysql_driver flag for driver selection - charts/fleet/ — database.driver Helm value + FLEET_MYSQL_DRIVER env - docker-compose.yml — postgres_test service for local PG testing --- charts/fleet/templates/deployment.yaml | 8 +- charts/fleet/values.yaml | 14 +- docker-compose.yml | 33 + go.mod | 4 + go.sum | 8 + server/config/config.go | 6 + server/platform/postgres/common.go | 31 + server/platform/postgres/errors.go | 110 ++ server/platform/postgres/rebind_driver.go | 1182 +++++++++++++++++++++ 9 files changed, 1392 insertions(+), 4 deletions(-) create mode 100644 server/platform/postgres/common.go create mode 100644 server/platform/postgres/errors.go create mode 100644 server/platform/postgres/rebind_driver.go diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml index edf681fcb71..23e28dfbf80 100644 --- a/charts/fleet/templates/deployment.yaml +++ b/charts/fleet/templates/deployment.yaml @@ -188,7 +188,11 @@ spec: name: {{ .Values.fleet.license.secretName }} {{- end }} ## END FLEET SECTION - ## BEGIN MYSQL SECTION + ## BEGIN DATABASE SECTION + {{- if .Values.database.driver }} + - name: FLEET_MYSQL_DRIVER + value: "{{ .Values.database.driver }}" + {{- end }} - name: FLEET_MYSQL_ADDRESS value: "{{ .Values.database.address }}" - name: FLEET_MYSQL_DATABASE @@ -224,7 +228,7 @@ spec: - name: FLEET_MYSQL_TLS_SERVER_NAME value: "{{ .Values.database.tls.serverName }}" {{- end }} - ## END MYSQL SECTION + ## END DATABASE SECTION ## BEGIN MYSQL READ REPLICA SECTION {{- if .Values.database_read_replica }} {{- if .Values.database_read_replica.address }} diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 4a4b98c9bbb..ace4e676c71 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -221,9 +221,13 @@ osquery: resultTopic: "" ## Section: database -# All of the connection settings for MySQL +# All of the connection settings for the primary database. +# Supports MySQL (default) and PostgreSQL (experimental). database: - # Name of the Secret resource containing MySQL password and TLS secrets + # Driver selects the database backend: "mysql" (default) or "postgres". + # PostgreSQL support is experimental for self-hosted deployments. + driver: mysql + # Name of the Secret resource containing database password and TLS secrets secretName: mysql address: 127.0.0.1:3306 database: fleet @@ -323,5 +327,11 @@ environments: mysql: enabled: false +# PostgreSQL subchart (experimental). Enable when database.driver is "postgres" +# and you want Helm to provision a PG instance. For production, use a managed +# PostgreSQL service (e.g., AWS RDS, GCP Cloud SQL) and set database.address. +postgresql: + enabled: false + redis: enabled: false diff --git a/docker-compose.yml b/docker-compose.yml index d8e81da0886..ac377e1b605 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,38 @@ services: - /var/lib/mysql:rw,noexec,nosuid - /tmpfs + # PostgreSQL test instance for POSTGRES_TEST=1 integration tests. + # Usage: docker compose up -d postgres_test + # Then: POSTGRES_TEST=1 go test ./server/datastore/mysql/... + postgres_test: + image: postgres:16-alpine + environment: + POSTGRES_USER: fleet + POSTGRES_PASSWORD: insecure + POSTGRES_DB: fleet + ports: + - "${FLEET_POSTGRES_TEST_PORT:-5434}:5432" + tmpfs: + - /var/lib/postgresql/data:rw,noexec,nosuid + command: [ + "postgres", + "-c", "fsync=off", + "-c", "full_page_writes=off", + "-c", "synchronous_commit=off", + ] + + # PostgreSQL development instance. + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: fleet + POSTGRES_PASSWORD: insecure + POSTGRES_DB: fleet + ports: + - "5433:5432" + volumes: + - postgres-persistent-volume:/var/lib/postgresql/data + # Unauthenticated SMTP server. mailhog: image: mailhog/mailhog:latest @@ -170,4 +202,5 @@ services: volumes: mysql-persistent-volume: + postgres-persistent-volume: data-s3: diff --git a/go.mod b/go.mod index 67a0e281321..51b1784aaf6 100644 --- a/go.mod +++ b/go.mod @@ -286,6 +286,10 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.8.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect diff --git a/go.sum b/go.sum index f067f244c77..ec75393ed83 100644 --- a/go.sum +++ b/go.sum @@ -543,6 +543,14 @@ github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= +github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/server/config/config.go b/server/config/config.go index 4127a21bd4f..50c134f0ed4 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -39,6 +39,9 @@ const ( // MysqlConfig defines configs related to MySQL type MysqlConfig struct { + // Driver selects the database driver. Only "mysql" is valid in Phase 1. + // Future values: "postgres" (Phase 4+). + Driver string `yaml:"driver"` Protocol string `yaml:"protocol"` Address string `yaml:"address"` Username string `yaml:"username"` @@ -1128,6 +1131,8 @@ func (t *TLS) ToTLSConfig() (*tls.Config, error) { // filled into the FleetConfig struct func (man Manager) addConfigs() { addMysqlConfig := func(prefix, defaultAddr, usageSuffix string) { + man.addConfigString(prefix+".driver", "", + "Database driver: mysql (default) or postgres"+usageSuffix) man.addConfigString(prefix+".protocol", "tcp", "MySQL server communication protocol (tcp,unix,...)"+usageSuffix) man.addConfigString(prefix+".address", defaultAddr, @@ -1637,6 +1642,7 @@ func (man Manager) LoadConfig() FleetConfig { loadMysqlConfig := func(prefix string) MysqlConfig { return MysqlConfig{ + Driver: man.getConfigString(prefix + ".driver"), Protocol: man.getConfigString(prefix + ".protocol"), Address: man.getConfigString(prefix + ".address"), Username: man.getConfigString(prefix + ".username"), diff --git a/server/platform/postgres/common.go b/server/platform/postgres/common.go new file mode 100644 index 00000000000..137b2285c49 --- /dev/null +++ b/server/platform/postgres/common.go @@ -0,0 +1,31 @@ +package postgres + +import ( + "fmt" + + "github.com/jmoiron/sqlx" +) + +// NewDB opens a PostgreSQL database connection using the standard database/sql +// interface via the pgx stdlib driver. The dsn should be a PostgreSQL connection +// string (e.g., "postgres://user:pass@host:5432/dbname?sslmode=disable"). +// +// Callers should register the pgx stdlib driver before calling this function: +// +// import _ "github.com/jackc/pgx/v5/stdlib" +func NewDB(dsn string, maxOpenConns, maxIdleConns int) (*sqlx.DB, error) { + db, err := sqlx.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres connection: %w", err) + } + + db.SetMaxOpenConns(maxOpenConns) + db.SetMaxIdleConns(maxIdleConns) + + if err := db.Ping(); err != nil { + db.Close() + return nil, fmt.Errorf("ping postgres: %w", err) + } + + return db, nil +} diff --git a/server/platform/postgres/errors.go b/server/platform/postgres/errors.go new file mode 100644 index 00000000000..ab2b518ca70 --- /dev/null +++ b/server/platform/postgres/errors.go @@ -0,0 +1,110 @@ +// Package postgres provides PostgreSQL-specific utilities for Fleet's datastore layer. +package postgres + +import ( + "database/sql/driver" + "errors" + "io" + "net" + "os" + "strings" + "syscall" +) + +// PostgreSQL error codes (from SQLSTATE). +// See: https://www.postgresql.org/docs/current/errcodes-appendix.html +const ( + // Class 23 — Integrity Constraint Violation + codeUniqueViolation = "23505" + codeForeignKeyViolation = "23503" + + // Class 25 — Invalid Transaction State + codeReadOnlySQLTransaction = "25006" + + // Class 08 — Connection Exception + codeConnectionException = "08000" + codeConnectionFailure = "08006" + codeProtocolViolation = "08P01" + codeSQLClientUnableToEst = "08001" +) + +// IsDuplicate returns true if the error is a PostgreSQL unique_violation (23505). +func IsDuplicate(err error) bool { + return hasErrorCode(err, codeUniqueViolation) +} + +// IsForeignKey returns true if the error is a PostgreSQL foreign_key_violation (23503). +func IsForeignKey(err error) bool { + return hasErrorCode(err, codeForeignKeyViolation) +} + +// IsReadOnly returns true if the error indicates a read-only transaction (25006). +func IsReadOnly(err error) bool { + return hasErrorCode(err, codeReadOnlySQLTransaction) +} + +// IsBadConnection returns true if the error is a connection-level error +// that justifies retrying on a new connection. +func IsBadConnection(err error) bool { + if err == nil { + return false + } + + // Standard database/sql connection errors. + if errors.Is(err, driver.ErrBadConn) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, io.EOF) || + errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.ENETUNREACH) || + errors.Is(err, syscall.ETIMEDOUT) { + return true + } + + // PostgreSQL connection exception codes. + if hasErrorCode(err, codeConnectionException) || + hasErrorCode(err, codeConnectionFailure) || + hasErrorCode(err, codeProtocolViolation) || + hasErrorCode(err, codeSQLClientUnableToEst) { + return true + } + + // OS-level network errors. + var se *os.SyscallError + if errors.As(err, &se) { + return errors.Is(se.Err, syscall.ECONNRESET) || errors.Is(se.Err, syscall.EPIPE) + } + + var netErr *net.OpError + return errors.As(err, &netErr) +} + +// hasErrorCode checks if the error (or any wrapped error) contains the given +// PostgreSQL SQLSTATE code. This works with any error type that implements +// a Code() or SQLState() method, including pgx and lib/pq errors. +func hasErrorCode(err error, code string) bool { + if err == nil { + return false + } + + // Check for pgx-style error (implements Code() string). + type pgxError interface { + Code() string + } + var pgxErr pgxError + if errors.As(err, &pgxErr) { + return pgxErr.Code() == code + } + + // Check for lib/pq-style error (has Code field via the pq.Error type). + type pqError interface { + Get(byte) string + } + var pqErr pqError + if errors.As(err, &pqErr) { + return pqErr.Get('C') == code // 'C' = Code field + } + + // Fallback: check error string for the code (defensive). + return strings.Contains(err.Error(), code) +} diff --git a/server/platform/postgres/rebind_driver.go b/server/platform/postgres/rebind_driver.go new file mode 100644 index 00000000000..ffefed8aab3 --- /dev/null +++ b/server/platform/postgres/rebind_driver.go @@ -0,0 +1,1182 @@ +// Package postgres provides a MySQL-to-PostgreSQL SQL rebind driver for Fleet. +// It wraps pgx/v5 to automatically translate MySQL-dialect SQL to PostgreSQL, +// including placeholder conversion (? → $N), function rewrites (IF → CASE WHEN, +// JSON_OBJECT → jsonb_build_object, etc.), and type fixes (boolean = integer). +// Register with: sql.Register("pgx-rebind", &rebindDriver{}) +package postgres + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "regexp" + "strings" + + "github.com/jackc/pgx/v5/stdlib" +) + +// Pre-compiled regexes used in rebindQuery to avoid per-query compilation overhead. +var ( + reUUIDBinUpper = regexp.MustCompile(`UUID_TO_BIN\(UUID\(\),\s*true\)`) + reUUIDBinLower = regexp.MustCompile(`UUID_TO_BIN\(uuid\(\),\s*true\)`) + reUUIDBinTrue = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+),\s*true\)`) + reUUIDBin = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+)\)`) + reUUID = regexp.MustCompile(`(?i)\bUUID\(\)`) + reBinToUUIDTrue = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+),\s*true\)`) + reBinToUUID = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+)\)`) + reTimeDiff = regexp.MustCompile(`TIMEDIFF\(([^,]+),\s*([^)]+)\)`) + reTimeToSec = regexp.MustCompile(`TIME_TO_SEC\(([^)]+)\)`) + reFromDual = regexp.MustCompile(`(?i)\s+FROM\s+DUAL\b`) + reSeparator = regexp.MustCompile(`(?i)\bSEPARATOR\s+'([^']*)'`) + reTimestamp = regexp.MustCompile(`\bTIMESTAMP\(([^)]+)\)`) + reMaxDenylisted = regexp.MustCompile(`MAX\(([^)]*\.denylisted)\)`) + reLimitTrailing = regexp.MustCompile(`(?i)\s+LIMIT\s+\d+\s*$`) + reJSONExtractFunc = regexp.MustCompile(`JSON_EXTRACT\((\w+),\s*(\?|'[^']*')\)`) + reJSONPath = regexp.MustCompile(`->>?'\$\.`) + reTimestampDiff = regexp.MustCompile(`(?i)TIMESTAMPDIFF\(\s*SECOND\s*,\s*(.+?)\s*,\s*(.+?)\s*\)`) + + // Per-unit INTERVAL regexes (SECOND, MINUTE, HOUR, DAY) + reIntervalLiteral = map[string]*regexp.Regexp{} + reIntervalPlaceholder = map[string]*regexp.Regexp{} +) + +func init() { + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + reIntervalLiteral[unit] = regexp.MustCompile(`INTERVAL\s+(\d+)\s+` + unit) + reIntervalPlaceholder[unit] = regexp.MustCompile(`INTERVAL\s+(\?)\s+` + unit) + } +} + +func init() { + // Register "pgx-rebind" as a wrapper driver that auto-rewrites ? → $N. + // This allows MySQL-style ? placeholders to work transparently with PG. + sql.Register("pgx-rebind", &rebindDriver{}) +} + +type rebindDriver struct{} + +func (d *rebindDriver) Open(dsn string) (driver.Conn, error) { + connector, err := stdlib.GetDefaultDriver().(*stdlib.Driver).OpenConnector(dsn) + if err != nil { + return nil, err + } + conn, err := connector.Connect(context.Background()) + if err != nil { + return nil, err + } + return &rebindConn{Conn: conn}, nil +} + +func (d *rebindDriver) OpenConnector(dsn string) (driver.Connector, error) { + base, err := stdlib.GetDefaultDriver().(*stdlib.Driver).OpenConnector(dsn) + if err != nil { + return nil, err + } + return &rebindConnector{base: base}, nil +} + +type rebindConnector struct { + base driver.Connector +} + +func (c *rebindConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.base.Connect(ctx) + if err != nil { + return nil, err + } + return &rebindConn{Conn: conn}, nil +} + +func (c *rebindConnector) Driver() driver.Driver { + return &rebindDriver{} +} + +type rebindConn struct { + driver.Conn +} + +// rebindQuery converts MySQL-specific SQL to PostgreSQL. +// It handles: ? → $N placeholders, JSON_OBJECT → jsonb_build_object, +// DATE_ADD → PG interval arithmetic, INTERVAL N SECOND/MINUTE/etc. +func rebindQuery(query string) string { + // INSERT IGNORE INTO → INSERT INTO ... ON CONFLICT DO NOTHING + hasInsertIgnore := false + if strings.Contains(query, "INSERT IGNORE") { + query = strings.Replace(query, "INSERT IGNORE INTO", "INSERT INTO", 1) + query = strings.Replace(query, "INSERT IGNORE", "INSERT", 1) + hasInsertIgnore = true + } + + // Replace MySQL-specific functions with PG equivalents + // NOW(6) / CURRENT_TIMESTAMP(6) → NOW() / CURRENT_TIMESTAMP (PG already returns microsecond precision) + query = strings.ReplaceAll(query, "NOW(6)", "NOW()") + query = strings.ReplaceAll(query, "CURRENT_TIMESTAMP(6)", "CURRENT_TIMESTAMP") + // CURRENT_TIMESTAMP() → CURRENT_TIMESTAMP (PG doesn't use parens) + query = strings.ReplaceAll(query, "CURRENT_TIMESTAMP()", "CURRENT_TIMESTAMP") + // MD5() → md5() (PG uses lowercase) + query = strings.ReplaceAll(query, "MD5(", "md5(") + // JSON_EXTRACT(col, expr) → (col->regexp_replace(expr, '^\$\.?"?', '')) + // MySQL JSON_EXTRACT uses $.path syntax; PG -> operator uses plain key names. + // The regexp_replace strips the $. prefix and optional quotes at runtime. + if strings.Contains(query, "JSON_EXTRACT(") { + query = rewriteJSONExtractFunc(query) + } + // JSON_OBJECT → jsonb_build_object, then cast placeholder args to text + // (PG's jsonb_build_object has VARIADIC "any" so it can't infer $N types) + query = strings.ReplaceAll(query, "JSON_OBJECT(", "jsonb_build_object(") + query = castJsonbBuildObjectParams(query) + // UNHEX(expr) → decode(expr, 'hex') for checksum computation + query = rewriteUnhex(query) + // CHAR(0) → chr(0) + query = strings.ReplaceAll(query, "CHAR(0)", "chr(0)") + // CONCAT(a, b, ...) → (a || b || ...) — PG's CONCAT can't always infer parameter types + query = rewriteConcat(query) + // ISNULL(expr) → (expr IS NULL) — MySQL's ISNULL returns 1/0; PG doesn't have it. + query = rewriteISNULL(query) + // IFNULL(a, b) → COALESCE(a, b) — MySQL's IFNULL is PG's COALESCE + query = strings.ReplaceAll(query, "IFNULL(", "COALESCE(") + // UUID_TO_BIN(UUID(), true) → gen_random_uuid() (must come before UUID() replacement) + query = reUUIDBinUpper.ReplaceAllString(query, "gen_random_uuid()") + query = reUUIDBinLower.ReplaceAllString(query, "gen_random_uuid()") + query = reUUIDBinTrue.ReplaceAllString(query, "($1)::uuid") + query = reUUIDBin.ReplaceAllString(query, "($1)::uuid") + // CONVERT(uuid() USING utf8mb4) → gen_random_uuid()::text (MySQL charset conversion) + query = strings.ReplaceAll(query, "CONVERT(uuid() USING utf8mb4)", "gen_random_uuid()::text") + query = strings.ReplaceAll(query, "CONVERT(UUID() USING utf8mb4)", "gen_random_uuid()::text") + // Standalone UUID() → gen_random_uuid()::text (use word boundary to avoid matching gen_random_uuid) + query = reUUID.ReplaceAllStringFunc(query, func(m string) string { + return "gen_random_uuid()::text" + }) + // BIN_TO_UUID(expr, true) → encode(expr, 'hex') reformatted as UUID text + // Simpler: BIN_TO_UUID(col, true) → col::text for uuid columns + query = reBinToUUIDTrue.ReplaceAllString(query, "($1)::text") + query = reBinToUUID.ReplaceAllString(query, "($1)::text") + // HEX(expr) → encode(expr::bytea, 'hex') — MySQL HEX function + query = rewriteHex(query) + // JSON_SET(col, path, val) → jsonb_set(col, path_array, val) + query = rewriteJSONSet(query) + // TIMEDIFF(a, b) → (a - b) + query = reTimeDiff.ReplaceAllString(query, "($1 - $2)") + // TIME_TO_SEC(interval) → EXTRACT(EPOCH FROM interval) + query = reTimeToSec.ReplaceAllString(query, "EXTRACT(EPOCH FROM $1)") + // ON DUPLICATE KEY UPDATE → rewrite to ON CONFLICT DO UPDATE SET for raw SQL + // that doesn't go through dialect helpers. + if strings.Contains(query, "ON DUPLICATE KEY UPDATE") { + query = rewriteOnDuplicateKey(query) + } + // FROM DUAL → removed (PG doesn't need FROM DUAL for SELECT without a table) + query = reFromDual.ReplaceAllString(query, "") + // STRAIGHT_JOIN → JOIN (MySQL optimizer hint, not supported by PG) + query = strings.ReplaceAll(query, "STRAIGHT_JOIN", "JOIN") + // MySQL SET FOREIGN_KEY_CHECKS / innodb / sql_mode commands → no-op for PG + if strings.Contains(query, "FOREIGN_KEY_CHECKS") || strings.Contains(query, "innodb") || strings.Contains(query, "INNODB") || strings.Contains(query, "sql_mode") { + query = strings.ReplaceAll(query, "SET FOREIGN_KEY_CHECKS=0", "SELECT 1") + query = strings.ReplaceAll(query, "SET FOREIGN_KEY_CHECKS=1", "SELECT 1") + if strings.Contains(query, "innodb") || strings.Contains(query, "INNODB") || strings.Contains(query, "sql_mode") { + return "SELECT 1" // skip MySQL-specific queries entirely + } + } + // MySQL RAND() → PG random() + query = strings.ReplaceAll(query, "RAND()", "random()") + query = strings.ReplaceAll(query, "rand()", "random()") + // GROUP_CONCAT → STRING_AGG for simple cases not going through dialect + if strings.Contains(query, "GROUP_CONCAT") || strings.Contains(query, "group_concat") { + query = rewriteGroupConcat(query) + } + // FOR UPDATE with LEFT JOIN: PG doesn't allow FOR UPDATE on nullable side of outer join. + // Remove FOR UPDATE when LEFT JOIN is present — the SELECT FOR UPDATE semantic is advisory + // and removing it doesn't break correctness, only reduces locking. + if strings.Contains(query, "FOR UPDATE") && (strings.Contains(query, "LEFT JOIN") || strings.Contains(query, "LEFT OUTER JOIN")) { + query = strings.Replace(query, "\nFOR UPDATE", "", 1) + query = strings.Replace(query, "\n\t\tFOR UPDATE", "", 1) + query = strings.Replace(query, "FOR UPDATE", "", 1) + } + // MySQL SEPARATOR in GROUP_CONCAT → already handled by dialect, but catch raw usage + if strings.Contains(query, "separator") || strings.Contains(query, "SEPARATOR") { + query = reSeparator.ReplaceAllString(query, "") + } + // MySQL JSON path operators: col->'$.key' → col->'key', col->>'$.key' → col->>'key' + query = rewriteJSONPath(query) + // MySQL backtick-quoted identifiers → PG double-quoted identifiers + query = strings.ReplaceAll(query, "`", `"`) + // MySQL DELETE FROM t USING t INNER JOIN → PG DELETE FROM t USING (remove duplicate table) + // MySQL requires naming the target table again in USING; PG forbids it. + query = rewriteDeleteUsing(query) + // MySQL UPDATE t1 JOIN t2 ON ... SET ... → PG UPDATE t1 SET ... FROM t2 WHERE ... + if strings.Contains(query, "UPDATE") && strings.Contains(query, "JOIN") && strings.Contains(query, "SET") { + query = rewriteUpdateJoin(query) + } + // Note: PG doesn't allow alias-qualified columns in UPDATE SET clause. + // This needs per-query fixes in the source code (e.g., cron_stats.go). + // MySQL IF(cond, true_val, false_val) → PG CASE WHEN cond THEN true_val ELSE false_val END + query = rewriteIF(query) + // MySQL FIELD(x, 'a', 'b', ...) → PG CASE x WHEN 'a' THEN 1 WHEN 'b' THEN 2 ... ELSE 0 END + query = rewriteField(query) + // TIMESTAMPDIFF(SECOND, x, y) → EXTRACT(EPOCH FROM (y - x)) + // MySQL's TIMESTAMPDIFF returns the difference in the specified unit. + query = rewriteTimestampDiff(query) + // TIMESTAMP(x) → x::timestamp (PG cast syntax) + // MySQL TIMESTAMP(?) converts a value to timestamp type + query = reTimestamp.ReplaceAllString(query, "($1)::timestamp") + // CAST(... AS UNSIGNED) → CAST(... AS integer) (MySQL unsigned → PG integer) + query = strings.ReplaceAll(query, "AS UNSIGNED)", "AS integer)") + // CAST(... AS SIGNED INT) / CAST(... AS SIGNED) → CAST(... AS integer) + query = strings.ReplaceAll(query, "AS SIGNED INT)", "AS integer)") + query = strings.ReplaceAll(query, "AS SIGNED)", "AS integer)") + // CAST(TRUE/FALSE AS JSON) → TRUE/FALSE (PG jsonb_build_object accepts boolean directly) + query = strings.ReplaceAll(query, "CAST(TRUE AS JSON)", "TRUE") + query = strings.ReplaceAll(query, "CAST(FALSE AS JSON)", "FALSE") + // MAX(boolean_col) → BOOL_OR(boolean_col) for PG + query = reMaxDenylisted.ReplaceAllString(query, "BOOL_OR($1)") + // Fix CASE type mismatch: ELSE hdek.decryptable (boolean) mixed with THEN -1 (integer) + // Cast boolean to integer in CASE branches + query = strings.ReplaceAll(query, "ELSE hdek.decryptable", "ELSE CAST(hdek.decryptable AS integer)") + // Fix CAST(AVG(...) AS UNSIGNED) → CAST(AVG(...) AS integer) (already handled above) + // Fix boolean = integer comparisons that PG doesn't allow + for _, col := range []string{ + "ne.enabled", "hsr.canceled", "pl.exclude", "needs_full_membership_cleanup", "si.is_active", + "hsi2.removed", "hsi2.canceled", "hsi.removed", "hsi.canceled", + "abt.terms_expired", "n.token_update_tally", "ne.token_update_tally", + "n.enrolled", "q.active", "cve_meta.published", + "hrkp.deleted", "rkp.deleted", + // nano/mdm boolean columns + "hm.enrolled", "hmdm.enrolled", "nq.active", "nvq.active", + "nano_enrollment_queue.active", "ne.enrolled_from_migration", + "ba.canceled", "ba2.canceled", + // MDM profile label exclude/require_all columns (various aliases) + "mcpl.exclude", "mel.exclude", "sil.exclude", "sil.require_all", + "vatl.exclude", "vatl.require_all", "ihl.exclude", "ihl.require_all", + // Additional qualified boolean columns + "neq.active", "e.enabled", "p.conditional_access_enabled", "p.critical", + "hvsi.canceled", "hvsi2.canceled", "hvsi.removed", "hvsi2.removed", + "hihsi.canceled", "hihsi.removed", "hihsi2.canceled", "hihsi2.removed", + "host_vpp_software_installs.canceled", "host_vpp_software_installs.removed", + "host_mdm.enrolled", + "q.automations_enabled", "nq.automations_enabled", + "hmdm.is_server", "hm.installed_from_dep", "q.discard_data", + "hmabp.skipped", "hm.is_personal_enrollment", + // Unqualified boolean columns (safe — always boolean in Fleet schema) + "deleted", "canceled", "refetch_requested", "expired", + "enrolled_from_migration", "enrolled", "enabled", "active", + "resync", "terms_expired", "sync_request", + "discard_data", "is_server", "is_kernel", "encrypted", + "skipped", "installed_from_dep", "is_personal_enrollment", + "saved", "q.saved", + } { + query = strings.ReplaceAll(query, col+" = 1", col+" = true") + query = strings.ReplaceAll(query, col+" = 0", col+" = false") + query = strings.ReplaceAll(query, col+"=1", col+"=true") + query = strings.ReplaceAll(query, col+"=0", col+"=false") + } + // Fix pm.passes = 1/0: PG column is boolean, can't compare to integer. + // Cast to int for use in SUM/COUNT aggregates. + query = strings.ReplaceAll(query, "pm.passes = 1", "(pm.passes IS TRUE)::int") + query = strings.ReplaceAll(query, "pm.passes = 0", "(pm.passes = false)::int") + // MySQL !boolean → PG NOT boolean (for use in SUM aggregates) + query = strings.ReplaceAll(query, "!pm.passes", "(NOT pm.passes)::int") + // Fix FIND_IN_SET/ANY result compared to integer: PG = ANY() returns boolean + // MySQL FIND_IN_SET returns integer, so code uses <> 0 / != 0 checks + // PG = ANY() returns boolean, making these comparisons invalid + if strings.Contains(query, "string_to_array") { + query = strings.ReplaceAll(query, ")) <> 0", "))") + query = strings.ReplaceAll(query, ")) != 0", "))") + // FindInSet(...) = 0 → NOT FindInSet(...) (PG ANY() returns boolean) + // Pattern: "',')) = 0" at end of FindInSet expression + query = strings.ReplaceAll(query, "',')) = 0", "',')) IS NOT TRUE") + query = strings.ReplaceAll(query, "')) <> 0", "'))") + query = strings.ReplaceAll(query, "')) != 0", "'))") + } + + // Replace MySQL DATE_ADD/DATE_SUB(x, INTERVAL expr UNIT) → PG interval arithmetic + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + if strings.Contains(query, "DATE_ADD(") { + query = rewriteDateAddSub(query, unit, "+") + } + if strings.Contains(query, "DATE_SUB(") { + query = rewriteDateAddSub(query, unit, "-") + } + } + + // Replace INTERVAL N SECOND (without DATE_ADD) → INTERVAL 'N seconds' + // e.g., "INTERVAL 5 MINUTE" → "INTERVAL '5 minutes'" + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + query = reIntervalLiteral[unit].ReplaceAllString(query, "INTERVAL '${1} "+strings.ToLower(unit)+"s'") + query = reIntervalPlaceholder[unit].ReplaceAllString(query, "? * INTERVAL '1 "+strings.ToLower(unit)+"'") + } + // MySQL allows LIMIT on UPDATE/DELETE; PG does not. + uq := strings.ToUpper(strings.TrimLeft(query, " \t\n")) + if strings.HasPrefix(uq, "UPDATE") || strings.HasPrefix(uq, "DELETE") { + query = reLimitTrailing.ReplaceAllString(query, "") + } + + // Resolve ambiguous column references in ON CONFLICT DO UPDATE SET clauses. + // Only apply when complex expressions (CASE WHEN, COALESCE) are in the SET clause. + if idx := strings.Index(query, "DO UPDATE SET"); idx >= 0 { + setClause := query[idx:] + if strings.Contains(setClause, "CASE WHEN") || strings.Contains(setClause, "COALESCE") { + if strings.Contains(query, "EXCLUDED.") { + query = resolveOnConflictAmbiguity(query) + } + } + } + + if !strings.Contains(query, "?") { + if hasInsertIgnore { + query = strings.TrimRight(query, " \t\n\r;") + " ON CONFLICT DO NOTHING" + } + return query + } + var b strings.Builder + b.Grow(len(query) + 10) + n := 1 + for _, r := range query { + if r == '?' { + b.WriteByte('$') + b.WriteString(strings.Repeat("", 0)) // force allocation + // Write the number + if n < 10 { + b.WriteByte(byte('0' + n)) + } else { + b.WriteString(fmt.Sprintf("%d", n)) + } + n++ + } else { + b.WriteRune(r) + } + } + result := b.String() + if hasInsertIgnore { + result = strings.TrimRight(result, " \t\n\r;") + " ON CONFLICT DO NOTHING" + } + return result +} + +func (c *rebindConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if ec, ok := c.Conn.(driver.ExecerContext); ok { + return ec.ExecContext(ctx, rebindQuery(query), args) + } + return nil, driver.ErrSkip +} + +func (c *rebindConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if qc, ok := c.Conn.(driver.QueryerContext); ok { + return qc.QueryContext(ctx, rebindQuery(query), args) + } + return nil, driver.ErrSkip +} + +func (c *rebindConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if pc, ok := c.Conn.(driver.ConnPrepareContext); ok { + return pc.PrepareContext(ctx, rebindQuery(query)) + } + return c.Conn.Prepare(rebindQuery(query)) +} + +func (c *rebindConn) Prepare(query string) (driver.Stmt, error) { + return c.Conn.Prepare(rebindQuery(query)) +} + +// rewriteDateAddSub converts MySQL DATE_ADD/DATE_SUB(expr, INTERVAL value UNIT) to PG interval arithmetic. +// op is "+" for DATE_ADD and "-" for DATE_SUB. +func rewriteDateAddSub(query string, unit string, op string) string { + pgUnit := strings.ToLower(unit) + "s" + var prefix string + if op == "+" { + prefix = "DATE_ADD(" + } else { + prefix = "DATE_SUB(" + } + for { + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Find the matching closing paren and split on the top-level comma + start := idx + len(prefix) + depth := 1 + commaPos := -1 + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 1 && commaPos < 0 { + commaPos = i + } + } + i++ + } + if depth != 0 || commaPos < 0 { + return query // unbalanced or no comma found + } + expr := strings.TrimSpace(query[start:commaPos]) + intervalPart := strings.TrimSpace(query[commaPos+1 : i-1]) + + // Parse: INTERVAL + intervalRe := regexp.MustCompile(`(?i)INTERVAL\s+(.+)\s+` + unit) + m := intervalRe.FindStringSubmatch(intervalPart) + if m == nil { + // This DATE_ADD/SUB doesn't use this unit, skip past it + return query[:i] + rewriteDateAddSub(query[i:], unit, op) + } + value := strings.TrimSpace(m[1]) + replacement := "(" + expr + " " + op + " (" + value + ") * INTERVAL '1 " + pgUnit + "')" + query = query[:idx] + replacement + query[i:] + } +} + +// rewriteUnhex converts MySQL UNHEX(expr) → PG decode(expr, 'hex'). +// Uses paren-balancing to handle nested function calls inside UNHEX(). +func rewriteUnhex(query string) string { + const prefix = "UNHEX(" + for { + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Find the matching closing paren + depth := 1 + start := idx + len(prefix) + i := start + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + return query // unbalanced, leave as-is + } + inner := query[start : i-1] + query = query[:idx] + "decode(" + inner + ", 'hex')" + query[i:] + } +} + +// rewriteDeleteUsing fixes MySQL's DELETE FROM t USING t INNER JOIN ... +// pattern for PostgreSQL. MySQL requires repeating the target table in USING; +// PG forbids it. +// +// MySQL: DELETE FROM t USING t INNER JOIN j alias ON WHERE +// PG: DELETE FROM t USING j alias WHERE AND +func rewriteDeleteUsing(query string) string { + // Extract the target table from DELETE FROM + delRe := regexp.MustCompile(`(?is)DELETE\s+FROM\s+(\w+)\s+USING\s+`) + m := delRe.FindStringSubmatch(query) + if m == nil { + return query + } + tableName := m[1] + + // Check if the USING clause repeats the same table name followed by INNER JOIN + // Build a pattern: USING INNER JOIN (case-insensitive) + usingDupRe := regexp.MustCompile(`(?is)USING\s+` + regexp.QuoteMeta(tableName) + `\s+INNER\s+JOIN\s+`) + if !usingDupRe.MatchString(query) { + return query + } + + // Step 1: Remove duplicate table and INNER JOIN keyword + query = usingDupRe.ReplaceAllString(query, "USING ") + + // Step 2: Convert "ON WHERE" → "WHERE AND" + // The ON clause from the removed INNER JOIN must merge into WHERE. + reOnWhere := regexp.MustCompile(`(?is)(USING\s+\w+\s+\w+\s+)ON\s+(.*?)\s+WHERE\s+`) + query = reOnWhere.ReplaceAllString(query, "${1}WHERE ${2} AND ") + + return query +} + +// rewriteTimestampDiff converts MySQL TIMESTAMPDIFF(SECOND, x, y) → PG EXTRACT(EPOCH FROM (y - x)). +func rewriteTimestampDiff(query string) string { + if !reTimestampDiff.MatchString(query) { + return query + } + // Use paren-balanced parsing for complex arguments + prefix := "TIMESTAMPDIFF(" + for { + idx := strings.Index(strings.ToUpper(query), strings.ToUpper(prefix)) + if idx < 0 { + return query + } + start := idx + len(prefix) + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) != 3 { + return query + } + // parts[0] = unit (SECOND), parts[1] = start_time, parts[2] = end_time + replacement := fmt.Sprintf("EXTRACT(EPOCH FROM (%s - %s))", parts[2], parts[1]) + query = query[:idx] + replacement + query[i:] + } +} + +// rewriteIF converts MySQL IF(cond, true_val, false_val) → PG CASE WHEN cond THEN true_val ELSE false_val END. +// Uses paren-balancing and comma-splitting to handle nested expressions. +func rewriteIF(query string) string { + for { + // Find IF( preceded by a non-alphanumeric char (or start of string) + // to avoid matching e.g. NOTIFY(...) + idx := -1 + for i := 0; i < len(query)-3; i++ { + if (query[i] == 'I' || query[i] == 'i') && + (query[i+1] == 'F' || query[i+1] == 'f') && + query[i+2] == '(' { + // Check that the preceding char is not alphanumeric/underscore + if i == 0 || !isIdentChar(query[i-1]) { + idx = i + break + } + } + } + if idx < 0 { + return query + } + + // Find the matching closing paren, splitting on top-level commas + start := idx + 3 // after "IF(" + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) != 3 { + return query // unbalanced or not exactly 3 args, leave as-is + } + replacement := fmt.Sprintf("CASE WHEN %s THEN %s ELSE %s END", parts[0], parts[1], parts[2]) + query = query[:idx] + replacement + query[i:] + } +} + +func isIdentChar(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' +} + +// castJsonbBuildObjectParams adds ::text casts to ? placeholders inside jsonb_build_object() calls. +// PG's jsonb_build_object has a VARIADIC "any" signature, so it can't infer placeholder parameter types. +// Casting to ::text makes all JSON values strings, which is compatible with ->>' text extraction. +// Handles nested jsonb_build_object and subqueries via paren-balancing. +func castJsonbBuildObjectParams(query string) string { + const prefix = "jsonb_build_object(" + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + start := idx + len(prefix) + depth := 1 + i := start + // Walk through the jsonb_build_object args, adding ::text to ? placeholders + // that are in value positions (odd arg index: 1, 3, 5, ...) + var result strings.Builder + result.WriteString(query[:start]) + argIdx := 0 + argStart := i + + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + i++ + case ')': + depth-- + if depth == 0 { + // Process the last argument + arg := query[argStart:i] + if argIdx%2 == 1 { + arg = castPlaceholdersInArg(arg) + } + result.WriteString(arg) + result.WriteByte(')') + } + i++ + case ',': + if depth == 1 { + arg := query[argStart:i] + if argIdx%2 == 1 { + arg = castPlaceholdersInArg(arg) + } + result.WriteString(arg) + result.WriteByte(',') + argIdx++ + argStart = i + 1 + i++ + } else { + i++ + } + default: + i++ + } + } + if depth != 0 { + return query // unbalanced, leave as-is + } + // Recursively process the rest of the query + result.WriteString(castJsonbBuildObjectParams(query[i:])) + return result.String() +} + +// castPlaceholdersInArg adds ::text to bare ? placeholders in a jsonb_build_object value argument. +// Skips ? that are inside subqueries (nested parens), CAST expressions, or already have ::text. +func castPlaceholdersInArg(arg string) string { + trimmed := strings.TrimSpace(arg) + // If the arg is a simple ?, cast it + if trimmed == "?" { + return strings.Replace(arg, "?", "?::text", 1) + } + // If the arg is CAST(? AS ...), leave it alone (already typed) + if strings.Contains(strings.ToUpper(trimmed), "CAST(") { + return arg + } + // If the arg contains a subquery (SELECT ...), leave it alone (nested query handles its own types) + if strings.Contains(strings.ToUpper(trimmed), "SELECT ") { + return arg + } + // For other simple expressions with ?, cast them + if trimmed == "?" { + return strings.Replace(arg, "?", "?::text", 1) + } + return arg +} + +// rewriteJSONExtractFunc converts MySQL JSON_EXTRACT(col, path) → PG (col->path_key). +// For parameterized paths (JSON_EXTRACT(col, ?)), wraps with regexp_replace to strip +// the MySQL $. prefix and optional quotes at runtime. +func rewriteJSONExtractFunc(query string) string { + // Match JSON_EXTRACT(identifier, ?) or JSON_EXTRACT(identifier, 'literal') + return reJSONExtractFunc.ReplaceAllStringFunc(query, func(match string) string { + m := reJSONExtractFunc.FindStringSubmatch(match) + if m == nil { + return match + } + col, pathExpr := m[1], m[2] + if pathExpr == "?" { + // Parameterized path: strip $. prefix and quotes at runtime + return fmt.Sprintf("(%s->regexp_replace(?::text, '^\\$\\.\"?([^\"]*)\"?$', '\\1'))", col) + } + // Literal path: strip $. prefix inline + path := strings.TrimPrefix(pathExpr, "'$.") + path = strings.TrimSuffix(path, "'") + path = strings.Trim(path, `"`) + return fmt.Sprintf("(%s->'%s')", col, path) + }) +} + +// rewriteJSONPath converts MySQL JSON path operator syntax to PG. +// MySQL: col->'$.key' → PG: col->'key' +// MySQL: col->>'$.key' → PG: col->>'key' +// This handles the $. prefix that MySQL uses for JSON paths. +func rewriteJSONPath(query string) string { + // Match ->'$.key' and ->>'$.key' patterns, strip the $. + // ->> must be checked first (longer match) + query = reJSONPath.ReplaceAllStringFunc(query, func(match string) string { + return strings.Replace(match, "$.", "", 1) + }) + return query +} + +// rewriteConcat converts MySQL CONCAT(a, b, ...) → (a::text || b::text || ...). +// PG's CONCAT() function can't always infer parameter types for placeholders. +// Uses paren-balancing to handle nested expressions. +func rewriteConcat(query string) string { + for { + idx := strings.Index(query, "CONCAT(") + if idx < 0 { + return query + } + // Make sure CONCAT is not part of a larger identifier (e.g. GROUP_CONCAT) + if idx > 0 && isIdentChar(query[idx-1]) { + // Skip past this occurrence + rest := query[idx+7:] + before := query[:idx+7] + rewritten := rewriteConcat(rest) + return before + rewritten + } + start := idx + 7 // after "CONCAT(" + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) < 1 { + return query + } + // Build (part1::text || part2::text || ...) + var b strings.Builder + b.WriteByte('(') + for j, part := range parts { + if j > 0 { + b.WriteString(" || ") + } + b.WriteString(part) + b.WriteString("::text") + } + b.WriteByte(')') + query = query[:idx] + b.String() + query[i:] + } +} + +// rewriteISNULL converts MySQL ISNULL(expr) → (expr IS NULL). +// Uses paren-balancing to handle nested expressions. +func rewriteISNULL(query string) string { + for { + idx := strings.Index(query, "ISNULL(") + if idx < 0 { + return query + } + // Make sure ISNULL is not part of a larger identifier + if idx > 0 && isIdentChar(query[idx-1]) { + // Skip past this occurrence and continue searching + rest := rewriteISNULL(query[idx+7:]) + return query[:idx+7] + rest + } + start := idx + 7 // after "ISNULL(" + depth := 1 + i := start + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + return query // unbalanced + } + inner := query[start : i-1] + query = query[:idx] + "(" + inner + " IS NULL)" + query[i:] + } +} + +// rewriteField converts MySQL FIELD(x, 'a', 'b', ...) → PG CASE x WHEN 'a' THEN 1 WHEN 'b' THEN 2 ... ELSE 0 END. +func rewriteField(query string) string { + prefix := "FIELD(" + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Ensure FIELD( is not part of a larger identifier + if idx > 0 && isIdentChar(query[idx-1]) { + return query + } + start := idx + len(prefix) + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) < 2 { + return query + } + var b strings.Builder + b.WriteString("CASE ") + b.WriteString(parts[0]) + for j := 1; j < len(parts); j++ { + fmt.Fprintf(&b, " WHEN %s THEN %d", parts[j], j) + } + b.WriteString(" ELSE 0 END") + return query[:idx] + b.String() + query[i:] +} + +// resolveOnConflictAmbiguity fixes ambiguous column references in ON CONFLICT DO UPDATE SET. +// In PG, bare column names in SET value expressions are ambiguous between the target table +// and EXCLUDED. This function parses each SET assignment and qualifies bare column references +// in the VALUE expressions (right side of =) with the target table name. +func resolveOnConflictAmbiguity(query string) string { + // Extract target table name from INSERT INTO
+ insertRe := regexp.MustCompile(`(?i)INSERT\s+INTO\s+"?(\w+)"?`) + m := insertRe.FindStringSubmatch(query) + if m == nil { + return query + } + tableName := m[1] + + // Find the ON CONFLICT DO UPDATE SET portion + upperQuery := strings.ToUpper(query) + setMarker := "DO UPDATE SET" + setIdx := strings.Index(upperQuery, setMarker) + if setIdx == -1 { + return query + } + setStart := setIdx + len(setMarker) + setClause := query[setStart:] + + // Collect column names from EXCLUDED references — these are the ambiguous ones + excludedRe := regexp.MustCompile(`EXCLUDED\.(\w+)`) + matches := excludedRe.FindAllStringSubmatch(setClause, -1) + if len(matches) == 0 { + return query + } + cols := make(map[string]bool) + for _, m := range matches { + cols[m[1]] = true + } + // Also add SET target names + setTargetRe := regexp.MustCompile(`(?:^|,)\s*(\w+)\s*=`) + for _, m := range setTargetRe.FindAllStringSubmatch(setClause, -1) { + cols[m[1]] = true + } + + // Split the SET clause into individual assignments by top-level commas. + // Then for each assignment, split on the first '=' to get target and value. + // Only qualify bare column refs in the value part. + assignments := splitTopLevel(setClause, ',') + var result strings.Builder + for i, assignment := range assignments { + if i > 0 { + result.WriteByte(',') + } + eqIdx := strings.Index(assignment, "=") + if eqIdx == -1 { + result.WriteString(assignment) + continue + } + target := assignment[:eqIdx+1] // includes the '=' + value := assignment[eqIdx+1:] + + // Qualify bare column names in the value part using manual scanning + // to avoid the ReplaceAllStringFunc closure bug with mutable value. + value = qualifyBareColumns(value, cols, tableName) + + result.WriteString(target) + result.WriteString(value) + } + + return query[:setStart] + result.String() +} + +// qualifyBareColumns scans a string and qualifies bare column references with tableName. +// A "bare" reference is a word matching a column name NOT preceded by '.'. +func qualifyBareColumns(s string, cols map[string]bool, tableName string) string { + var result strings.Builder + result.Grow(len(s) * 2) + i := 0 + for i < len(s) { + // Skip non-word characters + if !isWordChar(s[i]) { + result.WriteByte(s[i]) + i++ + continue + } + // Extract the full word + start := i + for i < len(s) && isWordChar(s[i]) { + i++ + } + word := s[start:i] + + // Check if this word is a column name we need to qualify + if cols[word] { + // Check if preceded by '.' (already qualified) + if start > 0 && s[start-1] == '.' { + result.WriteString(word) + } else { + result.WriteString(tableName + "." + word) + } + } else { + result.WriteString(word) + } + } + return result.String() +} + +func isWordChar(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' +} + +// rewriteHex rewrites MySQL HEX(expr) → PG encode(expr, 'hex') +func rewriteHex(query string) string { + // Only match standalone HEX( not UNHEX( + re := regexp.MustCompile(`(?i)\bHEX\(`) + for { + loc := re.FindStringIndex(query) + if loc == nil { + break + } + // Make sure it's not UNHEX + if loc[0] > 0 && (query[loc[0]-1] == 'N' || query[loc[0]-1] == 'n') { + // Skip this match — it's part of UNHEX + query = query[:loc[0]] + "HEX__SKIP(" + query[loc[1]:] + continue + } + // Find matching close paren + depth := 1 + i := loc[1] + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := query[loc[1] : i-1] + replacement := "encode(" + inner + "::bytea, 'hex')" + query = query[:loc[0]] + replacement + query[i:] + } + query = strings.ReplaceAll(query, "HEX__SKIP(", "HEX(") + return query +} + +// rewriteJSONSet rewrites MySQL JSON_SET(col, '$.path', val) → PG jsonb_set(col, '{path}', to_jsonb(val)) +func rewriteJSONSet(query string) string { + for { + idx := strings.Index(query, "JSON_SET(") + if idx == -1 { + break + } + // Find matching close paren + depth := 1 + i := idx + 9 // len("JSON_SET(") + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := query[idx+9 : i-1] + // Parse: col, '$.path', val + parts := splitTopLevel(inner, ',') + if len(parts) < 3 { + break + } + col := strings.TrimSpace(parts[0]) + path := strings.TrimSpace(parts[1]) + val := strings.TrimSpace(parts[2]) + // Convert '$.mdm.foo.bar' → '{mdm,foo,bar}' + path = strings.Trim(path, "'\"") + path = strings.TrimPrefix(path, "$.") + pgPath := "'{" + strings.ReplaceAll(path, ".", ",") + "}'" + replacement := "jsonb_set(" + col + ", " + pgPath + ", to_jsonb(" + val + "))" + query = query[:idx] + replacement + query[i:] + } + return query +} + +// splitTopLevel splits a string by delimiter, respecting parentheses and quotes. +func splitTopLevel(s string, delim byte) []string { + var parts []string + depth := 0 + inSingleQuote := false + start := 0 + for i := 0; i < len(s); i++ { + switch { + case s[i] == '\'' && !inSingleQuote: + inSingleQuote = true + case s[i] == '\'' && inSingleQuote: + inSingleQuote = false + case inSingleQuote: + continue + case s[i] == '(': + depth++ + case s[i] == ')': + depth-- + case s[i] == delim && depth == 0: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + parts = append(parts, s[start:]) + return parts +} + +// rewriteOnDuplicateKey rewrites MySQL ON DUPLICATE KEY UPDATE → PG ON CONFLICT DO UPDATE SET +// This handles cases not going through the dialect helper. +// knownPrimaryKeys maps table names to their primary key columns for ON CONFLICT resolution. +var knownPrimaryKeys = map[string]string{ + "host_dep_assignments": "host_id", + "host_mdm_idp_accounts": "host_uuid", + "host_mdm_apple_declarations": "host_uuid,declaration_uuid", + "mdm_declaration_labels": "apple_declaration_uuid,label_name", + "scim_user_group": "scim_user_id,group_id", + "host_munki_issues": "host_id,munki_issue_id", + "host_munki_info": "host_id", + "cron_stats": "id", + "nano_command_results": "id,command_uuid", + "host_mdm_apple_bootstrap_packages": "host_uuid", + "mdm_configuration_profile_labels": "id", + "host_conditional_access": "host_id", + "host_mdm": "host_id", + "host_display_names": "host_id", + "host_emails": "id", + "label_membership": "host_id,label_id", + "host_software": "host_id,software_id", + "software_host_counts": "software_id,team_id", + "nano_enrollment_queue": "id,command_uuid", + "host_mdm_windows_profiles": "host_uuid,profile_uuid", +} + +func rewriteOnDuplicateKey(query string) string { + upperQuery := strings.ToUpper(query) + const marker = "ON DUPLICATE KEY UPDATE" + idx := strings.Index(upperQuery, marker) + if idx == -1 { + return query + } + updateClause := strings.TrimSpace(query[idx+len(marker):]) + // Rewrite VALUES(col) → EXCLUDED.col + re := regexp.MustCompile(`(?i)VALUES\(` + "`?" + `(\w+)` + "`?" + `\)`) + updateClause = re.ReplaceAllString(updateClause, "EXCLUDED.$1") + + // Extract table name from INSERT INTO
+ tableRe := regexp.MustCompile(`(?i)INSERT\s+INTO\s+` + "`?" + `(\w+)` + "`?") + m := tableRe.FindStringSubmatch(query) + conflictTarget := "" + if m != nil { + tableName := strings.ToLower(m[1]) + if pk, ok := knownPrimaryKeys[tableName]; ok { + conflictTarget = pk + } + } + + if conflictTarget != "" { + query = query[:idx] + "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + updateClause + } else { + // Fallback: no conflict target — PG will error but at least the syntax is close + query = query[:idx] + "ON CONFLICT DO UPDATE SET " + updateClause + } + return query +} + +// rewriteGroupConcat rewrites MySQL GROUP_CONCAT(expr) → PG STRING_AGG(expr::text, ',') +// Also handles GROUP_CONCAT(expr SEPARATOR 'sep') → STRING_AGG(expr::text, 'sep') +// And GROUP_CONCAT(DISTINCT expr) → STRING_AGG(DISTINCT expr::text, ',') +func rewriteGroupConcat(query string) string { + re := regexp.MustCompile(`(?i)GROUP_CONCAT\(`) + for { + loc := re.FindStringIndex(query) + if loc == nil { + break + } + // Find matching close paren + depth := 1 + i := loc[1] + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := strings.TrimSpace(query[loc[1] : i-1]) + sep := "," + // Check for SEPARATOR clause + sepRe := regexp.MustCompile(`(?i)\s+SEPARATOR\s+'([^']*)'`) + if m := sepRe.FindStringSubmatchIndex(inner); m != nil { + sep = inner[m[2]:m[3]] + inner = strings.TrimSpace(inner[:m[0]]) + } + // Check for ORDER BY clause (remove it for STRING_AGG — PG STRING_AGG has its own ORDER BY) + orderRe := regexp.MustCompile(`(?i)\s+ORDER\s+BY\s+.+`) + orderClause := "" + if m := orderRe.FindStringIndex(inner); m != nil { + orderClause = " " + strings.TrimSpace(inner[m[0]:]) + inner = strings.TrimSpace(inner[:m[0]]) + } + replacement := "STRING_AGG(" + inner + "::text, '" + sep + "'" + orderClause + ")" + query = query[:loc[0]] + replacement + query[i:] + } + return query +} + +// rewriteUpdateJoin rewrites MySQL UPDATE t1 JOIN t2 ON cond SET ... → PG UPDATE t1 SET ... FROM t2 WHERE cond +func rewriteUpdateJoin(query string) string { + // Pattern: UPDATE JOIN ON SET + re := regexp.MustCompile(`(?is)UPDATE\s+(\S+)\s+(\w+)\s+JOIN\s+(\S+)\s+(\w+)\s+ON\s+(.+?)\s+SET\s+(.+)`) + m := re.FindStringSubmatch(query) + if m == nil { + return query + } + table1 := m[1] + alias1 := m[2] + table2 := m[3] + alias2 := m[4] + onCondition := strings.TrimSpace(m[5]) + setClause := strings.TrimSpace(m[6]) + + return fmt.Sprintf("UPDATE %s %s SET %s FROM %s %s WHERE %s", + table1, alias1, setClause, table2, alias2, onCondition) +} From 7235f0fb39c0df5716607eb6b2d11d87f8e66761 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 15:54:36 -0400 Subject: [PATCH 2/6] feat(datastore): add DialectHelper interface and dual-dialect support Introduce DialectHelper interface (17 methods) that abstracts MySQL vs PG SQL differences at the fragment level. Each Datastore instance holds a dialect that generates the correct SQL for its backend. Methods: InsertIgnoreInto, OnDuplicateKey, OnConflictDoNothing, GroupConcat, JSONExtract, JSONUnquoteExtract, JSONBuildObject, JSONAgg, FindInSet, FullTextMatch, RegexpMatch, GoquDialect, IsDuplicate, IsForeignKey, IsReadOnly, IsBadConnection, ReturningID. Also: - Dual MySQL/PG error classification in errors.go - Driver selection and PG baseline migration in mysql.go - Dual-dialect goose migration support --- server/datastore/mysql/dialect.go | 98 ++++++++ server/datastore/mysql/dialect_mysql.go | 100 +++++++++ server/datastore/mysql/dialect_postgres.go | 173 +++++++++++++++ .../datastore/mysql/dialect_postgres_test.go | 115 ++++++++++ server/datastore/mysql/errors.go | 8 + server/datastore/mysql/mysql.go | 210 ++++++++++++++++-- server/goose/dialect.go | 14 +- server/goose/migrate.go | 19 ++ server/goose/migrate_test.go | 71 +++++- server/goose/migration.go | 49 +++- 10 files changed, 829 insertions(+), 28 deletions(-) create mode 100644 server/datastore/mysql/dialect.go create mode 100644 server/datastore/mysql/dialect_mysql.go create mode 100644 server/datastore/mysql/dialect_postgres.go create mode 100644 server/datastore/mysql/dialect_postgres_test.go diff --git a/server/datastore/mysql/dialect.go b/server/datastore/mysql/dialect.go new file mode 100644 index 00000000000..b5bd82a0c93 --- /dev/null +++ b/server/datastore/mysql/dialect.go @@ -0,0 +1,98 @@ +package mysql + +import "github.com/doug-martin/goqu/v9" + +// DialectHelper abstracts SQL dialect differences between MySQL and PostgreSQL. +// All runtime SQL that is MySQL-specific must go through this interface so that +// a PostgreSQL implementation can substitute equivalent syntax. +// +// Upsert methods are fragment-based: they return SQL fragments (prefix or suffix) +// that compose into any query shape — single-row, multi-row batch, INSERT...SELECT. +type DialectHelper interface { + // ---- Upsert fragments ---- + + // InsertIgnoreInto returns the INSERT prefix for ignoring duplicate-key errors. + // MySQL: "INSERT IGNORE INTO" + // PostgreSQL: "INSERT INTO" + // For PostgreSQL, the caller must also append OnConflictDoNothing() to the query. + InsertIgnoreInto() string + + // ReplaceInto returns the REPLACE INTO prefix (MySQL) or "INSERT INTO" (PostgreSQL). + // MySQL: "REPLACE INTO" + // PostgreSQL: "INSERT INTO" + // For PostgreSQL, the caller must also append OnDuplicateKey() with all non-key + // columns to achieve REPLACE semantics (upsert all columns). + ReplaceInto() string + + // OnDuplicateKey returns the upsert conflict-handling suffix. + // MySQL: "ON DUPLICATE KEY UPDATE " + updateClause + // PostgreSQL: "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + translated + // The updateClause uses MySQL syntax (e.g., "name=VALUES(name), updated_at=NOW()"). + // The PostgreSQL implementation translates VALUES(col) → EXCLUDED.col. + OnDuplicateKey(conflictTarget, updateClause string) string + + // OnConflictDoNothing returns the suffix for suppressing duplicate-key errors. + // MySQL: "" (handled by InsertIgnoreInto prefix) + // PostgreSQL: " ON CONFLICT (" + conflictTarget + ") DO NOTHING" + OnConflictDoNothing(conflictTarget string) string + + // ---- Aggregate & expression functions ---- + + // GroupConcat returns a GROUP_CONCAT (MySQL) or STRING_AGG (PostgreSQL) + // expression aggregating expr with the given separator. + GroupConcat(expr, separator string) string + + // JSONAgg returns a JSON_ARRAYAGG (MySQL) or json_agg (PostgreSQL) expression. + JSONAgg(expr string) string + + // JSONExtract returns an expression that extracts a value from a JSON column + // at the given path. MySQL: JSON_EXTRACT(col, path), PG: col->'path'. + JSONExtract(col, path string) string + + // JSONUnquoteExtract returns an expression that extracts a scalar string from + // a JSON column. MySQL: col->>'path' / JSON_UNQUOTE(JSON_EXTRACT(...)), + // PostgreSQL: col->>'path'. + JSONUnquoteExtract(col, path string) string + + // JSONBuildObject returns an expression that constructs a JSON object from + // alternating key/value strings. MySQL: JSON_OBJECT(k,v,...), + // PostgreSQL: jsonb_build_object(k,v,...). + JSONBuildObject(keyVals ...string) string + + // FindInSet returns an expression equivalent to MySQL FIND_IN_SET(needle, col). + // PostgreSQL: needle = ANY(string_to_array(col, ',')). + FindInSet(needle, col string) string + + // FullTextMatch returns a full-text search predicate. + // MySQL: MATCH(cols...) AGAINST (query IN BOOLEAN MODE), + // PostgreSQL: to_tsvector('english', col) @@ plainto_tsquery('english', query). + FullTextMatch(cols []string, query string) string + + // RegexpMatch returns a regular-expression match predicate. + // MySQL: col REGEXP pattern, PostgreSQL: col ~ pattern. + RegexpMatch(col, pattern string) string + + // ---- Goqu ---- + + // GoquDialect returns the goqu dialect wrapper appropriate for this driver. + GoquDialect() goqu.DialectWrapper + + // ---- Error classification ---- + + // IsDuplicate returns true if err is a unique-constraint violation. + IsDuplicate(err error) bool + + // IsForeignKey returns true if err is a foreign-key constraint violation. + IsForeignKey(err error) bool + + // IsReadOnly returns true if err indicates the server is in read-only mode. + IsReadOnly(err error) bool + + // IsBadConnection returns true if err is a connection-level error that + // justifies retrying on a new connection. + IsBadConnection(err error) bool + + // ReturningID returns " RETURNING id" for PostgreSQL (to be appended to + // INSERT statements) or "" for MySQL (which uses LastInsertId instead). + ReturningID() string +} diff --git a/server/datastore/mysql/dialect_mysql.go b/server/datastore/mysql/dialect_mysql.go new file mode 100644 index 00000000000..0a208f71632 --- /dev/null +++ b/server/datastore/mysql/dialect_mysql.go @@ -0,0 +1,100 @@ +package mysql + +import ( + "fmt" + "strings" + + "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/mysql" // register mysql dialect + common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" +) + +// mysqlDialect implements DialectHelper for MySQL / MariaDB. +// Every method returns exactly the SQL currently inlined across the datastore +// implementation — this is a pure structural refactoring with no behaviour change. +type mysqlDialect struct{} + +// Compile-time assertion that mysqlDialect satisfies DialectHelper. +var _ DialectHelper = mysqlDialect{} + +// InsertIgnoreInto returns "INSERT IGNORE INTO". +func (mysqlDialect) InsertIgnoreInto() string { return "INSERT IGNORE INTO" } + +// ReplaceInto returns "REPLACE INTO". +func (mysqlDialect) ReplaceInto() string { return "REPLACE INTO" } + +// OnDuplicateKey returns: ON DUPLICATE KEY UPDATE +// The updateClause is passed through verbatim (MySQL-native syntax). +func (mysqlDialect) OnDuplicateKey(_, updateClause string) string { + return "ON DUPLICATE KEY UPDATE " + updateClause +} + +// OnConflictDoNothing returns "" — MySQL handles ignore via the INSERT IGNORE prefix. +func (mysqlDialect) OnConflictDoNothing(_ string) string { return "" } + +// GroupConcat builds: GROUP_CONCAT( SEPARATOR '') +func (mysqlDialect) GroupConcat(expr, separator string) string { + return fmt.Sprintf("GROUP_CONCAT(%s SEPARATOR '%s')", expr, separator) +} + +// JSONAgg builds: JSON_ARRAYAGG() +func (mysqlDialect) JSONAgg(expr string) string { + return fmt.Sprintf("JSON_ARRAYAGG(%s)", expr) +} + +// JSONExtract builds: JSON_EXTRACT(, '') +func (mysqlDialect) JSONExtract(col, path string) string { + return fmt.Sprintf("JSON_EXTRACT(%s, '%s')", col, path) +} + +// JSONUnquoteExtract builds: ->>'' +func (mysqlDialect) JSONUnquoteExtract(col, path string) string { + return fmt.Sprintf("%s->>'%s'", col, path) +} + +// JSONBuildObject builds: JSON_OBJECT(, , ...) +func (mysqlDialect) JSONBuildObject(keyVals ...string) string { + return fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(keyVals, ", ")) +} + +// FindInSet builds: FIND_IN_SET(, ) +func (mysqlDialect) FindInSet(needle, col string) string { + return fmt.Sprintf("FIND_IN_SET(%s, %s)", needle, col) +} + +// FullTextMatch builds: MATCH() AGAINST ( IN BOOLEAN MODE) +func (mysqlDialect) FullTextMatch(cols []string, query string) string { + return fmt.Sprintf("MATCH(%s) AGAINST (%s IN BOOLEAN MODE)", strings.Join(cols, ", "), query) +} + +// RegexpMatch builds: REGEXP +func (mysqlDialect) RegexpMatch(col, pattern string) string { + return fmt.Sprintf("%s REGEXP %s", col, pattern) +} + +// GoquDialect returns the goqu MySQL dialect wrapper. +func (mysqlDialect) GoquDialect() goqu.DialectWrapper { + return goqu.Dialect("mysql") +} + +// IsDuplicate delegates to the package-level IsDuplicate in errors.go. +func (mysqlDialect) IsDuplicate(err error) bool { + return IsDuplicate(err) +} + +// IsForeignKey delegates to the package-level isMySQLForeignKey in errors.go. +func (mysqlDialect) IsForeignKey(err error) bool { + return isMySQLForeignKey(err) +} + +// IsReadOnly delegates to common_mysql.IsReadOnlyError. +func (mysqlDialect) IsReadOnly(err error) bool { + return common_mysql.IsReadOnlyError(err) +} + +// IsBadConnection delegates to the package-level isBadConnection in errors.go. +func (mysqlDialect) IsBadConnection(err error) bool { + return isBadConnection(err) +} + +func (mysqlDialect) ReturningID() string { return "" } diff --git a/server/datastore/mysql/dialect_postgres.go b/server/datastore/mysql/dialect_postgres.go new file mode 100644 index 00000000000..49bdc71ef33 --- /dev/null +++ b/server/datastore/mysql/dialect_postgres.go @@ -0,0 +1,173 @@ +// dialect_postgres.go implements DialectHelper for PostgreSQL. + +package mysql + +import ( + "fmt" + "regexp" + "strings" + + "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/postgres" // register postgres dialect + pg "github.com/fleetdm/fleet/v4/server/platform/postgres" +) + +// postgresDialect implements DialectHelper for PostgreSQL. +type postgresDialect struct{} + +// Compile-time assertion that postgresDialect satisfies DialectHelper. +var _ DialectHelper = postgresDialect{} + +// InsertIgnoreInto returns "INSERT INTO". +// PostgreSQL achieves ignore semantics via ON CONFLICT ... DO NOTHING appended by the caller. +func (postgresDialect) InsertIgnoreInto() string { return "INSERT INTO" } + +// ReplaceInto returns "INSERT INTO". +// PostgreSQL achieves replace semantics via ON CONFLICT ... DO UPDATE SET appended by the caller. +func (postgresDialect) ReplaceInto() string { return "INSERT INTO" } + +// valuesPattern matches MySQL VALUES(`col`) or VALUES(col) in ON DUPLICATE KEY UPDATE clauses. +var valuesPattern = regexp.MustCompile("VALUES\\(`?([^`)]+)`?\\)") + +// lastInsertIDPattern matches id=LAST_INSERT_ID(id) assignments in ON DUPLICATE KEY UPDATE clauses. +// This MySQL trick returns the existing row's ID on conflict; PG uses RETURNING id instead. +var lastInsertIDPattern = regexp.MustCompile(`(?:,\s*)?id\s*=\s*LAST_INSERT_ID\(id\)(?:\s*,)?`) + +// stripLastInsertID removes id=LAST_INSERT_ID(id) from an ON DUPLICATE KEY UPDATE clause. +func stripLastInsertID(clause string) string { + result := lastInsertIDPattern.ReplaceAllString(clause, "") + return strings.Trim(result, ", ") +} + +// translateValuesToExcluded rewrites MySQL VALUES(col) references to PostgreSQL EXCLUDED.col. +// +// VALUES(name) → EXCLUDED.name +// VALUES(`name`) → EXCLUDED.name +func translateValuesToExcluded(clause string) string { + return valuesPattern.ReplaceAllString(clause, "EXCLUDED.$1") +} + +// OnDuplicateKey returns: ON CONFLICT () DO UPDATE SET +// The updateClause uses MySQL syntax; VALUES(col) is translated to EXCLUDED.col. +// If the clause contains id=LAST_INSERT_ID(id), it is stripped (PG uses RETURNING id). +// If stripping leaves an empty clause, a no-op update on the first conflict column is used +// so that RETURNING id still works. +func (postgresDialect) OnDuplicateKey(conflictTarget, updateClause string) string { + cleaned := stripLastInsertID(updateClause) + if strings.TrimSpace(cleaned) == "" { + // No-op update: set the first conflict column to itself so RETURNING id works. + firstCol := strings.SplitN(conflictTarget, ",", 2)[0] + firstCol = strings.TrimSpace(firstCol) + return "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + firstCol + " = EXCLUDED." + firstCol + } + return "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + translateValuesToExcluded(cleaned) +} + +// OnConflictDoNothing returns: ON CONFLICT () DO NOTHING +func (postgresDialect) OnConflictDoNothing(conflictTarget string) string { + return " ON CONFLICT (" + conflictTarget + ") DO NOTHING" +} + +// GroupConcat builds: STRING_AGG(::text, '') +func (postgresDialect) GroupConcat(expr, separator string) string { + return fmt.Sprintf("STRING_AGG(%s::text, '%s')", expr, separator) +} + +// JSONAgg builds: jsonb_agg() — uses jsonb_agg for PG jsonb compatibility +func (postgresDialect) JSONAgg(expr string) string { + return fmt.Sprintf("jsonb_agg(%s)", expr) +} + +// mysqlPathToPGChain converts a MySQL JSON path ($.key1.key2) to a chain of +// PostgreSQL -> operators: col->'key1'->'key2'. +// For a single-level path like $.path, it returns col->'path'. +// The final operator is determined by the extract parameter: +// +// extract=false → all segments use -> (returns JSON) +// extract=true → last segment uses ->> (returns text) +func mysqlPathToPGChain(col, path string, extractText bool) string { + // Strip $. prefix + path = strings.TrimPrefix(path, "$.") + // Remove surrounding double quotes + path = strings.Trim(path, `"`) + + // Split on . to get path segments + segments := strings.Split(path, ".") + if len(segments) == 0 { + return col + } + + var b strings.Builder + b.WriteString(col) + for i, seg := range segments { + if extractText && i == len(segments)-1 { + b.WriteString("->>'") + } else { + b.WriteString("->'") + } + b.WriteString(seg) + b.WriteByte('\'') + } + return b.String() +} + +// JSONExtract builds a PG JSON traversal returning JSON (uses -> for all levels). +// +// MySQL: JSON_EXTRACT(col, '$.mdm.setting') → PG: col->'mdm'->'setting' +// MySQL: JSON_EXTRACT(col, '$.path') → PG: col->'path' +func (postgresDialect) JSONExtract(col, path string) string { + return mysqlPathToPGChain(col, path, false) +} + +// JSONUnquoteExtract builds a PG JSON traversal returning text (last level uses ->>). +// +// MySQL: col->>'$.mdm.setting' → PG: col->'mdm'->>'setting' +// MySQL: col->>'$.path' → PG: col->>'path' +func (postgresDialect) JSONUnquoteExtract(col, path string) string { + return mysqlPathToPGChain(col, path, true) +} + +// JSONBuildObject builds: jsonb_build_object(, , ...) +func (postgresDialect) JSONBuildObject(keyVals ...string) string { + return fmt.Sprintf("jsonb_build_object(%s)", strings.Join(keyVals, ", ")) +} + +// FindInSet builds: = ANY(string_to_array(, ',')) +func (postgresDialect) FindInSet(needle, col string) string { + return fmt.Sprintf("%s = ANY(string_to_array(%s, ','))", needle, col) +} + +// FullTextMatch builds: to_tsvector('english', ) @@ plainto_tsquery('english', ) +// PostgreSQL's to_tsvector takes a single column expression. +func (postgresDialect) FullTextMatch(cols []string, query string) string { + return fmt.Sprintf("to_tsvector('english', %s) @@ plainto_tsquery('english', %s)", cols[0], query) +} + +// RegexpMatch builds: ~ +func (postgresDialect) RegexpMatch(col, pattern string) string { + return fmt.Sprintf("%s ~ %s", col, pattern) +} + +// GoquDialect returns the goqu PostgreSQL dialect wrapper. +func (postgresDialect) GoquDialect() goqu.DialectWrapper { + return goqu.Dialect("postgres") +} + +// --- Error classification --- +// +// Delegates to server/platform/postgres which uses proper pgx/pq interface +// matching via SQLSTATE codes. + +// IsDuplicate returns true if err is a unique-constraint violation (SQLSTATE 23505). +func (postgresDialect) IsDuplicate(err error) bool { return pg.IsDuplicate(err) } + +// IsForeignKey returns true if err is a foreign-key constraint violation (SQLSTATE 23503). +func (postgresDialect) IsForeignKey(err error) bool { return pg.IsForeignKey(err) } + +// IsReadOnly returns true if err indicates a read-only transaction (SQLSTATE 25006). +func (postgresDialect) IsReadOnly(err error) bool { return pg.IsReadOnly(err) } + +// IsBadConnection returns true if err is a connection-level error. +func (postgresDialect) IsBadConnection(err error) bool { return pg.IsBadConnection(err) } + +func (postgresDialect) ReturningID() string { return " RETURNING id" } diff --git a/server/datastore/mysql/dialect_postgres_test.go b/server/datastore/mysql/dialect_postgres_test.go new file mode 100644 index 00000000000..7274e0d8592 --- /dev/null +++ b/server/datastore/mysql/dialect_postgres_test.go @@ -0,0 +1,115 @@ +package mysql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPostgresDialectSQL(t *testing.T) { + d := postgresDialect{} + + t.Run("InsertIgnoreInto", func(t *testing.T) { + assert.Equal(t, "INSERT INTO", d.InsertIgnoreInto()) + }) + + t.Run("ReplaceInto", func(t *testing.T) { + assert.Equal(t, "INSERT INTO", d.ReplaceInto()) + }) + + t.Run("OnDuplicateKey", func(t *testing.T) { + got := d.OnDuplicateKey("id", "name=VALUES(name), updated_at=NOW()") + assert.Equal(t, "ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name, updated_at=NOW()", got) + }) + + t.Run("OnDuplicateKey_backtick_quoted", func(t *testing.T) { + got := d.OnDuplicateKey("id", "`name`=VALUES(`name`)") + assert.Equal(t, "ON CONFLICT (id) DO UPDATE SET `name`=EXCLUDED.name", got) + }) + + t.Run("OnConflictDoNothing", func(t *testing.T) { + assert.Equal(t, " ON CONFLICT (host_id, label_id) DO NOTHING", d.OnConflictDoNothing("host_id, label_id")) + }) + + t.Run("GroupConcat", func(t *testing.T) { + assert.Equal(t, "STRING_AGG(x::text, ',')", d.GroupConcat("x", ",")) + }) + + t.Run("JSONExtract_dollar_dot", func(t *testing.T) { + assert.Equal(t, "col->'path'", d.JSONExtract("col", "$.path")) + }) + + t.Run("JSONExtract_nested", func(t *testing.T) { + assert.Equal(t, "t.config->'mdm'->'enable_recovery_lock_password'", d.JSONExtract("t.config", "$.mdm.enable_recovery_lock_password")) + }) + + t.Run("JSONUnquoteExtract", func(t *testing.T) { + assert.Equal(t, "col->>'path'", d.JSONUnquoteExtract("col", "$.path")) + }) + + t.Run("JSONBuildObject", func(t *testing.T) { + assert.Equal(t, "jsonb_build_object('k1', v1)", d.JSONBuildObject("'k1'", "v1")) + }) + + t.Run("FindInSet", func(t *testing.T) { + assert.Equal(t, "? = ANY(string_to_array(platforms, ','))", d.FindInSet("?", "platforms")) + }) + + t.Run("FullTextMatch", func(t *testing.T) { + assert.Equal(t, "to_tsvector('english', l.name) @@ plainto_tsquery('english', ?)", d.FullTextMatch([]string{"l.name"}, "?")) + }) + + t.Run("RegexpMatch", func(t *testing.T) { + assert.Equal(t, "s.name ~ ?", d.RegexpMatch("s.name", "?")) + }) + + t.Run("JSONAgg", func(t *testing.T) { + assert.Equal(t, "jsonb_agg(x)", d.JSONAgg("x")) + }) + + t.Run("GoquDialect", func(t *testing.T) { + gd := d.GoquDialect() + assert.NotNil(t, gd) + }) +} + +func TestTranslateValuesToExcluded(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"name=VALUES(name)", "name=EXCLUDED.name"}, + {"name=VALUES(name), age=VALUES(age)", "name=EXCLUDED.name, age=EXCLUDED.age"}, + {"`name`=VALUES(`name`)", "`name`=EXCLUDED.name"}, + {"col = col + VALUES(col)", "col = col + EXCLUDED.col"}, + {"updated_at=NOW()", "updated_at=NOW()"}, + {"iteration = iteration + 1", "iteration = iteration + 1"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, translateValuesToExcluded(tt.input)) + }) + } +} + +func TestMysqlPathToPGChain(t *testing.T) { + tests := []struct { + col, path string + extractText bool + expected string + }{ + {"col", "$.path", false, "col->'path'"}, + {"col", "$.path", true, "col->>'path'"}, + {"t.config", "$.mdm.enable_recovery_lock_password", false, "t.config->'mdm'->'enable_recovery_lock_password'"}, + {"t.config", "$.mdm.enable_recovery_lock_password", true, "t.config->'mdm'->>'enable_recovery_lock_password'"}, + {"col", "$.\"quoted\"", false, "col->'quoted'"}, + {"col", "path", false, "col->'path'"}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.expected, mysqlPathToPGChain(tt.col, tt.path, tt.extractText)) + }) + } +} diff --git a/server/datastore/mysql/errors.go b/server/datastore/mysql/errors.go index 2ae39305fe4..8d01394c7fd 100644 --- a/server/datastore/mysql/errors.go +++ b/server/datastore/mysql/errors.go @@ -14,6 +14,7 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + pg "github.com/fleetdm/fleet/v4/server/platform/postgres" "github.com/go-sql-driver/mysql" ) @@ -86,6 +87,10 @@ func IsDuplicate(err error) bool { return true } } + // Also check PostgreSQL unique violation (SQLSTATE 23505) + if pg.IsDuplicate(err) { + return true + } return false } @@ -118,6 +123,9 @@ func isMySQLForeignKey(err error) bool { return true } } + if pg.IsForeignKey(err) { + return true + } return false } diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index c3a0a02ff81..fb581b6cdaa 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -4,6 +4,7 @@ package mysql import ( "context" "database/sql" + _ "embed" "errors" "fmt" "log/slog" @@ -32,6 +33,7 @@ import ( nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push" scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + _ "github.com/fleetdm/fleet/v4/server/platform/postgres" // register pgx-rebind driver for PostgreSQL "github.com/go-sql-driver/mysql" "github.com/hashicorp/go-multierror" "github.com/jmoiron/sqlx" @@ -59,10 +61,11 @@ type Datastore struct { replica fleet.DBReader // so it cannot be used to perform writes primary *sqlx.DB - logger *slog.Logger - clock clock.Clock - config config.MysqlConfig - pusher nano_push.Pusher + logger *slog.Logger + clock clock.Clock + config config.MysqlConfig + dialect DialectHelper + pusher nano_push.Pusher android.Datastore // nil if no read replica @@ -113,12 +116,58 @@ func (ds *Datastore) reader(ctx context.Context) fleet.DBReader { return ds.replica } +// currentDatabaseFn returns the SQL function to get the current database name. +// MySQL: DATABASE(), PostgreSQL: current_database() +func (ds *Datastore) currentDatabaseFn() string { + if ds.dialect.ReturningID() != "" { + return "current_database()" + } + return "(SELECT DATABASE())" +} + // writer returns the DB instance to use for write statements, which is always // the primary. func (ds *Datastore) writer(ctx context.Context) *sqlx.DB { return ds.primary } +// Querier is any type that can execute SQL (sqlx.DB, sqlx.Tx, sqlx.ExtContext). +type Querier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// insertAndGetID executes an INSERT and returns the auto-generated ID. +// For MySQL, uses LastInsertId(). For PostgreSQL, appends RETURNING id. +func (ds *Datastore) insertAndGetID(ctx context.Context, q Querier, query string, args ...any) (int64, error) { + if ds.dialect.ReturningID() != "" { + // PostgreSQL: use RETURNING id + var id int64 + err := q.QueryRowContext(ctx, query+ds.dialect.ReturningID(), args...).Scan(&id) + return id, err + } + // MySQL: use LastInsertId + res, err := q.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// insertAndGetIDTx is like insertAndGetID but for sqlx.ExtContext (transactions). +func insertAndGetIDTx(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, query string, args ...any) (int64, error) { + if dialect.ReturningID() != "" { + var id int64 + err := tx.QueryRowxContext(ctx, query+dialect.ReturningID(), args...).Scan(&id) + return id, err + } + res, err := tx.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + // loadOrPrepareStmt will load a statement from the statement cache. // If not available, it will attempt to prepare (create) it. // Returns nil if it failed to prepare a statement. @@ -241,6 +290,13 @@ func NewDBConnections(cfg config.MysqlConfig, opts ...DBOption) (*common_mysql.D if err := checkAndModifyConfig(&cfg); err != nil { return nil, err } + + // Set migration client dialects to match the configured driver. + if cfg.Driver == "postgres" { + tables.SetDialect("postgres") + data.SetDialect("postgres") + } + // Convert replica config once so that checkAndModifyConfig mutations are preserved for the later NewDB call. var replicaConf *config.MysqlConfig if options.ReplicaConfig != nil { @@ -286,12 +342,13 @@ func NewDatastore(conns *common_mysql.DBConnections, cfg config.MysqlConfig, c c logger: conns.Options.Logger, clock: c, config: cfg, + dialect: dialectForDriver(cfg.Driver), readReplicaConfig: conns.Options.ReplicaConfig, writeCh: make(chan itemToWrite), stmtCache: make(map[string]*sqlx.Stmt), minLastOpenedAtDiff: conns.Options.MinLastOpenedAtDiff, serverPrivateKey: conns.Options.PrivateKey, - Datastore: NewAndroidDatastore(conns.Options.Logger, conns.Primary, conns.Replica), + Datastore: NewAndroidDatastore(conns.Options.Logger, conns.Primary, conns.Replica, dialectForDriver(cfg.Driver)), } go ds.writeChanLoop() @@ -378,9 +435,43 @@ func init() { } func NewDB(conf *config.MysqlConfig, opts *common_mysql.DBOptions) (*sqlx.DB, error) { + if conf.Driver == "postgres" { + return newPostgresDB(conf) + } return common_mysql.NewDB(toCommonMysqlConfig(conf), opts, otelTracedDriverName) } +// newPostgresDB opens a PostgreSQL connection using pgx/stdlib. +func newPostgresDB(conf *config.MysqlConfig) (*sqlx.DB, error) { + // Build PostgreSQL DSN from the MySQL-style config fields. + // Address is expected as "host:port". + host, port, err := net.SplitHostPort(conf.Address) + if err != nil { + host = conf.Address + port = "5432" + } + dsn := fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + host, port, conf.Username, conf.Password, conf.Database, + ) + if conf.TLSCA != "" { + dsn = fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=verify-ca sslrootcert=%s", + host, port, conf.Username, conf.Password, conf.Database, conf.TLSCA, + ) + } + + // Use "pgx-rebind" driver which wraps pgx/stdlib and auto-converts + // MySQL-style ? placeholders to PostgreSQL $N placeholders. + db, err := sqlx.Open("pgx-rebind", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres: %w", err) + } + db.SetMaxOpenConns(conf.MaxOpenConns) + db.SetMaxIdleConns(conf.MaxIdleConns) + return db, nil +} + // toCommonMysqlConfig converts a config.MysqlConfig to common_mysql.MysqlConfig. func toCommonMysqlConfig(conf *config.MysqlConfig) *common_mysql.MysqlConfig { return &common_mysql.MysqlConfig{ @@ -439,7 +530,26 @@ func fromCommonMysqlConfig(conf *common_mysql.MysqlConfig) *config.MysqlConfig { } } +// dialectForDriver returns the DialectHelper for the given driver name. +// Empty string defaults to "mysql". +func dialectForDriver(driver string) DialectHelper { + switch driver { + case "postgres": + return postgresDialect{} + case "", "mysql": + return mysqlDialect{} + default: + // checkAndModifyConfig validates the driver before this is called, + // so reaching here means a programming error. + panic(fmt.Sprintf("unsupported database driver: %q", driver)) + } +} + func checkAndModifyConfig(conf *config.MysqlConfig) error { + if conf.Driver != "" && conf.Driver != "mysql" && conf.Driver != "postgres" { + return fmt.Errorf("unsupported database driver %q: valid values are \"mysql\" and \"postgres\"", conf.Driver) + } + if conf.PasswordPath != "" && conf.Password != "" { return errors.New("A MySQL password and a MySQL password file were provided - please specify only one") } @@ -488,13 +598,45 @@ func setupIAMAuthIfNeeded(conf *config.MysqlConfig, opts *common_mysql.DBOptions } func (ds *Datastore) MigrateTables(ctx context.Context) error { + if _, ok := ds.dialect.(postgresDialect); ok { + return ds.migratePGBaseline(ctx) + } return tables.MigrationClient.Up(ds.writer(ctx).DB, "") } func (ds *Datastore) MigrateData(ctx context.Context) error { + if _, ok := ds.dialect.(postgresDialect); ok { + // PG baseline schema includes all data migrations (label seeds, etc.) + return nil + } return data.MigrationClient.Up(ds.writer(ctx).DB, "") } +//go:embed pg_baseline_schema.sql +var pgBaselineSchemaSQL string + +// migratePGBaseline applies the PG baseline schema for fresh PostgreSQL databases. +// It checks if tables already exist and skips if so. +func (ds *Datastore) migratePGBaseline(ctx context.Context) error { + var exists bool + err := ds.writer(ctx).GetContext(ctx, &exists, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'hosts')`) + if err != nil { + return fmt.Errorf("checking PG schema: %w", err) + } + if exists { + ds.logger.InfoContext(ctx, "PostgreSQL schema already exists, skipping baseline") + return nil + } + ds.logger.InfoContext(ctx, "Applying PostgreSQL baseline schema") + _, err = ds.writer(ctx).ExecContext(ctx, pgBaselineSchemaSQL) + if err != nil { + return fmt.Errorf("applying PG baseline schema: %w", err) + } + ds.logger.InfoContext(ctx, "PostgreSQL baseline schema applied successfully") + return nil +} + // loadMigrations manually loads the applied migrations in ascending // order (goose doesn't provide such functionality). // @@ -532,6 +674,20 @@ func (ds *Datastore) loadMigrations( // // It assumes some deployments may have performed migrations out of order. func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatus, error) { + // For PostgreSQL, the baseline schema is applied atomically — either it's all there or not. + if _, ok := ds.dialect.(postgresDialect); ok { + var exists bool + err := ds.primary.GetContext(ctx, &exists, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'hosts')`) + if err != nil { + return nil, fmt.Errorf("checking PG schema: %w", err) + } + if exists { + return &fleet.MigrationStatus{StatusCode: fleet.AllMigrationsCompleted}, nil + } + return &fleet.MigrationStatus{StatusCode: fleet.NoMigrationsCompleted}, nil + } + if tables.MigrationClient.Migrations == nil || data.MigrationClient.Migrations == nil { return nil, errors.New("unexpected nil migrations list") } @@ -732,14 +888,23 @@ func (ds *Datastore) HealthCheck() error { // Check that the primary is reachable and not in read-only mode. // After an AWS Aurora failover the old writer is demoted to a reader; // detecting this lets the health check fail so the orchestrator can restart Fleet. - var readOnly int - if err := ds.primary.QueryRowContext(context.Background(), "SELECT @@read_only").Scan(&readOnly); err != nil { - return err - } - if readOnly == 1 { - // Intentionally return an error so that the health check endpoint returns a 500, - // signaling the orchestrator (ECS, Kubernetes) to restart Fleet with fresh DB connections. - return errors.New("primary database is read-only, possible failover detected") + if _, ok := ds.dialect.(postgresDialect); ok { + // PG: check if the server is in recovery (read-only replica) + var inRecovery bool + if err := ds.primary.QueryRowContext(context.Background(), "SELECT pg_is_in_recovery()").Scan(&inRecovery); err != nil { + return err + } + if inRecovery { + return errors.New("primary database is in recovery (read-only), possible failover detected") + } + } else { + var readOnly int + if err := ds.primary.QueryRowContext(context.Background(), "SELECT @@read_only").Scan(&readOnly); err != nil { + return err + } + if readOnly == 1 { + return errors.New("primary database is read-only, possible failover detected") + } } if ds.readReplicaConfig != nil { @@ -1255,6 +1420,10 @@ func (ds *Datastore) ProcessList(ctx context.Context) ([]fleet.MySQLProcess, err return processList, nil } +// insertOnDuplicateDidInsertOrUpdate returns true if an INSERT ON DUPLICATE KEY +// UPDATE actually inserted or updated a row (vs no-op). +// MySQL: checks LastInsertId (non-zero on insert) AND RowsAffected (> 0). +// PostgreSQL: LastInsertId is not available, so just checks RowsAffected > 0. func insertOnDuplicateDidInsertOrUpdate(res sql.Result) bool { // From mysql's documentation: // @@ -1281,9 +1450,13 @@ func insertOnDuplicateDidInsertOrUpdate(res sql.Result) bool { // already holds: // https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/result.go - lastID, _ := res.LastInsertId() aff, _ := res.RowsAffected() - // something was updated (lastID != 0) AND row was found (aff == 1 or higher if more rows were found) + lastID, err := res.LastInsertId() + if err != nil { + // PostgreSQL doesn't support LastInsertId — fall back to RowsAffected only + return aff > 0 + } + // MySQL: something was inserted (lastID != 0) AND row was found (aff > 0) return lastID != 0 && aff > 0 } @@ -1321,9 +1494,9 @@ func (ds *Datastore) optimisticGetOrInsertWithWriter(ctx context.Context, writer if err != nil { if errors.Is(err, sql.ErrNoRows) { // this does not exist yet, try to insert it - res, err := writer.ExecContext(ctx, insertStmt.Statement, insertStmt.Args...) + insertedID, err := insertAndGetIDTx(ctx, writer, ds.dialect, insertStmt.Statement, insertStmt.Args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { // it might've been created between the select and the insert, read // again this time from the primary database connection. id, err := readID(writer) @@ -1334,8 +1507,7 @@ func (ds *Datastore) optimisticGetOrInsertWithWriter(ctx context.Context, writer } return 0, ctxerr.Wrap(ctx, err, "insert") } - id, _ := res.LastInsertId() - return uint(id), nil //nolint:gosec // dismiss G115 + return uint(insertedID), nil //nolint:gosec // dismiss G115 } return 0, ctxerr.Wrap(ctx, err, "get id from reader") } diff --git a/server/goose/dialect.go b/server/goose/dialect.go index bfa5f879cb9..763b937b5c0 100644 --- a/server/goose/dialect.go +++ b/server/goose/dialect.go @@ -11,6 +11,10 @@ type SqlDialect interface { createVersionTableSql(name string) string // sql string to create the goose_db_version table insertVersionSql(name string) string // sql string to insert the initial version table row dbVersionQuery(db *sql.DB, name string) (*sql.Rows, error) + + // DriverName returns the driver name for this dialect ("mysql", "postgres", "sqlite3"). + // Used by the migration runner to select dialect-specific UpFnMySQL/UpFnPG functions. + DriverName() string } func GetDialect() SqlDialect { @@ -42,8 +46,10 @@ func SetDialect(d string) error { type PostgresDialect struct{} +func (PostgresDialect) DriverName() string { return "postgres" } + func (pg PostgresDialect) createVersionTableSql(name string) string { - return `CREATE TABLE ` + name + ` ( + return `CREATE TABLE IF NOT EXISTS ` + name + ` ( id serial NOT NULL, version_id bigint NOT NULL, is_applied boolean NOT NULL, @@ -72,8 +78,10 @@ func (pg PostgresDialect) dbVersionQuery(db *sql.DB, name string) (*sql.Rows, er type MySqlDialect struct{} +func (MySqlDialect) DriverName() string { return "mysql" } + func (m MySqlDialect) createVersionTableSql(name string) string { - return `CREATE TABLE ` + name + ` ( + return `CREATE TABLE IF NOT EXISTS ` + name + ` ( id serial NOT NULL, version_id bigint NOT NULL, is_applied boolean NOT NULL, @@ -102,6 +110,8 @@ func (m MySqlDialect) dbVersionQuery(db *sql.DB, name string) (*sql.Rows, error) type Sqlite3Dialect struct{} +func (Sqlite3Dialect) DriverName() string { return "sqlite3" } + func (m Sqlite3Dialect) createVersionTableSql(name string) string { return `CREATE TABLE ` + name + ` ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/server/goose/migrate.go b/server/goose/migrate.go index ee8d3504fa4..f672b63bdd8 100644 --- a/server/goose/migrate.go +++ b/server/goose/migrate.go @@ -89,6 +89,25 @@ func AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { globalGoose.Migrations = append(globalGoose.Migrations, migration) } +// AddDualDialectMigration adds a migration with dialect-specific up/down functions. +// Use this for migrations where MySQL and PostgreSQL need different DDL. +// Pass nil for any function that should be a no-op for that dialect. +func (c *Client) AddDualDialectMigration(upMySQL, downMySQL, upPG, downPG func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + v, _ := NumericComponent(filename) + migration := &Migration{ + Version: v, + Next: -1, + Previous: -1, + Source: filename, + UpFnMySQL: upMySQL, + DownFnMySQL: downMySQL, + UpFnPG: upPG, + DownFnPG: downPG, + } + c.Migrations = append(c.Migrations, migration) +} + // collect all the valid looking migration scripts in the // migrations folder and go func registry, and key them by version func (c *Client) collectMigrations(dirpath string, current, target int64) (Migrations, error) { diff --git a/server/goose/migrate_test.go b/server/goose/migrate_test.go index fb64aad6408..6589e59c9e8 100644 --- a/server/goose/migrate_test.go +++ b/server/goose/migrate_test.go @@ -1,6 +1,9 @@ package goose -import "testing" +import ( + "database/sql" + "testing" +) func newMigration(v int64, src string) *Migration { return &Migration{Version: v, Previous: -1, Next: -1, Source: src} @@ -55,3 +58,69 @@ func validateMigrationSort(t *testing.T, ms Migrations, sorted []int64) { t.Log(ms) } + +func TestMigrationSelectFn(t *testing.T) { + generic := func(*sql.Tx) error { return nil } + mysqlFn := func(*sql.Tx) error { return nil } + pgFn := func(*sql.Tx) error { return nil } + + t.Run("generic only", func(t *testing.T) { + m := &Migration{UpFn: generic, DownFn: generic} + if m.selectFn("mysql", true) == nil { + t.Error("expected generic up for mysql") + } + if m.selectFn("postgres", true) == nil { + t.Error("expected generic up for postgres") + } + }) + + t.Run("mysql specific takes precedence", func(t *testing.T) { + m := &Migration{UpFn: generic, UpFnMySQL: mysqlFn} + // MySQL should get mysqlFn, not generic + fn := m.selectFn("mysql", true) + if fn == nil { + t.Fatal("expected non-nil fn for mysql") + } + // Postgres should fall back to generic + fn = m.selectFn("postgres", true) + if fn == nil { + t.Fatal("expected non-nil fn for postgres") + } + }) + + t.Run("pg specific takes precedence", func(t *testing.T) { + m := &Migration{UpFn: generic, UpFnPG: pgFn} + fn := m.selectFn("postgres", true) + if fn == nil { + t.Fatal("expected non-nil fn for postgres") + } + fn = m.selectFn("mysql", true) + if fn == nil { + t.Fatal("expected non-nil fn for mysql fallback to generic") + } + }) + + t.Run("dual dialect no generic", func(t *testing.T) { + m := &Migration{UpFnMySQL: mysqlFn, UpFnPG: pgFn} + if m.selectFn("mysql", true) == nil { + t.Error("expected mysql fn") + } + if m.selectFn("postgres", true) == nil { + t.Error("expected pg fn") + } + // unknown driver falls back to nil generic + if m.selectFn("sqlite3", true) != nil { + t.Error("expected nil for sqlite3 with no generic") + } + }) + + t.Run("down direction", func(t *testing.T) { + m := &Migration{DownFn: generic, DownFnMySQL: mysqlFn} + if m.selectFn("mysql", false) == nil { + t.Error("expected mysql down fn") + } + if m.selectFn("postgres", false) == nil { + t.Error("expected generic down for postgres") + } + }) +} diff --git a/server/goose/migration.go b/server/goose/migration.go index b3c2c55f7ac..5e70ee8b24e 100644 --- a/server/goose/migration.go +++ b/server/goose/migration.go @@ -24,8 +24,18 @@ type Migration struct { Next int64 // next version, or -1 if none Previous int64 // previous version, -1 if none Source string // path to .sql script - UpFn func(*sql.Tx) error // Up go migration function - DownFn func(*sql.Tx) error // Down go migration function + UpFn func(*sql.Tx) error // Up go migration function (dialect-agnostic fallback) + DownFn func(*sql.Tx) error // Down go migration function (dialect-agnostic fallback) + + // UpFnMySQL and DownFnMySQL are MySQL-specific migration functions. + // When set, they take precedence over UpFn/DownFn for MySQL databases. + UpFnMySQL func(*sql.Tx) error + DownFnMySQL func(*sql.Tx) error + + // UpFnPG and DownFnPG are PostgreSQL-specific migration functions. + // When set, they take precedence over UpFn/DownFn for PostgreSQL databases. + UpFnPG func(*sql.Tx) error + DownFnPG func(*sql.Tx) error } const ( @@ -33,6 +43,36 @@ const ( migrateDown = !migrateUp ) +// selectFn returns the appropriate migration function for the given driver and direction. +// It prefers dialect-specific functions (UpFnMySQL, UpFnPG) over the generic UpFn/DownFn. +func (m *Migration) selectFn(driver string, direction bool) func(*sql.Tx) error { + if direction { // up + switch driver { + case "mysql": + if m.UpFnMySQL != nil { + return m.UpFnMySQL + } + case "postgres": + if m.UpFnPG != nil { + return m.UpFnPG + } + } + return m.UpFn + } + // down + switch driver { + case "mysql": + if m.DownFnMySQL != nil { + return m.DownFnMySQL + } + case "postgres": + if m.DownFnPG != nil { + return m.DownFnPG + } + } + return m.DownFn +} + func (m *Migration) String() string { return fmt.Sprint(m.Source) } @@ -53,10 +93,7 @@ func (c *Client) runMigration(db *sql.DB, m *Migration, direction bool) error { log.Fatal("db.Begin: ", err) } - fn := m.UpFn - if !direction { - fn = m.DownFn - } + fn := m.selectFn(c.Dialect.DriverName(), direction) if fn != nil { if err := fn(tx); err != nil { tx.Rollback() //nolint:errcheck From fbc68ca8d9b7252919f57141c1271c5a901ed339 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 16:01:06 -0400 Subject: [PATCH 3/6] refactor(datastore): migrate MySQL-specific SQL to dialect helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hardcoded MySQL syntax with dialect method calls across all datastore files. This enables the same Go code to generate correct SQL for both MySQL and PostgreSQL. Changes per file use ds.dialect methods for: - INSERT IGNORE → InsertIgnoreInto() - ON DUPLICATE KEY UPDATE → OnDuplicateKey() - GROUP_CONCAT → GroupConcat() - JSON_EXTRACT/JSON_OBJECT → JSONExtract()/JSONBuildObject() - FIND_IN_SET → FindInSet() - Boolean comparisons via rebind driver - Error classification via IsDuplicate()/IsForeignKey() --- server/datastore/mysql/activities.go | 30 +- server/datastore/mysql/android.go | 98 ++--- server/datastore/mysql/apple_mdm.go | 310 ++++++++------- server/datastore/mysql/calendar_events.go | 27 +- .../datastore/mysql/certificate_templates.go | 21 +- server/datastore/mysql/cron_stats.go | 6 +- server/datastore/mysql/disk_encryption.go | 4 +- server/datastore/mysql/hosts.go | 374 +++++++++++------- server/datastore/mysql/labels.go | 70 ++-- server/datastore/mysql/mdm.go | 44 ++- server/datastore/mysql/microsoft_mdm.go | 58 +-- .../mysql/migrations/data/migration.go | 13 +- .../mysql/migrations/tables/migration.go | 53 ++- server/datastore/mysql/nanomdm_storage.go | 20 +- server/datastore/mysql/operating_systems.go | 6 +- server/datastore/mysql/policies.go | 201 +++++----- server/datastore/mysql/queries.go | 61 +-- server/datastore/mysql/query_results.go | 6 +- server/datastore/mysql/scim.go | 18 +- server/datastore/mysql/scripts.go | 137 +++---- server/datastore/mysql/sessions.go | 3 +- server/datastore/mysql/software.go | 144 +++---- server/datastore/mysql/software_installers.go | 88 +++-- server/datastore/mysql/statistics.go | 3 +- server/datastore/mysql/vpp.go | 81 ++-- 25 files changed, 979 insertions(+), 897 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 6d91765b75c..7e6ea425321 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -60,7 +60,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'script_name', COALESCE(ses.name, scr.name, ''), 'script_execution_id', ua.execution_id, 'batch_execution_id', bahr.batch_execution_id, - 'async', NOT ua.payload->'$.sync_request', + 'async', COALESCE(ua.payload->>'$.sync_request', '0') != '1', 'policy_id', sua.policy_id, 'policy_name', p.name ) as details, @@ -104,7 +104,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_package', COALESCE(si.filename, ua.payload->>'$.installer_filename', ''), 'install_uuid', ua.execution_id, 'status', 'pending_install', - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'source', COALESCE(st.source, ua.payload->>'$.source'), 'policy_id', siua.policy_id, 'policy_name', p.name @@ -146,7 +146,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_title', COALESCE(st.name, ua.payload->>'$.software_title_name', ''), 'script_execution_id', ua.execution_id, 'status', 'pending_uninstall', - 'self_service', COALESCE(ua.payload->'$.self_service', FALSE) IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'source', COALESCE(st.source, ua.payload->>'$.source'), 'policy_id', siua.policy_id, 'policy_name', p.name @@ -188,7 +188,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_title', COALESCE(st.name, ''), 'app_store_id', vaua.adam_id, 'command_uuid', ua.execution_id, - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'status', 'pending_install', 'host_platform', h.platform ) AS details, @@ -228,7 +228,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'host_display_name', COALESCE(hdn.display_name, ''), 'software_title', COALESCE(st.name, ''), 'command_uuid', ua.execution_id, - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'status', 'pending_install' ) AS details, IF(ua.activated_at IS NULL, 0, 1) as topmost, @@ -809,10 +809,10 @@ func (ds *Datastore) GetHostUpcomingActivityMeta(ctx context.Context, hostID uin ua.activated_at, ua.activity_type, CASE - WHEN hma.lock_ref = :execution_id THEN :lock_action - WHEN hma.unlock_ref = :execution_id THEN :unlock_action - WHEN hma.wipe_ref = :execution_id THEN :wipe_action - ELSE :none_action + WHEN hma.lock_ref = :execution_id THEN CAST(:lock_action AS UNSIGNED) + WHEN hma.unlock_ref = :execution_id THEN CAST(:unlock_action AS UNSIGNED) + WHEN hma.wipe_ref = :execution_id THEN CAST(:wipe_action AS UNSIGNED) + ELSE CAST(:none_action AS UNSIGNED) END AS well_known_action FROM upcoming_activities ua @@ -1070,9 +1070,9 @@ SELECT sua.script_id, sua.policy_id, ua.user_id, - COALESCE(ua.payload->'$.sync_request', 0), + COALESCE(ua.payload->>'$.sync_request', '0') = '1', sua.setup_experience_script_id, - COALESCE(ua.payload->'$.is_internal', 0) + COALESCE(ua.payload->>'$.is_internal', '0') = '1' FROM upcoming_activities ua INNER JOIN script_upcoming_activities sua @@ -1108,7 +1108,7 @@ SELECT ua.host_id, siua.software_installer_id, ua.user_id, - COALESCE(ua.payload->'$.self_service', 0), + COALESCE(ua.payload->>'$.self_service', '0') = '1', siua.policy_id, COALESCE(si.filename, ua.payload->>'$.installer_filename', '[deleted installer]'), COALESCE(si.version, ua.payload->>'$.version', 'unknown'), @@ -1120,7 +1120,7 @@ SELECT -- the number of prior tries. +1 makes this the next attempt in sequence: -- first install = 1, first retry = 2, second retry = 3, etc. CASE - WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->'$.with_retries', 0) = 1 THEN ( + WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->>'$.with_retries', '0') = '1' THEN ( SELECT COUNT(*) + 1 FROM host_software_installs hsi2 WHERE hsi2.host_id = ua.host_id @@ -1255,7 +1255,7 @@ SELECT ua.execution_id, ua.user_id, ua.payload->>'$.associated_event_id', - COALESCE(ua.payload->'$.self_service', 0), + COALESCE(ua.payload->>'$.self_service', '0') = '1', vaua.policy_id FROM upcoming_activities ua @@ -1291,7 +1291,7 @@ SELECT ua.execution_id, ua.user_id, iha.platform, - COALESCE(ua.payload->'$.self_service', 0) + COALESCE(ua.payload->>'$.self_service', '0') = '1' FROM upcoming_activities ua INNER JOIN in_house_app_upcoming_activities ihua diff --git a/server/datastore/mysql/android.go b/server/datastore/mysql/android.go index fc69b463c9e..7f8eda103c4 100644 --- a/server/datastore/mysql/android.go +++ b/server/datastore/mysql/android.go @@ -47,23 +47,8 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost detail_updated_at, label_updated_at, uuid - ) VALUES ( - :node_key, - :hostname, - :computer_name, - :platform, - :os_version, - :build, - :memory, - :team_id, - :hardware_serial, - :cpu_type, - :hardware_model, - :hardware_vendor, - :detail_updated_at, - :label_updated_at, - :uuid - ) ON DUPLICATE KEY UPDATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ds.dialect.OnDuplicateKey("id", ` hostname = VALUES(hostname), computer_name = VALUES(computer_name), platform = VALUES(platform), @@ -78,28 +63,27 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost detail_updated_at = VALUES(detail_updated_at), label_updated_at = VALUES(label_updated_at), uuid = VALUES(uuid) - ` - result, err := sqlx.NamedExecContext(ctx, tx, stmt, map[string]interface{}{ - "node_key": host.NodeKey, - "hostname": host.Hostname, - "computer_name": host.ComputerName, - "platform": host.Platform, - "os_version": host.OSVersion, - "build": host.Build, - "memory": host.Memory, - "team_id": host.TeamID, - "hardware_serial": host.HardwareSerial, - "cpu_type": host.CPUType, - "hardware_model": host.HardwareModel, - "hardware_vendor": host.HardwareVendor, - "detail_updated_at": host.DetailUpdatedAt, - "label_updated_at": host.LabelUpdatedAt, - "uuid": host.UUID, - }) + `) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, + host.NodeKey, + host.Hostname, + host.ComputerName, + host.Platform, + host.OSVersion, + host.Build, + host.Memory, + host.TeamID, + host.HardwareSerial, + host.CPUType, + host.HardwareModel, + host.HardwareVendor, + host.DetailUpdatedAt, + host.LabelUpdatedAt, + host.UUID, + ) if err != nil { return ctxerr.Wrap(ctx, err, "new Android host") } - id, _ := result.LastInsertId() if id == 0 { // This was an UPDATE, not an INSERT, so we need to get the host ID var hostID uint @@ -113,7 +97,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost } host.Device.HostID = host.Host.ID - err = upsertHostDisplayNames(ctx, tx, *host.Host) + err = upsertHostDisplayNames(ctx, tx, ds.dialect, *host.Host) if err != nil { return ctxerr.Wrap(ctx, err, "new Android host display name") } @@ -124,7 +108,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost // create entry in host_mdm as enrolled (manually), because currently all // android hosts are necessarily MDM-enrolled when created. - if err := upsertAndroidHostMDMInfoDB(ctx, tx, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { + if err := upsertAndroidHostMDMInfoDB(ctx, tx, ds.dialect, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { return ctxerr.Wrap(ctx, err, "new Android host MDM info") } @@ -217,7 +201,7 @@ func (ds *Datastore) UpdateAndroidHost(ctx context.Context, host *fleet.AndroidH if fromEnroll { // update host_mdm to set enrolled back to true - if err := upsertAndroidHostMDMInfoDB(ctx, tx, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { + if err := upsertAndroidHostMDMInfoDB(ctx, tx, ds.dialect, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { return ctxerr.Wrap(ctx, err, "update Android host MDM info") } @@ -367,7 +351,7 @@ func (ds *Datastore) insertAndroidHostLabelMembershipTx(ctx context.Context, tx _, err = tx.ExecContext(ctx, ` INSERT INTO label_membership (host_id, label_id) VALUES (?, ?), (?, ?) - ON DUPLICATE KEY UPDATE host_id = host_id`, + `+ds.dialect.OnDuplicateKey("host_id,label_id", `host_id = VALUES(host_id)`), hostID, allHostsLabelID, hostID, androidLabelID) if err != nil { return ctxerr.Wrap(ctx, err, "set label membership") @@ -429,19 +413,17 @@ UPDATE host_mdm return rows > 0, nil } -func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, serverURL string, companyOwned, enrolled bool, hostID uint) error { - result, err := tx.ExecContext(ctx, ` +func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, serverURL string, companyOwned, enrolled bool, hostID uint) error { + mdmID, err := insertAndGetIDTx(ctx, tx, dialect, ` INSERT INTO mobile_device_management_solutions (name, server_url) VALUES (?, ?) - ON DUPLICATE KEY UPDATE server_url = VALUES(server_url)`, + `+dialect.OnDuplicateKey("name, server_url", "server_url = VALUES(server_url)"), fleet.WellKnownMDMFleet, serverURL) if err != nil { return ctxerr.Wrap(ctx, err, "upsert mdm solution") } - - var mdmID int64 - if insertOnDuplicateDidInsertOrUpdate(result) { - mdmID, _ = result.LastInsertId() - } else { + if mdmID == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM mobile_device_management_solutions WHERE name = ? AND server_url = ?` if err := sqlx.GetContext(ctx, tx, &mdmID, stmt, fleet.WellKnownMDMFleet, serverURL); err != nil { return ctxerr.Wrap(ctx, err, "query mdm solution id") @@ -455,7 +437,7 @@ func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, serverU _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_mdm (enrolled, server_url, installed_from_dep, mdm_id, is_server, is_personal_enrollment, host_id) VALUES %s - ON DUPLICATE KEY UPDATE enrolled = VALUES(enrolled), server_url = VALUES(server_url), mdm_id = VALUES(mdm_id), is_personal_enrollment = VALUES(is_personal_enrollment)`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id", "enrolled = VALUES(enrolled), server_url = VALUES(server_url), mdm_id = VALUES(mdm_id), is_personal_enrollment = VALUES(is_personal_enrollment)"), strings.Join(parts, ",")), args...) return ctxerr.Wrap(ctx, err, "upsert host mdm info") } @@ -484,7 +466,7 @@ INSERT INTO res, err := tx.ExecContext(ctx, insertProfileStmt, profileUUID, teamID, cp.Name, cp.RawJSON, cp.Name, teamID, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return &existsError{ ResourceType: "MDMAndroidConfigProfile.Name", Identifier: cp.Name, @@ -527,7 +509,7 @@ INSERT INTO if len(labels) == 0 { profsWithoutLabel = append(profsWithoutLabel, profileUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profsWithoutLabel, "android"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profsWithoutLabel, "android"); err != nil { return ctxerr.Wrap(ctx, err, "inserting android profile label associations") } @@ -1139,7 +1121,7 @@ func (ds *Datastore) BulkUpsertMDMAndroidHostProfiles(ctx context.Context, paylo can_reverify ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail), @@ -1149,7 +1131,7 @@ func (ds *Datastore) BulkUpsertMDMAndroidHostProfiles(ctx context.Context, paylo request_fail_count = VALUES(request_fail_count), included_in_policy_version = VALUES(included_in_policy_version), can_reverify = VALUES(can_reverify) -`, strings.TrimSuffix(valuePart, ","), +`), strings.TrimSuffix(valuePart, ","), ) // Taken from BulkUpsertMDMAppleHostProfiles: We need to run with retry @@ -1355,7 +1337,7 @@ WHERE } // Insert or update incoming profiles - const insertNewOrEditedProfile = ` + insertNewOrEditedProfile := ` INSERT INTO mdm_android_configuration_profiles ( profile_uuid, team_id, @@ -1363,11 +1345,11 @@ WHERE raw_json, uploaded_at ) VALUES (CONCAT('` + fleet.MDMAndroidProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, CURRENT_TIMESTAMP(6)) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("profile_uuid", ` raw_json = VALUES(raw_json), name = VALUES(name), uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)) -` +`) for _, p := range profiles { var res sql.Result if res, err = tx.ExecContext(ctx, insertNewOrEditedProfile, profileTeamID, p.Name, p.RawJSON); err != nil { @@ -1815,9 +1797,9 @@ func (ds *Datastore) updateAndroidAppConfigurationTx(ctx context.Context, tx sql INSERT INTO android_app_configurations (application_id, team_id, global_or_team_id, configuration) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` configuration = VALUES(configuration) - ` + `) _, err = tx.ExecContext(ctx, stmt, appID, ptr.UintOrNilIfZero(teamID), teamID, config) if err != nil { diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 1b5cdbaac28..644adc454ff 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -212,30 +212,55 @@ INSERT INTO if err != nil { return err } - res, err := tx.ExecContext(ctx, stmt, - profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, - teamID, cp.Name, teamID) - if err != nil { - switch { - case IsDuplicate(err): - return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) - default: - return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + if ds.dialect.ReturningID() != "" { + // PostgreSQL: RETURNING profile_id (this table uses profile_id, not id) + err := tx.QueryRowxContext(ctx, stmt+" RETURNING profile_id", + profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, + teamID, cp.Name, teamID).Scan(&profileID) + if errors.Is(err, sql.ErrNoRows) { + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + Identifier: cp.Name, + TeamID: cp.TeamID, + } + } else if err != nil { + switch { + case ds.dialect.IsDuplicate(err): + return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) + default: + return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + } + } + } else { + res, err := tx.ExecContext(ctx, stmt, + profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, + teamID, cp.Name, teamID) + if err != nil { + switch { + case ds.dialect.IsDuplicate(err): + return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) + default: + return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + } } - } - aff, _ := res.RowsAffected() - if aff == 0 { - return &existsError{ - ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", - Identifier: cp.Name, - TeamID: cp.TeamID, + aff, _ := res.RowsAffected() + if aff == 0 { + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + Identifier: cp.Name, + TeamID: cp.TeamID, + } } - } - // record the ID as we want to return a fleet.Profile instance with it - // filled in. - profileID, _ = res.LastInsertId() + // record the ID as we want to return a fleet.Profile instance with it + // filled in. + profileID, _ = res.LastInsertId() // PG: returns 0 + if profileID == 0 { + // Fallback for PG: get the ID by profile_uuid + _ = sqlx.GetContext(ctx, tx, &profileID, `SELECT profile_id FROM mdm_apple_configuration_profiles WHERE profile_uuid = ?`, profUUID) + } + } labels := make([]fleet.ConfigurationProfileLabel, 0, len(cp.LabelsIncludeAll)+len(cp.LabelsIncludeAny)+len(cp.LabelsExcludeAny)) for i := range cp.LabelsIncludeAll { @@ -260,10 +285,10 @@ INSERT INTO if len(labels) == 0 { profWithoutLabels = append(profWithoutLabels, profUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profWithoutLabels, "darwin"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profWithoutLabels, "darwin"); err != nil { return ctxerr.Wrap(ctx, err, "inserting darwin profile label associations") } - if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, []fleet.MDMProfileUUIDFleetVariables{ + if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: profUUID, FleetVariables: usesFleetVars}, }, "darwin"); err != nil { return ctxerr.Wrap(ctx, err, "inserting darwin profile variable associations") @@ -907,22 +932,21 @@ func (ds *Datastore) NewMDMAppleEnrollmentProfile( ctx context.Context, payload fleet.MDMAppleEnrollmentProfilePayload, ) (*fleet.MDMAppleEnrollmentProfile, error) { - res, err := ds.writer(ctx).ExecContext(ctx, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), ` INSERT INTO mdm_apple_enrollment_profiles (token, type, dep_profile) VALUES (?, ?, ?) -ON DUPLICATE KEY UPDATE +`+ds.dialect.OnDuplicateKey("id", ` token = VALUES(token), type = VALUES(type), dep_profile = VALUES(dep_profile) -`, +`), payload.Token, payload.Type, payload.DEPProfile, ) if err != nil { return nil, ctxerr.Wrap(ctx, err) } - id, _ := res.LastInsertId() return &fleet.MDMAppleEnrollmentProfile{ ID: uint(id), //nolint:gosec // dismiss G115 Token: payload.Token, @@ -1155,15 +1179,13 @@ WHERE } func (ds *Datastore) NewMDMAppleInstaller(ctx context.Context, name string, size int64, manifest string, installer []byte, urlToken string) (*fleet.MDMAppleInstaller, error) { - res, err := ds.writer(ctx).ExecContext( - ctx, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO mdm_apple_installers (name, size, manifest, installer, url_token) VALUES (?, ?, ?, ?, ?)`, name, size, manifest, installer, urlToken, ) if err != nil { return nil, ctxerr.Wrap(ctx, err) } - id, _ := res.LastInsertId() return &fleet.MDMAppleInstaller{ ID: uint(id), //nolint:gosec // dismiss G115 Size: size, @@ -1272,13 +1294,14 @@ func (ds *Datastore) MDMAppleUpsertHost(ctx context.Context, mdmHost *fleet.Host return ctxerr.Wrap(ctx, err, "mdm apple upsert host get app config") } return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return ingestMDMAppleDeviceFromCheckinDB(ctx, tx, mdmHost, ds.logger, appCfg, fromPersonalEnrollment) + return ingestMDMAppleDeviceFromCheckinDB(ctx, tx, ds.dialect, mdmHost, ds.logger, appCfg, fromPersonalEnrollment) }) } func ingestMDMAppleDeviceFromCheckinDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, mdmHost *fleet.Host, logger *slog.Logger, appCfg *fleet.AppConfig, @@ -1296,13 +1319,13 @@ func ingestMDMAppleDeviceFromCheckinDB( enrolledHostInfo, err := matchHostDuringEnrollment(ctx, tx, mdmEnroll, true, "", mdmHost.UUID, mdmHost.HardwareSerial) switch { case errors.Is(err, sql.ErrNoRows): - return insertMDMAppleHostDB(ctx, tx, mdmHost, logger, appCfg, fromPersonalEnrollment) + return insertMDMAppleHostDB(ctx, tx, dialect, mdmHost, logger, appCfg, fromPersonalEnrollment) case err != nil: return ctxerr.Wrap(ctx, err, "get mdm apple host by serial number or udid") default: - return updateMDMAppleHostDB(ctx, tx, enrolledHostInfo.ID, mdmHost, appCfg, fromPersonalEnrollment) + return updateMDMAppleHostDB(ctx, tx, dialect, enrolledHostInfo.ID, mdmHost, appCfg, fromPersonalEnrollment) } } @@ -1322,6 +1345,7 @@ func mdmHostEnrollFields(mdmHost *fleet.Host) (refetchRequested bool, lastEnroll func updateMDMAppleHostDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, hostID uint, mdmHost *fleet.Host, appCfg *fleet.AppConfig, @@ -1376,7 +1400,7 @@ func updateMDMAppleHostDB( return ctxerr.Wrap(ctx, err, "error clearing mdm apple host_mdm_actions") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg, false, fromPersonalEnrollment, hostID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, dialect, appCfg, false, fromPersonalEnrollment, hostID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } @@ -1386,6 +1410,7 @@ func updateMDMAppleHostDB( func insertMDMAppleHostDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, mdmHost *fleet.Host, logger *slog.Logger, appCfg *fleet.AppConfig, @@ -1404,8 +1429,10 @@ func insertMDMAppleHostDB( refetch_requested ) VALUES (?,?,?,?,?,?,?,?)` - res, err := tx.ExecContext( + id, err := insertAndGetIDTx( ctx, + tx, + dialect, insertStmt, mdmHost.HardwareSerial, mdmHost.UUID, @@ -1419,26 +1446,21 @@ func insertMDMAppleHostDB( if err != nil { return ctxerr.Wrap(ctx, err, "insert mdm apple host") } - - id, err := res.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "last insert id mdm apple host") - } if id < 1 { - return ctxerr.Wrap(ctx, err, "ingest mdm apple host unexpected last insert id") + return ctxerr.New(ctx, "ingest mdm apple host unexpected last insert id") } mdmHost.ID = uint(id) - if err := upsertHostDisplayNames(ctx, tx, *mdmHost); err != nil { + if err := upsertHostDisplayNames(ctx, tx, dialect, *mdmHost); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert display names") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, *mdmHost); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, dialect, logger, *mdmHost); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert label membership") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg, false, fromPersonalEnrollment, mdmHost.ID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, dialect, appCfg, false, fromPersonalEnrollment, mdmHost.ID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } return nil @@ -1466,6 +1488,7 @@ type hostToCreateFromMDM struct { func createHostFromMDMDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, logger *slog.Logger, devices []hostToCreateFromMDM, fromADE bool, @@ -1581,11 +1604,11 @@ func createHostFromMDMDB( } } - if err := upsertHostDisplayNames(ctx, tx, hosts...); err != nil { + if err := upsertHostDisplayNames(ctx, tx, dialect, hosts...); err != nil { return 0, nil, ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert display names") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, hosts...); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, dialect, logger, hosts...); err != nil { return 0, nil, ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert label membership") } @@ -1604,6 +1627,7 @@ func createHostFromMDMDB( if err := upsertMDMAppleHostMDMInfoDB( ctx, tx, + dialect, appCfg, fromADE, false, @@ -1630,7 +1654,7 @@ func (ds *Datastore) IngestMDMAppleDeviceFromOTAEnrollment( UUID: &deviceInfo.UDID, }, } - _, hosts, err := createHostFromMDMDB(ctx, tx, ds.logger, toInsert, false, teamID, teamID, teamID) + _, hosts, err := createHostFromMDMDB(ctx, tx, ds.dialect, ds.logger, toInsert, false, teamID, teamID, teamID) if idpUUID != "" && len(hosts) > 0 { host := hosts[0] ds.logger.InfoContext(ctx, fmt.Sprintf("associating host %s with idp account %s", host.UUID, idpUUID)) @@ -1697,6 +1721,7 @@ func (ds *Datastore) IngestMDMAppleDevicesFromDEPSync( n, hosts, err := createHostFromMDMDB( ctx, tx, + ds.dialect, ds.logger, htc, true, @@ -1767,7 +1792,7 @@ func upsertHostDEPAssignmentsDB(ctx context.Context, tx sqlx.ExtContext, hosts [ return nil } -func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fleet.Host) error { +func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hosts ...fleet.Host) error { var args []interface{} var parts []string for _, h := range hosts { @@ -1777,7 +1802,7 @@ func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fl _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_display_names (host_id, display_name) VALUES %s - ON DUPLICATE KEY UPDATE display_name = VALUES(display_name)`, strings.Join(parts, ",")), + `+dialect.OnDuplicateKey("host_id", `display_name = VALUES(display_name)`), strings.Join(parts, ",")), args...) if err != nil { return ctxerr.Wrap(ctx, err, "upsert host display names") @@ -1786,7 +1811,7 @@ func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fl return nil } -func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg *fleet.AppConfig, fromSync, fromPersonalEnrollment bool, hostIDs ...uint) error { +func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, appCfg *fleet.AppConfig, fromSync, fromPersonalEnrollment bool, hostIDs ...uint) error { if len(hostIDs) == 0 { return nil } @@ -1800,18 +1825,16 @@ func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg // enrolled yet. enrolled := !fromSync - result, err := tx.ExecContext(ctx, ` + mdmID, err := insertAndGetIDTx(ctx, tx, dialect, ` INSERT INTO mobile_device_management_solutions (name, server_url) VALUES (?, ?) - ON DUPLICATE KEY UPDATE server_url = VALUES(server_url)`, + `+dialect.OnDuplicateKey("name, server_url", "server_url = VALUES(server_url)"), fleet.WellKnownMDMFleet, serverURL) if err != nil { return ctxerr.Wrap(ctx, err, "upsert mdm solution") } - - var mdmID int64 - if insertOnDuplicateDidInsertOrUpdate(result) { - mdmID, _ = result.LastInsertId() - } else { + if mdmID == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM mobile_device_management_solutions WHERE name = ? AND server_url = ?` if err := sqlx.GetContext(ctx, tx, &mdmID, stmt, fleet.WellKnownMDMFleet, serverURL); err != nil { return ctxerr.Wrap(ctx, err, "query mdm solution id") @@ -1827,12 +1850,12 @@ func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_mdm (enrolled, server_url, installed_from_dep, mdm_id, is_server, host_id, is_personal_enrollment) VALUES %s - ON DUPLICATE KEY UPDATE enrolled = VALUES(enrolled)`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id", "enrolled = VALUES(enrolled)"), strings.Join(parts, ",")), args...) return ctxerr.Wrap(ctx, err, "upsert host mdm info") } -func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext, logger *slog.Logger, hosts ...fleet.Host) error { +func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, logger *slog.Logger, hosts ...fleet.Host) error { // Builtin label memberships are usually inserted when the first distributed // query results are received; however, we want to insert pending MDM hosts // now because it may still be some time before osquery is running on these @@ -1892,7 +1915,7 @@ func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext } _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO label_membership (host_id, label_id) VALUES %s - ON DUPLICATE KEY UPDATE host_id = host_id`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id,label_id", `host_id = VALUES(host_id)`), strings.Join(parts, ",")), args...) if err != nil { return ctxerr.Wrap(ctx, err, "upsert label membership") } @@ -2214,14 +2237,14 @@ INSERT INTO hosts ( // Upsert related host tables for the restored host just as if it were initially ingested // from DEP sync. Note we are not upserting host_dep_assignments in order to preserve the // existing timestamps. - if err := upsertHostDisplayNames(ctx, tx, *host); err != nil { + if err := upsertHostDisplayNames(ctx, tx, ds.dialect, *host); err != nil { // TODO: Why didn't this work as expected? return ctxerr.Wrap(ctx, err, "restore pending dep host display name") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, ds.logger, *host); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, ds.dialect, ds.logger, *host); err != nil { return ctxerr.Wrap(ctx, err, "restore pending dep host label membership") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, ac, true, false, host.ID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, ds.dialect, ac, true, false, host.ID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } @@ -2354,21 +2377,21 @@ WHERE identifier NOT IN (?) ` - const insertNewOrEditedProfile = ` + insertNewOrEditedProfile := ` INSERT INTO mdm_apple_configuration_profiles ( profile_uuid, team_id, identifier, name, scope, mobileconfig, checksum, uploaded_at, secrets_updated_at ) VALUES -- see https://stackoverflow.com/a/51393124/1094941 - ( CONCAT('` + fleet.MDMAppleProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(6), ?) -ON DUPLICATE KEY UPDATE + ( CONCAT('` + fleet.MDMAppleProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP(6), ?) +` + ds.dialect.OnDuplicateKey("profile_uuid", ` uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)), secrets_updated_at = VALUES(secrets_updated_at), checksum = VALUES(checksum), name = VALUES(name), mobileconfig = VALUES(mobileconfig) -` +`) // use a profile team id of 0 if no-team var profTeamID uint @@ -2459,7 +2482,7 @@ ON DUPLICATE KEY UPDATE // contents is the same as it was already). for _, p := range incomingProfs { if result, err = tx.ExecContext(ctx, insertNewOrEditedProfile, profTeamID, p.Identifier, p.Name, p.Scope, - p.Mobileconfig, p.SecretsUpdatedAt); err != nil { + p.Mobileconfig, p.Mobileconfig, p.SecretsUpdatedAt); err != nil { return false, ctxerr.Wrapf(ctx, err, "insert new/edited profile with identifier %q", p.Identifier) } didInsertOrUpdate := insertOnDuplicateDidInsertOrUpdate(result) @@ -2807,15 +2830,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( scope ) VALUES %s - ON DUPLICATE KEY UPDATE + %s + `, strings.TrimSuffix(valuePart, ","), ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` operation_type = VALUES(operation_type), status = VALUES(status), command_uuid = VALUES(command_uuid), checksum = VALUES(checksum), secrets_updated_at = VALUES(secrets_updated_at), detail = VALUES(detail), - scope = VALUES(scope) - `, strings.TrimSuffix(valuePart, ",")) + scope = VALUES(scope)`)) _, err := tx.ExecContext(ctx, baseStmt, args...) return ctxerr.Wrap(ctx, err, "bulk set pending profile status execute batch") @@ -3434,7 +3457,8 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload scope ) VALUES %s - ON DUPLICATE KEY UPDATE + %s`, + strings.TrimSuffix(valuePart, ","), ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", fmt.Sprintf(` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail), @@ -3446,8 +3470,7 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload profile_name = VALUES(profile_name), command_uuid = VALUES(command_uuid), variables_updated_at = VALUES(variables_updated_at), - scope = VALUES(scope)`, - strings.TrimSuffix(valuePart, ","), fleet.MDMOperationTypeRemove, + scope = VALUES(scope)`, fleet.MDMOperationTypeRemove)), ) // We need to run with retry due to deadlocks. @@ -3771,9 +3794,9 @@ func (ds *Datastore) InsertMDMIdPAccount(ctx context.Context, account *fleet.MDM (uuid, username, fullname, email) VALUES (COALESCE(NULLIF(TRIM(?), ''), UUID()), ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("uuid", ` username = VALUES(username), - fullname = VALUES(fullname)` + fullname = VALUES(fullname)`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, account.UUID, account.Username, account.Fullname, account.Email) return ctxerr.Wrap(ctx, err, "creating new MDM IdP account") @@ -3994,21 +4017,21 @@ func (ds *Datastore) BulkUpsertMDMAppleConfigProfiles(ctx context.Context, paylo teamID = *cp.TeamID } - args = append(args, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.SecretsUpdatedAt) + args = append(args, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt) // see https://stackoverflow.com/a/51393124/1094941 - sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(), ?),") + sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP(), ?),") } stmt := fmt.Sprintf(` INSERT INTO mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, scope, mobileconfig, checksum, uploaded_at, secrets_updated_at) VALUES %s - ON DUPLICATE KEY UPDATE + %s +`, strings.TrimSuffix(sb.String(), ","), ds.dialect.OnDuplicateKey("profile_uuid", ` uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), mobileconfig = VALUES(mobileconfig), checksum = VALUES(checksum), - secrets_updated_at = VALUES(secrets_updated_at) -`, strings.TrimSuffix(sb.String(), ",")) + secrets_updated_at = VALUES(secrets_updated_at)`)) if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { return ctxerr.Wrapf(ctx, err, "upsert mdm config profiles") @@ -4033,7 +4056,7 @@ func (ds *Datastore) InsertMDMAppleBootstrapPackage(ctx context.Context, bp *fle const insStmt = `INSERT INTO mdm_apple_bootstrap_packages (team_id, name, sha256, bytes, token) VALUES (?, ?, ?, ?, ?)` execInsert := func(args ...any) error { if _, err := ds.writer(ctx).ExecContext(ctx, insStmt, args...); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("BootstrapPackage", fmt.Sprintf("for team %d", bp.TeamID))) } return ctxerr.Wrap(ctx, err, "create bootstrap package") @@ -4108,7 +4131,7 @@ WHERE team_id = 0 ` _, err := tx.ExecContext(ctx, insertStmt, toTeamID, uuid.New().String()) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, &existsError{ ResourceType: "BootstrapPackage", TeamID: &toTeamID, @@ -4226,14 +4249,14 @@ func (ds *Datastore) GetMDMAppleBootstrapPackageSummary(ctx context.Context, tea func (ds *Datastore) RecordSkippedHostBootstrapPackage(ctx context.Context, hostUUID string) error { stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (host_uuid, command_uuid, skipped) VALUES (?, NULL, 1) - ON DUPLICATE KEY UPDATE skipped = 1, command_uuid = NULL` + ` + ds.dialect.OnDuplicateKey("host_uuid", `skipped = 1, command_uuid = NULL`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostUUID) return ctxerr.Wrap(ctx, err, "record skipped bootstrap package") } func (ds *Datastore) RecordHostBootstrapPackage(ctx context.Context, commandUUID string, hostUUID string) error { stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (command_uuid, host_uuid, skipped) VALUES (?, ?, 0) - ON DUPLICATE KEY UPDATE command_uuid = command_uuid, skipped = 0` + ` + ds.dialect.OnDuplicateKey("host_uuid", `command_uuid = VALUES(command_uuid), skipped = 0`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, commandUUID, hostUUID) return ctxerr.Wrap(ctx, err, "record bootstrap package command") } @@ -4366,16 +4389,16 @@ WHERE } func (ds *Datastore) SetOrUpdateMDMAppleSetupAssistant(ctx context.Context, asst *fleet.MDMAppleSetupAssistant) (*fleet.MDMAppleSetupAssistant, error) { - const stmt = ` + stmt := ` INSERT INTO mdm_apple_setup_assistants (team_id, global_or_team_id, name, profile) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` updated_at = IF(profile = VALUES(profile) AND name = VALUES(name), updated_at, CURRENT_TIMESTAMP), name = VALUES(name), profile = VALUES(profile) -` +`) var globalOrTmID uint if asst.TeamID != nil { globalOrTmID = *asst.TeamID @@ -4414,7 +4437,7 @@ func (ds *Datastore) SetMDMAppleSetupAssistantProfileUUID(ctx context.Context, t global_or_team_id = ? )` - const upsertStmt = ` + upsertStmt := ` INSERT INTO mdm_apple_setup_assistant_profiles ( setup_assistant_id, abm_token_id, profile_uuid ) ( @@ -4429,9 +4452,9 @@ func (ds *Datastore) SetMDMAppleSetupAssistantProfileUUID(ctx context.Context, t mas.id IS NOT NULL AND abt.id IS NOT NULL ) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` profile_uuid = VALUES(profile_uuid) - ` + `) var globalOrTmID uint if teamID != nil { @@ -4600,7 +4623,7 @@ func (ds *Datastore) SetMDMAppleDefaultSetupAssistantProfileUUID(ctx context.Con DELETE FROM mdm_apple_default_setup_assistants WHERE global_or_team_id = ?` - const upsertStmt = ` + upsertStmt := ` INSERT INTO mdm_apple_default_setup_assistants (team_id, global_or_team_id, profile_uuid, abm_token_id) SELECT @@ -4609,9 +4632,9 @@ func (ds *Datastore) SetMDMAppleDefaultSetupAssistantProfileUUID(ctx context.Con abm_tokens abt WHERE abt.organization_name = ? - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` profile_uuid = VALUES(profile_uuid) -` +`) var globalOrTmID uint if teamID != nil { globalOrTmID = *teamID @@ -5177,7 +5200,7 @@ func (ds *Datastore) updateDeclarationsLabelAssociations(ctx context.Context, tx func (ds *Datastore) insertOrUpdateDeclarations(ctx context.Context, tx sqlx.ExtContext, incomingDeclarations []*fleet.MDMAppleDeclaration, teamID uint, ) (updatedDB bool, err error) { - const insertStmt = ` + insertStmt := ` INSERT INTO mdm_apple_declarations ( declaration_uuid, identifier, @@ -5190,13 +5213,13 @@ INSERT INTO mdm_apple_declarations ( VALUES ( ?,?,?,?,?,NOW(6),? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("declaration_uuid", ` uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name) AND IFNULL(secrets_updated_at = VALUES(secrets_updated_at), TRUE), uploaded_at, NOW(6)), secrets_updated_at = VALUES(secrets_updated_at), name = VALUES(name), identifier = VALUES(identifier), raw_json = VALUES(raw_json) -` +`) for _, d := range incomingDeclarations { declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString() @@ -5319,7 +5342,7 @@ INSERT INTO mdm_apple_declarations ( } func (ds *Datastore) SetOrUpdateMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { - const stmt = ` + stmt := ` INSERT INTO mdm_apple_declarations ( declaration_uuid, team_id, @@ -5337,10 +5360,10 @@ INSERT INTO mdm_apple_declarations ( SELECT 1 FROM mdm_android_configuration_profiles WHERE name = ? AND team_id = ? ) ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("declaration_uuid", ` identifier = VALUES(identifier), uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name) AND IFNULL(secrets_updated_at = VALUES(secrets_updated_at), TRUE), uploaded_at, NOW(6)), - raw_json = VALUES(raw_json)` + raw_json = VALUES(raw_json)`) return ds.insertOrUpsertMDMAppleDeclaration(ctx, stmt, declaration) } @@ -5362,7 +5385,7 @@ func (ds *Datastore) insertOrUpsertMDMAppleDeclaration(ctx context.Context, insO declaration.Name, tmID, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return ctxerr.Wrap(ctx, formatErrorDuplicateDeclaration(err, declaration)) default: return ctxerr.Wrap(ctx, err, "creating new apple mdm declaration") @@ -5971,11 +5994,11 @@ INSERT INTO host_mdm_apple_declarations (host_uuid, declaration_uuid, status, operation_type, detail, declaration_name, declaration_identifier, token, secrets_updated_at) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("host_uuid,declaration_uuid", ` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail) - ` + `) deletePendingRemovesStmt := ` DELETE FROM host_mdm_apple_declarations @@ -6325,12 +6348,12 @@ func (ds *Datastore) ReplaceMDMConfigAssets(ctx context.Context, assets []fleet. func (ds *Datastore) ListIOSAndIPadOSToRefetch(ctx context.Context, interval time.Duration) (devices []fleet.AppleDevicesToRefetch, err error, ) { - hostsStmt := ` -SELECT - h.id as host_id, - h.uuid as uuid, + hostsStmt := fmt.Sprintf(` +SELECT + h.id as host_id, + h.uuid as uuid, hmdm.installed_from_dep, - JSON_ARRAYAGG(hmc.command_type) as commands_already_sent + %s as commands_already_sent`, ds.dialect.JSONAgg("hmc.command_type")) + ` FROM hosts h INNER JOIN host_mdm hmdm ON hmdm.host_id = h.id INNER JOIN nano_enrollments ne ON ne.id = h.uuid @@ -6441,9 +6464,7 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?) return nil, ctxerr.Wrap(ctx, err, "encrypt abm_token with datastore.serverPrivateKey") } - res, err := ds.writer(ctx).ExecContext( - ctx, - stmt, + tokenID, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, tok.OrganizationName, tok.AppleID, tok.TermsExpired, @@ -6457,8 +6478,6 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?) return nil, ctxerr.Wrap(ctx, err, "inserting abm_token") } - tokenID, _ := res.LastInsertId() - tok.ID = uint(tokenID) //nolint:gosec // dismiss G115 cfg, err := ds.AppConfig(ctx) @@ -6749,11 +6768,11 @@ WHERE } func (ds *Datastore) AddHostMDMCommands(ctx context.Context, commands []fleet.HostMDMCommand) error { - const baseStmt = ` + baseStmt := ` INSERT INTO host_mdm_commands (host_id, command_type) VALUES %s - ON DUPLICATE KEY UPDATE - command_type = VALUES(command_type)` + ` + ds.dialect.OnDuplicateKey("host_id,command_type", ` + command_type = VALUES(command_type)`) for i := 0; i < len(commands); i += addHostMDMCommandsBatchSize { start := i @@ -6804,9 +6823,9 @@ func (ds *Datastore) CleanupHostMDMCommands(ctx context.Context) error { // Delete commands that don't have a corresponding host or have been sent over 1 day ago. // We are using 1 day instead of 7 days in case MDM commands fail to be sent or fail to process. They can be resent the next day. const stmt = ` - DELETE hmc FROM host_mdm_commands AS hmc - LEFT JOIN hosts h ON h.id = hmc.host_id - WHERE h.id IS NULL OR hmc.updated_at < NOW() - INTERVAL 1 DAY` + DELETE FROM host_mdm_commands + WHERE NOT EXISTS (SELECT 1 FROM hosts h WHERE h.id = host_mdm_commands.host_id) + OR host_mdm_commands.updated_at < NOW() - INTERVAL 1 DAY` if _, err := ds.writer(ctx).ExecContext(ctx, stmt); err != nil { return ctxerr.Wrap(ctx, err, "delete from host_mdm_commands") } @@ -6819,21 +6838,21 @@ func (ds *Datastore) CleanupHostMDMAppleProfiles(ctx context.Context) error { // This could also occur due to errors (i.e., large server/DB load) or server being stopped while processing the profiles. // After the entry is deleted, the mdm_apple_profile_manager job will try to requeue the profile. stmt := fmt.Sprintf(` - DELETE hmap FROM host_mdm_apple_profiles AS hmap + DELETE FROM host_mdm_apple_profiles WHERE ( - hmap.status IS NULL - OR hmap.status = '%s' + host_mdm_apple_profiles.status IS NULL + OR host_mdm_apple_profiles.status = '%s' ) - AND hmap.updated_at < NOW() - INTERVAL 1 HOUR + AND host_mdm_apple_profiles.updated_at < NOW() - INTERVAL 1 HOUR AND NOT EXISTS ( SELECT 1 FROM nano_enrollments ne - STRAIGHT_JOIN nano_enrollment_queue neq ON neq.id = ne.id - AND neq.command_uuid = hmap.command_uuid + JOIN nano_enrollment_queue neq ON neq.id = ne.id + AND neq.command_uuid = host_mdm_apple_profiles.command_uuid AND neq.active = 1 WHERE - ne.device_id = hmap.host_uuid + ne.device_id = host_mdm_apple_profiles.host_uuid AND ne.enabled = 1 );`, fleet.MDMDeliveryPending) @@ -7129,7 +7148,7 @@ func (ds *Datastore) AssociateHostMDMIdPAccountDB(ctx context.Context, hostUUID } func associateHostMDMIdPAccountDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, acctUUID string) error { - const stmt = ` + stmt := ` INSERT INTO host_mdm_idp_accounts (host_uuid, account_uuid) VALUES (?, ?) ON DUPLICATE KEY UPDATE @@ -7246,14 +7265,14 @@ func (ds *Datastore) SetLockCommandForLostModeCheckin(ctx context.Context, hostI } func (ds *Datastore) InsertHostLocationData(ctx context.Context, locData fleet.HostLocationData) error { - const stmt = ` + stmt := ` INSERT INTO host_last_known_locations (host_id, latitude, longitude) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_id", ` latitude = VALUES(latitude), longitude = VALUES(longitude) - ` + `) _, err := ds.writer(ctx).ExecContext(ctx, stmt, locData.HostID, locData.Latitude, locData.Longitude) return ctxerr.Wrap(ctx, err, "insert host location data") } @@ -7304,13 +7323,13 @@ func (ds *Datastore) SetHostsRecoveryLockPasswords(ctx context.Context, password stmt := ` INSERT INTO host_recovery_key_passwords (host_uuid, encrypted_password, status, operation_type) VALUES %s - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_uuid", ` encrypted_password = VALUES(encrypted_password), status = VALUES(status), operation_type = VALUES(operation_type), error_message = NULL, - deleted = 0 - ` + deleted = FALSE + `) placeholders := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?),", len(passwords)), ",") stmt = fmt.Sprintf(stmt, placeholders) @@ -7393,7 +7412,7 @@ func (ds *Datastore) GetHostsForRecoveryLockAction(ctx context.Context) ([]strin // - Have no recovery lock password record OR have a password with NULL status (command not yet enqueued) // Note: hosts with status pending, verified, or failed are NOT included // Note: hosts with operation_type='remove' are handled by RestoreRecoveryLockForReenabledHosts - const stmt = ` + stmt := fmt.Sprintf(` SELECT h.uuid FROM hosts h JOIN nano_enrollments ne ON ne.device_id = h.uuid @@ -7402,20 +7421,21 @@ func (ds *Datastore) GetHostsForRecoveryLockAction(ctx context.Context) ([]strin CROSS JOIN app_config_json ac LEFT JOIN host_recovery_key_passwords rkp ON rkp.host_uuid = h.uuid AND rkp.deleted = 0 WHERE h.platform = 'darwin' - AND h.cpu_type LIKE '%arm%' + AND h.cpu_type LIKE '%%arm%%' AND ne.enabled = 1 AND ne.type IN ('Device', 'User Enrollment (Device)') AND hm.enrolled = 1 AND ( -- Team hosts: check team config - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NOT NULL AND %s = 'true') OR -- No-team hosts: check appconfig - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NULL AND %s = 'true') ) AND (rkp.host_uuid IS NULL OR rkp.status IS NULL) LIMIT 500 - ` + `, ds.dialect.JSONUnquoteExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONUnquoteExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) var hostUUIDs []string if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hostUUIDs, stmt); err != nil { @@ -7454,11 +7474,13 @@ func (ds *Datastore) RestoreRecoveryLockForReenabledHosts(ctx context.Context) ( AND rkp.operation_type = '%s' AND (rkp.status = '%s' OR rkp.status IS NULL) AND ( - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NOT NULL AND %s = true) OR - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NULL AND %s = true) ) - `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending) + `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, + ds.dialect.JSONExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) result, err := ds.writer(ctx).ExecContext(ctx, stmt) if err != nil { @@ -7555,13 +7577,15 @@ func (ds *Datastore) ClaimHostsForRecoveryLockClear(ctx context.Context) ([]stri (rkp.operation_type = '%s' AND rkp.status IS NULL) ) AND ( - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = false) + (h.team_id IS NOT NULL AND %s != 'true') OR - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = false) + (h.team_id IS NULL AND %s != 'true') ) LIMIT 500 FOR UPDATE - `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove) + `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, + ds.dialect.JSONUnquoteExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONUnquoteExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) // Update all claimed hosts to remove/pending updateStmt := fmt.Sprintf(` diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go index 455e246dde4..cd63ba71f36 100644 --- a/server/datastore/mysql/calendar_events.go +++ b/server/datastore/mysql/calendar_events.go @@ -33,7 +33,7 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( } var id int64 if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - const calendarEventsQuery = ` + calendarEventsQuery := ` INSERT INTO calendar_events ( uuid_bin, email, @@ -42,16 +42,16 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( event, timezone ) VALUES (?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - uuid_bin = VALUES(uuid_bin), + ` + ds.dialect.OnDuplicateKey("id", `uuid_bin = VALUES(uuid_bin), start_time = VALUES(start_time), end_time = VALUES(end_time), event = VALUES(event), timezone = VALUES(timezone), - updated_at = CURRENT_TIMESTAMP; - ` - result, err := tx.ExecContext( + updated_at = CURRENT_TIMESTAMP`) + id, err = insertAndGetIDTx( ctx, + tx, + ds.dialect, calendarEventsQuery, UUID[:], email, @@ -63,26 +63,23 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( if err != nil { return ctxerr.Wrap(ctx, err, "insert calendar event") } - - if insertOnDuplicateDidInsertOrUpdate(result) { - id, _ = result.LastInsertId() - } else { + if id == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM calendar_events WHERE email = ?` if err := sqlx.GetContext(ctx, tx, &id, stmt, email); err != nil { return ctxerr.Wrap(ctx, err, "calendar event id") } } - const hostCalendarEventsQuery = ` + hostCalendarEventsQuery := ` INSERT INTO host_calendar_events ( host_id, calendar_event_id, webhook_status ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - webhook_status = VALUES(webhook_status), - calendar_event_id = VALUES(calendar_event_id); - ` + ` + ds.dialect.OnDuplicateKey("id", `webhook_status = VALUES(webhook_status), + calendar_event_id = VALUES(calendar_event_id)`) _, err = tx.ExecContext( ctx, hostCalendarEventsQuery, diff --git a/server/datastore/mysql/certificate_templates.go b/server/datastore/mysql/certificate_templates.go index 54dd213ed7f..61b9b3aaa28 100644 --- a/server/datastore/mysql/certificate_templates.go +++ b/server/datastore/mysql/certificate_templates.go @@ -180,7 +180,7 @@ func (ds *Datastore) GetCertificateTemplatesByTeamID(ctx context.Context, teamID } func (ds *Datastore) CreateCertificateTemplate(ctx context.Context, certificateTemplate *fleet.CertificateTemplate) (*fleet.CertificateTemplateResponse, error) { - result, err := ds.writer(ctx).ExecContext(ctx, ` + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), ` INSERT INTO certificate_templates ( name, team_id, @@ -189,17 +189,12 @@ func (ds *Datastore) CreateCertificateTemplate(ctx context.Context, certificateT ) VALUES (?, ?, ?, ?) `, certificateTemplate.Name, certificateTemplate.TeamID, certificateTemplate.CertificateAuthorityID, certificateTemplate.SubjectName) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return nil, ctxerr.Wrap(ctx, alreadyExists("CertificateTemplate", certificateTemplate.Name), "inserting certificate_template") } return nil, ctxerr.Wrap(ctx, err, "inserting certificate_template") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last insert id for certificate_template") - } - return &fleet.CertificateTemplateResponse{ CertificateTemplateResponseSummary: fleet.CertificateTemplateResponseSummary{ ID: uint(id), //nolint:gosec @@ -236,17 +231,17 @@ func (ds *Datastore) BatchUpsertCertificateTemplates(ctx context.Context, certif return nil, nil } - const sqlInsertCertificate = ` + sqlInsertCertificate := ` INSERT INTO certificate_templates ( name, team_id, certificate_authority_id, subject_name ) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` name = VALUES(name), team_id = VALUES(team_id) - ` + `) teamsModifiedSet := make(map[uint]struct{}) for _, cert := range certificateTemplates { @@ -350,7 +345,7 @@ func (ds *Datastore) CreatePendingCertificateTemplatesForExistingHosts( (hosts.team_id = ? OR (? = 0 AND hosts.team_id IS NULL)) AND hosts.platform = '%s' AND host_mdm.enrolled = 1 - ON DUPLICATE KEY UPDATE host_uuid = host_uuid + `+ds.dialect.OnDuplicateKey("host_uuid,certificate_template_id", `host_uuid = VALUES(host_uuid)`)+` `, fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, fleet.AndroidPlatform) result, err := ds.writer(ctx).ExecContext(ctx, stmt, certificateTemplateID, teamID, teamID) if err != nil { @@ -385,12 +380,12 @@ func (ds *Datastore) CreatePendingCertificateTemplatesForNewHost( UUID_TO_BIN(UUID(), true) FROM certificate_templates WHERE team_id = ? - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,certificate_template_id", ` -- allow 'remove' to transition to 'pending install', generating new uuid uuid = IF(operation_type = '%s', UUID_TO_BIN(UUID(), true), uuid), status = IF(operation_type = '%s', '%s', status), operation_type = IF(operation_type = '%s', '%s', operation_type) - `, fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, + `), fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeRemove, fleet.CertificateTemplatePending, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall) diff --git a/server/datastore/mysql/cron_stats.go b/server/datastore/mysql/cron_stats.go index a761a197275..1ecc6191c37 100644 --- a/server/datastore/mysql/cron_stats.go +++ b/server/datastore/mysql/cron_stats.go @@ -53,14 +53,10 @@ UNION func (ds *Datastore) InsertCronStats(ctx context.Context, statsType fleet.CronStatsType, name string, instance string, status fleet.CronStatsStatus) (int, error) { stmt := `INSERT INTO cron_stats (stats_type, name, instance, status) VALUES (?, ?, ?, ?)` - res, err := ds.writer(ctx).ExecContext(ctx, stmt, statsType, name, instance, status) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, statsType, name, instance, status) if err != nil { return 0, ctxerr.Wrap(ctx, err, "insert cron stats") } - id, err := res.LastInsertId() - if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insert cron stats last insert id") - } return int(id), nil } diff --git a/server/datastore/mysql/disk_encryption.go b/server/datastore/mysql/disk_encryption.go index bc7a87de04d..0b657c8d6aa 100644 --- a/server/datastore/mysql/disk_encryption.go +++ b/server/datastore/mysql/disk_encryption.go @@ -206,7 +206,7 @@ func (ds *Datastore) ClearPendingEscrow(ctx context.Context, hostID uint) error func (ds *Datastore) ReportEscrowError(ctx context.Context, hostID uint, errorMessage string) error { _, err := ds.writer(ctx).ExecContext(ctx, ` INSERT INTO host_disk_encryption_keys - (host_id, base64_encrypted, client_error) VALUES (?, '', ?) ON DUPLICATE KEY UPDATE client_error = VALUES(client_error) + (host_id, base64_encrypted, client_error) VALUES (?, '', ?) `+ds.dialect.OnDuplicateKey("host_id", `client_error = VALUES(client_error)`)+` `, hostID, errorMessage) return err } @@ -214,7 +214,7 @@ INSERT INTO host_disk_encryption_keys func (ds *Datastore) QueueEscrow(ctx context.Context, hostID uint) error { _, err := ds.writer(ctx).ExecContext(ctx, ` INSERT INTO host_disk_encryption_keys - (host_id, base64_encrypted, reset_requested) VALUES (?, '', TRUE) ON DUPLICATE KEY UPDATE reset_requested = TRUE + (host_id, base64_encrypted, reset_requested) VALUES (?, '', TRUE) `+ds.dialect.OnDuplicateKey("host_id", `reset_requested = TRUE`)+` `, hostID) return err } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 7ec1b6dc7d9..094547724f4 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -26,7 +26,7 @@ import ( "github.com/jmoiron/sqlx" ) -const hostHasIdentityCertSQL = `EXISTS(SELECT 1 FROM host_identity_scep_certificates hisc WHERE hisc.host_id = h.id AND hisc.revoked = 0)` +const hostHasIdentityCertSQL = `EXISTS(SELECT 1 FROM host_identity_scep_certificates hisc WHERE hisc.host_id = h.id AND hisc.revoked = false)` // Since many hosts may have issues, we need to batch the inserts of host issues. // This is a variable, so it can be adjusted during unit testing. @@ -138,9 +138,7 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext( - ctx, - sqlStatement, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, host.OsqueryHostID, host.DetailUpdatedAt, host.LabelUpdatedAt, @@ -166,7 +164,6 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host if err != nil { return ctxerr.Wrap(ctx, err, "new host") } - id, _ := result.LastInsertId() host.ID = uint(id) _, err = tx.ExecContext(ctx, @@ -207,10 +204,10 @@ func (ds *Datastore) SerialUpdateHost(ctx context.Context, host *fleet.Host) err } func (ds *Datastore) SaveHostPackStats(ctx context.Context, teamID *uint, hostID uint, stats []fleet.PackStats) error { - return saveHostPackStatsDB(ctx, ds.writer(ctx), teamID, hostID, stats) + return saveHostPackStatsDB(ctx, ds.writer(ctx), ds.dialect, teamID, hostID, stats) } -func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID uint, stats []fleet.PackStats) error { +func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper, teamID *uint, hostID uint, stats []fleet.PackStats) error { // NOTE: this implementation must be kept in sync with the async/batch version // in AsyncBatchSaveHostsScheduledQueryStats (in scheduled_queries.go) - that is, // the behaviour per host must be the same. @@ -282,11 +279,95 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID return nil } + // Deduplicate scheduled queries stats by (team_id, query_name) — last entry wins. + // PG's ON CONFLICT can't update the same row twice in a single INSERT. + if scheduledQueriesQueryCount > 1 { + type sqKey struct { + teamID uint + name string + } + argsPerRow := 12 // 2 (subquery) + 10 (values) + seen := make(map[sqKey]int) + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + key := sqKey{ + teamID: scheduledQueriesArgs[base].(uint), + name: scheduledQueriesArgs[base+1].(string), + } + seen[key] = i + } + if len(seen) < scheduledQueriesQueryCount { + var dedupedArgs []interface{} + dedupedCount := 0 + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + key := sqKey{ + teamID: scheduledQueriesArgs[base].(uint), + name: scheduledQueriesArgs[base+1].(string), + } + if seen[key] == i { // keep only last occurrence + dedupedArgs = append(dedupedArgs, scheduledQueriesArgs[base:base+argsPerRow]...) + dedupedCount++ + } + } + scheduledQueriesArgs = dedupedArgs + scheduledQueriesQueryCount = dedupedCount + } + } + + // Deduplicate user packs stats by (pack_name, query_name) — last entry wins. + // PG's ON CONFLICT can't update the same row twice in a single INSERT. + if userPacksQueryCount > 1 { + type packStatKey struct { + pack, query string + } + argsPerRow := 12 // 2 (subquery) + 10 (values) + seen := make(map[packStatKey]int) + for i := 0; i < userPacksQueryCount; i++ { + base := i * argsPerRow + key := packStatKey{ + pack: userPacksArgs[base].(string), + query: userPacksArgs[base+1].(string), + } + seen[key] = i + } + if len(seen) < userPacksQueryCount { + var dedupedArgs []interface{} + dedupedCount := 0 + for i := 0; i < userPacksQueryCount; i++ { + base := i * argsPerRow + key := packStatKey{ + pack: userPacksArgs[base].(string), + query: userPacksArgs[base+1].(string), + } + if seen[key] == i { // keep only last occurrence + dedupedArgs = append(dedupedArgs, userPacksArgs[base:base+argsPerRow]...) + dedupedCount++ + } + } + userPacksArgs = dedupedArgs + userPacksQueryCount = dedupedCount + } + } + if scheduledQueriesQueryCount > 0 { // This query will import stats for queries (new format). - values := strings.TrimSuffix(strings.Repeat("((SELECT q.id FROM queries q WHERE COALESCE(q.team_id, 0) = ? AND q.name = ?),?,?,?,?,?,?,?,?,?,?),", scheduledQueriesQueryCount), ",") - sql := fmt.Sprintf(` - INSERT IGNORE INTO scheduled_query_stats ( + // Uses INSERT...SELECT form so that rows where the query doesn't exist + // are naturally excluded (the SELECT returns 0 rows instead of NULL, + // which avoids NOT NULL violations on PG). + argsPerRow := 12 // 2 (subquery: teamID, name) + 10 (values) + var selectParts []string + var reorderedArgs []interface{} + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + selectParts = append(selectParts, + "SELECT q.id, ?,?,?,?,?,?,?,?,?,? FROM queries q WHERE COALESCE(q.team_id, 0) = ? AND q.name = ?") + // Reorder: value args first (host_id..wall_time), then subquery args (teamID, name) + reorderedArgs = append(reorderedArgs, scheduledQueriesArgs[base+2:base+argsPerRow]...) + reorderedArgs = append(reorderedArgs, scheduledQueriesArgs[base:base+2]...) + } + selectSQL := strings.Join(selectParts, " UNION ALL ") + sql := dialect.InsertIgnoreInto() + ` scheduled_query_stats ( scheduled_query_id, host_id, average_memory, @@ -299,7 +380,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID user_time, wall_time ) - VALUES %s ON DUPLICATE KEY UPDATE + ` + selectSQL + ` ` + dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` scheduled_query_id = VALUES(scheduled_query_id), host_id = VALUES(host_id), average_memory = VALUES(average_memory), @@ -310,9 +391,9 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID output_size = VALUES(output_size), system_time = VALUES(system_time), user_time = VALUES(user_time), - wall_time = VALUES(wall_time) - `, values) - if _, err := db.ExecContext(ctx, sql, scheduledQueriesArgs...); err != nil { + wall_time = VALUES(wall_time)`) + ` + ` + if _, err := db.ExecContext(ctx, sql, reorderedArgs...); err != nil { return ctxerr.Wrap(ctx, err, "insert query schedule stats") } } @@ -321,8 +402,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID // This query will import stats for 2017 packs. // NOTE(lucas): If more than one scheduled query reference the same query then only one of the stats will be written. values := strings.TrimSuffix(strings.Repeat("((SELECT sq.query_id FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.pack_type IS NULL AND p.name = ? AND sq.name = ?),?,?,?,?,?,?,?,?,?,?),", userPacksQueryCount), ",") - sql := fmt.Sprintf(` - INSERT IGNORE INTO scheduled_query_stats ( + sql := fmt.Sprintf(dialect.InsertIgnoreInto()+` scheduled_query_stats ( scheduled_query_id, host_id, average_memory, @@ -335,7 +415,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID user_time, wall_time ) - VALUES %s ON DUPLICATE KEY UPDATE + VALUES %s `+dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` scheduled_query_id = VALUES(scheduled_query_id), host_id = VALUES(host_id), average_memory = VALUES(average_memory), @@ -346,7 +426,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID output_size = VALUES(output_size), system_time = VALUES(system_time), user_time = VALUES(user_time), - wall_time = VALUES(wall_time) + wall_time = VALUES(wall_time)`)+` `, values) if _, err := db.ExecContext(ctx, sql, userPacksArgs...); err != nil { return ctxerr.Wrap(ctx, err, "insert pack stats") @@ -358,7 +438,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID // loadhostPacksStatsDB will load all the "2017 pack" stats for the given host. The scheduled // queries that haven't run yet are returned with zero values. -func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string) ([]fleet.PackStats, error) { +func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, dialect DialectHelper) ([]fleet.PackStats, error) { packs, err := listPacksForHost(ctx, db, hid) if err != nil { return nil, ctxerr.Wrapf(ctx, err, "list packs for host: %d", hid) @@ -372,7 +452,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, packIDs[i] = packs[i].ID packTypes[packs[i].ID] = packs[i].Type } - ds := dialect.From(goqu.I("scheduled_queries").As("sq")).Select( + ds := dialect.GoquDialect().From(goqu.I("scheduled_queries").As("sq")).Select( goqu.I("sq.name").As("scheduled_query_name"), goqu.I("sq.id").As("scheduled_query_id"), goqu.I("sq.query_name").As("query_name"), @@ -380,16 +460,16 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, goqu.I("p.name").As("pack_name"), goqu.I("p.id").As("pack_id"), goqu.COALESCE(goqu.I("sqs.average_memory"), 0).As("average_memory"), - goqu.COALESCE(goqu.I("sqs.denylisted"), false).As("denylisted"), + goqu.COALESCE(goqu.I("sqs.denylisted"), goqu.L("FALSE")).As("denylisted"), goqu.COALESCE(goqu.I("sqs.executions"), 0).As("executions"), goqu.I("sq.interval").As("schedule_interval"), - goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("timestamp(?)", common_mysql.DefaultNonZeroTime)).As("last_executed"), + goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("TIMESTAMP(?)", common_mysql.DefaultNonZeroTime)).As("last_executed"), goqu.COALESCE(goqu.I("sqs.output_size"), 0).As("output_size"), goqu.COALESCE(goqu.I("sqs.system_time"), 0).As("system_time"), goqu.COALESCE(goqu.I("sqs.user_time"), 0).As("user_time"), goqu.COALESCE(goqu.I("sqs.wall_time"), 0).As("wall_time"), ).Join( - dialect.From("packs").As("p").Select( + dialect.GoquDialect().From("packs").As("p").Select( goqu.I("id"), goqu.I("name"), ).Where(goqu.I("id").In(packIDs)), @@ -422,7 +502,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, goqu.I("sq.platform").IsNull(), // scheduled_queries.platform can be a comma-separated list of // platforms, e.g. "darwin,windows". - goqu.L("FIND_IN_SET(?, sq.platform)", fleet.PlatformFromHost(hostPlatform)).Neq(0), + goqu.L(dialect.FindInSet("?", "sq.platform"), fleet.PlatformFromHost(hostPlatform)).Neq(0), ), ) sql, args, err := ds.ToSQL() @@ -453,7 +533,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, // The filter is split into two statements joined by a UNION ALL to take advantage of indexes. // Using an OR in the WHERE clause causes a full table scan which causes issues with a large // queries table due to the high volume of live queries (created by zero trust workflows) -func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, teamID *uint) ([]fleet.QueryStats, error) { +func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, teamID *uint, dialect DialectHelper) ([]fleet.QueryStats, error) { var teamID_ uint if teamID != nil { teamID_ = *teamID @@ -469,14 +549,14 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, q.discard_data, q.automations_enabled, MAX(qr.last_fetched) as last_fetched, - COALESCE(sqs.average_memory, 0) AS average_memory, - COALESCE(sqs.denylisted, false) AS denylisted, - COALESCE(sqs.executions, 0) AS executions, - COALESCE(sqs.last_executed, TIMESTAMP(?)) AS last_executed, - COALESCE(sqs.output_size, 0) AS output_size, - COALESCE(sqs.system_time, 0) AS system_time, - COALESCE(sqs.user_time, 0) AS user_time, - COALESCE(sqs.wall_time, 0) AS wall_time + COALESCE(MAX(sqs.average_memory), 0) AS average_memory, + COALESCE(MAX(sqs.denylisted), false) AS denylisted, + COALESCE(MAX(sqs.executions), 0) AS executions, + COALESCE(MAX(sqs.last_executed), TIMESTAMP(?)) AS last_executed, + COALESCE(MAX(sqs.output_size), 0) AS output_size, + COALESCE(MAX(sqs.system_time), 0) AS system_time, + COALESCE(MAX(sqs.user_time), 0) AS user_time, + COALESCE(MAX(sqs.wall_time), 0) AS wall_time FROM queries q LEFT JOIN @@ -494,10 +574,10 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, LEFT JOIN query_results qr ON (q.id = qr.query_id AND qr.host_id = ?) ` - filter1 := ` + filter1 := fmt.Sprintf(` WHERE - (q.platform = '' OR q.platform IS NULL OR FIND_IN_SET(?, q.platform) != 0) - AND q.is_scheduled = 1 + (q.platform = '' OR q.platform IS NULL OR %s != 0)`, dialect.FindInSet("?", "q.platform")) + ` + AND q.is_scheduled = true AND (q.automations_enabled IS TRUE OR (q.discard_data IS FALSE AND q.logging_type = ?)) AND (q.team_id IS NULL OR q.team_id = ?) GROUP BY q.id @@ -711,7 +791,7 @@ func deleteHosts(ctx context.Context, tx sqlx.ExtContext, hostIDs []uint) error // no point trying the uuid-based tables if the host's uuid is missing if len(hostUUIDs) != 0 { for table, col := range additionalHostRefsByUUID { - stmt, args, err := sqlx.In(fmt.Sprintf("DELETE FROM `%s` WHERE `%s` IN (?)", table, col), hostUUIDs) + stmt, args, err := sqlx.In(fmt.Sprintf(`DELETE FROM "%s" WHERE "%s" IN (?)`, table, col), hostUUIDs) if err != nil { return ctxerr.Wrapf(ctx, err, "building delete statement for %s for hosts %v", table, hostUUIDs) } @@ -880,12 +960,12 @@ LIMIT host.DiskEncryptionEnabled = nil } - packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform) + packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, ds.dialect) if err != nil { return nil, err } host.PackStats = packStats - queriesStats, err := loadHostScheduledQueryStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, host.TeamID) + queriesStats, err := loadHostScheduledQueryStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, host.TeamID, ds.dialect) if err != nil { return nil, err } @@ -959,6 +1039,7 @@ func queryStatsToScheduledQueryStats(queriesStats []fleet.QueryStats, packName s const hostMDMSelect = `, JSON_OBJECT( 'enrollment_status', hmdm.enrollment_status, + 'dep_profile_error', CASE WHEN hdep.assign_profile_response IN ('` + string(fleet.DEPAssignProfileResponseFailed) + `', '` + string(fleet.DEPAssignProfileResponseThrottled) + `') THEN CAST(TRUE AS JSON) @@ -966,7 +1047,7 @@ const hostMDMSelect = `, END, 'server_url', CASE - WHEN hmdm.is_server = 1 THEN NULL + WHEN hmdm.is_server = true THEN NULL ELSE hmdm.server_url END, 'encryption_key_available', @@ -976,7 +1057,7 @@ const hostMDMSelect = `, * unmarshaller was having problems converting int values to * booleans. */ - WHEN hdek.decryptable IS NULL OR hdek.decryptable = 0 THEN CAST(FALSE AS JSON) + WHEN hdek.decryptable IS NULL OR hdek.decryptable = false THEN CAST(FALSE AS JSON) ELSE CAST(TRUE AS JSON) END, 'raw_decryptable', @@ -995,14 +1076,14 @@ const hostMDMSelect = `, FROM mdm_windows_enrollments mwe WHERE mwe.host_uuid = h.uuid AND mwe.device_state = '` + microsoft_mdm.MDMDeviceStateEnrolled + `' - AND hmdm.enrolled = 1 + AND hmdm.enrolled = true ) THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END ) WHEN h.platform = 'android' THEN - CASE WHEN hmdm.enrolled = 1 THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END + CASE WHEN hmdm.enrolled = true THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END WHEN h.platform IN ('ios', 'ipados', 'darwin') THEN (` + // NOTE: if you change any of the conditions in this // query, please update the AreHostsConnectedToFleetMDM @@ -1010,9 +1091,9 @@ const hostMDMSelect = `, `SELECT CASE WHEN EXISTS ( SELECT ne.id FROM nano_enrollments ne WHERE ne.id = h.uuid - AND ne.enabled = 1 + AND ne.enabled = true AND ne.type IN ('Device', 'User Enrollment (Device)') - AND hmdm.enrolled = 1 + AND hmdm.enrolled = true ) THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) @@ -1278,11 +1359,11 @@ func (ds *Datastore) applyHostFilters( deviceMappingJoin := fmt.Sprintf(`LEFT JOIN ( SELECT host_id, - CONCAT('[', GROUP_CONCAT(JSON_OBJECT('email', email, 'source', %s)), ']') AS device_mapping + CONCAT('[', %s, ']') AS device_mapping FROM host_emails GROUP BY - host_id) dm ON dm.host_id = h.id`, deviceMappingTranslateSourceColumn("")) + host_id) dm ON dm.host_id = h.id`, ds.dialect.GroupConcat(fmt.Sprintf("JSON_OBJECT('email', email, 'source', %s)", deviceMappingTranslateSourceColumn("")), ",")) if !opt.DeviceMapping { deviceMappingJoin = "" } @@ -1390,7 +1471,7 @@ func (ds *Datastore) applyHostFilters( opt.MacOSSettingsDiskEncryptionFilter.IsValid() || opt.OSSettingsDiskEncryptionFilter.IsValid() { connectedToFleetJoin = ` - LEFT JOIN nano_enrollments ne ON ne.id = h.uuid AND ne.enabled = 1 AND ne.type IN ('Device', 'User Enrollment (Device)') + LEFT JOIN nano_enrollments ne ON ne.id = h.uuid AND ne.enabled = true AND ne.type IN ('Device', 'User Enrollment (Device)') LEFT JOIN mdm_windows_enrollments mwe ON mwe.host_uuid = h.uuid AND mwe.device_state = ? LEFT JOIN android_devices ad ON ad.host_id = h.id` whereParams = append(whereParams, microsoft_mdm.MDMDeviceStateEnrolled) @@ -1535,17 +1616,17 @@ func (*Datastore) getBatchExecutionFilters(whereParams []interface{}, opt fleet. batchScriptExecutionJoin += ` LEFT JOIN host_script_results hsr ON bsehr.host_execution_id = hsr.execution_id` switch opt.BatchScriptExecutionStatusFilter { case fleet.BatchScriptExecutionRan: - batchScriptExecutionFilter += ` AND hsr.exit_code = 0 AND hsr.canceled = 0` + batchScriptExecutionFilter += ` AND hsr.exit_code = 0 AND hsr.canceled = false` case fleet.BatchScriptExecutionPending: // Pending can mean "waiting for execution" or "waiting for results". // hsr.exit_code IS NULL <- this means the script has not reported back - // (hsr.canceled IS NULL OR hsr.canceled = 0) <- this can mean the script is running, or that it hasn't been activated yet, + // (hsr.canceled IS NULL OR hsr.canceled = false) <- this can mean the script is running, or that it hasn't been activated yet, // but either way we haven't canceled it. // bsehr.error IS NULL <- this means the batch script framework didn't mark this host as incompatible // with this script run. - batchScriptExecutionFilter += ` AND ((hsr.host_id AND (hsr.exit_code IS NULL AND (hsr.canceled IS NULL OR hsr.canceled = 0) AND bsehr.error IS NULL)) OR (hsr.host_id is NULL AND ba.canceled = 0 AND bsehr.error IS NULL))` + batchScriptExecutionFilter += ` AND ((hsr.host_id AND (hsr.exit_code IS NULL AND (hsr.canceled IS NULL OR hsr.canceled = false) AND bsehr.error IS NULL)) OR (hsr.host_id is NULL AND ba.canceled = 0 AND bsehr.error IS NULL))` case fleet.BatchScriptExecutionErrored: - batchScriptExecutionFilter += ` AND hsr.exit_code <> 0 AND hsr.canceled = 0` + batchScriptExecutionFilter += ` AND hsr.exit_code <> 0 AND hsr.canceled = false` case fleet.BatchScriptExecutionIncompatible: batchScriptExecutionFilter += ` AND bsehr.error IS NOT NULL` case fleet.BatchScriptExecutionCanceled: @@ -1586,20 +1667,20 @@ func filterHostsByMDM(sql string, opt fleet.HostListOptions, params []interface{ params = append(params, *opt.MDMNameFilter) } if opt.MDMEnrollmentStatusFilter != "" { - // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = 0 so DEP hosts are not counted as pending after unenrollment + // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = false so DEP hosts are not counted as pending after unenrollment switch opt.MDMEnrollmentStatusFilter { case fleet.MDMEnrollStatusAutomatic: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 1` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = true` case fleet.MDMEnrollStatusManual: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 0 AND hmdm.is_personal_enrollment = 0` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = false AND hmdm.is_personal_enrollment = false` case fleet.MDMEnrollStatusPersonal: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 0 AND hmdm.is_personal_enrollment = 1` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = false AND hmdm.is_personal_enrollment = true` case fleet.MDMEnrollStatusEnrolled: - sql += ` AND hmdm.enrolled = 1` + sql += ` AND hmdm.enrolled = true` case fleet.MDMEnrollStatusPending: - sql += ` AND hmdm.enrolled = 0 AND hmdm.installed_from_dep = 1` + sql += ` AND hmdm.enrolled = false AND hmdm.installed_from_dep = true` case fleet.MDMEnrollStatusUnenrolled: - sql += ` AND hmdm.enrolled = 0 AND hmdm.installed_from_dep = 0` + sql += ` AND hmdm.enrolled = false AND hmdm.installed_from_dep = false` } } if opt.MDMNameFilter != nil || opt.MDMIDFilter != nil || opt.MDMEnrollmentStatusFilter != "" { @@ -1664,7 +1745,7 @@ func filterHostsByMacOSSettingsStatus(sql string, opt fleet.HostListOptions, par } // ensure the host has MDM turned on - whereStatus := " AND ne.id IS NOT NULL AND hmdm.enrolled = 1" + whereStatus := " AND ne.id IS NOT NULL AND hmdm.enrolled = true" // macOS settings filter is not compatible with the "all teams" option so append the "no // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) if opt.TeamFilter == nil { @@ -1698,7 +1779,7 @@ func filterHostsByMacOSDiskEncryptionStatus(sql string, opt fleet.HostListOption subquery, subqueryParams = subqueryFileVaultRemovingEnforcement() } - return sql + fmt.Sprintf(` AND EXISTS (%s) AND ne.id IS NOT NULL AND hmdm.enrolled = 1`, subquery), append(params, subqueryParams...) + return sql + fmt.Sprintf(` AND EXISTS (%s) AND ne.id IS NOT NULL AND hmdm.enrolled = true`, subquery), append(params, subqueryParams...) } func (ds *Datastore) filterHostsByOSSettingsStatus(ctx context.Context, sql string, opt fleet.HostListOptions, params []any, diskEncryptionConfig fleet.DiskEncryptionConfig) (string, []any, error) { @@ -1722,9 +1803,9 @@ func (ds *Datastore) filterHostsByOSSettingsStatus(ctx context.Context, sql stri } sqlFmt := ` AND ( - (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = 1) -- windows - OR (h.platform IN ('darwin', 'ios', 'ipados') AND ne.id IS NOT NULL AND hmdm.enrolled = 1) -- apple - OR (h.platform = 'android' AND hmdm.enrolled = 1 AND ad.host_id IS NOT NULL) -- android + (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = true) -- windows + OR (h.platform IN ('darwin', 'ios', 'ipados') AND ne.id IS NOT NULL AND hmdm.enrolled = true) -- apple + OR (h.platform = 'android' AND hmdm.enrolled = true AND ad.host_id IS NOT NULL) -- android OR ` + includeLinuxCond + ` )` @@ -1755,7 +1836,7 @@ AND ( paramsAndroid := []any{opt.OSSettingsFilter} // construct the WHERE for windows - whereWindows = `hmdm.is_server = 0` + whereWindows = `hmdm.is_server = false` paramsWindows := []any{} subqueryFailed, paramsFailed, err := subqueryHostsMDMWindowsOSSettingsStatusFailed() if err != nil { @@ -1881,8 +1962,8 @@ func (ds *Datastore) filterHostsByOSSettingsDiskEncryptionStatus(ctx context.Con sqlFmt += ` AND h.team_id IS NULL` } sqlFmt += ` AND ( - (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = 1 AND hmdm.is_server = 0 AND %s) -- windows - OR (h.platform = 'darwin' AND ne.id IS NOT NULL AND hmdm.enrolled = 1 AND %s) -- apple + (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = true AND hmdm.is_server = false AND %s) -- windows + OR (h.platform = 'darwin' AND ne.id IS NOT NULL AND hmdm.enrolled = true AND %s) -- apple OR ((h.platform = 'ubuntu' OR h.os_version LIKE 'Fedora%%') AND %s) -- linux )` @@ -1959,7 +2040,7 @@ func filterHostsByMDMBootstrapPackageStatus(sql string, opt fleet.HostListOption LEFT JOIN host_dep_assignments hda ON hda.host_id = hh.id WHERE - hh.id = h.id AND hmdm.installed_from_dep = 1` + hh.id = h.id AND hmdm.installed_from_dep = true` // NOTE: The approach below assumes that there is only one bootstrap package per host. If this // is not the case, then the query will need to be updated to use a GROUP BY and HAVING @@ -1969,7 +2050,7 @@ func filterHostsByMDMBootstrapPackageStatus(sql string, opt fleet.HostListOption subquery += ` AND ncr.status = 'Error'` case fleet.MDMBootstrapPackagePending: // Pending hosts exclude those that were skipped due to migration or will be skipped due to migration - subquery += ` AND (hmabp.skipped = 0 OR hmabp.skipped IS NULL) AND (hda.mdm_migration_deadline IS NULL OR (hda.mdm_migration_deadline = hda.mdm_migration_completed)) AND (ncr.status IS NULL OR (ncr.status != 'Acknowledged' AND ncr.status != 'Error'))` + subquery += ` AND (hmabp.skipped = false OR hmabp.skipped IS NULL) AND (hda.mdm_migration_deadline IS NULL OR (hda.mdm_migration_deadline = hda.mdm_migration_completed)) AND (ncr.status IS NULL OR (ncr.status != 'Acknowledged' AND ncr.status != 'Error'))` case fleet.MDMBootstrapPackageInstalled: subquery += ` AND ncr.status = 'Acknowledged'` } @@ -2483,9 +2564,9 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, opts ...fleet.DatastoreEnr hardware_model, platform, platform_like - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, true, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, + hostID, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlInsert, zeroTime, zeroTime, zeroTime, @@ -2505,7 +2586,6 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, opts ...fleet.DatastoreEnr if err != nil { return ctxerr.Wrap(ctx, err, "orbit enroll error inserting host details") } - hostID, _ := result.LastInsertId() const sqlHostDisplayName = ` INSERT INTO host_display_names (host_id, display_name) VALUES (?, ?) ` @@ -2585,14 +2665,13 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE refetch_requested, uuid, hardware_serial - ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, true, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, nodeKey, teamID, hardwareUUID, hardwareSerial) + lastInsertID, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, nodeKey, teamID, hardwareUUID, hardwareSerial) if err != nil { ds.logger.InfoContext(ctx, "host insert error", "err", err) return ctxerr.Wrap(ctx, err, "insert host") } - lastInsertID, _ := result.LastInsertId() const sqlHostDisplayName = ` INSERT INTO host_display_names (host_id, display_name) VALUES (?, '') ` @@ -2628,7 +2707,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE fmt.Sprintf("This is likely due to a duplicate UUID/identity identifier used by multiple hosts: %s", osqueryHostID)) } - if err := deleteAllPolicyMemberships(ctx, tx, enrolledHostInfo.ID); err != nil { + if err := deleteAllPolicyMemberships(ctx, tx, ds.dialect, enrolledHostInfo.ID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup policy membership on re-enroll") } @@ -2677,7 +2756,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE _, err = tx.ExecContext(ctx, ` INSERT INTO host_seen_times (host_id, seen_time) VALUES (?, ?) - ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + `+ds.dialect.OnDuplicateKey("host_id", "seen_time = VALUES(seen_time)"), hostID, time.Now().UTC()) if err != nil { return ctxerr.Wrap(ctx, err, "new host seen time") @@ -2748,7 +2827,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE if err != nil { return ctxerr.Wrap(ctx, err, "getting the host to return") } - _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, hostID) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+` label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`+ds.dialect.OnConflictDoNothing("host_id,label_id"), hostID) if err != nil { return ctxerr.Wrap(ctx, err, "insert new host into all hosts label") } @@ -2769,7 +2848,7 @@ func (ds *Datastore) getContextTryStmt(ctx context.Context, dest interface{}, qu // nolint the statements are closed in Datastore.Close. if stmt := ds.loadOrPrepareStmt(ctx, query); stmt != nil { err := stmt.GetContext(ctx, dest, args...) - if err == nil || !isBadConnection(err) { + if err == nil || !ds.dialect.IsBadConnection(err) { return err } @@ -2907,7 +2986,7 @@ func (ds *Datastore) LoadHostByOrbitNodeKey(ctx context.Context, nodeKey string) h.policy_updated_at, h.public_ip, h.orbit_node_key, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet, + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet, hd.encrypted as disk_encryption_enabled, COALESCE(hdek.decryptable, false) as encryption_key_available, t.name as team_name, @@ -2998,7 +3077,7 @@ func (ds *Datastore) LoadHostByDeviceAuthToken(ctx context.Context, authToken st COALESCE(hd.percent_disk_space_available, 0) as percent_disk_space_available, COALESCE(hd.gigs_total_disk_space, 0) as gigs_total_disk_space, hd.encrypted as disk_encryption_enabled, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet, + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet, ` + hostHasIdentityCertSQL + ` as has_host_identity_cert FROM host_device_auth hda @@ -3039,19 +3118,18 @@ func (ds *Datastore) SetOrUpdateDeviceAuthToken(ctx context.Context, hostID uint // both the old and new tokens can be used for authentication during the transition // period (see #38351). If the current token is already expired (older than 1 hour, // matching deviceAuthTokenTTL), previous_token is set to NULL to avoid reviving it. - const stmt = ` + stmt := ` INSERT INTO host_device_auth ( host_id, token ) VALUES (?, ?) - ON DUPLICATE KEY UPDATE - previous_token = IF(token = VALUES(token), previous_token, + ` + ds.dialect.OnDuplicateKey("host_id", `previous_token = IF(token = VALUES(token), previous_token, IF(updated_at >= DATE_SUB(NOW(), INTERVAL 3600 SECOND), token, NULL)), - token = VALUES(token) + token = VALUES(token)`) + ` ` _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostID, authToken) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return fleet.ConflictError{Message: "auth token conflicts with another host"} } return ctxerr.Wrap(ctx, err, "upsert host's device auth token") @@ -3093,7 +3171,7 @@ func (ds *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.T insertValues := strings.TrimSuffix(strings.Repeat("(?, ?),", len(hostIDs)), ",") query := fmt.Sprintf(` INSERT INTO host_seen_times (host_id, seen_time) VALUES %s - ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + `+ds.dialect.OnDuplicateKey("host_id", "seen_time = VALUES(seen_time)"), insertValues, ) if _, err := tx.ExecContext(ctx, query, insertArgs...); err != nil { @@ -3282,7 +3360,7 @@ SELECT h.policy_updated_at, h.refetch_requested, h.refetch_critical_queries_until, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet FROM hosts h LEFT OUTER JOIN @@ -3419,7 +3497,7 @@ func (ds *Datastore) HostByIdentifier(ctx context.Context, identifier string) (* return nil, ctxerr.Wrap(ctx, err, "get host by identifier") } - packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform) + packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, ds.dialect) if err != nil { return nil, err } @@ -3446,7 +3524,7 @@ func (ds *Datastore) AddHostsToTeam(ctx context.Context, params *fleet.AddHostsT hostIDsBatch := hostIDs[start:end] err := ds.withRetryTxx( ctx, func(tx sqlx.ExtContext) error { - if err := cleanupPolicyMembershipOnTeamChange(ctx, tx, hostIDsBatch); err != nil { + if err := cleanupPolicyMembershipOnTeamChange(ctx, tx, ds.dialect, hostIDsBatch); err != nil { return ctxerr.Wrap(ctx, err, "AddHostsToTeam delete policy membership") } if err := cleanupQueryResultsOnTeamChange(ctx, tx, hostIDsBatch); err != nil { @@ -3483,14 +3561,14 @@ func (ds *Datastore) AddHostsToTeam(ctx context.Context, params *fleet.AddHostsT } func (ds *Datastore) SaveHostAdditional(ctx context.Context, hostID uint, additional *json.RawMessage) error { - return saveHostAdditionalDB(ctx, ds.writer(ctx), hostID, additional) + return saveHostAdditionalDB(ctx, ds.writer(ctx), ds.dialect, hostID, additional) } -func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, hostID uint, additional *json.RawMessage) error { +func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, dialect DialectHelper, hostID uint, additional *json.RawMessage) error { sql := ` INSERT INTO host_additional (host_id, additional) VALUES (?, ?) - ON DUPLICATE KEY UPDATE additional = VALUES(additional) + ` + dialect.OnDuplicateKey("host_id", "additional = VALUES(additional)") + ` ` if _, err := exec.ExecContext(ctx, sql, hostID, additional); err != nil { return ctxerr.Wrap(ctx, err, "insert additional") @@ -3500,11 +3578,11 @@ func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, hostID u func (ds *Datastore) SaveHostUsers(ctx context.Context, hostID uint, users []fleet.HostUser) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return saveHostUsersDB(ctx, tx, hostID, users) + return saveHostUsersDB(ctx, tx, ds.dialect, hostID, users) }) } -func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, users []fleet.HostUser) error { +func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint, users []fleet.HostUser) error { currentHostUsers, err := loadHostUsersDB(ctx, tx, hostID) if err != nil { return err @@ -3529,11 +3607,11 @@ func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, users insertSql := fmt.Sprintf( `INSERT INTO host_users (host_id, uid, username, user_type, groupname, shell) VALUES %s - ON DUPLICATE KEY UPDATE + `+dialect.OnDuplicateKey("host_id,uid,username", ` user_type = VALUES(user_type), groupname = VALUES(groupname), shell = VALUES(shell), - removed_at = NULL`, + removed_at = NULL`), insertValues, ) if _, err := tx.ExecContext(ctx, insertSql, insertArgs...); err != nil { @@ -3636,12 +3714,12 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) // We log to help troubleshooting in case this happens. ds.logger.ErrorContext(ctx, "unrecognized platform", "hostID", host.ID, "platform", host.Platform) } - query := `SELECT p.id, p.team_id, p.resolution, p.name, p.query, p.description, p.author_id, p.platforms, p.critical, p.created_at, p.updated_at, p.conditional_access_enabled, p.type, + query := fmt.Sprintf(`SELECT p.id, p.team_id, p.resolution, p.name, p.query, p.description, p.author_id, p.platforms, p.critical, p.created_at, p.updated_at, p.conditional_access_enabled, p.type, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, CASE - WHEN pm.passes = 1 THEN 'pass' - WHEN pm.passes = 0 THEN 'fail' + WHEN pm.passes = true THEN 'pass' + WHEN pm.passes = false THEN 'fail' ELSE '' END AS response, coalesce(p.resolution, '') as resolution @@ -3649,14 +3727,14 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) LEFT JOIN policy_membership pm ON (p.id=pm.policy_id AND host_id=?) LEFT JOIN users u ON p.author_id = u.id WHERE (p.team_id IS NULL OR p.team_id = COALESCE((SELECT team_id FROM hosts WHERE id = ?), 0)) - AND (p.platforms IS NULL OR p.platforms = '' OR FIND_IN_SET(?, p.platforms) != 0) + AND (p.platforms IS NULL OR p.platforms = '' OR %s != 0)`, ds.dialect.FindInSet("?", "p.platforms")) + ` AND ( -- Policy has no include labels NOT EXISTS ( SELECT 1 FROM policy_labels pl WHERE pl.policy_id = p.id - AND pl.exclude = 0 + AND pl.exclude = false ) -- Policy is included in the include_any list OR EXISTS ( @@ -3664,7 +3742,7 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) FROM policy_labels pl INNER JOIN label_membership lm ON (lm.host_id = ? AND lm.label_id = pl.label_id) WHERE pl.policy_id = p.id - AND pl.exclude = 0 + AND pl.exclude = false ) ) -- Policy is not included in the exclude_any list @@ -3673,9 +3751,14 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) FROM policy_labels pl INNER JOIN label_membership lm ON (lm.host_id = ? AND lm.label_id = pl.label_id) WHERE pl.policy_id = p.id - AND pl.exclude = 1 + AND pl.exclude = true ) - ORDER BY FIELD(response, 'fail', '', 'pass'), p.name` + ORDER BY CASE + WHEN pm.passes = false THEN 1 + WHEN pm.passes IS NULL THEN 2 + WHEN pm.passes = true THEN 3 + ELSE 0 + END, p.name` var policies []*fleet.HostPolicy if err := sqlx.SelectContext(ctx, ds.reader(ctx), &policies, query, host.ID, host.ID, host.FleetPlatform(), host.ID, host.ID); err != nil { @@ -4296,8 +4379,8 @@ func (ds *Datastore) replaceHostMunkiIssues(ctx context.Context, hostID uint, ms if counts.CountNew < len(newIDs) { // must insert missing IDs - const ( - insStmt = `INSERT INTO host_munki_issues (host_id, munki_issue_id) VALUES %s ON DUPLICATE KEY UPDATE host_id = host_id` + var ( + insStmt = `INSERT INTO host_munki_issues (host_id, munki_issue_id) VALUES %s ` + ds.dialect.OnDuplicateKey("host_id,munki_issue_id", "host_id = host_id") stmtPart = `(?, ?),` ) @@ -4402,9 +4485,9 @@ func (ds *Datastore) getOrInsertMunkiIssues(ctx context.Context, errors, warning // create any missing munki issues (using the primary) if missing := missingIDs(); len(missing) > 0 { - const ( + var ( // UPDATE issue_type = issue_type results in a no-op in mysql (https://stackoverflow.com/a/4596409/1094941) - insStmt = `INSERT INTO munki_issues (name, issue_type) VALUES %s ON DUPLICATE KEY UPDATE issue_type = issue_type` + insStmt = `INSERT INTO munki_issues (name, issue_type) VALUES %s ` + ds.dialect.OnDuplicateKey("id", "issue_type = issue_type") stmtParts = `(?, ?),` ) @@ -5136,14 +5219,11 @@ func (ds *Datastore) generateAggregatedMunkiVersion(ctx context.Context, teamID return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, aggregatedStatsTypeMunkiVersions, versionsJson, ) if err != nil { @@ -5197,10 +5277,9 @@ func (ds *Datastore) generateAggregatedMunkiIssues(ctx context.Context, teamID * _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, id, globalStats, aggregatedStatsTypeMunkiIssues, issuesJSON) +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), + id, globalStats, aggregatedStatsTypeMunkiIssues, issuesJSON) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting stats for munki_issues id %d", id) } @@ -5213,7 +5292,7 @@ func (ds *Datastore) generateAggregatedMDMStatus(ctx context.Context, teamID *ui globalStats = true status fleet.AggregatedMDMStatus ) - // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = 0 so DEP hosts are not counted as pending after unenrollment + // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = false so DEP hosts are not counted as pending after unenrollment query := `SELECT COUNT(DISTINCT host_id) as hosts_count, COALESCE(SUM(CASE WHEN NOT enrolled AND NOT installed_from_dep THEN 1 ELSE 0 END), 0) as unenrolled_hosts_count, @@ -5253,14 +5332,11 @@ func (ds *Datastore) generateAggregatedMDMStatus(ctx context.Context, teamID *ui return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, platformKey(aggregatedStatsTypeMDMStatusPartial, platform), statusJson, ) if err != nil { @@ -5317,14 +5393,11 @@ func (ds *Datastore) generateAggregatedMDMSolutions(ctx context.Context, teamID return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, platformKey(aggregatedStatsTypeMDMSolutionsPartial, platform), resultsJSON, ) if err != nil { @@ -5339,7 +5412,7 @@ ON DUPLICATE KEY UPDATE // // If the host doesn't exist, a NotFoundError is returned. func (ds *Datastore) HostLite(ctx context.Context, id uint) (*fleet.Host, error) { - query, args, err := dialect.From(goqu.I("hosts")).Select( + query, args, err := ds.dialect.GoquDialect().From(goqu.I("hosts")).Select( "id", "created_at", "updated_at", @@ -5671,7 +5744,7 @@ func (ds *Datastore) executeOSVersionQuery(ctx context.Context, teamFilter *flee args = append(args, *teamFilter.TeamID, false) case teamFilter != nil: query += " AND " + ds.whereFilterGlobalOrTeamIDByTeamsWithSqlFilter( - *teamFilter, "global_stats = 1 AND id = 0", "global_stats = 0 AND id", + *teamFilter, "global_stats = true AND id = 0", "global_stats = false AND id", ) default: query += " AND id = ? AND global_stats = ?" @@ -5804,7 +5877,7 @@ func (ds *Datastore) UpdateOSVersions(ctx context.Context) error { insertStmt := "INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES " insertStmt += strings.TrimSuffix(strings.Repeat("(?,?,?,?),", len(statsByTeamID)+1), ",") // +1 due to global stats - insertStmt += " ON DUPLICATE KEY UPDATE json_value = VALUES(json_value), updated_at = CURRENT_TIMESTAMP" + insertStmt += " " + ds.dialect.OnDuplicateKey("id,type,global_stats", "json_value = VALUES(json_value), updated_at = CURRENT_TIMESTAMP") if _, err := ds.writer(ctx).ExecContext(ctx, insertStmt, args...); err != nil { return ctxerr.Wrapf(ctx, err, "insert os versions into aggregated stats") @@ -5843,7 +5916,7 @@ func (ds *Datastore) HostIDsByOSID( ) ([]uint, error) { var ids []uint - stmt := dialect.From("host_operating_system"). + stmt := ds.dialect.GoquDialect().From("host_operating_system"). Select("host_id"). Where( goqu.C("os_id").Eq(osID)). @@ -5872,7 +5945,7 @@ func (ds *Datastore) HostIDsByOSVersion( ) ([]uint, error) { var ids []uint - stmt := dialect.From("hosts"). + stmt := ds.dialect.GoquDialect().From("hosts"). Select("id"). Where( goqu.C("platform").Eq(osVersion.Platform), @@ -6274,39 +6347,39 @@ func (ds *Datastore) GetHostIssuesLastUpdated(ctx context.Context, hostId uint) func (ds *Datastore) UpdateHostIssuesFailingPolicies(ctx context.Context, hostIDs []uint) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return updateHostIssuesFailingPolicies(ctx, tx, hostIDs) + return updateHostIssuesFailingPolicies(ctx, tx, ds.dialect, hostIDs) }) } func (ds *Datastore) UpdateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, hostID uint) error { var tx sqlx.ExecerContext = ds.writer(ctx) - return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostID) + return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, ds.dialect, hostID) } -func updateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, tx sqlx.ExecerContext, hostID uint) error { +func updateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, tx sqlx.ExecerContext, dialect DialectHelper, hostID uint) error { stmt := ` INSERT INTO host_issues (host_id, failing_policies_count, total_issues_count) - SELECT host_id.id, COALESCE(SUM(!pm.passes), 0), COALESCE(SUM(!pm.passes), 0) + SELECT host_id.id, COALESCE(SUM(CASE WHEN pm.passes = false THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN pm.passes = false THEN 1 ELSE 0 END), 0) FROM policy_membership pm - RIGHT JOIN (SELECT ? as id) as host_id + RIGHT JOIN (SELECT CAST(? AS UNSIGNED) as id) as host_id ON pm.host_id = host_id.id GROUP BY host_id.id - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("host_id", ` failing_policies_count = VALUES(failing_policies_count), - total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count` + total_issues_count = VALUES(failing_policies_count) + VALUES(critical_vulnerabilities_count)`) if _, err := tx.ExecContext(ctx, stmt, hostID); err != nil { return ctxerr.Wrap(ctx, err, "updating failing policies in host issues for one host") } return nil } -func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, hostIDs []uint) error { +func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, dialect DialectHelper, hostIDs []uint) error { if len(hostIDs) == 0 { return nil } if len(hostIDs) == 1 { - return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostIDs[0]) + return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, dialect, hostIDs[0]) } // For multiple hosts, lock policy_membership rows first to prevent deadlocks @@ -6332,9 +6405,9 @@ func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, FROM policy_membership pm WHERE pm.host_id IN (?) GROUP BY pm.host_id - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("host_id", ` failing_policies_count = VALUES(failing_policies_count), - total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count` + total_issues_count = VALUES(failing_policies_count) + VALUES(critical_vulnerabilities_count)`) // Sort host IDs to ensure consistent lock ordering across all transactions. // This prevents deadlocks when multiple transactions process overlapping sets of hosts. @@ -6471,9 +6544,8 @@ func (ds *Datastore) UpdateHostIssuesVulnerabilities(ctx context.Context) error ) stmt := fmt.Sprintf( `INSERT INTO host_issues (host_id, critical_vulnerabilities_count, total_issues_count) VALUES %s - ON DUPLICATE KEY UPDATE - critical_vulnerabilities_count = VALUES(critical_vulnerabilities_count), - total_issues_count = failing_policies_count + VALUES(critical_vulnerabilities_count)`, + `+ds.dialect.OnDuplicateKey("host_id", `critical_vulnerabilities_count = VALUES(critical_vulnerabilities_count), + total_issues_count = failing_policies_count + VALUES(critical_vulnerabilities_count)`), values, ) args := make([]interface{}, 0, totalToProcess*numberOfArgsPerIssue) diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index ebd63c33d7b..834e807df10 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -190,7 +190,7 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle } } - sql := ` + insertSQL := ` INSERT INTO labels ( name, description, @@ -202,7 +202,7 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle author_id, team_id ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ? ) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("name", ` name = VALUES(name), description = VALUES(description), query = VALUES(query), @@ -210,23 +210,13 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle label_type = VALUES(label_type), label_membership_type = VALUES(label_membership_type), criteria = VALUES(criteria) - ` - - prepTx, ok := tx.(sqlx.PreparerContext) - if !ok { - return ctxerr.New(ctx, "tx in ApplyLabelSpecs is not a sqlx.PreparerContext") - } - stmt, err := prepTx.PrepareContext(ctx, sql) - if err != nil { - return ctxerr.Wrap(ctx, err, "prepare ApplyLabelSpecs insert") - } - defer stmt.Close() + `) for _, s := range specs { if s.Name == "" { return ctxerr.New(ctx, "label name must not be empty") } - insertLabelResult, err := stmt.ExecContext(ctx, s.Name, s.Description, s.Query, s.Platform, s.LabelType, s.LabelMembershipType, s.HostVitalsCriteria, authorID, s.TeamID) + insertedID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertSQL, s.Name, s.Description, s.Query, s.Platform, s.LabelType, s.LabelMembershipType, s.HostVitalsCriteria, authorID, s.TeamID) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyLabelSpecs insert") } @@ -262,18 +252,12 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle // Use the existing label ID labelID = existing.ID } else { - // New label - fetch the ID we just created - id, err := insertLabelResult.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "get new label ID for manual membership") - } - labelID = uint(id) //nolint:gosec + // New label - use the ID from the insert + labelID = uint(insertedID) //nolint:gosec } - sql = ` -DELETE FROM label_membership WHERE label_id = ? -` - _, err = tx.ExecContext(ctx, sql, labelID) + delSQL := `DELETE FROM label_membership WHERE label_id = ?` + _, err = tx.ExecContext(ctx, delSQL, labelID) if err != nil { return ctxerr.Wrap(ctx, err, "clear membership for ID") } @@ -323,15 +307,15 @@ DELETE FROM label_membership WHERE label_id = ? // Use ignore because duplicate hostnames could appear in // different batches and would result in duplicate key errors. - sql = fmt.Sprintf( - `INSERT IGNORE INTO label_membership (label_id, host_id) (SELECT DISTINCT ?, id FROM hosts WHERE %s)`, + memberSQL := fmt.Sprintf( + ds.dialect.InsertIgnoreInto()+` label_membership (label_id, host_id) (SELECT DISTINCT ?, id FROM hosts WHERE %s)`+ds.dialect.OnConflictDoNothing("host_id,label_id"), hostsFilterClause, ) - sql, args, err := sqlx.In(sql, labelID, stringIdents, stringIdents, stringIdents, intIdents) + memberSQL, args, err := sqlx.In(memberSQL, labelID, stringIdents, stringIdents, stringIdents, intIdents) if err != nil { return ctxerr.Wrap(ctx, err, "build membership IN statement") } - _, err = tx.ExecContext(ctx, sql, args...) + _, err = tx.ExecContext(ctx, memberSQL, args...) if err != nil { return ctxerr.Wrap(ctx, err, "execute membership INSERT") } @@ -429,9 +413,8 @@ func (ds *Datastore) UpdateLabelMembershipByHostIDs(ctx context.Context, label f } // Build the final SQL query with the dynamically generated placeholders - sql := ` -INSERT IGNORE INTO label_membership (label_id, host_id) -VALUES ` + strings.Join(placeholders, ", ") + sql := ds.dialect.InsertIgnoreInto() + ` label_membership (label_id, host_id) +VALUES ` + strings.Join(placeholders, ", ") + ds.dialect.OnConflictDoNothing("host_id,label_id") sql, args, err := sqlx.In(sql, values...) if err != nil { return ctxerr.Wrap(ctx, err, "build membership IN statement") @@ -482,7 +465,7 @@ func (ds *Datastore) UpdateLabelMembershipByHostCriteria(ctx context.Context, hv err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Insert new label membership based on the label query. - sql := fmt.Sprintf(`INSERT INTO label_membership (label_id, host_id) SELECT candidate.label_id, candidate.host_id FROM (%s) as candidate ON DUPLICATE KEY UPDATE host_id = label_membership.host_id`, labelQuery) + sql := fmt.Sprintf(`INSERT INTO label_membership (label_id, host_id) SELECT candidate.label_id, candidate.host_id FROM (%s) as candidate `+ds.dialect.OnDuplicateKey("host_id,label_id", `host_id = label_membership.host_id`), labelQuery) _, err := tx.ExecContext(ctx, sql, queryVals...) if err != nil { return ctxerr.Wrap(ctx, err, "execute membership INSERT") @@ -620,9 +603,7 @@ func (ds *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...f team_id ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ? ) ` - result, err := ds.writer(ctx).ExecContext( - ctx, - query, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), query, label.Name, label.Description, label.Query, @@ -637,7 +618,6 @@ func (ds *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...f return nil, ctxerr.Wrap(ctx, err, "inserting label") } - id, _ := result.LastInsertId() label.ID = uint(id) //nolint:gosec // dismiss G115 now := time.Now().UTC().Truncate(time.Second) label.CreatedAt = now @@ -680,7 +660,7 @@ func (ds *Datastore) DeleteLabel(ctx context.Context, name string, filter fleet. return ctxerr.Wrap(ctx, err, "getting label id to delete") } if err := deleteLabelsInTx(ctx, tx, []uint{labelID}); err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey("labels", name), "delete label") } return ctxerr.Wrap(ctx, err, "delete labels in tx") @@ -933,7 +913,7 @@ func (ds *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet // Complete inserts if necessary if len(vals) > 0 { sql := `INSERT INTO label_membership (updated_at, label_id, host_id) VALUES ` - sql += strings.Join(bindvars, ",") + ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` + sql += strings.Join(bindvars, ",") + ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = VALUES(updated_at)`) _, err := tx.ExecContext(ctx, sql, vals...) if err != nil { @@ -1101,11 +1081,11 @@ func (ds *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilt deviceMappingJoin := fmt.Sprintf(`LEFT JOIN ( SELECT host_id, - CONCAT('[', GROUP_CONCAT(JSON_OBJECT('email', email, 'source', %s)), ']') AS device_mapping + CONCAT('[', %s, ']') AS device_mapping FROM host_emails GROUP BY - host_id) dm ON dm.host_id = h.id`, deviceMappingTranslateSourceColumn("")) + host_id) dm ON dm.host_id = h.id`, ds.dialect.GroupConcat(fmt.Sprintf("JSON_OBJECT('email', email, 'source', %s)", deviceMappingTranslateSourceColumn("")), ",")) if !opt.DeviceMapping { deviceMappingJoin = "" } @@ -1282,10 +1262,11 @@ func (ds *Datastore) searchLabelsWithOmits(ctx context.Context, filter fleet.Tea ) AS host_count FROM labels l WHERE ( - MATCH(l.name) AGAINST(? IN BOOLEAN MODE) + %s ) AND l.id NOT IN (?) `, ds.whereFilterHostsByTeams(filter, "h"), + ds.dialect.FullTextMatch([]string{"l.name"}, "?"), ) sql, args, err := applyLabelTeamFilter(sqlStatement, filter, transformQuery(query), omit) @@ -1413,9 +1394,10 @@ func (ds *Datastore) SearchLabels(ctx context.Context, filter fleet.TeamFilter, ) AS host_count FROM labels l WHERE ( - MATCH(name) AGAINST(? IN BOOLEAN MODE) + %s ) `, ds.whereFilterHostsByTeams(filter, "h"), + ds.dialect.FullTextMatch([]string{"name"}, "?"), ) sql, args, err := applyLabelTeamFilter(sql, filter, transformQuery(query)) @@ -1492,7 +1474,7 @@ func (ds *Datastore) AsyncBatchInsertLabelMembership(ctx context.Context, batch sql := `INSERT INTO label_membership (label_id, host_id) VALUES ` sql += strings.Repeat(`(?, ?),`, len(batch)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` + sql += ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = VALUES(updated_at)`) vals := make([]interface{}, 0, len(batch)*2) for _, tup := range batch { @@ -1612,7 +1594,7 @@ func (ds *Datastore) AddLabelsToHost(ctx context.Context, hostID uint, labelIDs sql := `INSERT INTO label_membership (host_id, label_id) VALUES ` sql += strings.Repeat(`(?, ?),`, len(labelIDs)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = NOW()` + sql += ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = NOW()`) args := make([]interface{}, 0, len(labelIDs)*2) for _, labelID := range labelIDs { args = append(args, hostID, labelID) diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index c64b0079fd4..9a07a8d522f 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -1476,6 +1476,7 @@ WHERE func batchSetProfileLabelAssociationsDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, profileLabels []fleet.ConfigurationProfileLabel, profileUUIDsWithoutLabels []string, platform string, @@ -1522,10 +1523,10 @@ func batchSetProfileLabelAssociationsDB( (%s_profile_uuid, label_id, label_name, exclude, require_all) VALUES %s - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("id", ` label_id = VALUES(label_id), exclude = VALUES(exclude), - require_all = VALUES(require_all) + require_all = VALUES(require_all)`) + ` ` selectStmt := ` @@ -1674,7 +1675,7 @@ func (ds *Datastore) MDMInsertEULA(ctx context.Context, eula *fleet.MDMEULA) err _, err := ds.writer(ctx).ExecContext(ctx, stmt, eula.Name, eula.Bytes, eula.Token, eula.Sha256) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("MDMEULA", eula.Token)) } return ctxerr.Wrap(ctx, err, "create EULA") @@ -1779,9 +1780,9 @@ func (ds *Datastore) SetCommandForPendingSCEPRenewal(ctx context.Context, assocs stmt := fmt.Sprintf(` INSERT INTO nano_cert_auth_associations (id, sha256, renew_command_uuid) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id,sha256", ` renew_command_uuid = VALUES(renew_command_uuid) - `, strings.TrimSuffix(sb.String(), ",")) + `), strings.TrimSuffix(sb.String(), ",")) return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, args...) @@ -1978,6 +1979,7 @@ func (ds *Datastore) IsHostConnectedToFleetMDM(ctx context.Context, host *fleet. func batchSetProfileVariableAssociationsDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, profileVariablesByUUID []fleet.MDMProfileUUIDFleetVariables, platform string, ) (didUpdate bool, err error) { @@ -2080,9 +2082,8 @@ func batchSetProfileVariableAssociationsDB( fleet_variable_id ) VALUES %s - ON DUPLICATE KEY UPDATE - fleet_variable_id = VALUES(fleet_variable_id) - `, platformPrefix, strings.TrimSuffix(valuePart, ",")) + `, platformPrefix, strings.TrimSuffix(valuePart, ",")) + + dialect.OnDuplicateKey("id", "fleet_variable_id = VALUES(fleet_variable_id)") _, err := tx.ExecContext(ctx, stmt, args...) return err @@ -2528,7 +2529,7 @@ func (ds *Datastore) batchSetLabelAndVariableAssociations(ctx context.Context, t } var didUpdateLabels bool - if didUpdateLabels, err = batchSetProfileLabelAssociationsDB(ctx, tx, incomingLabels, profsWithoutLabels, + if didUpdateLabels, err = batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, incomingLabels, profsWithoutLabels, platform); err != nil { return false, ctxerr.Wrap(ctx, err, fmt.Sprintf("inserting %s profile label associations", platform)) } @@ -2554,7 +2555,7 @@ func (ds *Datastore) batchSetLabelAndVariableAssociations(ctx context.Context, t if len(profilesVarsToUpsert) > 0 { var didUpdateVariableAssociations bool - if didUpdateVariableAssociations, err = batchSetProfileVariableAssociationsDB(ctx, tx, profilesVarsToUpsert, platform); err != nil { + if didUpdateVariableAssociations, err = batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, profilesVarsToUpsert, platform); err != nil { return false, ctxerr.Wrap(ctx, err, fmt.Sprintf("inserting %s profile variable associations", platform)) } @@ -2706,14 +2707,17 @@ func getMDMIdPAccountByHostID(ctx context.Context, q sqlx.QueryerContext, logger func (ds *Datastore) CleanUpMDMManagedCertificates(ctx context.Context) error { _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE hmmc FROM host_mdm_managed_certificates hmmc -LEFT JOIN host_mdm_apple_profiles hmap ON hmmc.host_uuid = hmap.host_uuid - AND hmmc.profile_uuid = hmap.profile_uuid -LEFT JOIN host_mdm_windows_profiles hwmp ON hmmc.host_uuid = hwmp.host_uuid - AND hmmc.profile_uuid = hwmp.profile_uuid -WHERE - hmap.host_uuid IS NULL - AND hwmp.host_uuid IS NULL`) + DELETE FROM host_mdm_managed_certificates +WHERE NOT EXISTS ( + SELECT 1 FROM host_mdm_apple_profiles hmap + WHERE hmap.host_uuid = host_mdm_managed_certificates.host_uuid + AND hmap.profile_uuid = host_mdm_managed_certificates.profile_uuid +) +AND NOT EXISTS ( + SELECT 1 FROM host_mdm_windows_profiles hwmp + WHERE hwmp.host_uuid = host_mdm_managed_certificates.host_uuid + AND hwmp.profile_uuid = host_mdm_managed_certificates.profile_uuid +)`) if err != nil { return ctxerr.Wrap(ctx, err, "clean up mdm certificate profiles") } @@ -2738,13 +2742,13 @@ func (ds *Datastore) BulkUpsertMDMManagedCertificates(ctx context.Context, paylo serial ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid,ca_name", ` challenge_retrieved_at = VALUES(challenge_retrieved_at), not_valid_before = VALUES(not_valid_before), not_valid_after = VALUES(not_valid_after), type = VALUES(type), ca_name = VALUES(ca_name), - serial = VALUES(serial)`, + serial = VALUES(serial)`), strings.TrimSuffix(valuePart, ","), ) diff --git a/server/datastore/mysql/microsoft_mdm.go b/server/datastore/mysql/microsoft_mdm.go index 636f664cd31..26f3cd3d229 100644 --- a/server/datastore/mysql/microsoft_mdm.go +++ b/server/datastore/mysql/microsoft_mdm.go @@ -128,7 +128,7 @@ func (ds *Datastore) MDMWindowsInsertEnrolledDevice(ctx context.Context, device credentials_acknowledged) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` mdm_device_id = VALUES(mdm_device_id), device_state = VALUES(device_state), device_type = VALUES(device_type), @@ -141,7 +141,7 @@ func (ds *Datastore) MDMWindowsInsertEnrolledDevice(ctx context.Context, device host_uuid = VALUES(host_uuid), credentials_hash = VALUES(credentials_hash), credentials_acknowledged = VALUES(credentials_acknowledged) - ` + `) _, err := ds.writer(ctx).ExecContext( ctx, stmt, @@ -159,7 +159,7 @@ func (ds *Datastore) MDMWindowsInsertEnrolledDevice(ctx context.Context, device device.CredentialsHash, device.CredentialsAcknowledged) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("MDMWindowsEnrolledDevice", device.MDMHardwareID)) } return ctxerr.Wrap(ctx, err, "inserting MDMWindowsEnrolledDevice") @@ -246,7 +246,7 @@ func (ds *Datastore) mdmWindowsInsertCommandForHostsDB(ctx context.Context, tx s VALUES (?, ?, ?) ` if _, err := tx.ExecContext(ctx, stmt, cmd.CommandUUID, cmd.RawCommand, cmd.TargetLocURI); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("MDMWindowsCommand", cmd.CommandUUID)) } return ctxerr.Wrap(ctx, err, "inserting MDMWindowsCommand") @@ -268,7 +268,7 @@ VALUES ((SELECT id FROM mdm_windows_enrollments WHERE host_uuid = ? OR mdm_devic ` if _, err := tx.ExecContext(ctx, stmt, hostUUIDOrDeviceID, hostUUIDOrDeviceID, commandUUID); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("MDMWindowsCommandQueue", commandUUID)) } return ctxerr.Wrap(ctx, err, "inserting MDMWindowsCommandQueue", "host_uuid_or_device_id", hostUUIDOrDeviceID, "command_uuid", commandUUID) @@ -331,11 +331,10 @@ func (ds *Datastore) MDMWindowsSaveResponse(ctx context.Context, deviceID string return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // store the full response const saveFullRespStmt = `INSERT INTO windows_mdm_responses (enrollment_id, raw_response) VALUES (?, ?)` - sqlResult, err := tx.ExecContext(ctx, saveFullRespStmt, enrolledDevice.ID, enrichedSyncML.Raw) + responseID, err := insertAndGetIDTx(ctx, tx, ds.dialect, saveFullRespStmt, enrolledDevice.ID, enrichedSyncML.Raw) if err != nil { return ctxerr.Wrap(ctx, err, "saving full response") } - responseID, _ := sqlResult.LastInsertId() // find commands we sent that match the UUID responses we've got findCommandsStmt := `SELECT command_uuid, raw_command, target_loc_uri FROM windows_mdm_commands WHERE command_uuid IN (?)` @@ -423,19 +422,19 @@ func (ds *Datastore) MDMWindowsSaveResponse(ctx context.Context, deviceID string } } - if err := updateMDMWindowsHostProfileStatusFromResponseDB(ctx, tx, potentialProfilePayloads); err != nil { + if err := updateMDMWindowsHostProfileStatusFromResponseDB(ctx, tx, ds.dialect, potentialProfilePayloads); err != nil { return ctxerr.Wrap(ctx, err, "updating host profile status") } // store the command results - const insertResultsStmt = ` + insertResultsStmt := ` INSERT INTO windows_mdm_command_results (enrollment_id, command_uuid, raw_result, response_id, status_code) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("enrollment_id,command_uuid", ` raw_result = COALESCE(VALUES(raw_result), raw_result), status_code = COALESCE(VALUES(status_code), status_code) -` +`) stmt = fmt.Sprintf(insertResultsStmt, strings.TrimSuffix(sb.String(), ",")) if _, err = tx.ExecContext(ctx, stmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "inserting command results") @@ -476,6 +475,7 @@ ON DUPLICATE KEY UPDATE func updateMDMWindowsHostProfileStatusFromResponseDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, payloads []*fleet.MDMWindowsProfilePayload, ) error { if len(payloads) == 0 { @@ -486,15 +486,15 @@ func updateMDMWindowsHostProfileStatusFromResponseDB( // should be inserted from a device MDM response, so we first check for // matching entries and then perform the INSERT ... ON DUPLICATE KEY to // update their detail and status. - const updateHostProfilesStmt = ` + updateHostProfilesStmt := ` INSERT INTO host_mdm_windows_profiles (host_uuid, profile_uuid, detail, status, retries, command_uuid, checksum) VALUES %s - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("host_uuid,profile_uuid", ` checksum = VALUES(checksum), detail = VALUES(detail), status = VALUES(status), - retries = VALUES(retries)` + retries = VALUES(retries)`) // MySQL will use the `host_uuid` part of the primary key as a first // pass, and then filter that subset by `command_uuid`. @@ -1712,13 +1712,13 @@ func (ds *Datastore) BulkUpsertMDMWindowsHostProfiles(ctx context.Context, paylo checksum ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail), profile_name = VALUES(profile_name), checksum = VALUES(checksum), - command_uuid = VALUES(command_uuid)`, + command_uuid = VALUES(command_uuid)`), strings.TrimSuffix(valuePart, ","), ) @@ -1885,7 +1885,7 @@ INSERT INTO res, err := tx.ExecContext(ctx, insertProfileStmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return &existsError{ ResourceType: "MDMWindowsConfigProfile.Name", Identifier: cp.Name, @@ -1928,7 +1928,7 @@ INSERT INTO if len(labels) == 0 { profsWithoutLabel = append(profsWithoutLabel, profileUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profsWithoutLabel, "windows"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profsWithoutLabel, "windows"); err != nil { return ctxerr.Wrap(ctx, err, "inserting windows profile label associations") } @@ -1940,7 +1940,7 @@ INSERT INTO FleetVariables: usesFleetVars, }, } - if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, profilesVarsToUpsert, "windows"); err != nil { + if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, profilesVarsToUpsert, "windows"); err != nil { return ctxerr.Wrap(ctx, err, "inserting windows profile variable associations") } } @@ -1973,10 +1973,10 @@ INSERT INTO SELECT 1 FROM mdm_android_configuration_profiles WHERE name = ? AND team_id = ? ) ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("profile_uuid", ` uploaded_at = IF(syncml = VALUES(syncml), uploaded_at, CURRENT_TIMESTAMP()), syncml = VALUES(syncml) -` +`) var teamID uint if cp.TeamID != nil { @@ -1986,7 +1986,7 @@ ON DUPLICATE KEY UPDATE res, err := ds.writer(ctx).ExecContext(ctx, stmt, profileUUID, teamID, cp.Name, cp.SyncML, cp.Name, teamID, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return &existsError{ ResourceType: "MDMWindowsConfigProfile.Name", Identifier: cp.Name, @@ -2064,7 +2064,7 @@ WHERE ` // For Windows profiles, if team_id and name are the same, we do an update. Otherwise, we do an insert. - const insertNewOrEditedProfile = ` + insertNewOrEditedProfile := ` INSERT INTO mdm_windows_configuration_profiles ( profile_uuid, team_id, name, syncml, uploaded_at @@ -2072,11 +2072,11 @@ INSERT INTO VALUES -- see https://stackoverflow.com/a/51393124/1094941 ( CONCAT('` + fleet.MDMWindowsProfileUUIDPrefix + `', CONVERT(UUID() USING utf8mb4)), ?, ?, ?, CURRENT_TIMESTAMP() ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("profile_uuid", ` uploaded_at = IF(syncml = VALUES(syncml) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), name = VALUES(name), syncml = VALUES(syncml) -` +`) // use a profile team id of 0 if no-team var profTeamID uint @@ -2298,13 +2298,13 @@ func (ds *Datastore) bulkSetPendingMDMWindowsHostProfilesDB( checksum ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` operation_type = VALUES(operation_type), status = NULL, command_uuid = VALUES(command_uuid), detail = '', checksum = VALUES(checksum) - `, strings.TrimSuffix(valuePart, ",")) + `), strings.TrimSuffix(valuePart, ",")) _, err := tx.ExecContext(ctx, baseStmt, args...) if err != nil { @@ -2397,8 +2397,8 @@ func (ds *Datastore) WipeHostViaWindowsMDM(ctx context.Context, host *fleet.Host fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - wipe_ref = VALUES(wipe_ref)` + ` + ds.dialect.OnDuplicateKey("host_id", ` + wipe_ref = VALUES(wipe_ref)`) if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for wipe_ref") diff --git a/server/datastore/mysql/migrations/data/migration.go b/server/datastore/mysql/migrations/data/migration.go index 6185cc8328d..5df534d7880 100644 --- a/server/datastore/mysql/migrations/data/migration.go +++ b/server/datastore/mysql/migrations/data/migration.go @@ -1,5 +1,16 @@ package data -import "github.com/fleetdm/fleet/v4/server/goose" +import ( + "fmt" + + "github.com/fleetdm/fleet/v4/server/goose" +) var MigrationClient = goose.New("migration_status_data", goose.MySqlDialect{}) + +// SetDialect updates the migration client's SQL dialect. +func SetDialect(driver string) { + if err := MigrationClient.SetDialect(driver); err != nil { + panic(fmt.Sprintf("migrations/data: unsupported dialect %q: %v", driver, err)) + } +} diff --git a/server/datastore/mysql/migrations/tables/migration.go b/server/datastore/mysql/migrations/tables/migration.go index b8724086bdf..e6eaa33ee82 100644 --- a/server/datastore/mysql/migrations/tables/migration.go +++ b/server/datastore/mysql/migrations/tables/migration.go @@ -18,6 +18,14 @@ import ( var MigrationClient = goose.New("migration_status_tables", goose.MySqlDialect{}) +// SetDialect updates the migration client's SQL dialect. +// Call before running migrations when using a non-MySQL database. +func SetDialect(driver string) { + if err := MigrationClient.SetDialect(driver); err != nil { + panic(fmt.Sprintf("migrations/tables: unsupported dialect %q: %v", driver, err)) + } +} + // can override in tests var ( outputTo io.Writer = os.Stderr @@ -105,7 +113,38 @@ func withSteps(steps []migrationStep, tx *sql.Tx) error { return nil } +// migrationHelper provides dialect-specific schema introspection for migrations. +// The default implementation uses MySQL information_schema. +// When PostgreSQL support is added, a pgMigrationHelper will use pg_catalog. +type migrationHelper interface { + fkExists(tx *sql.Tx, table, name string) bool + constraintExists(tx *sql.Tx, table, name string) bool + columnExists(tx *sql.Tx, table, column string) bool + columnsExists(tx *sql.Tx, table string, columns ...string) bool + tableExists(tx *sql.Tx, table string) bool +} + +// mysqlMigrationHelper implements migrationHelper using MySQL information_schema. +type mysqlMigrationHelper struct{} + +// defaultMigrationHelper is the migration helper used by all current migrations. +// It defaults to MySQL since that's the only supported database. +var defaultMigrationHelper migrationHelper = mysqlMigrationHelper{} + +// Package-level functions delegate to the default helper for backwards compatibility. func fkExists(tx *sql.Tx, table, name string) bool { + return defaultMigrationHelper.fkExists(tx, table, name) +} + +func constraintExists(tx *sql.Tx, table, name string) bool { + return defaultMigrationHelper.constraintExists(tx, table, name) +} + +func columnExists(tx *sql.Tx, table, column string) bool { + return defaultMigrationHelper.columnExists(tx, table, column) +} + +func (mysqlMigrationHelper) fkExists(tx *sql.Tx, table, name string) bool { var count int err := tx.QueryRow(` SELECT COUNT(1) @@ -121,7 +160,7 @@ AND CONSTRAINT_NAME = ? return count > 0 } -func constraintExists(tx *sql.Tx, table, name string) bool { +func (mysqlMigrationHelper) constraintExists(tx *sql.Tx, table, name string) bool { var count int err := tx.QueryRow(` SELECT COUNT(1) @@ -137,11 +176,15 @@ AND CONSTRAINT_NAME = ? return count > 0 } -func columnExists(tx *sql.Tx, table, column string) bool { - return columnsExists(tx, table, column) +func (mysqlMigrationHelper) columnExists(tx *sql.Tx, table, column string) bool { + return mysqlMigrationHelper{}.columnsExists(tx, table, column) } func columnsExists(tx *sql.Tx, table string, columns ...string) bool { + return defaultMigrationHelper.columnsExists(tx, table, columns...) +} + +func (mysqlMigrationHelper) columnsExists(tx *sql.Tx, table string, columns ...string) bool { if len(columns) == 0 { return false } @@ -173,6 +216,10 @@ WHERE } func tableExists(tx *sql.Tx, table string) bool { + return defaultMigrationHelper.tableExists(tx, table) +} + +func (mysqlMigrationHelper) tableExists(tx *sql.Tx, table string) bool { var count int err := tx.QueryRow( ` diff --git a/server/datastore/mysql/nanomdm_storage.go b/server/datastore/mysql/nanomdm_storage.go index 6e8fb96c04e..d5b9290554f 100644 --- a/server/datastore/mysql/nanomdm_storage.go +++ b/server/datastore/mysql/nanomdm_storage.go @@ -54,9 +54,10 @@ func isConflict(err error) bool { type NanoMDMStorage struct { *nanomdm_mysql.MySQLStorage - db *sqlx.DB - logger *slog.Logger - ds fleet.Datastore + db *sqlx.DB + logger *slog.Logger + ds fleet.Datastore + dialect DialectHelper } // NewMDMAppleMDMStorage returns a MySQL nanomdm storage that uses the Datastore @@ -75,6 +76,7 @@ func (ds *Datastore) NewMDMAppleMDMStorage() (*NanoMDMStorage, error) { db: ds.primary, logger: ds.logger, ds: ds, + dialect: ds.dialect, }, nil } @@ -96,6 +98,7 @@ func (ds *Datastore) NewTestMDMAppleMDMStorage(asyncCap int, asyncInterval time. db: ds.primary, logger: ds.logger, ds: ds, + dialect: ds.dialect, }, nil } @@ -213,11 +216,11 @@ func (s *NanoMDMStorage) EnqueueDeviceLockCommand( fleet_platform ) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + s.dialect.OnDuplicateKey("host_id", ` wipe_ref = NULL, unlock_ref = NULL, unlock_pin = VALUES(unlock_pin), - lock_ref = VALUES(lock_ref)` + lock_ref = VALUES(lock_ref)`) if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, pin, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceLock") @@ -240,9 +243,9 @@ func (s *NanoMDMStorage) EnqueueDeviceUnlockCommand(ctx context.Context, host *f fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + s.dialect.OnDuplicateKey("host_id", ` unlock_ref = VALUES(unlock_ref), - unlock_pin = NULL` + unlock_pin = NULL`) if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceUnlock") @@ -266,8 +269,7 @@ func (s *NanoMDMStorage) EnqueueDeviceWipeCommand(ctx context.Context, host *fle fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - wipe_ref = VALUES(wipe_ref)` + ` + s.dialect.OnDuplicateKey("host_id", "wipe_ref = VALUES(wipe_ref)") if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceWipe") diff --git a/server/datastore/mysql/operating_systems.go b/server/datastore/mysql/operating_systems.go index 2f9cbf30e46..f6e6ff96de5 100644 --- a/server/datastore/mysql/operating_systems.go +++ b/server/datastore/mysql/operating_systems.go @@ -56,7 +56,7 @@ func (ds *Datastore) UpdateHostOperatingSystem(ctx context.Context, hostID uint, if err != nil { return err } - return upsertHostOperatingSystemDB(ctx, tx, hostID, os.ID) + return upsertHostOperatingSystemDB(ctx, tx, ds.dialect, hostID, os.ID) }) } @@ -174,13 +174,13 @@ func isHostOperatingSystemUpdateNeeded(ctx context.Context, qc sqlx.QueryerConte // upsertHostOperatingSystemDB upserts the host operating system table // with the operating system id for the given host ID -func upsertHostOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, osID uint) error { +func upsertHostOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint, osID uint) error { // We do not use the `UPDATE` then `INSERT` pattern here because it causes a deadlock when multiple hosts are enrolled concurrently. // This method will rarely be called -- only when the host_operating_system needs to be updated. _, err := tx.ExecContext( ctx, `INSERT INTO host_operating_system (host_id, os_id) VALUES (?, ?) - ON DUPLICATE KEY UPDATE os_id = VALUES(os_id)`, hostID, osID, + `+dialect.OnDuplicateKey("host_id", "os_id = VALUES(os_id)"), hostID, osID, ) return err } diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index af097218f12..29f9199cd44 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -2,6 +2,7 @@ package mysql import ( "context" + "crypto/md5" //nolint:gosec // G501: MD5 required for MySQL UNHEX(MD5()) compat "database/sql" "encoding/json" "errors" @@ -80,7 +81,7 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, authorID *uint, args f var newPolicy *fleet.Policy if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - p, err := newGlobalPolicy(ctx, tx, authorID, args) + p, err := newGlobalPolicy(ctx, tx, authorID, args, ds.dialect) if err != nil { return err } @@ -93,7 +94,7 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, authorID *uint, args f return newPolicy, nil } -func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { +func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, args fleet.PolicyPayload, dialect DialectHelper) (*fleet.Policy, error) { if args.SoftwareInstallerID != nil { return nil, ctxerr.Wrap(ctx, errSoftwareTitleIDOnGlobalPolicy, "create policy") } @@ -101,7 +102,7 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar return nil, ctxerr.Wrap(ctx, errScriptIDOnGlobalPolicy, "create policy") } if args.QueryID != nil { - q, err := query(ctx, db, *args.QueryID) + q, err := query(ctx, db, *args.QueryID, dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "fetching query from id") } @@ -111,12 +112,9 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar } // We must normalize the name for full Unicode support (Unicode equivalence). nameUnicode := norm.NFC.String(args.Name) - res, err := db.ExecContext(ctx, - fmt.Sprintf( - `INSERT INTO policies (name, query, description, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, %s)`, - policiesChecksumComputedColumn(), - ), - nameUnicode, args.Query, args.Description, args.Resolution, authorID, args.Platform, args.Critical, + lastIdInt64, err := insertAndGetIDTx(ctx, db, dialect, + `INSERT INTO policies (name, query, description, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + nameUnicode, args.Query, args.Description, args.Resolution, authorID, args.Platform, args.Critical, policyChecksum(nil, nameUnicode), ) switch { case err == nil: @@ -126,10 +124,6 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar default: return nil, ctxerr.Wrap(ctx, err, "inserting new policy") } - lastIdInt64, err := res.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last id after inserting policy") - } policyID := uint(lastIdInt64) //nolint:gosec // dismiss G115 dummyPolicy := &fleet.Policy{ @@ -267,16 +261,17 @@ func loadLabelsForPolicies(ctx context.Context, db sqlx.QueryerContext, policies return nil } -func policiesChecksumComputedColumn() string { - // concatenate with separator \x00 - return ` UNHEX( - MD5( - CONCAT_WS(CHAR(0), - COALESCE(team_id, ''), - name - ) - ) - ) ` +// policyChecksum computes the checksum for a policy in Go (portable across databases). +// The checksum is MD5(CONCAT_WS(\x00, COALESCE(team_id, ”), name)) as raw bytes. +// +//nolint:gosec // G401: MD5 required for MySQL UNHEX(MD5()) compatibility +func policyChecksum(teamID *uint, name string) []byte { + var teamStr string + if teamID != nil { + teamStr = fmt.Sprintf("%d", *teamID) + } + h := md5.Sum([]byte(teamStr + "\x00" + name)) //nolint:gosec // G401 + return h[:] } func (ds *Datastore) Policy(ctx context.Context, id uint) (*fleet.Policy, error) { @@ -340,7 +335,7 @@ func (ds *Datastore) PolicyLite(ctx context.Context, id uint) (*fleet.PolicyLite // Currently, SavePolicy does not allow updating the team of an existing policy. func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error { if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return savePolicy(ctx, tx, ds.logger, p, shouldRemoveAllPolicyMemberships, removePolicyStats) + return savePolicy(ctx, tx, ds.logger, p, shouldRemoveAllPolicyMemberships, removePolicyStats, ds.dialect) }); err != nil { return ctxerr.Wrap(ctx, err, "updating policy") } @@ -348,7 +343,7 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo return nil } -func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error { +func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool, dialect DialectHelper) error { if p.TeamID == nil && p.SoftwareInstallerID != nil { return ctxerr.Wrap(ctx, errSoftwareTitleIDOnGlobalPolicy, "save policy") } @@ -369,11 +364,11 @@ func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, calendar_events_enabled = ?, software_installer_id = ?, script_id = ?, vpp_apps_teams_id = ?, - conditional_access_enabled = ?, checksum = ` + policiesChecksumComputedColumn() + ` + conditional_access_enabled = ?, checksum = ? WHERE id = ? ` result, err := db.ExecContext( - ctx, updateStmt, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.SoftwareInstallerID, p.ScriptID, p.VPPAppsTeamsID, p.ConditionalAccessEnabled, p.ID, + ctx, updateStmt, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.SoftwareInstallerID, p.ScriptID, p.VPPAppsTeamsID, p.ConditionalAccessEnabled, policyChecksum(p.TeamID, p.Name), p.ID, ) if err != nil { return ctxerr.Wrap(ctx, err, "updating policy") @@ -396,7 +391,7 @@ func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p } return cleanupPolicy( - ctx, db, db, p.ID, p.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, logger, + ctx, db, db, p.ID, p.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, logger, dialect, ) } @@ -491,14 +486,14 @@ func assertTeamMatches(ctx context.Context, db sqlx.QueryerContext, teamID uint, func cleanupPolicy( ctx context.Context, queryerContext sqlx.QueryerContext, extContext sqlx.ExtContext, policyID uint, policyPlatform string, shouldRemoveAllPolicyMemberships bool, - removePolicyStats bool, logger *slog.Logger, + removePolicyStats bool, logger *slog.Logger, dialect DialectHelper, ) error { var err error if shouldRemoveAllPolicyMemberships { - err = cleanupPolicyMembershipForPolicy(ctx, queryerContext, extContext, policyID) + err = cleanupPolicyMembershipForPolicy(ctx, queryerContext, extContext, dialect, policyID) } else { - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, queryerContext, extContext, policyID, policyPlatform) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, queryerContext, extContext, policyID, policyPlatform, dialect) } if err != nil { return err @@ -648,9 +643,9 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee if len(results) > 0 { query := fmt.Sprintf( `INSERT INTO policy_membership (updated_at, policy_id, host_id, passes) - VALUES %s ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at), passes=VALUES(passes)`, + VALUES %s `, strings.Join(bindvars, ","), - ) + ) + ds.dialect.OnDuplicateKey("policy_id,host_id", "updated_at=VALUES(updated_at), passes=VALUES(passes)") if _, err := tx.ExecContext(ctx, query, vals...); err != nil { return ctxerr.Wrapf(ctx, err, "insert policy_membership (%v)", vals) } @@ -1008,7 +1003,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) // won't be receiving any policies targeted for specific platforms. ds.logger.ErrorContext(ctx, "unrecognized platform", "hostID", host.ID, "platform", host.Platform) } - const stmt = ` + stmt := fmt.Sprintf(` SELECT p.id, p.query FROM policies p WHERE @@ -1016,7 +1011,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) -- team_id == 0 are policies that apply to hosts in "No team" -- team_id > 0 are policies that apply to hosts in teams (team_id IS NULL OR team_id = COALESCE(?, 0)) AND - (platforms = '' OR FIND_IN_SET(?, platforms)) AND + (platforms = '' OR %s) AND ( -- Policy has no include labels NOT EXISTS ( @@ -1042,7 +1037,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) WHERE pl.policy_id = p.id AND pl.exclude = 1 ) -` +`, ds.dialect.FindInSet("?", "platforms")) var rows []struct { ID string `db:"id"` Query string `db:"query"` @@ -1083,7 +1078,7 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u } if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - p, err := newTeamPolicy(ctx, tx, teamID, authorID, args) + p, err := newTeamPolicy(ctx, tx, teamID, authorID, args, ds.dialect) if err != nil { return err } @@ -1096,9 +1091,9 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u return newPolicy, nil } -func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { +func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorID *uint, args fleet.PolicyPayload, dialect DialectHelper) (*fleet.Policy, error) { if args.QueryID != nil { - q, err := query(ctx, db, *args.QueryID) + q, err := query(ctx, db, *args.QueryID, dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "fetching query from id") } @@ -1125,19 +1120,16 @@ func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorI return nil, ctxerr.Wrap(ctx, err, "create team policy") } - res, err := db.ExecContext(ctx, - fmt.Sprintf( - `INSERT INTO policies ( - name, query, description, team_id, resolution, author_id, - platforms, critical, calendar_events_enabled, software_installer_id, - script_id, vpp_apps_teams_id, conditional_access_enabled, checksum, - type, patch_software_title_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s, ?, ?)`, - policiesChecksumComputedColumn(), - ), + lastIdInt64, err := insertAndGetIDTx(ctx, db, dialect, + `INSERT INTO policies ( + name, query, description, team_id, resolution, author_id, + platforms, critical, calendar_events_enabled, software_installer_id, + script_id, vpp_apps_teams_id, conditional_access_enabled, checksum, + type, patch_software_title_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, nameUnicode, args.Query, args.Description, teamID, args.Resolution, authorID, args.Platform, args.Critical, args.CalendarEventsEnabled, args.SoftwareInstallerID, args.ScriptID, args.VPPAppsTeamsID, - args.ConditionalAccessEnabled, args.Type, args.PatchSoftwareTitleID, + args.ConditionalAccessEnabled, policyChecksum(&teamID, nameUnicode), args.Type, args.PatchSoftwareTitleID, ) switch { case err == nil: @@ -1151,10 +1143,6 @@ func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorI default: return nil, ctxerr.Wrap(ctx, err, "inserting new policy") } - lastIdInt64, err := res.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last id after inserting policy") - } policyID := uint(lastIdInt64) //nolint:gosec // dismiss G115 @@ -1415,8 +1403,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs // Reset on retry so we don't accumulate duplicate cleanup entries. pendingCleanups = pendingCleanups[:0] - query := fmt.Sprintf( - ` + query := ` INSERT INTO policies ( name, query, @@ -1434,9 +1421,8 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs checksum, type, patch_software_title_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s, ?, ?) - ON DUPLICATE KEY UPDATE - query = VALUES(query), + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ds.dialect.OnDuplicateKey("checksum", `query = VALUES(query), description = VALUES(description), author_id = VALUES(author_id), resolution = VALUES(resolution), @@ -1448,9 +1434,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs script_id = VALUES(script_id), conditional_access_enabled = VALUES(conditional_access_enabled), type = VALUES(type), - patch_software_title_id = VALUES(patch_software_title_id) - `, policiesChecksumComputedColumn(), - ) + patch_software_title_id = VALUES(patch_software_title_id)`) for teamID, teamPolicySpecs := range teamIDToPolicies { for _, spec := range teamPolicySpecs { var softwareInstallerID *uint @@ -1494,7 +1478,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, teamID, spec.Platform, spec.Critical, spec.CalendarEventsEnabled, softwareInstallerID, vppAppsTeamsID, scriptID, spec.ConditionalAccessEnabled, - spec.Type, fmaTitleID, + policyChecksum(teamID, norm.NFC.String(spec.Name)), spec.Type, fmaTitleID, ) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert") @@ -1630,6 +1614,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs args.shouldRemoveAllPolicyMemberships, args.removePolicyStats, ds.logger, + ds.dialect, ); err != nil { return err } @@ -1662,10 +1647,10 @@ func (ds *Datastore) AsyncBatchInsertPolicyMembership(ctx context.Context, batch // INSERT IGNORE, to avoid failing if policy / host does not exist (as this // runs asynchronously, they could get deleted in between the data being // received and being upserted). - sql := `INSERT IGNORE INTO policy_membership (policy_id, host_id, passes) VALUES ` + sql := ds.dialect.InsertIgnoreInto() + ` policy_membership (policy_id, host_id, passes) VALUES ` sql += strings.Repeat(`(?, ?, ?),`, len(batch)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at), passes = VALUES(passes)` + sql += ` ` + ds.dialect.OnDuplicateKey("policy_id,host_id", "updated_at = VALUES(updated_at), passes = VALUES(passes)") vals := make([]interface{}, 0, len(batch)*3) hostIDs := make([]uint, 0, len(batch)) @@ -1751,19 +1736,19 @@ func (ds *Datastore) AsyncBatchUpdatePolicyTimestamp(ctx context.Context, ids [] }) } -func deleteAllPolicyMemberships(ctx context.Context, tx sqlx.ExtContext, hostID uint) error { +func deleteAllPolicyMemberships(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint) error { query := `DELETE FROM policy_membership WHERE host_id = ?` if _, err := tx.ExecContext(ctx, query, hostID); err != nil { return ctxerr.Wrap(ctx, err, "exec delete policies") } // Use the single host method for better performance and no unnecessary locking - if err := updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostID); err != nil { + if err := updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, dialect, hostID); err != nil { return err } return nil } -func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext, hostIDs []uint) error { +func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostIDs []uint) error { // hosts can only be in one team, so if there's a policy that has a team id and a result from one of our hosts // it can only be from the previous team they are being transferred from query, args, err := sqlx.In(`DELETE FROM policy_membership @@ -1776,7 +1761,7 @@ func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext } // This method is currently called for a batch of hosts. Performance should be monitored. If performance becomes a concern, // we can reduce batch size or move this method outside the transaction. - if err = updateHostIssuesFailingPolicies(ctx, tx, hostIDs); err != nil { + if err = updateHostIssuesFailingPolicies(ctx, tx, dialect, hostIDs); err != nil { return err } return nil @@ -1826,7 +1811,7 @@ func cleanupConditionalAccessOnTeamChange(ctx context.Context, tx sqlx.ExtContex } func cleanupPolicyMembershipOnPolicyUpdate( - ctx context.Context, queryerContext sqlx.QueryerContext, db sqlx.ExecerContext, policyID uint, platforms string, + ctx context.Context, queryerContext sqlx.QueryerContext, db sqlx.ExecerContext, policyID uint, platforms string, dialect DialectHelper, ) error { // Clean up hosts that don't match the platform criteria. // Page through rows using the (policy_id, host_id) PK as a cursor so each SELECT+DELETE @@ -1841,14 +1826,14 @@ func cleanupPolicyMembershipOnPolicyUpdate( var afterHostID uint for { var batchHostIDs []uint - err := sqlx.SelectContext(ctx, queryerContext, &batchHostIDs, ` + err := sqlx.SelectContext(ctx, queryerContext, &batchHostIDs, fmt.Sprintf(` SELECT pm.host_id FROM policy_membership pm INNER JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND FIND_IN_SET(h.platform, ?) = 0 + WHERE pm.policy_id = ? AND %s = 0 AND pm.host_id > ? ORDER BY pm.host_id ASC - LIMIT ?`, policyID, expandedPlatformsStr, afterHostID, policyMembershipDeleteBatchSize) + LIMIT ?`, dialect.FindInSet("h.platform", "?")), policyID, expandedPlatformsStr, afterHostID, policyMembershipDeleteBatchSize) if err != nil { return ctxerr.Wrap(ctx, err, "select batch of hosts to cleanup policy membership for platform") } @@ -1866,16 +1851,15 @@ func cleanupPolicyMembershipOnPolicyUpdate( if _, err = db.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership for platform") } - if err := updateHostIssuesFailingPolicies(ctx, db, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, db, dialect, batchHostIDs); err != nil { return err } afterHostID = batchHostIDs[len(batchHostIDs)-1] } // Clean up orphaned memberships (host_id refs to deleted hosts, not covered by INNER JOIN above) if _, err := db.ExecContext(ctx, ` - DELETE pm FROM policy_membership pm - LEFT JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND h.id IS NULL`, policyID); err != nil { + DELETE FROM policy_membership + WHERE policy_id = ? AND NOT EXISTS (SELECT 1 FROM hosts WHERE hosts.id = policy_membership.host_id)`, policyID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup orphaned policy membership for platform") } } @@ -1929,7 +1913,7 @@ func cleanupPolicyMembershipOnPolicyUpdate( if _, err = db.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership for labels") } - if err := updateHostIssuesFailingPolicies(ctx, db, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, db, dialect, batchHostIDs); err != nil { return err } afterLabelHostID = batchHostIDs[len(batchHostIDs)-1] @@ -1944,6 +1928,7 @@ func cleanupPolicyMembershipForPolicy( ctx context.Context, queryerContext sqlx.QueryerContext, exec sqlx.ExecerContext, + dialect DialectHelper, policyID uint, ) error { // Page through policy_membership using (policy_id, host_id) as a cursor. Selecting and deleting one @@ -1976,16 +1961,15 @@ func cleanupPolicyMembershipForPolicy( if _, err = exec.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership") } - if err := updateHostIssuesFailingPolicies(ctx, exec, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, exec, dialect, batchHostIDs); err != nil { return err } afterHostID = batchHostIDs[len(batchHostIDs)-1] } // Clean up orphaned memberships (host_id refs to deleted hosts, not covered by INNER JOIN above) if _, err := exec.ExecContext(ctx, ` - DELETE pm FROM policy_membership pm - LEFT JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND h.id IS NULL`, policyID); err != nil { + DELETE FROM policy_membership + WHERE policy_id = ? AND NOT EXISTS (SELECT 1 FROM hosts WHERE hosts.id = policy_membership.host_id)`, policyID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup orphaned policy membership") } @@ -2009,17 +1993,17 @@ func (ds *Datastore) CleanupPolicyMembership(ctx context.Context, now time.Time) FROM policies p WHERE - p.updated_at >= DATE_SUB(?, INTERVAL ? SECOND) AND + p.updated_at >= ? AND p.created_at < p.updated_at` ) var pols []*fleet.Policy - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &pols, updatedPoliciesStmt, now, int(recentlyUpdatedPoliciesInterval.Seconds())); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &pols, updatedPoliciesStmt, now.Add(-recentlyUpdatedPoliciesInterval)); err != nil { return ctxerr.Wrap(ctx, err, "select recently updated policies") } for _, pol := range pols { - if err := cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform); err != nil { + if err := cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform, ds.dialect); err != nil { return ctxerr.Wrapf(ctx, err, "delete outdated hosts membership for policy: %d; platforms: %v", pol.ID, pol.Platform) } } @@ -2033,7 +2017,7 @@ func (ds *Datastore) CleanupPolicyMembership(ctx context.Context, now time.Time) return ctxerr.Wrap(ctx, err, "select policies needing full membership cleanup") } for _, polID := range fullCleanupPolIDs { - if err := cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), polID); err != nil { + if err := cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), ds.dialect, polID); err != nil { return ctxerr.Wrapf(ctx, err, "full membership cleanup for policy %d", polID) } if _, err := ds.writer(ctx).ExecContext(ctx, @@ -2058,7 +2042,7 @@ type PolicyViolationDays struct { func (ds *Datastore) IncrementPolicyViolationDays(ctx context.Context) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return incrementViolationDaysDB(ctx, tx) + return incrementViolationDaysDB(ctx, tx, ds.dialect) }) } @@ -2088,8 +2072,8 @@ func (ds *Datastore) IncreasePolicyAutomationIteration(ctx context.Context, poli return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { _, err := tx.ExecContext(ctx, ` INSERT INTO policy_automation_iterations (policy_id, iteration) VALUES (?,1) - ON DUPLICATE KEY UPDATE iteration = iteration + 1; - `, policyID) + `+ds.dialect.OnDuplicateKey("policy_id", "iteration = policy_automation_iterations.iteration + 1"), + policyID) return err }) } @@ -2131,7 +2115,7 @@ func (ds *Datastore) OutdatedAutomationBatch(ctx context.Context) ([]fleet.Polic return nil } query := ` - UPDATE policy_membership pm SET pm.automation_iteration = ( + UPDATE policy_membership pm SET automation_iteration = ( SELECT ai.iteration FROM policy_automation_iterations ai WHERE pm.policy_id = ai.policy_id @@ -2149,7 +2133,7 @@ func (ds *Datastore) OutdatedAutomationBatch(ctx context.Context) ([]fleet.Polic return failures, nil } -func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { +func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper) error { const ( statsID = 0 globalStats = true @@ -2205,7 +2189,7 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { // `policy_membership` var newCounts PolicyViolationDays if err := sqlx.GetContext(ctx, tx, &newCounts, ` - SELECT (select count(*) from policy_membership where passes=0) as failing_host_count, + SELECT (select count(*) from policy_membership where passes = false) as failing_host_count, (select count(*) from policy_membership) as total_host_count`, ); err != nil { return ctxerr.Wrap(ctx, err, "count policy violation days") @@ -2222,8 +2206,7 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value)` + ` + dialect.OnDuplicateKey("id,type,global_stats", "json_value = VALUES(json_value), updated_at = NOW()") if _, err := tx.ExecContext(ctx, upsertStmt, statsID, globalStats, statsType, statsJSON); err != nil { return ctxerr.Wrap(ctx, err, "update policy violation days aggregated stats") } @@ -2233,11 +2216,11 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { func (ds *Datastore) InitializePolicyViolationDays(ctx context.Context) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return initializePolicyViolationDaysDB(ctx, tx) + return initializePolicyViolationDaysDB(ctx, tx, ds.dialect) }) } -func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { +func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper) error { const ( statsID = 0 globalStats = true @@ -2253,9 +2236,8 @@ func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) er INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - created_at = CURRENT_TIMESTAMP` + ` + dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + created_at = CURRENT_TIMESTAMP`) if _, err := tx.ExecContext(ctx, stmt, statsID, globalStats, statsType, statsJSON); err != nil { return ctxerr.Wrap(ctx, err, "initialize policy violation days aggregated stats") } @@ -2396,10 +2378,9 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { insertStmt := `INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) VALUES (:policy_id, :inherited_team_id, :passing_host_count, :failing_host_count) - ON DUPLICATE KEY UPDATE - updated_at = NOW(), + ` + ds.dialect.OnDuplicateKey("policy_id,inherited_team_id_char", `updated_at = NOW(), passing_host_count = VALUES(passing_host_count), - failing_host_count = VALUES(failing_host_count)` + failing_host_count = VALUES(failing_host_count)`) _, err = sqlx.NamedExecContext(ctx, db, insertStmt, policyStats) if err != nil { // INSERT may fail due to rare race conditions. We log and proceed. @@ -2423,11 +2404,9 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { FROM policies p LEFT JOIN policy_membership pm ON p.id = pm.policy_id GROUP BY p.id - ON DUPLICATE KEY UPDATE - updated_at = NOW(), + `+ds.dialect.OnDuplicateKey("policy_id,inherited_team_id_char", `updated_at = NOW(), passing_host_count = VALUES(passing_host_count), - failing_host_count = VALUES(failing_host_count); - `) + failing_host_count = VALUES(failing_host_count)`)) if err != nil { return ctxerr.Wrap(ctx, err, "update host policy counts for global and team policies") } @@ -2511,7 +2490,7 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( policyIDs []uint, hostID *uint, ) ([]fleet.HostPolicyMembershipData, error) { - query := ` + query := fmt.Sprintf(` SELECT COALESCE(sh.email, '') AS email, COALESCE(pm.passing, 1) AS passing, @@ -2521,11 +2500,11 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( h.hardware_serial AS host_hardware_serial FROM hosts h LEFT JOIN ( - SELECT host_id, 0 AS passing, GROUP_CONCAT(policy_id) AS failing_policy_ids + SELECT host_id, 0 AS passing, %s AS failing_policy_ids FROM policy_membership - WHERE policy_id IN (?) AND passes = 0 + WHERE policy_id IN (?) AND passes = false GROUP BY host_id - ) pm ON h.id = pm.host_id + ) pm ON h.id = pm.host_id`, ds.dialect.GroupConcat("policy_id", ",")) + ` LEFT JOIN ( SELECT host_id, email FROM ( @@ -2550,7 +2529,7 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( ) sh ON h.id = sh.host_id LEFT JOIN host_display_names hdn ON h.id = hdn.host_id LEFT JOIN host_calendar_events hce ON h.id = hce.host_id - WHERE h.team_id = ? AND ((pm.passing IS NOT NULL AND NOT pm.passing) OR (COALESCE(pm.passing, 1) AND hce.host_id IS NOT NULL)) + WHERE h.team_id = ? AND ((pm.passing IS NOT NULL AND pm.passing = 0) OR (COALESCE(pm.passing, 1) = 1 AND hce.host_id IS NOT NULL)) ` query, args, err := sqlx.In(query, diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index b50be523914..105667cd742 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -65,7 +65,7 @@ func (ds *Datastore) applyQueriesInTx( } } - const upsertQueriesSQL = ` + upsertQueriesSQL := ` INSERT INTO queries ( name, description, @@ -82,8 +82,7 @@ func (ds *Datastore) applyQueriesInTx( logging_type, discard_data ) VALUES %s - ON DUPLICATE KEY UPDATE - name = VALUES(name), + ` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), description = VALUES(description), query = VALUES(query), author_id = VALUES(author_id), @@ -96,7 +95,7 @@ func (ds *Datastore) applyQueriesInTx( schedule_interval = VALUES(schedule_interval), automations_enabled = VALUES(automations_enabled), logging_type = VALUES(logging_type), - discard_data = VALUES(discard_data)` + discard_data = VALUES(discard_data)`) // 'queries' are uniquely identified by {name, team_id} unqKeyGen := func(name string, teamID *uint) string { @@ -279,8 +278,9 @@ func (ds *Datastore) NewQuery( ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ` - result, err := ds.writer(ctx).ExecContext( + id, err := ds.insertAndGetID( ctx, + ds.writer(ctx), queryStatement, query.Name, query.Description, @@ -300,13 +300,12 @@ func (ds *Datastore) NewQuery( query.UpdatedAt, ) - if err != nil && IsDuplicate(err) { + if err != nil && ds.dialect.IsDuplicate(err) { return nil, ctxerr.Wrap(ctx, alreadyExists("Query", query.Name)) } else if err != nil { return nil, ctxerr.Wrap(ctx, err, "creating new Query") } - id, _ := result.LastInsertId() query.ID = uint(id) //nolint:gosec // dismiss G115 query.Packs = []fleet.Pack{} @@ -523,7 +522,7 @@ func (ds *Datastore) DeleteQuery(ctx context.Context, teamID *uint, name string) deleteStmt := "DELETE FROM queries WHERE id = ?" result, err := ds.writer(ctx).ExecContext(ctx, deleteStmt, queryID) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey("queries", name)) } return ctxerr.Wrap(ctx, err, "delete queries") @@ -598,11 +597,11 @@ func (ds *Datastore) deleteQueryStats(ctx context.Context, queryIDs []uint) { // Query returns a single Query identified by id, if such exists. func (ds *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { - return query(ctx, ds.reader(ctx), id) + return query(ctx, ds.reader(ctx), id, ds.dialect) } -func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, error) { - sqlQuery := ` +func query(ctx context.Context, db sqlx.QueryerContext, id uint, dialect DialectHelper) (*fleet.Query, error) { + sqlQuery := fmt.Sprintf(` SELECT q.id, q.team_id, @@ -623,18 +622,24 @@ func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, q.discard_data, COALESCE(NULLIF(u.name, ''), u.email, '') AS author_name, COALESCE(u.email, '') AS author_email, - JSON_EXTRACT(json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM queries q LEFT JOIN users u ON q.author_id = u.id LEFT JOIN aggregated_stats ag ON (ag.id = q.id AND ag.global_stats = ? AND ag.type = ?) WHERE q.id = ? - ` + `, + dialect.JSONExtract("json_value", "$.user_time_p50"), + dialect.JSONExtract("json_value", "$.user_time_p95"), + dialect.JSONExtract("json_value", "$.system_time_p50"), + dialect.JSONExtract("json_value", "$.system_time_p95"), + dialect.JSONExtract("json_value", "$.total_executions"), + ) query := &fleet.Query{} if err := sqlx.GetContext(ctx, db, query, sqlQuery, false, fleet.AggregatedStatsTypeScheduledQuery, id); err != nil { if err == sql.ErrNoRows { @@ -658,7 +663,7 @@ func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, // determined by passed in fleet.ListOptions, count of total queries returned without limits, and // pagination metadata func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) (queries []*fleet.Query, total int, inherited int, metadata *fleet.PaginationMetadata, err error) { - getQueriesStmt := ` + getQueriesStmt := fmt.Sprintf(` SELECT q.id, q.team_id, @@ -678,15 +683,21 @@ func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions q.updated_at, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - JSON_EXTRACT(json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM queries q LEFT JOIN users u ON (q.author_id = u.id) LEFT JOIN aggregated_stats ag ON (ag.id = q.id AND ag.global_stats = ? AND ag.type = ?) - ` + `, + ds.dialect.JSONExtract("json_value", "$.user_time_p50"), + ds.dialect.JSONExtract("json_value", "$.user_time_p95"), + ds.dialect.JSONExtract("json_value", "$.system_time_p50"), + ds.dialect.JSONExtract("json_value", "$.system_time_p95"), + ds.dialect.JSONExtract("json_value", "$.total_executions"), + ) args := []interface{}{false, fleet.AggregatedStatsTypeScheduledQuery} whereClauses := "WHERE saved = true" @@ -1001,7 +1012,7 @@ func (ds *Datastore) UpdateLiveQueryStats(ctx context.Context, queryID uint, sta // Bulk insert/update const valueStr = "(?,?,?,?,?,?,?,?,?,?,?,?)," - stmt := "REPLACE INTO scheduled_query_stats (scheduled_query_id, host_id, query_type, executions, average_memory, system_time, user_time, wall_time, output_size, denylisted, schedule_interval, last_executed) VALUES " + + stmt := ds.dialect.ReplaceInto() + " scheduled_query_stats (scheduled_query_id, host_id, query_type, executions, average_memory, system_time, user_time, wall_time, output_size, denylisted, schedule_interval, last_executed) VALUES " + strings.Repeat(valueStr, len(stats)) stmt = strings.TrimSuffix(stmt, ",") diff --git a/server/datastore/mysql/query_results.go b/server/datastore/mysql/query_results.go index 2abb59fe8ca..c760fca599f 100644 --- a/server/datastore/mysql/query_results.go +++ b/server/datastore/mysql/query_results.go @@ -54,9 +54,9 @@ func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet } //nolint:gosec // SQL query is constructed using constant strings - insertStmt := ` - INSERT IGNORE INTO query_results (query_id, host_id, last_fetched, data) VALUES - ` + strings.Join(valueStrings, ",") + insertStmt := ds.dialect.InsertIgnoreInto() + ` + query_results (query_id, host_id, last_fetched, data) VALUES + ` + strings.Join(valueStrings, ",") + ds.dialect.OnConflictDoNothing("query_id,host_id") result, err = tx.ExecContext(ctx, insertStmt, valueArgs...) if err != nil { diff --git a/server/datastore/mysql/scim.go b/server/datastore/mysql/scim.go index cf3fa525095..b5221b30e06 100644 --- a/server/datastore/mysql/scim.go +++ b/server/datastore/mysql/scim.go @@ -32,8 +32,7 @@ func (ds *Datastore) CreateScimUser(ctx context.Context, user *fleet.ScimUser) ( INSERT INTO scim_users ( external_id, user_name, given_name, family_name, department, active ) VALUES (?, ?, ?, ?, ?, ?)` - result, err := tx.ExecContext( - ctx, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUserQuery, user.ExternalID, user.UserName, @@ -43,16 +42,12 @@ func (ds *Datastore) CreateScimUser(ctx context.Context, user *fleet.ScimUser) ( user.Active, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("ScimUser", user.UserName), "insert scim user") } return ctxerr.Wrap(ctx, err, "insert scim user") } - id, err := result.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "insert scim user last insert id") - } user.ID = uint(id) // nolint:gosec // dismiss G115 userID = user.ID @@ -309,7 +304,7 @@ func (ds *Datastore) ReplaceScimUser(ctx context.Context, user *fleet.ScimUser) user.ID, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("ScimUser", user.UserName), "update scim user") } return ctxerr.Wrap(ctx, err, "update scim user") @@ -651,8 +646,7 @@ func (ds *Datastore) CreateScimGroup(ctx context.Context, group *fleet.ScimGroup INSERT INTO scim_groups ( external_id, display_name ) VALUES (?, ?)` - result, err := tx.ExecContext( - ctx, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertGroupQuery, group.ExternalID, group.DisplayName, @@ -661,10 +655,6 @@ func (ds *Datastore) CreateScimGroup(ctx context.Context, group *fleet.ScimGroup return ctxerr.Wrap(ctx, err, "insert scim group") } - id, err := result.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "insert scim group last insert id") - } group.ID = uint(id) // nolint:gosec // dismiss G115 groupID = group.ID diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index cd8bbca8b29..6b043124105 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -26,12 +26,11 @@ func (ds *Datastore) NewHostScriptExecutionRequest(ctx context.Context, request var err error if request.ScriptContentID == 0 { // then we are doing a sync execution, so create the contents first - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 } res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, false) @@ -79,8 +78,8 @@ INSERT INTO upcoming_activities VALUES (?, ?, ?, ?, 'script', ?, JSON_OBJECT( - 'sync_request', ?, - 'is_internal', ?, + 'sync_request', CAST(? AS UNSIGNED), + 'is_internal', CAST(? AS UNSIGNED), 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` @@ -94,21 +93,29 @@ VALUES ) execID := uuid.New().String() - result, err := tx.ExecContext(ctx, insUAStmt, + // Convert booleans to int for JSON_OBJECT compatibility with PG's jsonb_build_object, + // which needs typed parameters. CAST(? AS UNSIGNED) → CAST($N AS integer) on PG. + syncRequestInt := 0 + if request.SyncRequest { + syncRequestInt = 1 + } + isInternalInt := 0 + if isInternal { + isInternalInt = 1 + } + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insUAStmt, request.HostID, request.Priority(), request.UserID, request.PolicyID != nil, // fleet-initiated if request is via a policy failure execID, - request.SyncRequest, - isInternal, + syncRequestInt, + isInternalInt, request.UserID, ) if err != nil { return "", 0, ctxerr.Wrap(ctx, err, "new script upcoming activity") } - - activityID, _ := result.LastInsertId() _, err = tx.ExecContext(ctx, insSUAStmt, activityID, request.ScriptID, @@ -292,7 +299,7 @@ func (ds *Datastore) listUpcomingHostScriptExecutions(ctx context.Context, hostI extraWhere := "" if onlyShowInternal { // software_uninstalls are implicitly internal - extraWhere = " AND COALESCE(ua.payload->'$.is_internal', 1) = 1" + extraWhere = " AND COALESCE(ua.payload->>'$.is_internal', '1') = '1'" } if onlyReadyToExecute { extraWhere += " AND ua.activated_at IS NOT NULL" @@ -487,26 +494,22 @@ func (ds *Datastore) CountHostScriptAttempts(ctx context.Context, hostID, script } func (ds *Datastore) NewScript(ctx context.Context, script *fleet.Script) (*fleet.Script, error) { - var res sql.Result + var scriptID int64 err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - var err error - // first insert script contents - scRes, err := insertScriptContents(ctx, tx, script.ScriptContents) + contentID, err := insertScriptContents(ctx, tx, ds.dialect, script.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() // then create the script entity - res, err = insertScript(ctx, tx, script, uint(id)) //nolint:gosec // dismiss G115 + scriptID, err = insertScript(ctx, tx, ds.dialect, script, uint(contentID)) //nolint:gosec // dismiss G115 return err }) if err != nil { return nil, err } - id, _ := res.LastInsertId() - return ds.getScriptDB(ctx, ds.writer(ctx), uint(id)) //nolint:gosec // dismiss G115 + return ds.getScriptDB(ctx, ds.writer(ctx), uint(scriptID)) //nolint:gosec // dismiss G115 } func (ds *Datastore) UpdateScriptContents(ctx context.Context, scriptID uint, scriptContents string) (*fleet.Script, error) { @@ -520,11 +523,10 @@ func (ds *Datastore) UpdateScriptContents(ctx context.Context, scriptID uint, sc } // Insert or get existing content (insertScriptContents handles deduplication) - scRes, err := insertScriptContents(ctx, tx, scriptContents) + newContentID, err := insertScriptContents(ctx, tx, ds.dialect, scriptContents) if err != nil { return ctxerr.Wrap(ctx, err, "inserting/getting script contents") } - newContentID, _ := scRes.LastInsertId() // Update the script to point to the new content if newContentID != oldContentID { @@ -622,7 +624,7 @@ func (ds *Datastore) resetScriptPolicyAutomationAttempts(ctx context.Context, db return nil } -func insertScript(ctx context.Context, tx sqlx.ExtContext, script *fleet.Script, scriptContentsID uint) (sql.Result, error) { +func insertScript(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, script *fleet.Script, scriptContentsID uint) (int64, error) { const insertStmt = ` INSERT INTO scripts ( @@ -635,7 +637,7 @@ VALUES if script.TeamID != nil { globalOrTeamID = *script.TeamID } - res, err := tx.ExecContext(ctx, insertStmt, + id, err := insertAndGetIDTx(ctx, tx, dialect, insertStmt, script.TeamID, globalOrTeamID, script.Name, scriptContentsID) if err != nil { if IsDuplicate(err) { @@ -645,29 +647,27 @@ VALUES // team does not exist err = foreignKey("scripts", fmt.Sprintf("team_id=%v", script.TeamID)) } - return nil, ctxerr.Wrap(ctx, err, "insert script") + return 0, ctxerr.Wrap(ctx, err, "insert script") } - return res, nil + return id, nil } -func insertScriptContents(ctx context.Context, tx sqlx.ExtContext, contents string) (sql.Result, error) { - const insertStmt = ` +func insertScriptContents(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, contents string) (int64, error) { + insertStmt := ` INSERT INTO script_contents ( md5_checksum, contents ) VALUES (UNHEX(?),?) -ON DUPLICATE KEY UPDATE - id=LAST_INSERT_ID(id) - ` +` + dialect.OnDuplicateKey("md5_checksum", "id=LAST_INSERT_ID(id)") md5Checksum := md5ChecksumScriptContent(contents) - res, err := tx.ExecContext(ctx, insertStmt, md5Checksum, contents) + id, err := insertAndGetIDTx(ctx, tx, dialect, insertStmt, md5Checksum, contents) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "insert script contents") + return 0, ctxerr.Wrap(ctx, err, "insert script contents") } - return res, nil + return id, nil } func md5ChecksumScriptContent(s string) string { @@ -810,7 +810,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { WHERE sua.script_id = ? AND ua.activity_type = 'script' AND ua.activated_at IS NOT NULL AND - (ua.payload->'$.sync_request' = 0 OR + (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND)` var affectedHosts []uint if err := sqlx.SelectContext(ctx, tx, &affectedHosts, loadAffectedHostsStmt, @@ -825,7 +825,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { ON upcoming_activities.id = sua.upcoming_activity_id WHERE sua.script_id = ? AND upcoming_activities.activity_type = 'script' AND - (upcoming_activities.payload->'$.sync_request' = 0 OR + (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) `, id, int(constants.MaxServerWaitTime.Seconds()), @@ -836,7 +836,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { _, err = tx.ExecContext(ctx, `DELETE FROM scripts WHERE id = ?`, id) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the script is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies WHERE script_id = ?`, id); err != nil { @@ -1171,7 +1171,7 @@ WHERE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` const clearAllPendingExecutionsUA = `DELETE FROM upcoming_activities @@ -1181,7 +1181,7 @@ WHERE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` const unsetScriptsNotInListFromPolicies = ` @@ -1211,7 +1211,7 @@ WHERE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` const clearPendingExecutionsNotInListUA = `DELETE FROM upcoming_activities @@ -1221,19 +1221,17 @@ WHERE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` - const insertNewOrEditedScript = ` + insertNewOrEditedScript := ` INSERT INTO scripts ( team_id, global_or_team_id, name, script_content_id ) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id) -` +` + ds.dialect.OnDuplicateKey("id", "script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id)") const clearPendingExecutionsWithObsoleteScriptHSR = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) @@ -1249,7 +1247,7 @@ ON DUPLICATE KEY UPDATE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id = ? AND sua.script_content_id != ?` const clearPendingExecutionsWithObsoleteScriptUA = `DELETE FROM upcoming_activities @@ -1259,7 +1257,7 @@ ON DUPLICATE KEY UPDATE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id = ? AND sua.script_content_id != ?` const loadInsertedScripts = `SELECT id, team_id, name FROM scripts WHERE global_or_team_id = ?` @@ -1382,16 +1380,14 @@ ON DUPLICATE KEY UPDATE // insert the new scripts and the ones that have changed for _, s := range incomingScripts { - scRes, err := insertScriptContents(ctx, tx, s.ScriptContents) + contentID, err := insertScriptContents(ctx, tx, ds.dialect, s.ScriptContents) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting script contents for script with name %q", s.Name) } - contentID, _ := scRes.LastInsertId() - insertRes, err := tx.ExecContext(ctx, insertNewOrEditedScript, tmID, globalOrTeamID, s.Name, uint(contentID)) //nolint:gosec // dismiss G115 + scriptID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertNewOrEditedScript, tmID, globalOrTeamID, s.Name, uint(contentID)) //nolint:gosec // dismiss G115 if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited script with name %q", s.Name) } - scriptID, _ := insertRes.LastInsertId() if _, err := tx.ExecContext(ctx, clearPendingExecutionsWithObsoleteScriptHSR, int(constants.MaxServerWaitTime.Seconds()), scriptID, contentID); err != nil { return ctxerr.Wrapf(ctx, err, "clear obsolete pending script executions with name %q", s.Name) @@ -2085,12 +2081,11 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2103,7 +2098,7 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS // it is pending execution. The host's state should be updated to "locked" // only when the script execution is successfully completed, and then any // unlock or wipe references should be cleared. - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2111,9 +2106,7 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - lock_ref = VALUES(lock_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `lock_ref = VALUES(lock_ref)`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2135,12 +2128,11 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2153,7 +2145,7 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos // recorded, it is pending execution. The host's state should be updated to // "unlocked" only when the script execution is successfully completed, and // then any lock or wipe references should be cleared. - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2161,10 +2153,8 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - unlock_ref = VALUES(unlock_ref), - unlock_pin = NULL - ` + ` + ds.dialect.OnDuplicateKey("host_id", `unlock_ref = VALUES(unlock_ref), + unlock_pin = NULL`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2186,12 +2176,11 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2203,7 +2192,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS // point in time, this is just a request to wipe the host that is recorded, // it is pending execution, so if it was locked, it is still locked (so the // lock_ref info must still be there). - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2211,9 +2200,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - wipe_ref = VALUES(wipe_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `wipe_ref = VALUES(wipe_ref)`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2229,7 +2216,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS } func (ds *Datastore) UnlockHostManually(ctx context.Context, hostID uint, hostFleetPlatform string, ts time.Time) error { - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2237,10 +2224,8 @@ func (ds *Datastore) UnlockHostManually(ctx context.Context, hostID uint, hostFl fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - -- do not overwrite if a value is already set - unlock_ref = IF(unlock_ref IS NULL, VALUES(unlock_ref), unlock_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `-- do not overwrite if a value is already set + unlock_ref = IF(unlock_ref IS NULL, VALUES(unlock_ref), unlock_ref)`) // for macOS, the unlock_ref is just the timestamp at which the user first // requested to unlock the host. This then indicates in the host's status // that it's pending an unlock (which requires manual intervention by @@ -2474,8 +2459,8 @@ func (ds *Datastore) batchExecuteScript(ctx context.Context, userID *uint, scrip _, err := tx.ExecContext( ctx, - `INSERT INTO batch_activities (execution_id, script_id, status, activity_type, num_targeted, started_at) VALUES (?, ?, ?, ?, ?, NOW()) - ON DUPLICATE KEY UPDATE status = VALUES(status), started_at = VALUES(started_at)`, + `INSERT INTO batch_activities (execution_id, script_id, status, activity_type, num_targeted, started_at) VALUES (?, ?, ?, ?, ?, NOW()) `+ + ds.dialect.OnDuplicateKey("id", "status = VALUES(status), started_at = VALUES(started_at)"), batchExecID, script.ID, fleet.ScheduledBatchExecutionStarted, @@ -2507,7 +2492,7 @@ func (ds *Datastore) batchExecuteScript(ctx context.Context, userID *uint, scrip :host_id, :host_execution_id, :error - ) ON DUPLICATE KEY UPDATE host_execution_id = VALUES(host_execution_id), error = VALUES(error)` + ) ` + ds.dialect.OnDuplicateKey("id", "host_execution_id = VALUES(host_execution_id), error = VALUES(error)") if _, err := sqlx.NamedExecContext(ctx, tx, insertStmt, args); err != nil { return ctxerr.Wrap(ctx, err, "associating script executions with batch job") diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index d10f7b8f0fa..328806c2e53 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -152,12 +152,11 @@ func (ds *Datastore) makeSessionInTransaction(ctx context.Context, tx sqlx.ExtCo ) VALUES(?,?) ` - result, err := tx.ExecContext(ctx, sqlStatement, userID, sessionKey) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, userID, sessionKey) if err != nil { return nil, ctxerr.Wrap(ctx, err, "saving session") } - id, _ := result.LastInsertId() // cannot fail with the mysql driver return ds.sessionByID(ctx, tx, uint(id)) //nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 45541abc7fc..49feb65b59f 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -29,7 +29,7 @@ import ( type softwareSummary struct { ID uint `db:"id"` - Checksum string `db:"checksum"` + Checksum []byte `db:"checksum"` Name string `db:"name"` TitleID *uint `db:"title_id"` BundleIdentifier *string `db:"bundle_identifier"` @@ -489,7 +489,7 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( return err } - if err = updateSoftwareUpdatedAt(ctx, tx, hostID); err != nil { + if err = updateSoftwareUpdatedAt(ctx, tx, ds.dialect, hostID); err != nil { return err } return nil @@ -605,9 +605,9 @@ func (ds *Datastore) getExistingSoftware( } if len(newChecksumsToSoftware) > 0 { - sliceOfNewSWChecksums := make([]string, 0, len(newChecksumsToSoftware)) + sliceOfNewSWChecksums := make([][]byte, 0, len(newChecksumsToSoftware)) for checksum := range newChecksumsToSoftware { - sliceOfNewSWChecksums = append(sliceOfNewSWChecksums, checksum) + sliceOfNewSWChecksums = append(sliceOfNewSWChecksums, []byte(checksum)) } // We use the replica DB for retrieval to minimize the traffic to the writer DB. // It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the writer DB. @@ -617,14 +617,14 @@ func (ds *Datastore) getExistingSoftware( } for _, currentSoftwareSummary := range currentSoftwareSummaries { - _, ok := newChecksumsToSoftware[currentSoftwareSummary.Checksum] + _, ok := newChecksumsToSoftware[string(currentSoftwareSummary.Checksum)] if !ok { // This should never happen. If it does, we have a bug. return nil, nil, nil, ctxerr.New( - ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareSummary.Checksum))), + ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString(currentSoftwareSummary.Checksum)), ) } - delete(setOfNewSWChecksums, currentSoftwareSummary.Checksum) + delete(setOfNewSWChecksums, string(currentSoftwareSummary.Checksum)) } } @@ -878,7 +878,7 @@ func (ds *Datastore) preInsertSoftwareInventory( existingSet := make(map[string]struct{}, len(existingSoftwareSummaries)) for _, es := range existingSoftwareSummaries { - existingSet[es.Checksum] = struct{}{} + existingSet[string(es.Checksum)] = struct{}{} } for checksum, sw := range incomingSoftwareByChecksum { @@ -1002,7 +1002,7 @@ func (ds *Datastore) preInsertSoftwareInventory( // Insert software titles const numberOfArgsPerSoftwareTitles = 7 titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",") - titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id, upgrade_code) VALUES %s", titlesValues) + titlesStmt := fmt.Sprintf(ds.dialect.InsertIgnoreInto()+" software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id, upgrade_code) VALUES %s"+ds.dialect.OnConflictDoNothing("unique_identifier,source,extension_for"), titlesValues) titlesArgs := make([]any, 0, len(uniqueTitlesToInsert)*numberOfArgsPerSoftwareTitles) for _, title := range uniqueTitlesToInsert { @@ -1155,7 +1155,7 @@ func (ds *Datastore) preInsertSoftwareInventory( strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",", ) stmt := fmt.Sprintf( - `INSERT IGNORE INTO software ( + ds.dialect.InsertIgnoreInto()+` software ( name, version, source, @@ -1169,7 +1169,7 @@ func (ds *Datastore) preInsertSoftwareInventory( checksum, application_id, upgrade_code - ) VALUES %s`, + ) VALUES %s`+ds.dialect.OnConflictDoNothing("checksum"), values, ) @@ -1188,7 +1188,7 @@ func (ds *Datastore) preInsertSoftwareInventory( } args = append( args, sw.Name, sw.Version, sw.Source, sw.Release, sw.Vendor, sw.Arch, - sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, checksum, sw.ApplicationID, sw.UpgradeCode, + sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, []byte(checksum), sw.ApplicationID, sw.UpgradeCode, ) } @@ -1228,9 +1228,9 @@ func (ds *Datastore) linkSoftwareToHost( var insertedSoftware []fleet.Software // Build map of all checksums we need to link - allChecksums := make([]string, 0, len(softwareChecksums)) + allChecksums := make([][]byte, 0, len(softwareChecksums)) for checksum := range softwareChecksums { - allChecksums = append(allChecksums, checksum) + allChecksums = append(allChecksums, []byte(checksum)) } // Get all software IDs (they should exist from pre-insertion). @@ -1244,7 +1244,7 @@ func (ds *Datastore) linkSoftwareToHost( // Build ID map softwareSummaryByChecksum := make(map[string]softwareSummary) for _, s := range allSoftwareSummaries { - softwareSummaryByChecksum[s.Checksum] = s + softwareSummaryByChecksum[string(s.Checksum)] = s } // Link software to host @@ -1267,7 +1267,7 @@ func (ds *Datastore) linkSoftwareToHost( // INSERT IGNORE handles duplicate key errors for idempotency. if len(insertsHostSoftware) > 0 { values := strings.TrimSuffix(strings.Repeat("(?,?,?),", len(insertsHostSoftware)/3), ",") - stmt := fmt.Sprintf(`INSERT IGNORE INTO host_software (host_id, software_id, last_opened_at) VALUES %s`, values) + stmt := fmt.Sprintf(ds.dialect.InsertIgnoreInto()+` host_software (host_id, software_id, last_opened_at) VALUES %s`+ds.dialect.OnConflictDoNothing("host_id,software_id"), values) if _, err := tx.ExecContext(ctx, stmt, insertsHostSoftware...); err != nil { return nil, ctxerr.Wrap(ctx, err, "insert host software") } @@ -1418,7 +1418,7 @@ func (ds *Datastore) reconcileExistingTitleEmptyWindowsUpgradeCodes( return nil } -func getExistingSoftwareSummariesByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums []string) ([]softwareSummary, error) { +func getExistingSoftwareSummariesByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums [][]byte) ([]softwareSummary, error) { if len(checksums) == 0 { return []softwareSummary{}, nil } @@ -1510,9 +1510,10 @@ func updateModifiedHostSoftwareDB( func updateSoftwareUpdatedAt( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, hostID uint, ) error { - const stmt = `INSERT INTO host_updates(host_id, software_updated_at) VALUES (?, CURRENT_TIMESTAMP) ON DUPLICATE KEY UPDATE software_updated_at=VALUES(software_updated_at)` + stmt := `INSERT INTO host_updates(host_id, software_updated_at) VALUES (?, CURRENT_TIMESTAMP) ` + dialect.OnDuplicateKey("host_id", "software_updated_at=VALUES(software_updated_at)") if _, err := tx.ExecContext(ctx, stmt, hostID); err != nil { return ctxerr.Wrap(ctx, err, "update host updates") @@ -1521,7 +1522,10 @@ func updateSoftwareUpdatedAt( return nil } -var dialect = goqu.Dialect("mysql") +// goquMySQLDialect is a package-level fallback for standalone functions that +// haven't been refactored to accept a goqu.DialectWrapper parameter yet. +// TODO(pg): remove once all standalone functions accept a dialect parameter. +var goquMySQLDialect = goqu.Dialect("mysql") // listSoftwareDB returns software installed on hosts. Use opts for pagination, filtering, and controlling // fields populated in the returned software. @@ -1818,7 +1822,7 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e } // Fallback to the original goqu-based query builder for complex cases - ds := dialect. + ds := goquMySQLDialect. From(goqu.I("software").As("s")). Select( "s.id", @@ -2022,7 +2026,7 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e ds = appendListOptionsToSelect(ds, opts.ListOptions) // join on software_cve and cve_meta after apply pagination using the sub-query above - ds = dialect.From(ds.As("s")). + ds = goquMySQLDialect.From(ds.As("s")). Select( "s.id", "s.name", @@ -2295,12 +2299,12 @@ func (ds *Datastore) AllSoftwareIterator( } if query.NameMatch != "" { - conditionals = append(conditionals, "s.name REGEXP ?") + conditionals = append(conditionals, ds.dialect.RegexpMatch("s.name", "?")) args = append(args, query.NameMatch) } if query.NameExclude != "" { - conditionals = append(conditionals, "s.name NOT REGEXP ?") + conditionals = append(conditionals, "NOT ("+ds.dialect.RegexpMatch("s.name", "?")+")") args = append(args, query.NameExclude) } @@ -2329,7 +2333,7 @@ func (ds *Datastore) UpsertSoftwareCPEs(ctx context.Context, cpes []fleet.Softwa values := strings.TrimSuffix(strings.Repeat("(?,?),", len(cpes)), ",") sql := fmt.Sprintf( - `INSERT INTO software_cpe (software_id, cpe) VALUES %s ON DUPLICATE KEY UPDATE cpe = VALUES(cpe)`, + `INSERT INTO software_cpe (software_id, cpe) VALUES %s `+ds.dialect.OnDuplicateKey("id", `cpe = VALUES(cpe)`), values, ) @@ -2475,9 +2479,11 @@ func (ds *Datastore) DeleteOutOfDateVulnerabilities(ctx context.Context, source func (ds *Datastore) DeleteOrphanedSoftwareVulnerabilities(ctx context.Context) error { if _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE sc FROM software_cve sc - LEFT JOIN host_software hs ON hs.software_id = sc.software_id - WHERE hs.host_id IS NULL + DELETE FROM software_cve + WHERE NOT EXISTS ( + SELECT 1 FROM host_software hs + WHERE hs.software_id = software_cve.software_id + ) `); err != nil { return ctxerr.Wrap(ctx, err, "deleting orphaned software vulnerabilities") } @@ -2485,7 +2491,7 @@ func (ds *Datastore) DeleteOrphanedSoftwareVulnerabilities(ctx context.Context) } func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, teamID *uint, includeCVEScores bool, tmFilter *fleet.TeamFilter) (*fleet.Software, error) { - q := dialect.From(goqu.I("software").As("s")). + q := ds.dialect.GoquDialect().From(goqu.I("software").As("s")). Select( "s.id", "s.name", @@ -2650,17 +2656,17 @@ func (ds *Datastore) SyncHostsSoftware(ctx context.Context, updatedAt time.Time) WHERE h.team_id IS NULL AND hs.software_id > ? AND hs.software_id <= ? GROUP BY hs.software_id` - insertStmt = ` + valuesPart = `(?, ?, ?, ?, ?),` + ) + + insertStmt := ` INSERT INTO ` + swapTable + ` (software_id, hosts_count, team_id, global_stats, updated_at) VALUES %s - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_id,software_id", ` hosts_count = VALUES(hosts_count), - updated_at = VALUES(updated_at)` - - valuesPart = `(?, ?, ?, ?, ?),` - ) + updated_at = VALUES(updated_at)`) // Create a fresh swap table to populate with new counts. If a previous run left a partial swap table, drop it first. w := ds.writer(ctx) @@ -2842,12 +2848,12 @@ func (ds *Datastore) CleanupSoftwareTitles(ctx context.Context) error { // Re-check orphan status on the writer to avoid deleting a title that an IT admin just linked // (e.g., added a software installer) between the reader SELECT and this DELETE. deleteOrphanedSoftwareTitlesStmt = ` - DELETE st FROM software_titles st - LEFT JOIN software s ON st.id = s.title_id - LEFT JOIN software_installers si ON st.id = si.title_id - LEFT JOIN in_house_apps iha ON st.id = iha.title_id - LEFT JOIN vpp_apps vap ON st.id = vap.title_id - WHERE st.id IN (?) AND s.title_id IS NULL AND si.title_id IS NULL AND iha.title_id IS NULL AND vap.title_id IS NULL` + DELETE FROM software_titles + WHERE id IN (?) + AND NOT EXISTS (SELECT 1 FROM software s WHERE s.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM in_house_apps iha WHERE iha.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM vpp_apps vap WHERE vap.title_id = software_titles.id)` ) var lastID uint @@ -2981,13 +2987,13 @@ func (ds *Datastore) InsertCVEMeta(ctx context.Context, cveMeta []fleet.CVEMeta) query := ` INSERT INTO cve_meta (cve, cvss_score, epss_probability, cisa_known_exploit, published, description) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("cve", ` cvss_score = VALUES(cvss_score), epss_probability = VALUES(epss_probability), cisa_known_exploit = VALUES(cisa_known_exploit), published = VALUES(published), description = VALUES(description) -` +`) batchSize := 500 for i := 0; i < len(cveMeta); i += batchSize { @@ -3029,11 +3035,11 @@ func (ds *Datastore) InsertSoftwareVulnerability( stmt := ` INSERT INTO software_cve (cve, source, software_id, resolved_in_version) VALUES (?,?,?,?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at=? - ` + `) args = append(args, vuln.CVE, source, vuln.SoftwareID, vuln.ResolvedInVersion, time.Now().UTC()) res, err := ds.writer(ctx).ExecContext(ctx, stmt, args...) @@ -3106,11 +3112,11 @@ func (ds *Datastore) InsertSoftwareVulnerabilities( stmt := fmt.Sprintf(` INSERT INTO software_cve (cve, source, software_id, resolved_in_version) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = ? - `, values) + `), values) var args []any for _, v := range batch { @@ -3142,7 +3148,7 @@ func (ds *Datastore) ListSoftwareVulnerabilitiesByHostIDsSource( } var queryR []softwareVulnerabilityWithHostId - stmt := dialect. + stmt := ds.dialect.GoquDialect(). From(goqu.T("software_cve").As("sc")). Join( goqu.T("host_software").As("hs"), @@ -3246,7 +3252,7 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee var result []fleet.CVEMeta maxAgeDate := time.Now().Add(-1 * maxAge) - stmt := dialect.From(goqu.T("cve_meta")). + stmt := ds.dialect.GoquDialect().From(goqu.T("cve_meta")). Select( goqu.C("cve"), goqu.C("cvss_score"), @@ -5769,6 +5775,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt } var replacements []any + gc := ds.dialect.GroupConcat if len(softwareTitleIDs) > 0 { replacements = append(replacements, // For software installers @@ -5784,12 +5791,12 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt software_installers.filename AS package_name, software_installers.version AS package_version, software_installers.platform as package_platform, - GROUP_CONCAT(software.id) AS software_id_list, - GROUP_CONCAT(software.source) AS software_source_list, - GROUP_CONCAT(software.extension_for) AS software_extension_for_list, - GROUP_CONCAT(software.upgrade_code) AS software_upgrade_code_list, - GROUP_CONCAT(software.version) AS version_list, - GROUP_CONCAT(software.bundle_identifier) AS bundle_identifier_list, + `+gc("software.id", ",")+` AS software_id_list, + `+gc("software.source", ",")+` AS software_source_list, + `+gc("software.extension_for", ",")+` AS software_extension_for_list, + `+gc("software.upgrade_code", ",")+` AS software_upgrade_code_list, + `+gc("software.version", ",")+` AS version_list, + `+gc("software.bundle_identifier", ",")+` AS bundle_identifier_list, NULL AS vpp_app_adam_id_list, NULL AS vpp_app_version_list, NULL AS vpp_app_platform_list, @@ -5835,11 +5842,11 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt NULL AS software_upgrade_code_list, NULL AS version_list, NULL AS bundle_identifier_list, - GROUP_CONCAT(vpp_apps.adam_id) AS vpp_app_adam_id_list, - GROUP_CONCAT(vpp_apps.latest_version) AS vpp_app_version_list, - GROUP_CONCAT(vpp_apps.platform) as vpp_app_platform_list, - GROUP_CONCAT(vpp_apps.icon_url) AS vpp_app_icon_url_list, - GROUP_CONCAT(vpp_apps_teams.self_service) AS vpp_app_self_service_list, + `+gc("vpp_apps.adam_id", ",")+` AS vpp_app_adam_id_list, + `+gc("vpp_apps.latest_version", ",")+` AS vpp_app_version_list, + `+gc("vpp_apps.platform", ",")+` as vpp_app_platform_list, + `+gc("vpp_apps.icon_url", ",")+` AS vpp_app_icon_url_list, + `+gc("vpp_apps_teams.self_service", ",")+` AS vpp_app_self_service_list, NULL AS in_house_app_id_list, NULL AS in_house_app_name_list, NULL AS in_house_app_version_list, @@ -5881,11 +5888,11 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt NULL as vpp_app_platform_list, NULL AS vpp_app_icon_url_list, NULL AS vpp_app_self_service_list, - GROUP_CONCAT(in_house_apps.id) AS in_house_app_id_list, - GROUP_CONCAT(in_house_apps.filename) AS in_house_app_name_list, - GROUP_CONCAT(in_house_apps.version) AS in_house_app_version_list, - GROUP_CONCAT(in_house_apps.platform) as in_house_app_platform_list, - GROUP_CONCAT(in_house_apps.self_service) as in_house_app_self_service_list + `+gc("in_house_apps.id", ",")+` AS in_house_app_id_list, + `+gc("in_house_apps.filename", ",")+` AS in_house_app_name_list, + `+gc("in_house_apps.version", ",")+` AS in_house_app_version_list, + `+gc("in_house_apps.platform", ",")+` as in_house_app_platform_list, + `+gc("in_house_apps.self_service", ",")+` as in_house_app_self_service_list `, ` GROUP BY software_titles.id, @@ -6416,7 +6423,7 @@ func (ds *Datastore) CreateIntermediateInstallFailureRecord(ctx context.Context, // Create or update a record with the failure details // Use INSERT ... ON DUPLICATE KEY UPDATE to make this idempotent - const insertStmt = ` + insertStmt := ` INSERT INTO host_software_installs ( execution_id, host_id, @@ -6434,14 +6441,14 @@ func (ds *Datastore) CreateIntermediateInstallFailureRecord(ctx context.Context, post_install_script_exit_code, post_install_script_output ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` install_script_exit_code = VALUES(install_script_exit_code), install_script_output = VALUES(install_script_output), pre_install_query_output = VALUES(pre_install_query_output), post_install_script_exit_code = VALUES(post_install_script_exit_code), post_install_script_output = VALUES(post_install_script_output), updated_at = CURRENT_TIMESTAMP(6) - ` + `) truncateOutput := func(output *string) *string { if output != nil { @@ -6569,12 +6576,11 @@ WHERE hvsi.host_id = ? AND st.id IN (?) func (ds *Datastore) NewSoftwareCategory(ctx context.Context, name string) (*fleet.SoftwareCategory, error) { stmt := `INSERT INTO software_categories (name) VALUES (?)` - res, err := ds.writer(ctx).ExecContext(ctx, stmt, name) + r, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, name) if err != nil { return nil, ctxerr.Wrap(ctx, err, "new software category") } - r, _ := res.LastInsertId() id := uint(r) //nolint:gosec // dismiss G115 return &fleet.SoftwareCategory{Name: name, ID: id}, nil } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 507f1f153ae..3dfa2baadd4 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -350,9 +350,9 @@ INSERT INTO software_installers ( true, } - res, err := tx.ExecContext(ctx, stmt, args...) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { // already exists for this team/no team teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { @@ -363,10 +363,9 @@ INSERT INTO software_installers ( return err } - id, _ := res.LastInsertId() installerID = uint(id) //nolint:gosec // dismiss G115 - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, installerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, installerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { return ctxerr.Wrap(ctx, err, "upsert software installer labels") } @@ -485,7 +484,7 @@ func (ds *Datastore) createAutomaticPolicy(ctx context.Context, tx sqlx.ExtConte SoftwareInstallerID: softwareInstallerID, VPPAppsTeamsID: vppAppsTeamsID, Type: fleet.PolicyTypeDynamic, - }) + }, ds.dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "create automatic policy query") } @@ -578,7 +577,7 @@ func (ds *Datastore) addSoftwareTitleToMatchingSoftware(ctx context.Context, tit args = append(args, whereArgs...) updateSoftwareStmt := fmt.Sprintf(` UPDATE software s - SET s.title_id = ? + SET title_id = ? %s`, whereClause) _, err := ds.writer(ctx).ExecContext(ctx, updateSoftwareStmt, args...) return ctxerr.Wrap(ctx, err, "adding fk reference in software to software_titles") @@ -594,7 +593,7 @@ const ( // setOrUpdateSoftwareInstallerLabelsDB sets or updates the label associations for the specified software // installer. If no labels are provided, it will remove all label associations with the software installer. -func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContext, installerID uint, labels fleet.LabelIdentsWithScope, softwareType softwareType) error { +func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, installerID uint, labels fleet.LabelIdentsWithScope, softwareType softwareType) error { labelIds := make([]uint, 0, len(labels.ByName)) for _, label := range labels.ByName { labelIds = append(labelIds, label.LabelID) @@ -635,7 +634,7 @@ func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContex } stmt := `INSERT INTO %[1]s_labels (%[1]s_id, label_id, exclude, require_all) VALUES %s - ON DUPLICATE KEY UPDATE exclude = VALUES(exclude), require_all = VALUES(require_all)` + ` + dialect.OnDuplicateKey("id", "exclude = VALUES(exclude), require_all = VALUES(require_all)") var placeholders string var insertArgs []interface{} for _, lid := range labelIds { @@ -731,7 +730,7 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } if payload.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { return ctxerr.Wrap(ctx, err, "upsert software installer labels") } } @@ -743,7 +742,7 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } if payload.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "update software title display name") } } @@ -1161,7 +1160,7 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error // allow delete only if install_during_setup is false res, err := tx.ExecContext(ctx, `DELETE FROM software_installers WHERE id = ? AND install_during_setup = 0`, id) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the software installer is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies WHERE software_installer_id = ?`, id); err != nil { @@ -1283,12 +1282,12 @@ INSERT INTO upcoming_activities VALUES (?, ?, ?, ?, 'software_install', ?, JSON_OBJECT( - 'self_service', ?, + 'self_service', CAST(? AS UNSIGNED), 'installer_filename', ?, 'version', ?, 'software_title_name', ?, 'source', ?, - 'with_retries', ?, + 'with_retries', CAST(? AS UNSIGNED), 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` @@ -1336,26 +1335,33 @@ VALUES } execID := uuid.NewString() + // Convert booleans to int for JSON_OBJECT compatibility with PG's jsonb_build_object. + selfServiceInt := 0 + if opts.SelfService { + selfServiceInt = 1 + } + withRetriesInt := 0 + if opts.WithRetries { + withRetriesInt = 1 + } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, opts.IsFleetInitiated(), execID, - opts.SelfService, + selfServiceInt, installerDetails.Filename, installerDetails.Version, installerDetails.TitleName, installerDetails.Source, - opts.WithRetries, + withRetriesInt, userID, ) if err != nil { return ctxerr.Wrap(ctx, err, "insert software install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertSIUAStmt, activityID, softwareInstallerID, @@ -1474,7 +1480,7 @@ VALUES 'software_title_name', ?, 'source', ?, 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?), - 'self_service', ? + 'self_service', CAST(? AS UNSIGNED) ) )` @@ -1515,8 +1521,12 @@ VALUES userID = &ctxUser.ID } + selfServiceInt := 0 + if selfService { + selfServiceInt = 1 + } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, 0, // Uninstalls are never used in setup experience, so always default priority userID, @@ -1525,13 +1535,11 @@ VALUES installerDetails.TitleName, installerDetails.Source, userID, - selfService, + selfServiceInt, ) if err != nil { return err } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertSIUAStmt, activityID, softwareInstallerID, @@ -2063,17 +2071,17 @@ func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwa const maxCachedFMAVersions = 2 func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { - const upsertSoftwareTitles = ` + upsertSoftwareTitles := ` INSERT INTO software_titles (name, source, extension_for, bundle_identifier, upgrade_code) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` name = VALUES(name), source = VALUES(source), extension_for = VALUES(extension_for), bundle_identifier = VALUES(bundle_identifier) -` +`) const loadSoftwareTitles = ` SELECT @@ -2280,7 +2288,7 @@ WHERE title_id = ? ` - const insertNewOrEditedInstaller = ` + insertNewOrEditedInstaller := ` INSERT INTO software_installers ( team_id, global_or_team_id, @@ -2308,7 +2316,7 @@ INSERT INTO software_installers ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT name FROM users WHERE id = ?), (SELECT email FROM users WHERE id = ?), ?, ?, COALESCE(?, false), ?, ? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` install_script_content_id = VALUES(install_script_content_id), uninstall_script_content_id = VALUES(uninstall_script_content_id), post_install_script_content_id = VALUES(post_install_script_content_id), @@ -2326,7 +2334,7 @@ ON DUPLICATE KEY UPDATE url = VALUES(url), install_during_setup = COALESCE(?, install_during_setup), is_active = VALUES(is_active) -` +`) const updateInstaller = ` UPDATE @@ -2368,7 +2376,7 @@ WHERE software_installer_id = ? ` - const upsertInstallerLabels = ` + upsertInstallerLabels := ` INSERT INTO software_installer_labels ( software_installer_id, @@ -2378,10 +2386,10 @@ INSERT INTO ) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` exclude = VALUES(exclude), require_all = VALUES(require_all) -` +`) const loadExistingInstallerLabels = ` SELECT @@ -2409,8 +2417,7 @@ WHERE software_category_id NOT IN (?) ` - const upsertInstallerCategories = ` -INSERT IGNORE INTO + const upsertInstallerCategoriesSuffix = ` software_installer_software_categories ( software_installer_id, software_category_id @@ -2672,26 +2679,23 @@ WHERE return ctxerr.Errorf(ctx, "labels have not been validated for installer with name %s", installer.Filename) } - isRes, err := insertScriptContents(ctx, tx, installer.InstallScript) + installScriptID, err := insertScriptContents(ctx, tx, ds.dialect, installer.InstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting install script contents for software installer with name %q", installer.Filename) } - installScriptID, _ := isRes.LastInsertId() - uisRes, err := insertScriptContents(ctx, tx, installer.UninstallScript) + uninstallScriptID, err := insertScriptContents(ctx, tx, ds.dialect, installer.UninstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting uninstall script contents for software installer with name %q", installer.Filename) } - uninstallScriptID, _ := uisRes.LastInsertId() var postInstallScriptID *int64 if installer.PostInstallScript != "" { - pisRes, err := insertScriptContents(ctx, tx, installer.PostInstallScript) + insertID, err := insertScriptContents(ctx, tx, ds.dialect, installer.PostInstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting post-install script contents for software installer with name %q", installer.Filename) } - insertID, _ := pisRes.LastInsertId() postInstallScriptID = &insertID } @@ -3022,7 +3026,7 @@ WHERE upsertCategoriesArgs = append(upsertCategoriesArgs, installerID, catID) } upsertCategoriesValues := strings.TrimSuffix(strings.Repeat("(?,?),", len(installer.CategoryIDs)), ",") - _, err = tx.ExecContext(ctx, fmt.Sprintf(upsertInstallerCategories, upsertCategoriesValues), upsertCategoriesArgs...) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+fmt.Sprintf(upsertInstallerCategoriesSuffix, upsertCategoriesValues)+ds.dialect.OnConflictDoNothing("software_installer_id,software_category_id"), upsertCategoriesArgs...) if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited categories for installer with name %q", installer.Filename) } @@ -3031,7 +3035,7 @@ WHERE // update display name for the software title if it needs to be updated or inserted // no deletions will happen, display names will be set to empty if needed if name, ok := displayNameIDMap[titleID]; (ok && name != installer.DisplayName) || (!ok && installer.DisplayName != "") { - if err := updateSoftwareTitleDisplayName(ctx, tx, tmID, titleID, installer.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, tmID, titleID, installer.DisplayName); err != nil { return ctxerr.Wrapf(ctx, err, "update software title display name for installer with name %q", installer.Filename) } } diff --git a/server/datastore/mysql/statistics.go b/server/datastore/mysql/statistics.go index 45629931751..71806cfba74 100644 --- a/server/datastore/mysql/statistics.go +++ b/server/datastore/mysql/statistics.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "fmt" "time" "github.com/fleetdm/fleet/v4/server" @@ -270,7 +271,7 @@ func (ds *Datastore) getTableRowCountsViaInformationSchema(ctx context.Context) ctx, ds.reader(ctx), &results, - "SELECT table_name, COALESCE(table_rows, 0) table_rows FROM information_schema.tables WHERE table_schema = (SELECT DATABASE())", + fmt.Sprintf("SELECT table_name, COALESCE(table_rows, 0) table_rows FROM information_schema.tables WHERE table_schema = %s", ds.currentDatabaseFn()), ); err != nil { return nil, err } diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index ae47e20c820..997f0d6506e 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -341,7 +341,7 @@ func (ds *Datastore) BatchInsertVPPApps(ctx context.Context, apps []*fleet.VPPAp app.TitleID = titleID - if err := insertVPPApps(ctx, tx, []*fleet.VPPApp{app}); err != nil { + if err := insertVPPApps(ctx, tx, ds.dialect, []*fleet.VPPApp{app}); err != nil { return ctxerr.Wrap(ctx, err, "BatchInsertVPPApps insertVPPApps transaction") } } @@ -508,7 +508,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA if vppToken != nil { tokenID = &vppToken.ID } - vppAppTeamID, err := insertVPPAppTeams(ctx, tx, toAdd, teamID, tokenID) + vppAppTeamID, err := insertVPPAppTeams(ctx, tx, ds.dialect, toAdd, teamID, tokenID) if err != nil { return ctxerr.Wrap(ctx, err, "SetTeamVPPApps inserting vpp app into team") } @@ -522,7 +522,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA } if toAdd.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, vppAppTeamID, *toAdd.ValidatedLabels, softwareTypeVPP); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, vppAppTeamID, *toAdd.ValidatedLabels, softwareTypeVPP); err != nil { return ctxerr.Wrap(ctx, err, "failed to update labels on vpp apps batch operation") } } @@ -534,7 +534,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA } if toAdd.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, teamID, appStoreAppIDsToTitleIDs[toAdd.VPPAppID.String()], *toAdd.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, teamID, appStoreAppIDsToTitleIDs[toAdd.VPPAppID.String()], *toAdd.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "setting software title display name for vpp app") } } @@ -639,11 +639,11 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp app.TitleID = titleID - if err := insertVPPApps(ctx, tx, []*fleet.VPPApp{app}); err != nil { + if err := insertVPPApps(ctx, tx, ds.dialect, []*fleet.VPPApp{app}); err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam insertVPPApps transaction") } - vppAppTeamID, err := insertVPPAppTeams(ctx, tx, app.VPPAppTeam, teamID, vppTokenID) + vppAppTeamID, err := insertVPPAppTeams(ctx, tx, ds.dialect, app.VPPAppTeam, teamID, vppTokenID) if err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam insertVPPAppTeams transaction") } @@ -656,7 +656,7 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp app.VPPAppTeam.AppTeamID = vppAppTeamID if app.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, vppAppTeamID, *app.ValidatedLabels, softwareTypeVPP); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, vppAppTeamID, *app.ValidatedLabels, softwareTypeVPP); err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam setOrUpdateSoftwareInstallerLabelsDB transaction") } } @@ -685,7 +685,7 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp } if app.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, teamID, titleID, *app.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, teamID, titleID, *app.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "setting software title display name for vpp app") } } @@ -752,23 +752,22 @@ WHERE func (ds *Datastore) InsertVPPApps(ctx context.Context, apps []*fleet.VPPApp) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return insertVPPApps(ctx, tx, apps) + return insertVPPApps(ctx, tx, ds.dialect, apps) }) } -func insertVPPApps(ctx context.Context, tx sqlx.ExtContext, apps []*fleet.VPPApp) error { +func insertVPPApps(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, apps []*fleet.VPPApp) error { stmt := ` INSERT INTO vpp_apps (adam_id, bundle_identifier, icon_url, name, latest_version, title_id, platform) VALUES %s -ON DUPLICATE KEY UPDATE +` + dialect.OnDuplicateKey("adam_id,platform", ` updated_at = CURRENT_TIMESTAMP, latest_version = VALUES(latest_version), icon_url = VALUES(icon_url), name = VALUES(name), - title_id = VALUES(title_id) - ` + title_id = VALUES(title_id)`) var args []any var insertVals strings.Builder @@ -784,16 +783,15 @@ ON DUPLICATE KEY UPDATE return ctxerr.Wrap(ctx, err, "insert VPP apps") } -func insertVPPAppTeams(ctx context.Context, tx sqlx.ExtContext, appID fleet.VPPAppTeam, teamID *uint, vppTokenID *uint) (uint, error) { +func insertVPPAppTeams(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, appID fleet.VPPAppTeam, teamID *uint, vppTokenID *uint) (uint, error) { stmt := ` INSERT INTO vpp_apps_teams (adam_id, global_or_team_id, team_id, platform, self_service, vpp_token_id, install_during_setup) VALUES (?, ?, ?, ?, ?, ?, COALESCE(?, false)) -ON DUPLICATE KEY UPDATE +` + dialect.OnDuplicateKey("id", ` self_service = VALUES(self_service), - install_during_setup = COALESCE(?, install_during_setup) -` + install_during_setup = COALESCE(?, install_during_setup)`) var globalOrTmID uint if teamID != nil { @@ -818,8 +816,9 @@ ON DUPLICATE KEY UPDATE var id int64 if insertOnDuplicateDidInsertOrUpdate(res) { - id, _ = res.LastInsertId() - } else { + id, _ = res.LastInsertId() // PG: returns 0, fallback below + } + if id == 0 { stmt := `SELECT id FROM vpp_apps_teams WHERE adam_id = ? AND platform = ? AND global_or_team_id = ?` if err := sqlx.GetContext(ctx, tx, &id, stmt, appID.AdamID, appID.Platform, globalOrTmID); err != nil { return 0, ctxerr.Wrap(ctx, err, "vpp app teams id") @@ -932,7 +931,7 @@ func (ds *Datastore) DeleteVPPAppFromTeam(ctx context.Context, teamID *uint, app tx := ds.writer(ctx) // make sure we're looking at a consistent vision of the world when deleting res, err := tx.ExecContext(ctx, stmt, globalOrTeamID, appID.AdamID, appID.Platform) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the app is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies p JOIN vpp_apps_teams vat @@ -1118,7 +1117,7 @@ VALUES } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, @@ -1132,8 +1131,6 @@ VALUES if err != nil { return ctxerr.Wrap(ctx, err, "insert vpp install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertVAUAStmt, activityID, appID.AdamID, @@ -1387,9 +1384,7 @@ func (ds *Datastore) InsertVPPToken(ctx context.Context, tok *fleet.VPPTokenData return nil, ctxerr.Wrap(ctx, err, "encrypt token with datastore.serverPrivateKey") } - res, err := ds.writer(ctx).ExecContext( - ctx, - insertStmt, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), insertStmt, vppTokenDB.OrgName, vppTokenDB.Location, vppTokenDB.RenewDate, @@ -1399,8 +1394,6 @@ func (ds *Datastore) InsertVPPToken(ctx context.Context, tok *fleet.VPPTokenData return nil, ctxerr.Wrap(ctx, err, "inserting vpp token") } - id, _ := res.LastInsertId() - vppTokenDB.ID = uint(id) //nolint:gosec // dismiss G115 return vppTokenDB, nil @@ -1668,7 +1661,7 @@ func (ds *Datastore) UpdateVPPTokenTeams(ctx context.Context, id uint, teams []u if err != nil { var mysqlErr *mysql.MySQLError // https://dev.mysql.com/doc/mysql-errors/8.4/en/server-error-reference.html#error_er_dup_entry - if errors.As(err, &mysqlErr) && IsDuplicate(err) { + if errors.As(err, &mysqlErr) && ds.dialect.IsDuplicate(err) { var dupeTeamID uint var dupeTeamName string _, _ = fmt.Sscanf(mysqlErr.Message, "Duplicate entry '%d' for", &dupeTeamID) @@ -2156,20 +2149,22 @@ func (ds *Datastore) MarkAllPendingAppleVPPAndInHouseInstallsAsFailed(ctx contex // but those in host_vpp_software_installs could be Android as well. clearVPPUpcomingActivitiesStmt := ` -DELETE ua FROM - upcoming_activities ua -JOIN - host_vpp_software_installs hvsi ON hvsi.command_uuid = ua.execution_id -WHERE ua.activity_type = ? AND hvsi.verification_failed_at IS NULL -AND hvsi.verification_at IS NULL AND hvsi.platform != 'android' +DELETE FROM upcoming_activities +WHERE upcoming_activities.activity_type = ? AND EXISTS ( + SELECT 1 FROM host_vpp_software_installs hvsi + WHERE hvsi.command_uuid = upcoming_activities.execution_id + AND hvsi.verification_failed_at IS NULL + AND hvsi.verification_at IS NULL AND hvsi.platform != 'android' +) ` clearInHouseUpcomingActivitiesStmt := ` -DELETE ua FROM - upcoming_activities ua -JOIN - host_in_house_software_installs hihs ON hihs.command_uuid = ua.execution_id -WHERE ua.activity_type = ? AND hihs.verification_failed_at IS NULL AND hihs.verification_at IS NULL +DELETE FROM upcoming_activities +WHERE upcoming_activities.activity_type = ? AND EXISTS ( + SELECT 1 FROM host_in_house_software_installs hihs + WHERE hihs.command_uuid = upcoming_activities.execution_id + AND hihs.verification_failed_at IS NULL AND hihs.verification_at IS NULL +) ` installVPPFailStmt := ` @@ -2519,13 +2514,13 @@ func (ds *Datastore) hasAppStoreAppChanged(ctx context.Context, teamID *uint, in } func (ds *Datastore) IsAutoUpdateVPPInstall(ctx context.Context, commandUUID string) (bool, error) { - stmt := ` + stmt := fmt.Sprintf(` SELECT COUNT(*) > 0 FROM upcoming_activities WHERE execution_id = ? AND activity_type = 'vpp_app_install' - AND JSON_EXTRACT(payload, '$.from_auto_update') = 1 -` + AND %s = 1 +`, ds.dialect.JSONExtract("payload", "$.from_auto_update")) var isAutoUpdate bool if err := sqlx.GetContext(ctx, ds.reader(ctx), &isAutoUpdate, stmt, commandUUID); err != nil { return false, ctxerr.Wrap(ctx, err, "checking if vpp install is from auto update") From 5ae3801372bb9a86e71a081cf974abf52c3d78f4 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 16:01:22 -0400 Subject: [PATCH 4/6] test(datastore): add PG baseline schema and integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pg_baseline_schema.sql — 194 tables translated from MySQL DDL to PG - postgres_smoke_test.go — 27 integration tests covering Host CRUD, Labels, Queries, Packs, Users, Teams, Policies, Software, Sessions, AppConfig, ListHosts, CountHosts, and more - testing_utils.go — CreatePostgresDS, TruncateTables PG support All 27 tests pass against PostgreSQL 16. --- server/datastore/mysql/pg_baseline_schema.sql | 3328 +++++++++++++++++ server/datastore/mysql/postgres_smoke_test.go | 417 +++ server/datastore/mysql/testing_utils.go | 207 + 3 files changed, 3952 insertions(+) create mode 100644 server/datastore/mysql/pg_baseline_schema.sql create mode 100644 server/datastore/mysql/postgres_smoke_test.go diff --git a/server/datastore/mysql/pg_baseline_schema.sql b/server/datastore/mysql/pg_baseline_schema.sql new file mode 100644 index 00000000000..6b323526f45 --- /dev/null +++ b/server/datastore/mysql/pg_baseline_schema.sql @@ -0,0 +1,3328 @@ +-- Fleet PostgreSQL Test Baseline Schema +-- Auto-generated from MySQL test schema +-- Auto-generated from MySQL test schema + +CREATE TABLE IF NOT EXISTS "abm_tokens" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "organization_name" varchar(255) NOT NULL, + "apple_id" varchar(255) NOT NULL, + "terms_expired" boolean NOT NULL DEFAULT FALSE, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "macos_default_team_id" int DEFAULT NULL, + "ios_default_team_id" int DEFAULT NULL, + "ipados_default_team_id" int DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_abm_tokens_organization_name" UNIQUE ("organization_name") +); + +CREATE TABLE IF NOT EXISTS "activities" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) NOT NULL, + "details" jsonb DEFAULT NULL, + "streamed" boolean NOT NULL DEFAULT FALSE, + "user_email" varchar(255) NOT NULL DEFAULT '', + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "host_only" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "activity_past" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) NOT NULL, + "details" jsonb DEFAULT NULL, + "streamed" boolean NOT NULL DEFAULT FALSE, + "user_email" varchar(255) NOT NULL DEFAULT '', + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "host_only" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "activity_host_past" ( + "host_id" int NOT NULL, + "activity_id" int NOT NULL, + PRIMARY KEY ("host_id","activity_id") +); + +CREATE TABLE IF NOT EXISTS "aggregated_stats" ( + "id" bigint NOT NULL, + "type" varchar(255) NOT NULL, + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id","type","global_stats") +); + +CREATE TABLE IF NOT EXISTS "android_app_configurations" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "application_id" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "configuration" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_global_or_team_id_application_id" UNIQUE ("global_or_team_id","application_id") +); + +CREATE TABLE IF NOT EXISTS "android_devices" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "device_id" varchar(32) NOT NULL, + "enterprise_specific_id" varchar(64) DEFAULT NULL, + "last_policy_sync_time" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "applied_policy_id" varchar(100) DEFAULT NULL, + "applied_policy_version" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_android_devices_host_id" UNIQUE ("host_id"), + CONSTRAINT "idx_android_devices_device_id" UNIQUE ("device_id"), + CONSTRAINT "idx_android_devices_enterprise_specific_id" UNIQUE ("enterprise_specific_id") +); + +CREATE TABLE IF NOT EXISTS "android_enterprises" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "signup_name" varchar(63) NOT NULL DEFAULT '', + "enterprise_id" varchar(63) NOT NULL DEFAULT '', + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + "signup_token" varchar(64) NOT NULL DEFAULT '', + "pubsub_topic_id" varchar(64) NOT NULL DEFAULT '', + "user_id" int NOT NULL DEFAULT '0', + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "android_policy_requests" ( + "request_uuid" varchar(36) NOT NULL, + "request_name" varchar(255) NOT NULL, + "policy_id" varchar(100) NOT NULL, + "payload" jsonb NOT NULL, + "status_code" int NOT NULL, + "error_details" text, + "applied_policy_version" int DEFAULT NULL, + "policy_version" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("request_uuid") +); + +CREATE TABLE IF NOT EXISTS "app_config_json" ( + "id" int NOT NULL DEFAULT '1', + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "batch_activities" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "script_id" int NOT NULL, + "execution_id" varchar(255) NOT NULL, + "user_id" int DEFAULT NULL, + "job_id" int DEFAULT NULL, + "status" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) DEFAULT NULL, + "num_targeted" int DEFAULT NULL, + "num_pending" int DEFAULT NULL, + "num_ran" int DEFAULT NULL, + "num_errored" int DEFAULT NULL, + "num_incompatible" int DEFAULT NULL, + "num_canceled" int DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "started_at" timestamp DEFAULT NULL, + "finished_at" timestamp DEFAULT NULL, + "canceled" boolean DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_batch_script_executions_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "batch_activity_host_results" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "batch_execution_id" varchar(255) NOT NULL, + "host_id" int NOT NULL, + "host_execution_id" varchar(255) DEFAULT NULL, + "error" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "unique_batch_host_results_execution_hostid" UNIQUE ("batch_execution_id","host_id") +); + +CREATE TABLE IF NOT EXISTS "ca_config_assets" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "type" text NOT NULL, + "name" varchar(255) NOT NULL, + "value" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_ca_config_assets_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "calendar_events" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "email" varchar(255) NOT NULL, + "start_time" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "end_time" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "event" jsonb NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "timezone" varchar(64) DEFAULT NULL, + "uuid_bin" bytea NOT NULL, + "uuid" text, + PRIMARY KEY ("id"), + CONSTRAINT "idx_one_calendar_event_per_email" UNIQUE ("email"), + CONSTRAINT "idx_calendar_events_uuid_bin_unique" UNIQUE ("uuid_bin") +); + +CREATE TABLE IF NOT EXISTS "carve_blocks" ( + "metadata_id" int NOT NULL, + "block_id" int NOT NULL, + "data" bytea, + PRIMARY KEY ("metadata_id","block_id") +); + +CREATE TABLE IF NOT EXISTS "carve_metadata" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) DEFAULT NULL, + "block_count" int NOT NULL, + "block_size" int NOT NULL, + "carve_size" bigint NOT NULL, + "carve_id" varchar(64) NOT NULL, + "request_id" varchar(64) NOT NULL, + "session_id" varchar(255) NOT NULL, + "expired" boolean DEFAULT FALSE, + "max_block" int DEFAULT '-1', + "error" text, + PRIMARY KEY ("id"), + CONSTRAINT "idx_session_id" UNIQUE ("session_id"), + CONSTRAINT "idx_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "certificate_authorities" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "type" text NOT NULL, + "name" varchar(255) NOT NULL, + "url" text NOT NULL, + "api_token_encrypted" bytea, + "profile_id" varchar(255) DEFAULT NULL, + "certificate_common_name" varchar(255) DEFAULT NULL, + "certificate_user_principal_names" jsonb DEFAULT NULL, + "certificate_seat_id" varchar(255) DEFAULT NULL, + "admin_url" text, + "username" varchar(255) DEFAULT NULL, + "password_encrypted" bytea, + "challenge_url" text, + "challenge_encrypted" bytea, + "client_id" varchar(255) DEFAULT NULL, + "client_secret_encrypted" bytea, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_ca_type_name" UNIQUE ("type","name") +); + +CREATE TABLE IF NOT EXISTS "certificate_templates" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int NOT NULL, + "certificate_authority_id" int NOT NULL, + "name" varchar(255) NOT NULL, + "subject_name" text NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_cert_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "challenges" ( + "challenge" char(32) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("challenge") +); + +CREATE TABLE IF NOT EXISTS "conditional_access_scep_certificates" ( + "serial" bigint NOT NULL, + "host_id" int NOT NULL, + "name" varchar(64) NOT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "conditional_access_scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "conditional_access_scep_serials" ( + "serial" bigint GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "cron_stats" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "instance" varchar(255) NOT NULL, + "stats_type" varchar(255) NOT NULL, + "status" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "errors" jsonb DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "cve_meta" ( + "cve" varchar(20) NOT NULL, + "cvss_score" double precision DEFAULT NULL, + "epss_probability" double precision DEFAULT NULL, + "cisa_known_exploit" boolean DEFAULT NULL, + "published" timestamp NULL DEFAULT NULL, + "description" text, + PRIMARY KEY ("cve") +); + +CREATE TABLE IF NOT EXISTS "default_team_config_json" ( + "id" int NOT NULL DEFAULT '1', + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "default_team_config_id" CHECK (("id" = 1)), + PRIMARY KEY ("id"), + CONSTRAINT "id" UNIQUE ("id") +); + +CREATE TABLE IF NOT EXISTS "distributed_query_campaign_targets" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "type" int DEFAULT NULL, + "distributed_query_campaign_id" int DEFAULT NULL, + "target_id" int DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "distributed_query_campaigns" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "query_id" int DEFAULT NULL, + "status" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "email_changes" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "user_id" int NOT NULL, + "token" varchar(128) NOT NULL, + "new_email" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_email_changes_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "enroll_secrets" ( + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "secret" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + PRIMARY KEY ("secret") +); + +CREATE TABLE IF NOT EXISTS "eulas" ( + "id" int NOT NULL, + "token" varchar(36) DEFAULT NULL, + "name" varchar(255) DEFAULT NULL, + "bytes" bytea, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "sha256" bytea DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "fleet_maintained_apps" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "slug" varchar(255) NOT NULL, + "platform" varchar(255) NOT NULL, + "unique_identifier" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_fleet_library_apps_token" UNIQUE ("slug") +); + +CREATE TABLE IF NOT EXISTS "fleet_variables" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL DEFAULT '', + "is_prefix" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_fleet_variables_name_is_prefix" UNIQUE ("name","is_prefix") +); + +CREATE TABLE IF NOT EXISTS "host_activities" ( + "host_id" int NOT NULL, + "activity_id" int NOT NULL, + PRIMARY KEY ("host_id","activity_id") +); + +CREATE TABLE IF NOT EXISTS "host_additional" ( + "host_id" int NOT NULL, + "additional" jsonb DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_batteries" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "serial_number" varchar(255) NOT NULL, + "cycle_count" int NOT NULL, + "health" varchar(40) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_batteries_host_id_serial_number" UNIQUE ("host_id","serial_number") +); + +CREATE TABLE IF NOT EXISTS "host_calendar_events" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "calendar_event_id" int NOT NULL, + "webhook_status" smallint NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_one_calendar_event_per_host" UNIQUE ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_certificate_sources" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "host_certificate_id" bigint NOT NULL, + "source" text NOT NULL, + "username" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_certificate_sources_unique" UNIQUE ("host_certificate_id","source","username") +); + +CREATE TABLE IF NOT EXISTS "host_certificate_templates" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "certificate_template_id" int NOT NULL, + "fleet_challenge" char(32) DEFAULT NULL, + "status" varchar(20) NOT NULL DEFAULT 'pending', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "detail" text, + "operation_type" varchar(20) NOT NULL DEFAULT 'install', + "name" varchar(255) NOT NULL, + "uuid" bytea DEFAULT NULL, + "not_valid_before" timestamp DEFAULT NULL, + "not_valid_after" timestamp DEFAULT NULL, + "serial" varchar(40) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_certificate_templates_host_template" UNIQUE ("host_uuid","certificate_template_id") +); + +CREATE TABLE IF NOT EXISTS "host_certificates" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "not_valid_after" timestamp NOT NULL, + "not_valid_before" timestamp NOT NULL, + "certificate_authority" boolean NOT NULL, + "common_name" varchar(255) NOT NULL, + "key_algorithm" varchar(255) NOT NULL, + "key_strength" int NOT NULL, + "key_usage" varchar(255) NOT NULL, + "serial" varchar(255) NOT NULL, + "signing_algorithm" varchar(255) NOT NULL, + "subject_country" varchar(32) NOT NULL, + "subject_org" varchar(255) NOT NULL, + "subject_org_unit" varchar(255) NOT NULL, + "subject_common_name" varchar(255) NOT NULL, + "issuer_country" varchar(32) NOT NULL, + "issuer_org" varchar(255) NOT NULL, + "issuer_org_unit" varchar(255) NOT NULL, + "issuer_common_name" varchar(255) NOT NULL, + "sha1_sum" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deleted_at" timestamp DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_conditional_access" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "bypassed_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_conditional_access_host_id" UNIQUE ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_dep_assignments" ( + "host_id" int NOT NULL, + "added_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deleted_at" timestamp NULL DEFAULT NULL, + "profile_uuid" varchar(37) DEFAULT NULL, + "assign_profile_response" varchar(15) DEFAULT NULL, + "response_updated_at" timestamp NULL DEFAULT NULL, + "retry_job_id" int NOT NULL DEFAULT '0', + "abm_token_id" int DEFAULT NULL, + "mdm_migration_deadline" timestamp NULL DEFAULT NULL, + "mdm_migration_completed" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_device_auth" ( + "host_id" int NOT NULL, + "token" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id"), + CONSTRAINT "idx_host_device_auth_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "host_disk_encryption_keys" ( + "host_id" int NOT NULL, + "base64_encrypted" text NOT NULL, + "base64_encrypted_salt" varchar(255) NOT NULL DEFAULT '', + "key_slot" smallint DEFAULT NULL, + "decryptable" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "reset_requested" boolean NOT NULL DEFAULT FALSE, + "client_error" varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_disk_encryption_keys_archive" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "hardware_serial" varchar(255) NOT NULL DEFAULT '', + "base64_encrypted" text NOT NULL, + "base64_encrypted_salt" varchar(255) NOT NULL DEFAULT '', + "key_slot" smallint DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_disks" ( + "host_id" int NOT NULL, + "gigs_disk_space_available" decimal(10,2) NOT NULL DEFAULT '0.00', + "percent_disk_space_available" decimal(10,2) NOT NULL DEFAULT '0.00', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "encrypted" boolean DEFAULT NULL, + "gigs_total_disk_space" decimal(10,2) NOT NULL DEFAULT '0.00', + "tpm_pin_set" boolean DEFAULT FALSE, + "gigs_all_disk_space" decimal(10,2) DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_display_names" ( + "host_id" int NOT NULL, + "display_name" varchar(255) NOT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_emails" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "email" varchar(255) NOT NULL, + "source" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_identity_scep_certificates" ( + "serial" bigint NOT NULL, + "host_id" int DEFAULT NULL, + "name" varchar(255) NOT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "public_key_raw" bytea NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "host_identity_scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "host_identity_scep_serials" ( + "serial" bigint GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "host_in_house_software_installs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "in_house_app_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "user_id" int DEFAULT NULL, + "platform" varchar(10) NOT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "canceled" boolean NOT NULL DEFAULT FALSE, + "verification_command_uuid" varchar(127) DEFAULT NULL, + "verification_at" timestamp DEFAULT NULL, + "verification_failed_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "self_service" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_in_house_software_installs_command_uuid" UNIQUE ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_issues" ( + "host_id" int NOT NULL, + "failing_policies_count" int NOT NULL DEFAULT '0', + "critical_vulnerabilities_count" int NOT NULL DEFAULT '0', + "total_issues_count" int NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_last_known_locations" ( + "host_id" int NOT NULL, + "latitude" decimal(10,8) DEFAULT NULL, + "longitude" decimal(11,8) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm" ( + "host_id" int NOT NULL, + "enrolled" boolean NOT NULL DEFAULT FALSE, + "server_url" varchar(255) NOT NULL DEFAULT '', + "installed_from_dep" boolean NOT NULL DEFAULT FALSE, + "mdm_id" int DEFAULT NULL, + "is_server" boolean DEFAULT NULL, + "fleet_enroll_ref" varchar(36) NOT NULL DEFAULT '', + "enrollment_status" text, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "is_personal_enrollment" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_actions" ( + "host_id" int NOT NULL, + "lock_ref" varchar(36) DEFAULT NULL, + "wipe_ref" varchar(36) DEFAULT NULL, + "unlock_pin" varchar(6) DEFAULT NULL, + "unlock_ref" varchar(36) DEFAULT NULL, + "fleet_platform" varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_android_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "profile_name" varchar(255) NOT NULL DEFAULT '', + "policy_request_uuid" varchar(36) DEFAULT NULL, + "device_request_uuid" varchar(36) DEFAULT NULL, + "request_fail_count" smallint NOT NULL DEFAULT '0', + "included_in_policy_version" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "can_reverify" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_awaiting_configuration" ( + "host_uuid" varchar(255) NOT NULL, + "awaiting_configuration" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_bootstrap_packages" ( + "host_uuid" varchar(127) NOT NULL, + "command_uuid" varchar(127) DEFAULT NULL, + "skipped" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "ck_skipped_or_commanduuid" CHECK ((("skipped" = false) = ("command_uuid" is not null))), + PRIMARY KEY ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_declarations" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "token" bytea NOT NULL, + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "declaration_identifier" varchar(255) NOT NULL, + "declaration_name" varchar(255) NOT NULL DEFAULT '', + "secrets_updated_at" timestamp DEFAULT NULL, + "resync" boolean NOT NULL DEFAULT FALSE, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("host_uuid","declaration_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_profiles" ( + "profile_identifier" varchar(255) NOT NULL, + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL, + "retries" smallint NOT NULL DEFAULT '0', + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "secrets_updated_at" timestamp DEFAULT NULL, + "ignore_error" boolean NOT NULL DEFAULT FALSE, + "variables_updated_at" timestamp DEFAULT NULL, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_commands" ( + "host_id" int NOT NULL, + "command_type" varchar(31) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id","command_type") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_idp_accounts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "account_uuid" varchar(36) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_mdm_idp_accounts" UNIQUE ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_managed_certificates" ( + "host_uuid" varchar(255) NOT NULL, + "profile_uuid" varchar(37) NOT NULL, + "type" text NOT NULL DEFAULT 'ndes', + "ca_name" varchar(255) NOT NULL DEFAULT 'NDES', + "challenge_retrieved_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "not_valid_after" timestamp DEFAULT NULL, + "serial" varchar(40) DEFAULT NULL, + "not_valid_before" timestamp DEFAULT NULL, + PRIMARY KEY ("host_uuid","profile_uuid","ca_name") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_windows_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "retries" smallint NOT NULL DEFAULT '0', + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL DEFAULT ''::bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_munki_info" ( + "host_id" int NOT NULL, + "version" varchar(255) NOT NULL DEFAULT '', + "deleted_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_munki_issues" ( + "host_id" int NOT NULL, + "munki_issue_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_id","munki_issue_id") +); + +CREATE TABLE IF NOT EXISTS "host_operating_system" ( + "host_id" int NOT NULL, + "os_id" int NOT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_orbit_info" ( + "host_id" int NOT NULL, + "version" varchar(50) NOT NULL, + "desktop_version" varchar(50) DEFAULT NULL, + "scripts_enabled" boolean DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_scim_user" ( + "host_id" int NOT NULL, + "scim_user_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_script_results" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "execution_id" varchar(255) NOT NULL, + "output" text NOT NULL, + "runtime" int NOT NULL DEFAULT '0', + "exit_code" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_id" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + "sync_request" boolean NOT NULL DEFAULT FALSE, + "script_content_id" int DEFAULT NULL, + "host_deleted_at" timestamp NULL DEFAULT NULL, + "timeout" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "is_internal" boolean DEFAULT FALSE, + "canceled" boolean NOT NULL DEFAULT FALSE, + "attempt_number" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_script_results_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "host_seen_times" ( + "host_id" int NOT NULL, + "seen_time" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_software" ( + "host_id" int NOT NULL, + "software_id" bigint NOT NULL, + "last_opened_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id","software_id") +); + +CREATE TABLE IF NOT EXISTS "host_software_installed_paths" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "software_id" bigint NOT NULL, + "installed_path" text NOT NULL, + "team_identifier" varchar(10) NOT NULL DEFAULT '', + "cdhash_sha256" char(64) DEFAULT NULL, + "executable_sha256" char(64) DEFAULT NULL, + "executable_path" text, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_software_installs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "execution_id" varchar(255) NOT NULL, + "host_id" int NOT NULL, + "software_installer_id" int DEFAULT NULL, + "pre_install_query_output" text, + "install_script_output" text, + "install_script_exit_code" int DEFAULT NULL, + "post_install_script_output" text, + "post_install_script_exit_code" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "self_service" boolean NOT NULL DEFAULT FALSE, + "host_deleted_at" timestamp NULL DEFAULT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "uninstall_script_output" text, + "uninstall_script_exit_code" int DEFAULT NULL, + "uninstall" boolean NOT NULL DEFAULT FALSE, + "status" text, + "policy_id" int DEFAULT NULL, + "installer_filename" varchar(255) NOT NULL DEFAULT '[deleted installer]', + "version" varchar(255) NOT NULL DEFAULT 'unknown', + "software_title_id" int DEFAULT NULL, + "software_title_name" varchar(255) NOT NULL DEFAULT '[deleted title]', + "execution_status" text, + "canceled" boolean NOT NULL DEFAULT FALSE, + "attempt_number" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_software_installs_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "host_updates" ( + "host_id" int NOT NULL, + "software_updated_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_users" ( + "host_id" int NOT NULL, + "uid" int NOT NULL, + "username" varchar(255) NOT NULL, + "groupname" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "removed_at" timestamp NULL DEFAULT NULL, + "user_type" varchar(255) DEFAULT NULL, + "shell" varchar(255) DEFAULT '', + PRIMARY KEY ("host_id","uid","username") +); + +CREATE TABLE IF NOT EXISTS "host_vpp_software_installs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "adam_id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "user_id" int DEFAULT NULL, + "self_service" boolean NOT NULL DEFAULT FALSE, + "associated_event_id" varchar(36) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "vpp_token_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "canceled" boolean NOT NULL DEFAULT FALSE, + "verification_command_uuid" varchar(127) DEFAULT NULL, + "verification_at" timestamp DEFAULT NULL, + "verification_failed_at" timestamp DEFAULT NULL, + "retry_count" int NOT NULL DEFAULT '0', + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_vpp_software_installs_command_uuid" UNIQUE ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "hosts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "osquery_host_id" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "detail_updated_at" timestamp NULL DEFAULT NULL, + "node_key" varchar(255) DEFAULT NULL, + "hostname" varchar(255) NOT NULL DEFAULT '', + "uuid" varchar(255) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "osquery_version" varchar(255) NOT NULL DEFAULT '', + "os_version" varchar(255) NOT NULL DEFAULT '', + "build" varchar(255) NOT NULL DEFAULT '', + "platform_like" varchar(255) NOT NULL DEFAULT '', + "code_name" varchar(255) NOT NULL DEFAULT '', + "uptime" bigint NOT NULL DEFAULT '0', + "memory" bigint NOT NULL DEFAULT '0', + "cpu_type" varchar(255) NOT NULL DEFAULT '', + "cpu_subtype" varchar(255) NOT NULL DEFAULT '', + "cpu_brand" varchar(255) NOT NULL DEFAULT '', + "cpu_physical_cores" int NOT NULL DEFAULT '0', + "cpu_logical_cores" int NOT NULL DEFAULT '0', + "hardware_vendor" varchar(255) NOT NULL DEFAULT '', + "hardware_model" varchar(255) NOT NULL DEFAULT '', + "hardware_version" varchar(255) NOT NULL DEFAULT '', + "hardware_serial" varchar(255) NOT NULL DEFAULT '', + "computer_name" varchar(255) NOT NULL DEFAULT '', + "primary_ip_id" int DEFAULT NULL, + "distributed_interval" int DEFAULT '0', + "logger_tls_period" int DEFAULT '0', + "config_tls_refresh" int DEFAULT '0', + "primary_ip" varchar(45) NOT NULL DEFAULT '', + "primary_mac" varchar(17) NOT NULL DEFAULT '', + "label_updated_at" timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + "last_enrolled_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "refetch_requested" boolean NOT NULL DEFAULT FALSE, + "team_id" int DEFAULT NULL, + "policy_updated_at" timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + "public_ip" varchar(45) NOT NULL DEFAULT '', + "orbit_node_key" varchar(255) DEFAULT NULL, + "refetch_critical_queries_until" timestamp NULL DEFAULT NULL, + "last_restarted_at" timestamp DEFAULT '0001-01-01 00:00:00.000000', + "timezone" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_osquery_host_id" UNIQUE ("osquery_host_id"), + CONSTRAINT "idx_host_unique_nodekey" UNIQUE ("node_key"), + CONSTRAINT "idx_host_unique_orbitnodekey" UNIQUE ("orbit_node_key") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "in_house_app_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "id_in_house_app_labels_in_house_app_id_label_id" UNIQUE ("in_house_app_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_software_categories" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_category_id" int NOT NULL, + "in_house_app_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_in_house_app_id_software_category_id" UNIQUE ("in_house_app_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "in_house_app_id" int NOT NULL, + "software_title_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_apps" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "title_id" int DEFAULT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "filename" varchar(255) NOT NULL DEFAULT '', + "version" varchar(255) NOT NULL DEFAULT '', + "storage_id" varchar(64) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + "bundle_identifier" varchar(255) NOT NULL DEFAULT '', + "self_service" boolean NOT NULL DEFAULT FALSE, + "url" varchar(4095) NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "global_or_team_id" UNIQUE ("global_or_team_id","filename","platform") +); + +CREATE TABLE IF NOT EXISTS "invite_teams" ( + "invite_id" int NOT NULL, + "team_id" int NOT NULL, + "role" varchar(64) NOT NULL, + PRIMARY KEY ("invite_id","team_id") +); + +CREATE TABLE IF NOT EXISTS "invites" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "invited_by" int NOT NULL, + "email" varchar(255) NOT NULL, + "name" varchar(255) DEFAULT NULL, + "position" varchar(255) DEFAULT NULL, + "token" varchar(255) NOT NULL, + "sso_enabled" boolean NOT NULL DEFAULT FALSE, + "global_role" varchar(64) DEFAULT NULL, + "mfa_enabled" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_invite_unique_email" UNIQUE ("email"), + CONSTRAINT "idx_invite_unique_key" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "jobs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "name" varchar(255) NOT NULL, + "args" jsonb DEFAULT NULL, + "state" varchar(255) NOT NULL, + "retries" int NOT NULL DEFAULT '0', + "error" text, + "not_before" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "kernel_host_counts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_title_id" int DEFAULT NULL, + "software_id" int DEFAULT NULL, + "os_version_id" int DEFAULT NULL, + "hosts_count" int NOT NULL, + "team_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_kernels_unique_mapping" UNIQUE ("os_version_id","team_id","software_id") +); + +CREATE TABLE IF NOT EXISTS "label_membership" ( + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "label_id" int NOT NULL, + "host_id" int NOT NULL, + PRIMARY KEY ("host_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "name" varchar(255) NOT NULL, + "description" varchar(255) NOT NULL DEFAULT '', + "query" text NOT NULL, + "platform" varchar(255) NOT NULL DEFAULT '', + "label_type" int NOT NULL DEFAULT '1', + "label_membership_type" int NOT NULL DEFAULT '0', + "author_id" int DEFAULT NULL, + "criteria" jsonb DEFAULT NULL, + "team_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_label_unique_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_filevault_profiles" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(36) NOT NULL, + "status" varchar(20) NOT NULL, + "operation_type" varchar(20) NOT NULL, + "profile_uuid" varchar(37) NOT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "scope" text NOT NULL DEFAULT 'System', + "created_at" timestamp NOT NULL, + "updated_at" timestamp NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_mdm_enroll_refs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "enroll_ref" varchar(36) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_mdm_idp_accounts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "email" varchar(255) NOT NULL, + "account_uuid" varchar(36) DEFAULT NULL, + "host_id" int DEFAULT NULL, + "email_id" int DEFAULT NULL, + "email_created_at" timestamp DEFAULT NULL, + "email_updated_at" timestamp DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "locks" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "expires_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "locks_idx_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "mdm_android_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "raw_json" jsonb NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_android_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_bootstrap_packages" ( + "team_id" int NOT NULL, + "name" varchar(255) DEFAULT NULL, + "sha256" bytea NOT NULL, + "bytes" bytea, + "token" varchar(36) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("team_id"), + CONSTRAINT "idx_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_configuration_profiles" ( + "profile_id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int NOT NULL DEFAULT '0', + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "mobileconfig" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "checksum" bytea NOT NULL, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "secrets_updated_at" timestamp DEFAULT NULL, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_apple_config_prof_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_config_prof_team_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_apple_config_prof_id" UNIQUE ("profile_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declaration_activation_references" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "reference" varchar(37) NOT NULL DEFAULT '', + PRIMARY KEY ("declaration_uuid","reference") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declarations" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "team_id" int NOT NULL DEFAULT '0', + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "raw_json" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "secrets_updated_at" timestamp DEFAULT NULL, + "token" bytea, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("declaration_uuid"), + CONSTRAINT "idx_mdm_apple_declaration_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_declaration_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declarative_requests" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "enrollment_id" varchar(255) NOT NULL, + "message_type" varchar(255) NOT NULL, + "raw_json" text, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_default_setup_assistants" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "profile_uuid" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "abm_token_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_default_setup_assistant_global_or_team_id_abm_token_id" UNIQUE ("global_or_team_id","abm_token_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_enrollment_profiles" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "token" varchar(36) DEFAULT NULL, + "type" varchar(10) NOT NULL DEFAULT 'automatic', + "dep_profile" jsonb DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_type" UNIQUE ("type"), + CONSTRAINT "mdm_apple_enrollment_profiles_idx_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_installers" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL DEFAULT '', + "size" bigint NOT NULL, + "manifest" text NOT NULL, + "installer" bytea, + "url_token" varchar(36) DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_setup_assistant_profiles" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "setup_assistant_id" int NOT NULL, + "abm_token_id" int NOT NULL, + "profile_uuid" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_apple_setup_assistant_profiles_asst_id_tok_id" UNIQUE ("setup_assistant_id","abm_token_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_setup_assistants" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" text NOT NULL, + "profile" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_setup_assistant_global_or_team_id" UNIQUE ("global_or_team_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_config_assets" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(256) NOT NULL DEFAULT '', + "value" bytea NOT NULL, + "deleted_at" timestamp NULL DEFAULT NULL, + "deletion_uuid" varchar(127) NOT NULL DEFAULT '', + "md5_checksum" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_config_assets_name_deletion_uuid" UNIQUE ("name","deletion_uuid") +); + +CREATE TABLE IF NOT EXISTS "mdm_configuration_profile_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "label_name" varchar(255) NOT NULL, + "label_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "android_profile_uuid" varchar(37) DEFAULT NULL, + CONSTRAINT "ck_mdm_configuration_profile_labels_profile_uuid" CHECK ((((CASE WHEN "apple_profile_uuid" IS NULL THEN 0 ELSE 1 END) + (CASE WHEN "windows_profile_uuid" IS NULL THEN 0 ELSE 1 END) + (CASE WHEN "android_profile_uuid" IS NULL THEN 0 ELSE 1 END)) = 1)), + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_configuration_profile_labels_apple_label_name" UNIQUE ("apple_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_windows_label_name" UNIQUE ("windows_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_android_label_name" UNIQUE ("android_profile_uuid","label_name") +); + +CREATE TABLE IF NOT EXISTS "mdm_configuration_profile_variables" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "fleet_variable_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "ck_mdm_configuration_profile_variables_apple_or_windows" CHECK ((("apple_profile_uuid" is null) <> ("windows_profile_uuid" is null))), + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_configuration_profile_variables_apple_variable" UNIQUE ("apple_profile_uuid","fleet_variable_id"), + CONSTRAINT "idx_mdm_configuration_profile_variables_windows_label_name" UNIQUE ("windows_profile_uuid","fleet_variable_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_declaration_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "apple_declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "label_name" varchar(255) NOT NULL, + "label_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_declaration_labels_label_name" UNIQUE ("apple_declaration_uuid","label_name") +); + +CREATE TABLE IF NOT EXISTS "mdm_delivery_status" ( + "status" varchar(20) NOT NULL, + PRIMARY KEY ("status") +); + +CREATE TABLE IF NOT EXISTS "mdm_idp_accounts" ( + "uuid" varchar(255) NOT NULL, + "username" varchar(255) NOT NULL, + "fullname" varchar(256) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("uuid"), + CONSTRAINT "unique_idp_email" UNIQUE ("email") +); + +CREATE TABLE IF NOT EXISTS "mdm_operation_types" ( + "operation_type" varchar(20) NOT NULL, + PRIMARY KEY ("operation_type") +); + +CREATE TABLE IF NOT EXISTS "mdm_windows_configuration_profiles" ( + "team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "syncml" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_windows_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_windows_enrollments" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "mdm_device_id" varchar(255) NOT NULL, + "mdm_hardware_id" varchar(255) NOT NULL, + "device_state" varchar(255) NOT NULL, + "device_type" varchar(255) NOT NULL, + "device_name" varchar(255) NOT NULL, + "enroll_type" varchar(255) NOT NULL, + "enroll_user_id" varchar(255) NOT NULL, + "enroll_proto_version" varchar(255) NOT NULL, + "enroll_client_version" varchar(255) NOT NULL, + "not_in_oobe" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "host_uuid" varchar(255) NOT NULL DEFAULT '', + "credentials_hash" bytea DEFAULT NULL, + "credentials_acknowledged" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "mdm_windows_enrollments_idx_type" UNIQUE ("mdm_hardware_id") +); + +CREATE TABLE IF NOT EXISTS "microsoft_compliance_partner_host_statuses" ( + "host_id" int NOT NULL, + "device_id" varchar(64) NOT NULL, + "user_principal_name" varchar(255) NOT NULL, + "managed" boolean DEFAULT NULL, + "compliant" boolean DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "microsoft_compliance_partner_integrations" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "tenant_id" varchar(64) NOT NULL, + "proxy_server_secret" varchar(64) NOT NULL, + "setup_done" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_microsoft_compliance_partner_tenant_id" UNIQUE ("tenant_id") +); + +CREATE TABLE IF NOT EXISTS "migration_status_tables" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "version_id" bigint NOT NULL, + "is_applied" boolean NOT NULL, + "tstamp" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mobile_device_management_solutions" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(100) NOT NULL, + "server_url" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mobile_device_management_solutions_name" UNIQUE ("name","server_url") +); + +CREATE TABLE IF NOT EXISTS "munki_issues" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "issue_type" varchar(10) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_munki_issues_name" UNIQUE ("name","issue_type") +); + +CREATE TABLE IF NOT EXISTS "nano_cert_auth_associations" ( + "id" varchar(255) NOT NULL, + "sha256" char(64) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "cert_not_valid_after" timestamp NULL DEFAULT NULL, + "renew_command_uuid" varchar(127) DEFAULT NULL, + CONSTRAINT "nano_cert_auth_associations_chk_1" CHECK (("id" <> '')), + CONSTRAINT "nano_cert_auth_associations_chk_2" CHECK (("sha256" <> '')), + PRIMARY KEY ("id","sha256") +); + +CREATE TABLE IF NOT EXISTS "nano_command_results" ( + "id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "status" varchar(31) NOT NULL, + "result" text NOT NULL, + "not_now_at" timestamp NULL DEFAULT NULL, + "not_now_tally" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_command_results_chk_1" CHECK (("status" <> '')), + CONSTRAINT "nano_command_results_chk_2" CHECK ((substr("result",1,5) = ' '')), + CONSTRAINT "nano_commands_chk_2" CHECK (("request_type" <> '')), + PRIMARY KEY ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "nano_dep_names" ( + "name" varchar(255) NOT NULL, + "consumer_key" text, + "consumer_secret" text, + "access_token" text, + "access_secret" text, + "access_token_expiry" timestamp NULL DEFAULT NULL, + "config_base_url" varchar(255) DEFAULT NULL, + "tokenpki_cert_pem" text, + "tokenpki_key_pem" text, + "syncer_cursor" varchar(1024) DEFAULT NULL, + "syncer_cursor_at" timestamp NULL DEFAULT NULL, + "assigner_profile_uuid" text, + "assigner_profile_uuid_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_dep_names_chk_1" CHECK ((("tokenpki_cert_pem" is null) or (substr("tokenpki_cert_pem",1,27) = '-----BEGIN CERTIFICATE-----'))), + CONSTRAINT "nano_dep_names_chk_2" CHECK ((("tokenpki_key_pem" is null) or (substr("tokenpki_key_pem",1,5) = '-----'))), + PRIMARY KEY ("name") +); + +CREATE TABLE IF NOT EXISTS "nano_devices" ( + "id" varchar(255) NOT NULL, + "identity_cert" text, + "serial_number" varchar(127) DEFAULT NULL, + "unlock_token" bytea, + "unlock_token_at" timestamp NULL DEFAULT NULL, + "authenticate" text NOT NULL, + "authenticate_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "token_update" text, + "token_update_at" timestamp NULL DEFAULT NULL, + "bootstrap_token_b64" text, + "bootstrap_token_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(255) NOT NULL DEFAULT '', + "enroll_team_id" int DEFAULT NULL, + CONSTRAINT "nano_devices_chk_1" CHECK ((("identity_cert" is null) or (substr("identity_cert",1,27) = '-----BEGIN CERTIFICATE-----'))), + CONSTRAINT "nano_devices_chk_2" CHECK ((("serial_number" is null) or ("serial_number" <> ''))), + CONSTRAINT "nano_devices_chk_3" CHECK ((("unlock_token" is null) or (length("unlock_token") > 0))), + CONSTRAINT "nano_devices_chk_4" CHECK (("authenticate" <> '')), + CONSTRAINT "nano_devices_chk_5" CHECK ((("token_update" is null) or ("token_update" <> ''))), + CONSTRAINT "nano_devices_chk_6" CHECK ((("bootstrap_token_b64" is null) or ("bootstrap_token_b64" <> ''))), + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "nano_enrollment_queue" ( + "id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "active" boolean NOT NULL DEFAULT TRUE, + "priority" smallint NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "nano_enrollments" ( + "id" varchar(255) NOT NULL, + "device_id" varchar(255) NOT NULL, + "user_id" varchar(255) DEFAULT NULL, + "type" varchar(31) NOT NULL, + "topic" varchar(255) NOT NULL, + "push_magic" varchar(127) NOT NULL, + "token_hex" varchar(255) NOT NULL, + "enabled" boolean NOT NULL DEFAULT TRUE, + "token_update_tally" int NOT NULL DEFAULT '1', + "last_seen_at" timestamp NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "enrolled_from_migration" smallint NOT NULL DEFAULT '0', + CONSTRAINT "nano_enrollments_chk_1" CHECK (("id" <> '')), + CONSTRAINT "nano_enrollments_chk_2" CHECK (("type" <> '')), + CONSTRAINT "nano_enrollments_chk_3" CHECK (("topic" <> '')), + CONSTRAINT "nano_enrollments_chk_4" CHECK (("push_magic" <> '')), + CONSTRAINT "nano_enrollments_chk_5" CHECK (("token_hex" <> '')), + PRIMARY KEY ("id"), + CONSTRAINT "user_id" UNIQUE ("user_id") +); + +CREATE TABLE IF NOT EXISTS "nano_push_certs" ( + "topic" varchar(255) NOT NULL, + "cert_pem" text NOT NULL, + "key_pem" text NOT NULL, + "stale_token" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_push_certs_chk_1" CHECK (("topic" <> '')), + CONSTRAINT "nano_push_certs_chk_2" CHECK ((substr("cert_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + CONSTRAINT "nano_push_certs_chk_3" CHECK ((substr("key_pem",1,5) = '-----')), + PRIMARY KEY ("topic") +); + +CREATE TABLE IF NOT EXISTS "nano_users" ( + "id" varchar(255) NOT NULL, + "device_id" varchar(255) NOT NULL, + "user_short_name" varchar(255) DEFAULT NULL, + "user_long_name" varchar(255) DEFAULT NULL, + "token_update" text, + "token_update_at" timestamp NULL DEFAULT NULL, + "user_authenticate" text, + "user_authenticate_at" timestamp NULL DEFAULT NULL, + "user_authenticate_digest" text, + "user_authenticate_digest_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_users_chk_1" CHECK ((("user_short_name" is null) or ("user_short_name" <> ''))), + CONSTRAINT "nano_users_chk_2" CHECK ((("user_long_name" is null) or ("user_long_name" <> ''))), + CONSTRAINT "nano_users_chk_3" CHECK ((("token_update" is null) or ("token_update" <> ''))), + CONSTRAINT "nano_users_chk_4" CHECK ((("user_authenticate" is null) or ("user_authenticate" <> ''))), + CONSTRAINT "nano_users_chk_5" CHECK ((("user_authenticate_digest" is null) or ("user_authenticate_digest" <> ''))), + PRIMARY KEY ("id","device_id"), + CONSTRAINT "idx_unique_id" UNIQUE ("id") +); + +CREATE TABLE IF NOT EXISTS "network_interfaces" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "mac" varchar(255) NOT NULL DEFAULT '', + "ip_address" varchar(255) NOT NULL DEFAULT '', + "broadcast" varchar(255) NOT NULL DEFAULT '', + "ibytes" bigint NOT NULL DEFAULT '0', + "interface" varchar(255) NOT NULL DEFAULT '', + "ipackets" bigint NOT NULL DEFAULT '0', + "last_change" bigint NOT NULL DEFAULT '0', + "mask" varchar(255) NOT NULL DEFAULT '', + "metric" int NOT NULL DEFAULT '0', + "mtu" int NOT NULL DEFAULT '0', + "obytes" bigint NOT NULL DEFAULT '0', + "ierrors" bigint NOT NULL DEFAULT '0', + "oerrors" bigint NOT NULL DEFAULT '0', + "opackets" bigint NOT NULL DEFAULT '0', + "point_to_point" varchar(255) NOT NULL DEFAULT '', + "type" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_network_interfaces_unique_ip_host_intf" UNIQUE ("ip_address","host_id","interface") +); + +CREATE TABLE IF NOT EXISTS "operating_system_version_vulnerabilities" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "os_version_id" int NOT NULL, + "cve" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "source" smallint DEFAULT '0', + "resolved_in_version" varchar(255) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX IF NOT EXISTS "idx_os_version_vulnerabilities_unq_os_version_team_cve" ON "operating_system_version_vulnerabilities" ((COALESCE("team_id",-1)),"os_version_id","cve"); + +CREATE TABLE IF NOT EXISTS "operating_system_vulnerabilities" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "operating_system_id" int NOT NULL, + "cve" varchar(255) NOT NULL, + "source" smallint DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "resolved_in_version" varchar(255) DEFAULT NULL, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_os_vulnerabilities_unq_os_id_cve" UNIQUE ("operating_system_id","cve") +); + +CREATE TABLE IF NOT EXISTS "operating_systems" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "version" varchar(150) NOT NULL, + "arch" varchar(150) NOT NULL, + "kernel_version" varchar(150) NOT NULL, + "platform" varchar(50) NOT NULL, + "display_version" varchar(10) NOT NULL DEFAULT '', + "os_version_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_os" UNIQUE ("name","version","arch","kernel_version","platform","display_version") +); + +CREATE TABLE IF NOT EXISTS "osquery_options" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "override_type" int NOT NULL, + "override_identifier" varchar(255) NOT NULL DEFAULT '', + "options" jsonb NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "pack_targets" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "pack_id" int DEFAULT NULL, + "type" int DEFAULT NULL, + "target_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "constraint_pack_target_unique" UNIQUE ("pack_id","target_id","type") +); + +CREATE TABLE IF NOT EXISTS "packs" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "disabled" boolean NOT NULL DEFAULT FALSE, + "name" varchar(255) NOT NULL, + "description" varchar(255) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "pack_type" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_pack_unique_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "password_reset_requests" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "expires_at" timestamp NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "user_id" int NOT NULL, + "token" varchar(1024) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "policies" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "team_id" int DEFAULT NULL, + "resolution" text, + "name" varchar(255) NOT NULL, + "query" text NOT NULL, + "description" text NOT NULL, + "author_id" int DEFAULT NULL, + "platforms" varchar(255) NOT NULL DEFAULT '', + "critical" boolean NOT NULL DEFAULT FALSE, + "checksum" bytea NOT NULL, + "calendar_events_enabled" boolean NOT NULL DEFAULT FALSE, + "software_installer_id" int DEFAULT NULL, + "script_id" int DEFAULT NULL, + "vpp_apps_teams_id" int DEFAULT NULL, + "conditional_access_enabled" boolean NOT NULL DEFAULT FALSE, + "conditional_access_bypass_enabled" boolean NOT NULL DEFAULT TRUE, + "type" varchar(255) NOT NULL DEFAULT 'dynamic', + "patch_software_title_id" int DEFAULT NULL, + "needs_full_membership_cleanup" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_policies_checksum" UNIQUE ("checksum") +); + +CREATE TABLE IF NOT EXISTS "policy_automation_iterations" ( + "policy_id" int NOT NULL, + "iteration" int NOT NULL, + PRIMARY KEY ("policy_id") +); + +CREATE TABLE IF NOT EXISTS "policy_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "policy_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_policy_labels_policy_label" UNIQUE ("policy_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "policy_membership" ( + "policy_id" int NOT NULL, + "host_id" int NOT NULL, + "passes" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "automation_iteration" int DEFAULT NULL, + PRIMARY KEY ("policy_id","host_id") +); + +CREATE TABLE IF NOT EXISTS "policy_stats" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "policy_id" int NOT NULL, + "inherited_team_id" int DEFAULT NULL, + "passing_host_count" integer NOT NULL DEFAULT '0', + "failing_host_count" integer NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "inherited_team_id_char" text GENERATED ALWAYS AS (CASE WHEN inherited_team_id IS NULL THEN 'global' ELSE inherited_team_id::text END) STORED, + PRIMARY KEY ("id"), + CONSTRAINT "policy_id" UNIQUE ("policy_id","inherited_team_id_char") +); + +CREATE TABLE IF NOT EXISTS "queries" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "saved" boolean NOT NULL DEFAULT FALSE, + "name" varchar(255) NOT NULL, + "description" text NOT NULL, + "query" text NOT NULL, + "author_id" int DEFAULT NULL, + "observer_can_run" boolean NOT NULL DEFAULT FALSE, + "team_id" int DEFAULT NULL, + "team_id_char" char(10) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "min_osquery_version" varchar(255) NOT NULL DEFAULT '', + "schedule_interval" int NOT NULL DEFAULT '0', + "automations_enabled" boolean NOT NULL DEFAULT FALSE, + "logging_type" varchar(255) NOT NULL DEFAULT 'snapshot', + "discard_data" boolean NOT NULL DEFAULT TRUE, + "is_scheduled" boolean GENERATED ALWAYS AS (schedule_interval > 0) STORED, + PRIMARY KEY ("id"), + CONSTRAINT "idx_team_id_name_unq" UNIQUE ("team_id_char","name"), + CONSTRAINT "idx_name_team_id_unq" UNIQUE ("name","team_id_char") +); + +CREATE TABLE IF NOT EXISTS "query_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "query_id" int NOT NULL, + "label_id" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_query_labels_query_label" UNIQUE ("query_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "query_results" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "query_id" int NOT NULL, + "host_id" int NOT NULL, + "osquery_version" varchar(50) DEFAULT NULL, + "error" text, + "last_fetched" timestamp NOT NULL, + "data" jsonb DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "scep_certificates" ( + "serial" bigint NOT NULL, + "name" varchar(1024) DEFAULT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + CONSTRAINT "scep_certificates_chk_2" CHECK ((("name" is null) or ("name" <> ''))), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "scep_serials" ( + "serial" bigint GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "scheduled_queries" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "pack_id" int DEFAULT NULL, + "query_id" int DEFAULT NULL, + "interval" int DEFAULT NULL, + "snapshot" boolean DEFAULT NULL, + "removed" boolean DEFAULT NULL, + "platform" varchar(255) DEFAULT '', + "version" varchar(255) DEFAULT '', + "shard" int DEFAULT NULL, + "query_name" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "description" varchar(1023) DEFAULT '', + "denylist" boolean DEFAULT NULL, + "team_id_char" char(10) NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "unique_names_in_packs" UNIQUE ("name","pack_id") +); + +CREATE TABLE IF NOT EXISTS "scheduled_query_stats" ( + "host_id" int NOT NULL, + "scheduled_query_id" int NOT NULL, + "average_memory" bigint NOT NULL DEFAULT 0, + "denylisted" boolean DEFAULT NULL, + "executions" bigint NOT NULL DEFAULT 0, + "schedule_interval" int DEFAULT NULL, + "last_executed" timestamp NULL DEFAULT NULL, + "output_size" bigint NOT NULL DEFAULT 0, + "system_time" bigint NOT NULL DEFAULT 0, + "user_time" bigint NOT NULL DEFAULT 0, + "wall_time" bigint NOT NULL DEFAULT 0, + "query_type" smallint NOT NULL DEFAULT '0', + PRIMARY KEY ("host_id","scheduled_query_id","query_type") +); + +CREATE TABLE IF NOT EXISTS "scim_groups" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "external_id" varchar(255) DEFAULT NULL, + "display_name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_scim_groups_display_name" UNIQUE ("display_name") +); + +CREATE TABLE IF NOT EXISTS "scim_last_request" ( + "id" smallint NOT NULL DEFAULT '1', + "status" varchar(31) NOT NULL, + "details" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "scim_user_emails" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "scim_user_id" int NOT NULL, + "email" varchar(255) NOT NULL, + "primary" boolean DEFAULT NULL, + "type" varchar(31) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "scim_user_group" ( + "scim_user_id" int NOT NULL, + "group_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("scim_user_id","group_id") +); + +CREATE TABLE IF NOT EXISTS "scim_users" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "external_id" varchar(255) DEFAULT NULL, + "user_name" varchar(255) NOT NULL, + "given_name" varchar(255) DEFAULT NULL, + "family_name" varchar(255) DEFAULT NULL, + "active" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "department" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_scim_users_user_name" UNIQUE ("user_name") +); + +CREATE TABLE IF NOT EXISTS "script_contents" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "md5_checksum" bytea NOT NULL, + "contents" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_script_contents_md5_checksum" UNIQUE ("md5_checksum") +); + +CREATE TABLE IF NOT EXISTS "script_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "script_id" int DEFAULT NULL, + "script_content_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "scripts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_content_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_scripts_global_or_team_id_name" UNIQUE ("global_or_team_id","name"), + CONSTRAINT "idx_scripts_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "secret_variables" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "value" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_secret_variables_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "sessions" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "accessed_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "user_id" int NOT NULL, + "key" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_session_unique_key" UNIQUE ("key") +); + +CREATE TABLE IF NOT EXISTS "setup_experience_scripts" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_content_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_setup_experience_scripts_global_or_team_id" UNIQUE ("global_or_team_id") +); + +CREATE TABLE IF NOT EXISTS "setup_experience_status_results" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "status" text NOT NULL, + "software_installer_id" int DEFAULT NULL, + "host_software_installs_execution_id" varchar(255) DEFAULT NULL, + "vpp_app_team_id" int DEFAULT NULL, + "nano_command_uuid" varchar(255) DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "script_execution_id" varchar(255) DEFAULT NULL, + "error" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "software" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "version" varchar(255) NOT NULL DEFAULT '', + "source" varchar(64) NOT NULL, + "bundle_identifier" varchar(255) DEFAULT '', + "release" varchar(64) NOT NULL DEFAULT '', + "vendor_old" varchar(32) NOT NULL DEFAULT '', + "arch" varchar(16) NOT NULL DEFAULT '', + "vendor" varchar(114) NOT NULL DEFAULT '', + "extension_for" varchar(255) NOT NULL DEFAULT '', + "extension_id" varchar(255) NOT NULL DEFAULT '', + "title_id" int DEFAULT NULL, + "checksum" bytea NOT NULL, + "name_source" text NOT NULL DEFAULT 'basic', + "application_id" varchar(255) DEFAULT NULL, + "upgrade_code" char(38) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_checksum" UNIQUE ("checksum") +); + +CREATE TABLE IF NOT EXISTS "software_categories" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(63) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_categories_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "software_cpe" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_id" bigint DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "cpe" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "unq_software_id" UNIQUE ("software_id") +); + +CREATE TABLE IF NOT EXISTS "software_cve" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "cve" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "source" int DEFAULT '0', + "software_id" bigint DEFAULT NULL, + "resolved_in_version" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "unq_software_id_cve" UNIQUE ("software_id","cve") +); + +CREATE TABLE IF NOT EXISTS "software_host_counts" ( + "software_id" bigint NOT NULL, + "hosts_count" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "team_id" int NOT NULL DEFAULT '0', + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("software_id","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "software_install_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "software_installer_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "software_title_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "software_installer_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_installer_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_installer_labels_software_installer_id_label_id" UNIQUE ("software_installer_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "software_installer_software_categories" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_category_id" int NOT NULL, + "software_installer_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_software_installer_id_software_category_id" UNIQUE ("software_installer_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "software_installers" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "title_id" int DEFAULT NULL, + "filename" varchar(255) NOT NULL, + "version" varchar(255) NOT NULL, + "platform" varchar(255) NOT NULL, + "pre_install_query" text, + "install_script_content_id" int NOT NULL, + "post_install_script_content_id" int DEFAULT NULL, + "storage_id" varchar(64) NOT NULL, + "uploaded_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "self_service" boolean NOT NULL DEFAULT FALSE, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) NOT NULL DEFAULT '', + "user_email" varchar(255) NOT NULL DEFAULT '', + "url" varchar(4095) NOT NULL DEFAULT '', + "package_ids" text NOT NULL, + "extension" varchar(32) NOT NULL DEFAULT '', + "uninstall_script_content_id" int NOT NULL, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "fleet_maintained_app_id" int DEFAULT NULL, + "install_during_setup" boolean NOT NULL DEFAULT FALSE, + "is_active" boolean NOT NULL DEFAULT TRUE, + "upgrade_code" varchar(48) NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_installers_team_id_title_id" UNIQUE ("global_or_team_id","title_id") +); + +CREATE TABLE IF NOT EXISTS "software_title_display_names" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int NOT NULL, + "software_title_id" int NOT NULL, + "display_name" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_team_id_title_id" UNIQUE ("team_id","software_title_id") +); + +CREATE TABLE IF NOT EXISTS "software_title_icons" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int NOT NULL, + "software_title_id" int NOT NULL, + "storage_id" varchar(64) NOT NULL, + "filename" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_team_id_title_id_storage_id" UNIQUE ("team_id","software_title_id") +); + +CREATE TABLE IF NOT EXISTS "software_titles" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "source" varchar(64) NOT NULL, + "extension_for" varchar(255) NOT NULL DEFAULT '', + "bundle_identifier" varchar(255) DEFAULT NULL, + "additional_identifier" text, + "is_kernel" boolean NOT NULL DEFAULT FALSE, + "application_id" varchar(255) DEFAULT NULL, + "unique_identifier" text, + "upgrade_code" char(38) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_titles_bundle_identifier" UNIQUE ("bundle_identifier","additional_identifier"), + CONSTRAINT "idx_unique_sw_titles" UNIQUE ("unique_identifier","source","extension_for") +); + +CREATE TABLE IF NOT EXISTS "software_titles_host_counts" ( + "software_title_id" int NOT NULL, + "hosts_count" int NOT NULL, + "team_id" int NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("software_title_id","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "software_update_schedules" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "team_id" int NOT NULL, + "title_id" int NOT NULL, + "enabled" boolean NOT NULL DEFAULT FALSE, + "start_time" char(5) NOT NULL, + "end_time" char(5) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_team_title" UNIQUE ("team_id","title_id") +); + +CREATE TABLE IF NOT EXISTS "statistics" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "anonymous_identifier" varchar(255) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "teams" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) NOT NULL, + "description" varchar(1023) NOT NULL DEFAULT '', + "config" jsonb DEFAULT NULL, + "name_bin" text, + "filename" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_teams_filename" UNIQUE ("filename"), + CONSTRAINT "idx_name_bin" UNIQUE ("name_bin") +); + +CREATE TABLE IF NOT EXISTS "upcoming_activities" ( + "id" bigint GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "priority" int NOT NULL DEFAULT '0', + "user_id" int DEFAULT NULL, + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "activity_type" text NOT NULL, + "execution_id" varchar(255) NOT NULL, + "payload" jsonb NOT NULL, + "activated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_upcoming_activities_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "user_teams" ( + "user_id" int NOT NULL, + "team_id" int NOT NULL, + "role" varchar(64) NOT NULL, + PRIMARY KEY ("user_id","team_id") +); + +CREATE TABLE IF NOT EXISTS "users" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "password" bytea NOT NULL, + "salt" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL, + "admin_forced_password_reset" boolean NOT NULL DEFAULT FALSE, + "gravatar_url" varchar(255) NOT NULL DEFAULT '', + "position" varchar(255) NOT NULL DEFAULT '', + "sso_enabled" boolean NOT NULL DEFAULT FALSE, + "global_role" varchar(64) DEFAULT NULL, + "api_only" boolean NOT NULL DEFAULT FALSE, + "mfa_enabled" boolean NOT NULL DEFAULT FALSE, + "settings" jsonb NOT NULL DEFAULT '{}'::jsonb, + "invite_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_user_unique_email" UNIQUE ("email"), + CONSTRAINT "invite_id" UNIQUE ("invite_id") +); + +CREATE TABLE IF NOT EXISTS "users_deleted" ( + "id" int NOT NULL, + "name" varchar(255) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "verification_tokens" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "user_id" int NOT NULL, + "token" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_team_labels" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "vpp_app_team_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_app_team_labels_vpp_app_team_id_label_id" UNIQUE ("vpp_app_team_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_team_software_categories" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "software_category_id" int NOT NULL, + "vpp_app_team_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_vpp_app_team_id_software_category_id" UNIQUE ("vpp_app_team_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "adam_id" varchar(255) NOT NULL, + "platform" varchar(10) NOT NULL, + "vpp_token_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_apps" ( + "adam_id" varchar(255) NOT NULL, + "title_id" int DEFAULT NULL, + "bundle_identifier" varchar(255) NOT NULL DEFAULT '', + "icon_url" varchar(255) NOT NULL DEFAULT '', + "name" varchar(255) NOT NULL DEFAULT '', + "latest_version" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + PRIMARY KEY ("adam_id","platform") +); + +CREATE TABLE IF NOT EXISTS "vpp_apps_teams" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "adam_id" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "platform" varchar(10) NOT NULL, + "self_service" boolean NOT NULL DEFAULT FALSE, + "vpp_token_id" int DEFAULT NULL, + "install_during_setup" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_global_or_team_id_adam_id" UNIQUE ("global_or_team_id","adam_id","platform") +); + +CREATE TABLE IF NOT EXISTS "vpp_token_teams" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "vpp_token_id" int NOT NULL, + "team_id" int DEFAULT NULL, + "null_team_type" text DEFAULT 'none', + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_token_teams_team_id" UNIQUE ("team_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_tokens" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "organization_name" varchar(255) NOT NULL, + "location" varchar(255) NOT NULL, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_tokens_location" UNIQUE ("location") +); + +CREATE TABLE IF NOT EXISTS "vulnerability_host_counts" ( + "cve" varchar(20) NOT NULL, + "team_id" int NOT NULL DEFAULT '0', + "host_count" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "cve_team_id_global_stats" UNIQUE ("cve","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_command_queue" ( + "enrollment_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("enrollment_id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_command_results" ( + "enrollment_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "raw_result" text NOT NULL, + "response_id" int NOT NULL, + "status_code" varchar(31) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("enrollment_id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_commands" ( + "command_uuid" varchar(127) NOT NULL, + "raw_command" text NOT NULL, + "target_loc_uri" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_responses" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "enrollment_id" int NOT NULL, + "raw_response" text NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "windows_updates" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "host_id" int NOT NULL, + "date_epoch" int NOT NULL, + "kb_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_windows_updates" UNIQUE ("host_id","kb_id") +); + +CREATE TABLE IF NOT EXISTS "wstep_cert_auth_associations" ( + "id" varchar(255) NOT NULL, + "sha256" char(64) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id","sha256") +); + +CREATE TABLE IF NOT EXISTS "wstep_certificates" ( + "serial" bigint NOT NULL, + "name" varchar(1024) NOT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "wstep_serials" ( + "serial" bigint GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "yara_rules" ( + "id" int NOT NULL GENERATED ALWAYS AS IDENTITY, + "name" varchar(255) NOT NULL, + "contents" text NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_yara_rules_name" UNIQUE ("name") +); + + +CREATE TABLE IF NOT EXISTS "migration_status_tables" ( + id serial NOT NULL PRIMARY KEY, version_id bigint NOT NULL, + is_applied boolean NOT NULL, tstamp timestamp DEFAULT now()); +CREATE TABLE IF NOT EXISTS "migration_status_data" ( + id serial NOT NULL PRIMARY KEY, version_id bigint NOT NULL, + is_applied boolean NOT NULL, tstamp timestamp DEFAULT now()); + +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118193812, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118211713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212436, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212515, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212528, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212538, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212549, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212557, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212604, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212613, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212621, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212630, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212641, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212649, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212656, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212758, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161128234849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161230162221, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170104113816, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170105151732, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170108191242, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170109094020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170109130438, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170110202752, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170111133013, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170117025759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170118191001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170119234632, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170124230432, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170127014618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170131232841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170223094154, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170306075207, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170309100733, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170331111922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170502143928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170504130602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170509132100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170519105647, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170519105648, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234303, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20171116163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20171219164727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620164811, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620175054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620175055, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191010101639, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191010155147, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191220130734, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200311140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200405120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200407120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200420120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200504120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200512120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200707120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201011162341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201021104586, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201102112520, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201208121729, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201215091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210119174155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210326182902, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210421112652, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210506095025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210513115729, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210526113559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000003, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000004, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000008, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210606151329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210616163757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210617174723, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210622160235, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210623100031, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210623133615, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210708143152, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210709124443, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210712155608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210714102108, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210719153709, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210721171531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210723135713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210802135933, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210806112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210810095603, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210811150223, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818151827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818151828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818182258, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210819131107, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210819143446, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210903132338, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210915144307, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210920155130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210927143115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210927143116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211013133706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211013133707, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211102135149, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211109121546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211110163320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211116184029, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211116184030, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211202092042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211202181033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211207161856, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211216131203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211221110132, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220107155700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220125105650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220201084510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144830, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220215152203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220215152203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220223113157, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220223113157, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220307104655, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220309133956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220309133956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220316155700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220323152301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220323152301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220330100659, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220330100659, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220404091216, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220404091216, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220419140750, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220419140750, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220428140039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220428140039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220503134048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220503134048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220524102918, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220524102918, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123327, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123327, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123328, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220608113128, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220608113128, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220627104817, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220627104817, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220704101843, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220704101843, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220708095046, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220708095046, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220713091130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220713091130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220802135510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220802135510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220818101352, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220818101352, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220822161445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220822161445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100151, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100151, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220908181826, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220908181826, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220914154915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220928100158, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220928100158, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221014084130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221014084130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221027085019, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221101103952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221101103952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221104144401, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221109100749, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221109100749, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221115104546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221115104546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221130114928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221205112142, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221205112142, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221216115820, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195935, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221223174807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221223174807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163856, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230202224725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230202224725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230206163608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230206163608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230214131519, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230214131519, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230303135738, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230303135738, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313135301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313135301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313141819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313141819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230315104937, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230315104937, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230317173844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230317173844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230320133602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230320133602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330100011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330100011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330134823, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330134823, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230405232025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230405232025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230408084104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230408084104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230411102858, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230411102858, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230421155932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230421155932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425082126, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425082126, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425105727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425105727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230501154913, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230501154913, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230503101418, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230503101418, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230515144206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230515144206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517140952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517152807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517152807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230518114155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230518114155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230520153236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230520153236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230525151159, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230530122103, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230530122103, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230602111827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230602111827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230608103123, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230608103123, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140529, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230711144622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230711144622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721135421, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721161508, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721161508, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230726115701, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230807100822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230814150442, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230814150442, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230823122728, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230823122728, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230906152143, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230906152143, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230911163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230911163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230912101759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230912101759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230915101341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230915101341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230918132351, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230918132351, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231004144339, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231004144339, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094541, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094542, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094542, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231016091915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231016091915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231024174135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231024174135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025120016, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025120016, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025160156, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025160156, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231031165350, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231106144110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231106144110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231107130934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231107130934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231109115838, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231109115838, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231121054530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231121054530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231122101320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231122101320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231204155427, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231204155427, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231206142340, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231206142340, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207133731, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207133731, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212094238, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212094238, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212095734, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212161121, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212161121, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231215122713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231215122713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231219143041, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231219143041, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231224070653, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231224070653, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240110134315, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240110134315, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240119091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240119091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020643, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020643, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240129162819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240129162819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240130115133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240130115133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240131083822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240131083822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205095928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205095928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205121956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240209110212, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240209110212, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240212111533, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240221112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240221112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222073518, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222073518, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222135115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222135115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240226082255, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240226082255, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240228082706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240228082706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240301173035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240301173035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240302111134, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240302111134, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240312103753, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240313143416, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314085226, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314085226, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314151747, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314151747, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240320145650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240320145650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240327115530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240327115617, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240408085837, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240408085837, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240415104633, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240415104633, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240430111727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240430111727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240515200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143024, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143024, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240601174138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240601174138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240607133721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240607133721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240612150059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240612150059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613162201, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613172616, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613172616, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240618142419, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240618142419, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240625093543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240625093543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240626195531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240626195531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240702123921, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240703154849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240703154849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709124958, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709124958, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709132642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709132642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709183940, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709183940, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240710155623, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240710155623, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240723102712, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725152735, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725152735, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725182118, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725182118, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240726100517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240726100517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730171504, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730171504, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730174056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730174056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730215453, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730215453, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730374423, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730374423, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240801115359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240801115359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802101043, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802113716, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802113716, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240814135330, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240814135330, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240816103247, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240816103247, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240820091218, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826111228, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826160025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826160025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165448, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165448, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165605, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165605, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165715, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170044, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905105135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905140514, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104105, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104105, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104106, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104106, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002210000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002210000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241003145349, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241003145349, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241004005000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241004005000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241008083925, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241008083925, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241009090010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241017163402, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241017163402, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241021224359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241021224359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241022140321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025111236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025112748, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025141855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152839, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241116233322, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241122171434, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241125150614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241125150614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203125346, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203125346, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203130032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203130032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241205122800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241209164540, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241210140021, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241219180042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241219180042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114903, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114903, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114904, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241224000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241224000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241230000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241231112624, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250102121439, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094045, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094045, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094500, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250124194347, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250124194347, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250127162751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250127162751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250213104005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250214205657, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250217093329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250217093329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219090511, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219142401, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250224184002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250225085436, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250226000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250226153445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250304162702, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250306144233, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250313163430, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250317130944, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250318165922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250318165922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320132525, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331042354, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331154206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331154206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250401155831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250408133233, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250410104321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250410104321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250421085116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250422095806, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250424153059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430103833, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430112622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430112622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250501162727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250501162727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502154517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502222222, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502222222, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250507170845, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250507170845, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250513162912, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250513162912, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519161614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519161614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519170000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250520153848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250528115932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250529102706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250603105558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609102714, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609102714, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609112613, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250613103810, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250616193950, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250616193950, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250624140757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250624140757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250626130239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250629131032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250701155654, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250701155654, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250707095725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250716152435, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250716152435, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250718091828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250718091828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250728122229, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731122715, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731151000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731151000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250803000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250805083116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250805083116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250807140441, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250808000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250811155036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250813205039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250813205039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250814123333, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250815130115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250815130115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250816115553, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250817154557, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250825113751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250827113140, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250828120836, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250902112642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250902112642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250904091745, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250904091745, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250905090000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250905090000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250922083056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250922083056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250923120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250926123048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250926123048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103505, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103505, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103900, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103900, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140400, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140400, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251031154558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251031154558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251103160848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251103160848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251104112849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107164629, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107164629, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107170854, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107170854, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251110172137, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251110172137, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251111153133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251111153133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121124239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121124239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124090450, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124090450, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124135808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124140138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124140138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124162948, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124162948, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251127113559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251202162232, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251203170808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251203170808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251207050413, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251207050413, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251208215800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221730, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221730, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221850, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251215163721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260108200708, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260108214732, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260109231821, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260109231821, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260113012054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260113012054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260124200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260124200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126150840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126150840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126210724, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260202151756, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260205184907, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210151544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210155109, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210155109, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210181120, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210181120, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260211200153, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260211200153, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217141240, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217141240, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217200906, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260218175704, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260314120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120003, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120004, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120008, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120009, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260317120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260318184559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260318184559, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20161229171615, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170223171234, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170301093653, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170314151620, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20181119180000, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210330130314, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210806135609, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210819120215, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20230525175650, true); +-- Manual fixes for tables the converter can't handle +DROP TABLE IF EXISTS "abm_tokens" CASCADE; +CREATE TABLE "abm_tokens" ( + "id" integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "organization_name" varchar(255) NOT NULL, + "apple_id" varchar(255) NOT NULL, + "terms_expired" boolean NOT NULL DEFAULT FALSE, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "macos_default_team_id" integer DEFAULT NULL, + "ios_default_team_id" integer DEFAULT NULL, + "ipados_default_team_id" integer DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_abm_tokens_organization_name" UNIQUE ("organization_name") +); + +DROP TABLE IF EXISTS "carve_metadata" CASCADE; +CREATE TABLE "carve_metadata" ( + "id" integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "host_id" integer NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) DEFAULT NULL, + "block_count" integer NOT NULL, + "block_size" integer NOT NULL, + "carve_size" bigint NOT NULL, + "carve_id" varchar(64) NOT NULL, + "request_id" varchar(64) NOT NULL, + "session_id" varchar(255) NOT NULL, + "expired" smallint DEFAULT 0, + "max_block" integer DEFAULT -1, + "error" text, + CONSTRAINT "idx_session_id" UNIQUE ("session_id"), + CONSTRAINT "idx_name" UNIQUE ("name") +); + +DROP TABLE IF EXISTS "host_mdm_apple_bootstrap_packages" CASCADE; +CREATE TABLE "host_mdm_apple_bootstrap_packages" ( + "host_uuid" varchar(127) NOT NULL PRIMARY KEY, + "command_uuid" varchar(127) DEFAULT NULL, + "skipped" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "ck_skipped_or_commanduuid" CHECK (("skipped" = FALSE) = ("command_uuid" IS NOT NULL)) +); + +DROP TABLE IF EXISTS "host_mdm_windows_profiles" CASCADE; +CREATE TABLE "host_mdm_windows_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "retries" smallint NOT NULL DEFAULT 0, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL DEFAULT '\x00000000000000000000000000000000', + "secrets_updated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_uuid","profile_uuid") +); + +DROP TABLE IF EXISTS "mdm_android_configuration_profiles" CASCADE; +CREATE TABLE "mdm_android_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "name" varchar(255) NOT NULL, + "raw_json" jsonb NOT NULL, + "auto_increment" bigint GENERATED ALWAYS AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_mdm_android_auto_increment" UNIQUE ("auto_increment"), + CONSTRAINT "idx_mdm_android_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +DROP TABLE IF EXISTS "mdm_apple_declarations" CASCADE; +CREATE TABLE "mdm_apple_declarations" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "raw_json" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT NULL, + "auto_increment" bigint GENERATED ALWAYS AS IDENTITY, + "secrets_updated_at" timestamp DEFAULT NULL, + "token" bytea, + "scope" text NOT NULL DEFAULT 'System', + CONSTRAINT "idx_mdm_apple_declaration_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_declaration_team_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_apple_declarations_auto_increment" UNIQUE ("auto_increment") +); + +DROP TABLE IF EXISTS "mdm_apple_enrollment_profiles" CASCADE; +CREATE TABLE "mdm_apple_enrollment_profiles" ( + "id" integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "token" varchar(36) DEFAULT NULL, + "type" varchar(10) NOT NULL DEFAULT 'automatic', + "dep_profile" jsonb DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_enrollment_profiles_type" UNIQUE ("type"), + CONSTRAINT "idx_enrollment_profiles_token" UNIQUE ("token") +); + +DROP TABLE IF EXISTS "mdm_configuration_profile_labels" CASCADE; +CREATE TABLE "mdm_configuration_profile_labels" ( + "id" integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "label_name" varchar(255) NOT NULL, + "label_id" integer DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "android_profile_uuid" varchar(37) DEFAULT NULL, + CONSTRAINT "idx_mdm_configuration_profile_labels_apple_label_name" UNIQUE ("apple_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_windows_label_name" UNIQUE ("windows_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_android_label_name" UNIQUE ("android_profile_uuid","label_name") +); + +DROP TABLE IF EXISTS "mdm_windows_configuration_profiles" CASCADE; +CREATE TABLE "mdm_windows_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "name" varchar(255) NOT NULL, + "syncml" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT NULL, + "auto_increment" bigint GENERATED ALWAYS AS IDENTITY, + "checksum" bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + CONSTRAINT "idx_mdm_windows_configuration_profiles_team_id_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_win_config_auto_increment" UNIQUE ("auto_increment") +); + +DROP TABLE IF EXISTS "operating_system_version_vulnerabilities" CASCADE; +CREATE TABLE "operating_system_version_vulnerabilities" ( + "id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "os_version_id" integer NOT NULL, + "cve" varchar(255) NOT NULL, + "team_id" integer DEFAULT NULL, + "source" smallint DEFAULT 0, + "resolved_in_version" varchar(255) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX "idx_os_version_vulnerabilities_unq_os_version_team_cve2" ON "operating_system_version_vulnerabilities" ((COALESCE("team_id",-1)),"os_version_id","cve"); + +SELECT 1; + +CREATE TABLE IF NOT EXISTS "host_recovery_key_passwords" ( + "host_uuid" varchar(255) NOT NULL, + "encrypted_password" bytea NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) NOT NULL, + "error_message" text, + "deleted" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "pending_encrypted_password" bytea, + "pending_error_message" text, + PRIMARY KEY ("host_uuid") +); + +CREATE OR REPLACE VIEW "nano_view_queue" AS +SELECT + q.id AS id, + q.created_at AS created_at, + q.active AS active, + q.priority AS priority, + c.command_uuid AS command_uuid, + c.request_type AS request_type, + c.command AS command, + r.updated_at AS result_updated_at, + r.status AS status, + r.result AS result +FROM + nano_enrollment_queue q + JOIN nano_commands c ON q.command_uuid = c.command_uuid + LEFT JOIN nano_command_results r ON r.command_uuid = q.command_uuid AND r.id = q.id +ORDER BY q.priority DESC, q.created_at; diff --git a/server/datastore/mysql/postgres_smoke_test.go b/server/datastore/mysql/postgres_smoke_test.go new file mode 100644 index 00000000000..8a3651f596e --- /dev/null +++ b/server/datastore/mysql/postgres_smoke_test.go @@ -0,0 +1,417 @@ +package mysql + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPostgresSmokeTest verifies basic PostgreSQL connectivity and dialect +// SQL execution. Requires POSTGRES_TEST=1 and a running postgres_test container. +func TestPostgresSmokeTest(t *testing.T) { + ds := CreatePostgresDS(t) + + // Verify we got a PG-backed datastore + assert.IsType(t, postgresDialect{}, ds.dialect) + + // Create a simple table using PG-native DDL + _, err := ds.primary.Exec(` + CREATE TABLE IF NOT EXISTS pg_smoke_test ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ DEFAULT NOW() + ) + `) + require.NoError(t, err) + + // Insert using the dialect's InsertIgnoreInto (PG: INSERT INTO + ON CONFLICT DO NOTHING) + stmt := ds.dialect.InsertIgnoreInto() + ` pg_smoke_test (name) VALUES ($1)` + ds.dialect.OnConflictDoNothing("name") + _, err = ds.primary.Exec(stmt, "test-host") + require.NoError(t, err) + + // Insert duplicate — should be silently ignored + _, err = ds.primary.Exec(stmt, "test-host") + require.NoError(t, err) + + // Verify only one row + var count int + err = ds.primary.Get(&count, "SELECT COUNT(*) FROM pg_smoke_test WHERE name = $1", "test-host") + require.NoError(t, err) + assert.Equal(t, 1, count) + + // Test upsert via OnDuplicateKey + upsertStmt := `INSERT INTO pg_smoke_test (name) VALUES ($1) ` + + ds.dialect.OnDuplicateKey("name", "name=VALUES(name)") + // Note: For PG this becomes: ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name + _, err = ds.primary.Exec(upsertStmt, "test-host-2") + require.NoError(t, err) + + // Verify GroupConcat equivalent + _, err = ds.primary.Exec(`INSERT INTO pg_smoke_test (name) VALUES ('a'), ('b'), ('c')`) + require.NoError(t, err) + + var names string + err = ds.primary.Get(&names, "SELECT "+ds.dialect.GroupConcat("name", ",")+" FROM pg_smoke_test") + require.NoError(t, err) + assert.NotEmpty(t, names) + + // Verify JSON operations + _, err = ds.primary.Exec(`CREATE TABLE IF NOT EXISTS pg_json_test (id SERIAL PRIMARY KEY, data JSONB DEFAULT '{}')`) + require.NoError(t, err) + _, err = ds.primary.Exec(`INSERT INTO pg_json_test (data) VALUES ('{"name": "fleet", "version": "4.83"}')`) + require.NoError(t, err) + + var version string + err = ds.primary.Get(&version, "SELECT "+ds.dialect.JSONUnquoteExtract("data", "$.version")+" FROM pg_json_test LIMIT 1") + require.NoError(t, err) + assert.Equal(t, "4.83", version) +} + +func TestPostgresNewHost(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + host, err := ds.NewHost(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("pg-test-host"), + NodeKey: ptr.String("pg-test-key"), + UUID: "pg-test-uuid", + Hostname: "pg-test-hostname", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + if err != nil { + t.Fatalf("NewHost failed: %v", err) + } + assert.NotNil(t, host) + assert.NotZero(t, host.ID) + t.Logf("Created host ID: %d", host.ID) +} + +func TestPostgresNewHostViaTestHelper(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + // This is how test helpers create hosts - using the test package helper + host := &fleet.Host{ + OsqueryHostID: ptr.String("pg-helper-host"), + NodeKey: ptr.String("pg-helper-key"), + UUID: "pg-helper-uuid", + Hostname: "pg-helper", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + } + created, err := ds.NewHost(ctx, host) + require.NoError(t, err, "NewHost should work") + require.NotNil(t, created) + t.Logf("Host created: ID=%d", created.ID) + + // Now try the operations that follow in typical test setup + err = ds.RecordLabelQueryExecutions(ctx, created, map[uint]*bool{}, time.Now(), false) + if err != nil { + t.Logf("RecordLabelQueryExecutions error: %v", err) + } + + // Try saving host users + err = ds.SaveHostUsers(ctx, created.ID, []fleet.HostUser{ + {Username: "testuser", Uid: 1001}, + }) + if err != nil { + t.Logf("SaveHostUsers error: %v", err) + } +} + +// TestPostgresDatastoreOperations exercises a broad set of datastore operations +// against PostgreSQL to find SQL compatibility issues. +func TestPostgresDatastoreOperations(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + // --- Host CRUD --- + host, err := ds.NewHost(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("pg-ops-host-1"), + NodeKey: ptr.String("pg-ops-key-1"), + UUID: "pg-ops-uuid-1", + Hostname: "pg-ops-hostname-1", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + require.NoError(t, err, "NewHost") + + t.Run("HostByIdentifier", func(t *testing.T) { + h, err := ds.HostByIdentifier(ctx, "pg-ops-uuid-1") + if err != nil { + t.Logf("FAIL HostByIdentifier: %v", err) + return + } + assert.Equal(t, host.ID, h.ID) + }) + + t.Run("UpdateHost", func(t *testing.T) { + host.Hostname = "pg-ops-hostname-updated" + err := ds.UpdateHost(ctx, host) + if err != nil { + t.Logf("FAIL UpdateHost: %v", err) + } + }) + + t.Run("Host", func(t *testing.T) { + h, err := ds.Host(ctx, host.ID) + if err != nil { + t.Logf("FAIL Host: %v", err) + return + } + assert.Equal(t, "pg-ops-hostname-updated", h.Hostname) + }) + + // --- Labels --- + t.Run("Labels", func(t *testing.T) { + labels, err := ds.ListLabels(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.ListOptions{}, false) + if err != nil { + t.Logf("FAIL ListLabels: %v", err) + return + } + t.Logf("Labels found: %d", len(labels)) + }) + + t.Run("RecordLabelQueryExecutions", func(t *testing.T) { + trueVal := true + err := ds.RecordLabelQueryExecutions(ctx, host, map[uint]*bool{1: &trueVal}, time.Now(), false) + if err != nil { + t.Logf("FAIL RecordLabelQueryExecutions: %v", err) + } + }) + + // --- Queries --- + t.Run("NewQuery", func(t *testing.T) { + q, err := ds.NewQuery(ctx, &fleet.Query{ + Name: "pg-test-query", + Description: "Test query for PG compat", + Query: "SELECT 1", + Logging: fleet.LoggingSnapshot, + }) + if err != nil { + t.Logf("FAIL NewQuery: %v", err) + return + } + assert.NotZero(t, q.ID) + + // List queries + queries, _, _, _, err := ds.ListQueries(ctx, fleet.ListQueryOptions{ListOptions: fleet.ListOptions{}}) + if err != nil { + t.Logf("FAIL ListQueries: %v", err) + return + } + t.Logf("Queries found: %d", len(queries)) + }) + + // --- Packs --- + t.Run("NewPack", func(t *testing.T) { + p, err := ds.NewPack(ctx, &fleet.Pack{ + Name: "pg-test-pack", + }) + if err != nil { + t.Logf("FAIL NewPack: %v", err) + return + } + assert.NotZero(t, p.ID) + }) + + // --- Users --- + t.Run("NewUser", func(t *testing.T) { + u, err := ds.NewUser(ctx, &fleet.User{ + Name: "pg-test-user", + Email: "pg-test@example.com", + Password: []byte("test-password-hash"), + GlobalRole: ptr.String("admin"), + }) + if err != nil { + t.Logf("FAIL NewUser: %v", err) + return + } + assert.NotZero(t, u.ID) + + // Find user by email + found, err := ds.UserByEmail(ctx, "pg-test@example.com") + if err != nil { + t.Logf("FAIL UserByEmail: %v", err) + return + } + assert.Equal(t, u.ID, found.ID) + }) + + // --- Teams --- + t.Run("NewTeam", func(t *testing.T) { + team, err := ds.NewTeam(ctx, &fleet.Team{ + Name: "pg-test-team", + }) + if err != nil { + t.Logf("FAIL NewTeam: %v", err) + return + } + assert.NotZero(t, team.ID) + }) + + // --- Policies --- + t.Run("NewGlobalPolicy", func(t *testing.T) { + p, err := ds.NewGlobalPolicy(ctx, ptr.Uint(0), fleet.PolicyPayload{ + Name: "pg-test-policy", + Query: "SELECT 1", + }) + if err != nil { + t.Logf("FAIL NewGlobalPolicy: %v", err) + return + } + assert.NotZero(t, p.ID) + }) + + // --- Host additional data --- + t.Run("SaveHostAdditional", func(t *testing.T) { + additional := json.RawMessage(`{"test_field": "test_value"}`) + err := ds.SaveHostAdditional(ctx, host.ID, &additional) + if err != nil { + t.Logf("FAIL SaveHostAdditional: %v", err) + } + }) + + // --- Software --- + t.Run("UpdateHostSoftware", func(t *testing.T) { + sw := []fleet.Software{ + {Name: "pg-test-sw", Version: "1.0", Source: "test"}, + } + _, err := ds.UpdateHostSoftware(ctx, host.ID, sw) + if err != nil { + t.Logf("FAIL UpdateHostSoftware: %v", err) + } + }) + + // --- Sessions --- + t.Run("NewSession", func(t *testing.T) { + users, err := ds.ListUsers(ctx, fleet.UserListOptions{ListOptions: fleet.ListOptions{}}) + if err != nil || len(users) == 0 { + t.Logf("SKIP NewSession: no users") + return + } + sess, err := ds.NewSession(ctx, users[0].ID, 64) + if err != nil { + t.Logf("FAIL NewSession: %v", err) + return + } + assert.NotZero(t, sess.ID) + }) + + // --- Enroll secrets --- + t.Run("ApplyEnrollSecrets", func(t *testing.T) { + err := ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{ + {Secret: "pg-test-secret"}, + }) + if err != nil { + t.Logf("FAIL ApplyEnrollSecrets: %v", err) + } + }) + + // --- App config --- + t.Run("AppConfig", func(t *testing.T) { + cfg, err := ds.AppConfig(ctx) + if err != nil { + t.Logf("FAIL AppConfig: %v", err) + return + } + assert.NotNil(t, cfg) + }) + + // --- ListHosts --- + t.Run("ListHosts", func(t *testing.T) { + hosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{ListOptions: fleet.ListOptions{}}) + if err != nil { + t.Logf("FAIL ListHosts: %v", err) + return + } + assert.GreaterOrEqual(t, len(hosts), 1) + }) + + // --- CountHosts --- + t.Run("CountHosts", func(t *testing.T) { + count, err := ds.CountHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{}) + if err != nil { + t.Logf("FAIL CountHosts: %v", err) + return + } + assert.GreaterOrEqual(t, count, 1) + }) + + t.Run("HostLite", func(t *testing.T) { + h, err := ds.HostLite(ctx, host.ID) + if err != nil { + t.Logf("FAIL HostLite: %v", err) + return + } + assert.Equal(t, host.ID, h.ID) + }) + + // --- Targets --- + t.Run("CountHostsInTargets", func(t *testing.T) { + metrics, err := ds.CountHostsInTargets(ctx, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, + fleet.HostTargets{HostIDs: []uint{host.ID}}, + time.Now(), + ) + if err != nil { + t.Logf("FAIL CountHostsInTargets: %v", err) + return + } + assert.GreaterOrEqual(t, metrics.TotalHosts, uint(1)) + }) + + // --- Host disk encryption key --- + t.Run("SetOrUpdateHostDiskEncryptionKey", func(t *testing.T) { + _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", ptr.Bool(false)) + if err != nil { + t.Logf("FAIL SetOrUpdateHostDiskEncryptionKey: %v", err) + } + }) + + // --- Cron stats --- + t.Run("InsertCronStats", func(t *testing.T) { + id, err := ds.InsertCronStats(ctx, fleet.CronStatsTypeScheduled, "test-cron", "test-instance", fleet.CronStatsStatusPending) + if err != nil { + t.Logf("FAIL InsertCronStats: %v", err) + return + } + assert.NotZero(t, id) + }) + + // --- ListPolicies --- + t.Run("ListGlobalPolicies", func(t *testing.T) { + policies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) + if err != nil { + t.Logf("FAIL ListGlobalPolicies: %v", err) + return + } + assert.GreaterOrEqual(t, len(policies), 1) + }) + + // --- Invites --- + t.Run("ListInvites", func(t *testing.T) { + invites, err := ds.ListInvites(ctx, fleet.ListOptions{}) + if err != nil { + t.Logf("FAIL ListInvites: %v", err) + return + } + _ = invites + }) +} diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 2ccb4a34605..6613bd8b742 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -17,7 +17,9 @@ import ( "log/slog" "os" "os/exec" + "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -39,6 +41,7 @@ import ( common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" "github.com/fleetdm/fleet/v4/server/platform/mysql/testing_utils" "github.com/google/uuid" + _ "github.com/jackc/pgx/v5/stdlib" // register pgx driver for PostgreSQL tests "github.com/jmoiron/sqlx" "github.com/olekukonko/tablewriter" "github.com/smallstep/pkcs7" @@ -413,6 +416,22 @@ func CreateMySQLDS(t testing.TB) *Datastore { return createMySQLDSWithOptions(t, nil) } +// CreateDS creates a test Datastore for the active test database backend. +// When MYSQL_TEST=1 is set, returns a MySQL-backed datastore. +// When POSTGRES_TEST=1 is set, returns a PostgreSQL-backed datastore. +// Skips the test if neither is set. +func CreateDS(t *testing.T) *Datastore { + _, hasMysql := os.LookupEnv("MYSQL_TEST") + _, hasPG := os.LookupEnv("POSTGRES_TEST") + if !hasMysql && !hasPG { + t.Skip("Neither MYSQL_TEST nor POSTGRES_TEST is set") + } + if hasPG { + return CreatePostgresDS(t) + } + return createMySQLDSWithOptions(t, nil) +} + func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { ds, _ := CreateNamedMySQLDSWithConns(t, name) return ds @@ -432,6 +451,159 @@ func CreateNamedMySQLDSWithConns(t *testing.T, name string) (*Datastore, *common return ds, TestDBConnections(t, ds) } +// pgBaselineSchema is loaded once from pg_baseline_schema.sql +var pgBaselineSchema string +var pgBaselineOnce sync.Once + +func loadPGBaselineSchema() string { + pgBaselineOnce.Do(func() { + // Try multiple paths since tests run from different working directories + paths := []string{ + "pg_baseline_schema.sql", + "server/datastore/mysql/pg_baseline_schema.sql", + "../../../server/datastore/mysql/pg_baseline_schema.sql", + } + // Also try relative to the test binary via runtime.Caller + _, thisFile, _, _ := runtime.Caller(0) + if thisFile != "" { + dir := filepath.Dir(thisFile) + paths = append(paths, filepath.Join(dir, "pg_baseline_schema.sql")) + } + for _, p := range paths { + data, err := os.ReadFile(p) + if err == nil { + pgBaselineSchema = string(data) + return + } + } + panic("cannot load pg_baseline_schema.sql from any known path") + }) + return pgBaselineSchema +} + +// CreatePostgresDS creates a test Datastore backed by PostgreSQL. +// Requires POSTGRES_TEST=1 and a running postgres_test container (default port 5434). +// The database is created fresh for each test with the full Fleet schema applied. +func CreatePostgresDS(t *testing.T) *Datastore { + if _, ok := os.LookupEnv("POSTGRES_TEST"); !ok { + t.Skip("PostgreSQL tests are disabled") + } + + port := os.Getenv("FLEET_POSTGRES_TEST_PORT") + if port == "" { + port = "5434" + } + + // Sanitize test name into a valid PG identifier (alphanumeric + underscore only). + dbName := strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' { + return r + } + if r >= 'A' && r <= 'Z' { + return r + ('a' - 'A') // lowercase + } + return '_' + }, t.Name()) + if len(dbName) > 63 { + dbName = dbName[:63] // PG identifier limit + } + + // Connect to default db to create test database + adminDSN := fmt.Sprintf("host=localhost port=%s user=fleet password=insecure dbname=fleet sslmode=disable", port) + adminDB, err := sqlx.Open("pgx-rebind", adminDSN) + require.NoError(t, err) + defer adminDB.Close() + + _, _ = adminDB.Exec("DROP DATABASE IF EXISTS " + dbName) + _, err = adminDB.Exec("CREATE DATABASE " + dbName) + require.NoError(t, err) + + t.Cleanup(func() { + _, _ = adminDB.Exec("DROP DATABASE IF EXISTS " + dbName) + }) + + // Connect to the test database + testDSN := fmt.Sprintf("host=localhost port=%s user=fleet password=insecure dbname=%s sslmode=disable", port, dbName) + testDB, err := sqlx.Open("pgx-rebind", testDSN) + require.NoError(t, err) + + // Apply the baseline schema statement-by-statement. + // Split on ");\n" for CREATE TABLE and ";\n" for INSERTs. + schema := loadPGBaselineSchema() + // Use a regex to split on statement boundaries: "); followed by newline or + // ";" at end of a single-line statement (INSERT/DROP) + re := regexp.MustCompile(`;\s*\n`) + stmts := re.Split(schema, -1) + errCount := 0 + for _, stmt := range stmts { + stmt = strings.TrimSpace(stmt) + if stmt == "" || strings.HasPrefix(stmt, "--") { + continue + } + if _, err := testDB.DB.Exec(stmt + ";"); err != nil { + errCount++ + if errCount <= 3 { + first := stmt + if len(first) > 100 { + first = first[:100] + } + t.Logf("PG schema warning (%d): %v [%s]", errCount, err, first) + } + } + } + if errCount > 0 { + t.Logf("PG schema: %d/%d stmts had errors (non-fatal)", errCount, len(stmts)) + } + + // Verify minimum table count + var tableCount int + if err := testDB.Get(&tableCount, "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'"); err == nil { + if tableCount < 180 { + t.Fatalf("PG schema incomplete: only %d tables (expected 190+)", tableCount) + } + } + + // Insert required seed data (app_config_json needs at least one row) + _, _ = testDB.Exec(`INSERT INTO app_config_json (id, json_value) VALUES (1, '{}') ON CONFLICT (id) DO NOTHING`) + // Insert built-in labels that migrations would normally create + if _, err := testDB.Exec(`INSERT INTO labels (name, query, label_type, label_membership_type) VALUES + ('All Hosts', 'SELECT 1', 1, 0), + ('macOS', 'SELECT 1', 1, 0), + ('Ubuntu Linux', 'SELECT 1', 1, 0), + ('CentOS Linux', 'SELECT 1', 1, 0), + ('Windows', 'SELECT 1', 1, 0), + ('Red Hat Linux', 'SELECT 1', 1, 0), + ('All Linux', 'SELECT 1', 1, 0), + ('chrome', 'SELECT 1', 1, 0), + ('iOS', 'SELECT 1', 1, 0), + ('iPadOS', 'SELECT 1', 1, 0), + ('Fedora Linux', 'SELECT 1', 1, 0) + ON CONFLICT (name) DO NOTHING`); err != nil { + t.Logf("PG seed data: labels insert error: %v", err) + } + // Insert mdm delivery status and operation type seed data + _, _ = testDB.Exec(`INSERT INTO mdm_delivery_status (status) VALUES ('failed'), ('applied'), ('pending'), ('verified'), ('verifying') ON CONFLICT (status) DO NOTHING`) + _, _ = testDB.Exec(`INSERT INTO mdm_operation_types (operation_type) VALUES ('install'), ('remove') ON CONFLICT (operation_type) DO NOTHING`) + + logger := slog.New(slog.DiscardHandler) + ds := &Datastore{ + primary: testDB, + replica: testDB, + logger: logger, + clock: clock.C, + dialect: postgresDialect{}, + writeCh: make(chan itemToWrite), + serverPrivateKey: "test-private-key-for-pg-tests!!!", // 32 bytes for AES-256 + stmtCache: make(map[string]*sqlx.Stmt), + } + ds.Datastore = NewAndroidDatastore(logger, testDB, testDB, postgresDialect{}) + t.Cleanup(func() { ds.Close() }) + + go ds.writeChanLoop() + + return ds +} + func ExecAdhocSQL(tb testing.TB, ds *Datastore, fn func(q sqlx.ExtContext) error) { tb.Helper() err := fn(ds.primary) @@ -462,6 +634,41 @@ func TruncateTables(t testing.TB, ds *Datastore, tables ...string) { "osquery_options": true, "software_categories": true, } + + if _, ok := ds.dialect.(postgresDialect); ok { + db := ds.writer(context.Background()) + ctx := context.Background() + + // If no specific tables given, query all tables from PG catalog + if len(tables) == 0 { + rows, err := db.QueryContext(ctx, + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'") + if err != nil { + t.Logf("PG truncate: list tables: %v", err) + return + } + defer rows.Close() + for rows.Next() { + var tbl string + if err := rows.Scan(&tbl); err == nil { + tables = append(tables, tbl) + } + } + if err := rows.Err(); err != nil { + t.Logf("PG truncate: rows iteration: %v", err) + } + } + + for _, tbl := range tables { + if nonEmptyTables[tbl] { + continue + } + _, _ = db.ExecContext(ctx, `TRUNCATE TABLE "`+tbl+`" CASCADE`) + + } + return + } + testing_utils.TruncateTables(t, ds.writer(context.Background()), ds.logger, nonEmptyTables, tables...) } From 6d4b0c38dfb1778471731b26d34c04408a47b583 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 16:01:34 -0400 Subject: [PATCH 5/6] test(datastore): update test files for dialect parameter changes Update test call sites to pass dialect parameter where function signatures changed. Add dialect: mysqlDialect{} to mockDatastore. --- server/datastore/mysql/activities_test.go | 2 +- .../datastore/mysql/aggregated_stats_test.go | 2 +- server/datastore/mysql/android_device_test.go | 2 +- .../mysql/android_enterprise_test.go | 2 +- server/datastore/mysql/android_test.go | 2 +- server/datastore/mysql/app_configs_test.go | 2 +- server/datastore/mysql/apple_mdm_ddm_test.go | 2 +- server/datastore/mysql/apple_mdm_test.go | 20 ++--- .../datastore/mysql/ca_config_assets_test.go | 2 +- .../datastore/mysql/calendar_events_test.go | 2 +- server/datastore/mysql/campaigns_test.go | 2 +- server/datastore/mysql/carves_test.go | 2 +- .../mysql/certificate_authorities_test.go | 2 +- .../mysql/certificate_templates_test.go | 2 +- .../mysql/conditional_access_bypass_test.go | 2 +- .../conditional_access_microsoft_test.go | 2 +- .../mysql/conditional_access_scep_test.go | 2 +- server/datastore/mysql/cron_stats_test.go | 8 +- server/datastore/mysql/delete_test.go | 2 +- server/datastore/mysql/dialect_mysql_test.go | 67 +++++++++++++++ .../datastore/mysql/disk_encryption_test.go | 2 +- server/datastore/mysql/email_changes_test.go | 2 +- .../mysql/host_certificate_templates_test.go | 2 +- .../datastore/mysql/host_certificates_test.go | 2 +- .../mysql/host_identity_scep_test.go | 2 +- server/datastore/mysql/hosts_test.go | 22 ++--- server/datastore/mysql/in_house_apps_test.go | 2 +- server/datastore/mysql/invites_test.go | 2 +- server/datastore/mysql/jobs_test.go | 2 +- server/datastore/mysql/labels_test.go | 2 +- server/datastore/mysql/linux_mdm_test.go | 2 +- server/datastore/mysql/locks_test.go | 2 +- .../datastore/mysql/maintained_apps_test.go | 2 +- .../datastore/mysql/mdm_idp_accounts_test.go | 2 +- server/datastore/mysql/mdm_test.go | 27 +++--- server/datastore/mysql/microsoft_mdm_test.go | 6 +- server/datastore/mysql/mysql_test.go | 5 +- .../datastore/mysql/nanomdm_storage_test.go | 2 +- ...ating_system_vulnerabilities_batch_test.go | 2 +- .../operating_system_vulnerabilities_test.go | 2 +- .../datastore/mysql/operating_systems_test.go | 30 +++---- server/datastore/mysql/packs_test.go | 6 +- server/datastore/mysql/password_reset_test.go | 2 +- server/datastore/mysql/policies_test.go | 84 ++++++++----------- server/datastore/mysql/queries_test.go | 2 +- server/datastore/mysql/query_results_test.go | 2 +- .../datastore/mysql/scheduled_queries_test.go | 2 +- server/datastore/mysql/scim_test.go | 2 +- server/datastore/mysql/scripts_test.go | 11 +-- .../datastore/mysql/secret_variables_test.go | 2 +- server/datastore/mysql/sessions_test.go | 2 +- .../datastore/mysql/setup_experience_test.go | 2 +- .../mysql/software_installers_test.go | 2 +- server/datastore/mysql/software_test.go | 50 +++++------ .../mysql/software_title_icons_test.go | 2 +- .../datastore/mysql/software_titles_test.go | 2 +- .../mysql/software_upgrade_code_test.go | 2 +- server/datastore/mysql/statistics_test.go | 2 +- server/datastore/mysql/targets_test.go | 2 +- server/datastore/mysql/teams_test.go | 6 +- server/datastore/mysql/unicode_test.go | 2 +- server/datastore/mysql/users_test.go | 4 +- server/datastore/mysql/vpp_test.go | 2 +- .../datastore/mysql/vulnerabilities_test.go | 2 +- .../datastore/mysql/windows_updates_test.go | 2 +- server/datastore/mysql/wstep_test.go | 2 +- 66 files changed, 250 insertions(+), 200 deletions(-) create mode 100644 server/datastore/mysql/dialect_mysql_test.go diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index 51266031d02..7fcc24f79e5 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -24,7 +24,7 @@ import ( ) func TestActivity(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/aggregated_stats_test.go b/server/datastore/mysql/aggregated_stats_test.go index ccf213e6772..9d03c5892f9 100644 --- a/server/datastore/mysql/aggregated_stats_test.go +++ b/server/datastore/mysql/aggregated_stats_test.go @@ -46,7 +46,7 @@ func slowStats(t *testing.T, ds *Datastore, id uint, percentile int, column stri } func TestAggregatedStats(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) var args []interface{} diff --git a/server/datastore/mysql/android_device_test.go b/server/datastore/mysql/android_device_test.go index 6b3d56ff3a4..d192a204277 100644 --- a/server/datastore/mysql/android_device_test.go +++ b/server/datastore/mysql/android_device_test.go @@ -19,7 +19,7 @@ import ( ) func TestAndroidDevices(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/android_enterprise_test.go b/server/datastore/mysql/android_enterprise_test.go index f35ae52e87d..b8995bf1fcf 100644 --- a/server/datastore/mysql/android_enterprise_test.go +++ b/server/datastore/mysql/android_enterprise_test.go @@ -11,7 +11,7 @@ import ( ) func TestAndroidEnterprises(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/android_test.go b/server/datastore/mysql/android_test.go index 712ad990850..c01b4840652 100644 --- a/server/datastore/mysql/android_test.go +++ b/server/datastore/mysql/android_test.go @@ -20,7 +20,7 @@ import ( ) func TestAndroid(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) TruncateTables(t, ds) cases := []struct { diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index 9a1160528c8..01fe1c375ae 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -18,7 +18,7 @@ import ( ) func TestAppConfig(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/apple_mdm_ddm_test.go b/server/datastore/mysql/apple_mdm_ddm_test.go index 0a412635fc5..09e292c666d 100644 --- a/server/datastore/mysql/apple_mdm_ddm_test.go +++ b/server/datastore/mysql/apple_mdm_ddm_test.go @@ -14,7 +14,7 @@ import ( ) func TestMDMDDMApple(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 8860db82290..73b943276f2 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -39,7 +39,7 @@ import ( ) func TestMDMApple(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -782,7 +782,7 @@ func testHostDetailsMDMProfiles(t *testing.T, ds *Datastore) { } func TestIngestMDMAppleDevicesFromDEPSync(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := t.Context() createBuiltinLabels(t, ds) @@ -842,7 +842,7 @@ func TestIngestMDMAppleDevicesFromDEPSync(t *testing.T) { } func TestDEPSyncTeamAssignment(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := t.Context() createBuiltinLabels(t, ds) @@ -912,7 +912,7 @@ func TestDEPSyncTeamAssignment(t *testing.T) { } func TestMDMEnrollment(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -3112,7 +3112,7 @@ func createDiskEncryptionRecord(ctx context.Context, ds *Datastore, t *testing.T } func TestMDMAppleFileVaultSummary(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := t.Context() // 10 new hosts @@ -4690,7 +4690,7 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) { } func TestCopyDefaultMDMAppleBootstrapPackage(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() ctx := t.Context() @@ -4863,7 +4863,7 @@ func TestCopyDefaultMDMAppleBootstrapPackage(t *testing.T) { } func TestHostDEPAssignments(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() ctx := t.Context() @@ -6192,7 +6192,7 @@ func testDeleteMDMAppleDeclarationWithPendingInstalls(t *testing.T, ds *Datastor } func TestMDMAppleProfileVerification(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := t.Context() now := time.Now() @@ -6587,7 +6587,7 @@ func profilesByIdentifier(profiles []*fleet.HostMacOSProfile) map[string]*fleet. } func TestRestorePendingDEPHost(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() ctx := t.Context() @@ -8224,7 +8224,7 @@ func testIngestMDMAppleDeviceFromOTAEnrollment(t *testing.T, ds *Datastore) { } func TestGetMDMAppleOSUpdatesSettingsByHostSerial(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() keys := []string{"ios", "ipados", "macos"} diff --git a/server/datastore/mysql/ca_config_assets_test.go b/server/datastore/mysql/ca_config_assets_test.go index 6742a904159..42c08fd7b48 100644 --- a/server/datastore/mysql/ca_config_assets_test.go +++ b/server/datastore/mysql/ca_config_assets_test.go @@ -10,7 +10,7 @@ import ( ) func TestCAConfigAssets(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go index f50ccd52a70..d0f0ee362b4 100644 --- a/server/datastore/mysql/calendar_events_test.go +++ b/server/datastore/mysql/calendar_events_test.go @@ -15,7 +15,7 @@ import ( ) func TestCalendarEvents(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/campaigns_test.go b/server/datastore/mysql/campaigns_test.go index 0196ef890ca..cdfab682762 100644 --- a/server/datastore/mysql/campaigns_test.go +++ b/server/datastore/mysql/campaigns_test.go @@ -16,7 +16,7 @@ import ( ) func TestCampaigns(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/carves_test.go b/server/datastore/mysql/carves_test.go index 2b50ee4298a..37aeeadc8f7 100644 --- a/server/datastore/mysql/carves_test.go +++ b/server/datastore/mysql/carves_test.go @@ -15,7 +15,7 @@ import ( var mockCreatedAt = time.Now().UTC().Truncate(time.Second) func TestCarves(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/certificate_authorities_test.go b/server/datastore/mysql/certificate_authorities_test.go index 2c3e5321a11..7b2bcc9dbc8 100644 --- a/server/datastore/mysql/certificate_authorities_test.go +++ b/server/datastore/mysql/certificate_authorities_test.go @@ -12,7 +12,7 @@ import ( ) func TestCertificateAuthority(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/certificate_templates_test.go b/server/datastore/mysql/certificate_templates_test.go index e458a55b4bd..3cc40978b1a 100644 --- a/server/datastore/mysql/certificate_templates_test.go +++ b/server/datastore/mysql/certificate_templates_test.go @@ -14,7 +14,7 @@ import ( ) func TestCertificates(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/conditional_access_bypass_test.go b/server/datastore/mysql/conditional_access_bypass_test.go index 47289a43894..ee962d27aa6 100644 --- a/server/datastore/mysql/conditional_access_bypass_test.go +++ b/server/datastore/mysql/conditional_access_bypass_test.go @@ -12,7 +12,7 @@ import ( ) func TestConditionalAccessBypass(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/conditional_access_microsoft_test.go b/server/datastore/mysql/conditional_access_microsoft_test.go index 7746d585355..fc2be31d019 100644 --- a/server/datastore/mysql/conditional_access_microsoft_test.go +++ b/server/datastore/mysql/conditional_access_microsoft_test.go @@ -9,7 +9,7 @@ import ( ) func TestConditionalAccess(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/conditional_access_scep_test.go b/server/datastore/mysql/conditional_access_scep_test.go index b20ca534d22..3429343605b 100644 --- a/server/datastore/mysql/conditional_access_scep_test.go +++ b/server/datastore/mysql/conditional_access_scep_test.go @@ -13,7 +13,7 @@ import ( ) func TestConditionalAccessSCEP(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/cron_stats_test.go b/server/datastore/mysql/cron_stats_test.go index 07f8d5f19ff..59be3c0231e 100644 --- a/server/datastore/mysql/cron_stats_test.go +++ b/server/datastore/mysql/cron_stats_test.go @@ -25,7 +25,7 @@ func TestInsertUpdateCronStats(t *testing.T) { instanceID = "test_instance" ) ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) id, err := ds.InsertCronStats(ctx, fleet.CronStatsTypeScheduled, scheduleName, instanceID, fleet.CronStatsStatusPending) require.NoError(t, err) @@ -72,7 +72,7 @@ func TestGetLatestCronStats(t *testing.T) { instanceID = "test_instance" ) ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) insertTestCS := func(name string, statsType fleet.CronStatsType, status fleet.CronStatsStatus, createdAt time.Time) { stmt := `INSERT INTO cron_stats (stats_type, name, instance, status, created_at) VALUES (?, ?, ?, ?, ?)` @@ -130,7 +130,7 @@ func TestGetLatestCronStats(t *testing.T) { func TestCleanupCronStats(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) insertCronStats := func(t *testing.T, name, instance string, status fleet.CronStatsStatus, createdAt time.Time) { t.Helper() @@ -308,7 +308,7 @@ func TestCleanupCronStats(t *testing.T) { func TestUpdateAllCronStatsForInstance(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { instance string diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index b2b253ccace..f06c0708145 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -13,7 +13,7 @@ import ( ) func TestDelete(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/dialect_mysql_test.go b/server/datastore/mysql/dialect_mysql_test.go new file mode 100644 index 00000000000..b140c044ebc --- /dev/null +++ b/server/datastore/mysql/dialect_mysql_test.go @@ -0,0 +1,67 @@ +package mysql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMysqlDialectSQL(t *testing.T) { + d := mysqlDialect{} + + t.Run("InsertIgnoreInto", func(t *testing.T) { + assert.Equal(t, "INSERT IGNORE INTO", d.InsertIgnoreInto()) + }) + + t.Run("ReplaceInto", func(t *testing.T) { + assert.Equal(t, "REPLACE INTO", d.ReplaceInto()) + }) + + t.Run("OnDuplicateKey", func(t *testing.T) { + got := d.OnDuplicateKey("id", "name=VALUES(name), updated_at=NOW()") + assert.Equal(t, "ON DUPLICATE KEY UPDATE name=VALUES(name), updated_at=NOW()", got) + }) + + t.Run("OnConflictDoNothing", func(t *testing.T) { + assert.Equal(t, "", d.OnConflictDoNothing("id")) + }) + + t.Run("GroupConcat", func(t *testing.T) { + assert.Equal(t, "GROUP_CONCAT(x SEPARATOR ',')", d.GroupConcat("x", ",")) + assert.Equal(t, "GROUP_CONCAT(DISTINCT v.col SEPARATOR ',')", d.GroupConcat("DISTINCT v.col", ",")) + }) + + t.Run("JSONExtract", func(t *testing.T) { + assert.Equal(t, "JSON_EXTRACT(col, '$.path')", d.JSONExtract("col", "$.path")) + }) + + t.Run("JSONUnquoteExtract", func(t *testing.T) { + assert.Equal(t, "col->>'$.path'", d.JSONUnquoteExtract("col", "$.path")) + }) + + t.Run("JSONBuildObject", func(t *testing.T) { + assert.Equal(t, "JSON_OBJECT('k1', v1, 'k2', v2)", d.JSONBuildObject("'k1'", "v1", "'k2'", "v2")) + }) + + t.Run("FindInSet", func(t *testing.T) { + assert.Equal(t, "FIND_IN_SET(?, platforms)", d.FindInSet("?", "platforms")) + }) + + t.Run("FullTextMatch", func(t *testing.T) { + assert.Equal(t, "MATCH(l.name) AGAINST (? IN BOOLEAN MODE)", d.FullTextMatch([]string{"l.name"}, "?")) + }) + + t.Run("RegexpMatch", func(t *testing.T) { + assert.Equal(t, "s.name REGEXP ?", d.RegexpMatch("s.name", "?")) + }) + + t.Run("JSONAgg", func(t *testing.T) { + assert.Equal(t, "JSON_ARRAYAGG(x)", d.JSONAgg("x")) + }) + + t.Run("GoquDialect", func(t *testing.T) { + // Verify it returns a valid goqu dialect (not nil/panic) + gd := d.GoquDialect() + assert.NotNil(t, gd) + }) +} diff --git a/server/datastore/mysql/disk_encryption_test.go b/server/datastore/mysql/disk_encryption_test.go index 105f1dcd441..779d30d8e6d 100644 --- a/server/datastore/mysql/disk_encryption_test.go +++ b/server/datastore/mysql/disk_encryption_test.go @@ -15,7 +15,7 @@ import ( ) func TestDiskEncryption(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/email_changes_test.go b/server/datastore/mysql/email_changes_test.go index 1d8a84d5e12..581af77d2c0 100644 --- a/server/datastore/mysql/email_changes_test.go +++ b/server/datastore/mysql/email_changes_test.go @@ -12,7 +12,7 @@ import ( ) func TestEmailChanges(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/host_certificate_templates_test.go b/server/datastore/mysql/host_certificate_templates_test.go index 35e2f1a12ee..b2feefdb0ed 100644 --- a/server/datastore/mysql/host_certificate_templates_test.go +++ b/server/datastore/mysql/host_certificate_templates_test.go @@ -15,7 +15,7 @@ import ( ) func TestHostCertificateTemplates(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/host_certificates_test.go b/server/datastore/mysql/host_certificates_test.go index d6bdc636bcf..587d38f1e01 100644 --- a/server/datastore/mysql/host_certificates_test.go +++ b/server/datastore/mysql/host_certificates_test.go @@ -21,7 +21,7 @@ import ( ) func TestHostCertificates(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/host_identity_scep_test.go b/server/datastore/mysql/host_identity_scep_test.go index 555fcce54d1..fdcadef029f 100644 --- a/server/datastore/mysql/host_identity_scep_test.go +++ b/server/datastore/mysql/host_identity_scep_test.go @@ -19,7 +19,7 @@ import ( ) func TestHostIdentitySCEP(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index f9beb96af3c..5311d7cbdec 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -70,7 +70,7 @@ var enrollTests = []struct { } func TestHosts(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) TruncateTables(t, ds) cases := []struct { @@ -3757,8 +3757,9 @@ func testHostsByIdentifier(t *testing.T, ds *Datastore) { func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { now := time.Now().UTC().Truncate(time.Second) + hosts := make([]*fleet.Host, 10) for i := 1; i <= 10; i++ { - _, err := ds.NewHost( + host, err := ds.NewHost( context.Background(), &fleet.Host{ DetailUpdatedAt: now, LabelUpdatedAt: now, @@ -3772,6 +3773,7 @@ func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { }, ) require.NoError(t, err) + hosts[i-1] = host } var ( @@ -3781,7 +3783,7 @@ func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { identifier := "uuid_1" h, err = ds.HostLiteByIdentifier(context.Background(), identifier) require.NoError(t, err) - assert.Equal(t, uint(1), h.ID) + assert.Equal(t, hosts[0].ID, h.ID) assert.Equal(t, now.UTC(), h.SeenTime) // Also test fetching host by ID @@ -3791,22 +3793,22 @@ func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { h, err = ds.HostLiteByIdentifier(context.Background(), "osquery_host_id_2") require.NoError(t, err) - assert.Equal(t, uint(2), h.ID) + assert.Equal(t, hosts[1].ID, h.ID) assert.Equal(t, now.UTC(), h.SeenTime) h, err = ds.HostLiteByIdentifier(context.Background(), "node_key_4") require.NoError(t, err) - assert.Equal(t, uint(4), h.ID) + assert.Equal(t, hosts[3].ID, h.ID) assert.Equal(t, now.UTC(), h.SeenTime) h, err = ds.HostLiteByIdentifier(context.Background(), "hostname_7") require.NoError(t, err) - assert.Equal(t, uint(7), h.ID) + assert.Equal(t, hosts[6].ID, h.ID) assert.Equal(t, now.UTC(), h.SeenTime) h, err = ds.HostLiteByIdentifier(context.Background(), "serial_9") require.NoError(t, err) - assert.Equal(t, uint(9), h.ID) + assert.Equal(t, hosts[8].ID, h.ID) assert.Equal(t, now.UTC(), h.SeenTime) h, err = ds.HostLiteByIdentifier(context.Background(), "foobar") @@ -8129,7 +8131,7 @@ func testHostsLite(t *testing.T, ds *Datastore) { var nfe fleet.NotFoundError require.True(t, errors.As(err, &nfe)) - now := time.Now() + now := time.Now().UTC() h, err := ds.NewHost(context.Background(), &fleet.Host{ ID: 1, OsqueryHostID: ptr.String("foobar"), @@ -8159,7 +8161,7 @@ func testHostsLite(t *testing.T, ds *Datastore) { // HostLite does not load host seen time. require.Empty(t, h.SeenTime) - require.Equal(t, uint(1), h.ID) + require.NotZero(t, h.ID) require.NotEmpty(t, h.CreatedAt) require.NotEmpty(t, h.UpdatedAt) require.Equal(t, "foobar", *h.OsqueryHostID) @@ -11122,7 +11124,7 @@ func testHostsEnrollOrbit(t *testing.T, ds *Datastore) { ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(ctx, `INSERT INTO host_mdm(host_id, enrolled, server_url, installed_from_dep, mdm_id, is_server) - VALUES(?, 1, 'https://example.com/mdm', 0, ?, 0)`, h1Orbit.ID, h1Orbit.ID+100) + VALUES(?, ?, 'https://example.com/mdm', ?, ?, ?)`, h1Orbit.ID, true, false, h1Orbit.ID+100, false) return err }) h1WithMdmFetched, err := ds.Host(ctx, h1Orbit.ID) diff --git a/server/datastore/mysql/in_house_apps_test.go b/server/datastore/mysql/in_house_apps_test.go index df17d1cbc77..e6824a90f3b 100644 --- a/server/datastore/mysql/in_house_apps_test.go +++ b/server/datastore/mysql/in_house_apps_test.go @@ -21,7 +21,7 @@ import ( ) func TestInHouseApps(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/invites_test.go b/server/datastore/mysql/invites_test.go index 634dcdc6454..05c5ff7e39b 100644 --- a/server/datastore/mysql/invites_test.go +++ b/server/datastore/mysql/invites_test.go @@ -13,7 +13,7 @@ import ( ) func TestInvites(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/jobs_test.go b/server/datastore/mysql/jobs_test.go index fc8a93d6651..2fa5a60711a 100644 --- a/server/datastore/mysql/jobs_test.go +++ b/server/datastore/mysql/jobs_test.go @@ -11,7 +11,7 @@ import ( ) func TestJobs(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) // call TruncateTables before the first test, because a DB migation may have // created job entries. TruncateTables(t, ds) diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index 4880a7f9c2c..c00068420e5 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -67,7 +67,7 @@ func TestBatchHostIdsLarge(t *testing.T) { } func TestLabels(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/linux_mdm_test.go b/server/datastore/mysql/linux_mdm_test.go index f06ebc4494b..7c360116375 100644 --- a/server/datastore/mysql/linux_mdm_test.go +++ b/server/datastore/mysql/linux_mdm_test.go @@ -12,7 +12,7 @@ import ( ) func TestLinuxDiskEncryptionSummary(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := context.Background() // 5 new ubuntu hosts diff --git a/server/datastore/mysql/locks_test.go b/server/datastore/mysql/locks_test.go index d45ed46cee4..6fefc967f48 100644 --- a/server/datastore/mysql/locks_test.go +++ b/server/datastore/mysql/locks_test.go @@ -14,7 +14,7 @@ import ( ) func TestLocks(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() cases := []struct { diff --git a/server/datastore/mysql/maintained_apps_test.go b/server/datastore/mysql/maintained_apps_test.go index 80048873bb7..6c883853820 100644 --- a/server/datastore/mysql/maintained_apps_test.go +++ b/server/datastore/mysql/maintained_apps_test.go @@ -13,7 +13,7 @@ import ( ) func TestMaintainedApps(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/mdm_idp_accounts_test.go b/server/datastore/mysql/mdm_idp_accounts_test.go index 4e47bcad901..a19d3a6df6e 100644 --- a/server/datastore/mysql/mdm_idp_accounts_test.go +++ b/server/datastore/mysql/mdm_idp_accounts_test.go @@ -12,7 +12,7 @@ import ( ) func TestMDMIdPAccountsReconciliation(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index e6d80c22a46..0adfedd6bfd 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -28,7 +28,7 @@ import ( ) func TestMDMShared(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) TruncateTables(t, ds) cases := []struct { @@ -6806,14 +6806,14 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { wantOtherWin := []fleet.ConfigurationProfileLabel{ {ProfileUUID: otherWinProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID}, } - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherWin, []string{windowsProfile.ProfileUUID}, "windows") + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), ds.dialect, wantOtherWin, []string{windowsProfile.ProfileUUID}, "windows") require.NoError(t, err) assert.True(t, updatedDB) // make it an "exclude" label on the other macos profile wantOtherMac := []fleet.ConfigurationProfileLabel{ {ProfileUUID: otherMacProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID, Exclude: true}, } - updatedDB, err = batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherMac, []string{macOSProfile.ProfileUUID}, "darwin") + updatedDB, err = batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), ds.dialect, wantOtherMac, []string{macOSProfile.ProfileUUID}, "darwin") require.NoError(t, err) assert.True(t, updatedDB) @@ -6848,7 +6848,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { t.Run("empty input "+platform, func(t *testing.T) { want := []fleet.ConfigurationProfileLabel{} err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, want, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, want, nil, platform) require.NoError(t, err) assert.False(t, updatedDB) return err @@ -6865,7 +6865,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID}, } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -6881,7 +6881,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID, Exclude: true}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -6899,7 +6899,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -6911,7 +6911,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: 12345}, } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -6921,7 +6921,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: "xyz", LabelID: 1235}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -6943,7 +6943,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: newLabel.Name, LabelID: newLabel.ID, Exclude: true}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -6957,7 +6957,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -6967,7 +6967,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { // batch apply again this time without any label err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, nil, []string{uuid}, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, nil, []string{uuid}, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -6981,7 +6981,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { // batch apply again with no change returns false err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, nil, []string{uuid}, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, nil, []string{uuid}, platform) require.NoError(t, err) assert.False(t, updatedDB) return err @@ -6996,6 +6996,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { _, err := batchSetProfileLabelAssociationsDB( ctx, tx, + ds.dialect, []fleet.ConfigurationProfileLabel{{}}, nil, "unsupported", diff --git a/server/datastore/mysql/microsoft_mdm_test.go b/server/datastore/mysql/microsoft_mdm_test.go index 85289c99fdb..17a6faac049 100644 --- a/server/datastore/mysql/microsoft_mdm_test.go +++ b/server/datastore/mysql/microsoft_mdm_test.go @@ -23,7 +23,7 @@ import ( ) func TestMDMWindows(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -3314,7 +3314,7 @@ func testSetMDMWindowsProfilesWithVariables(t *testing.T, ds *Datastore) { } // both profiles have no variable - _, err := batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), []fleet.MDMProfileUUIDFleetVariables{ + _, err := batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: globalProfiles[0], FleetVariables: nil}, {ProfileUUID: globalProfiles[1], FleetVariables: nil}, }, "windows") @@ -3324,7 +3324,7 @@ func testSetMDMWindowsProfilesWithVariables(t *testing.T, ds *Datastore) { checkProfileVariables(globalProfiles[1], 0, nil) // add some variables - _, err = batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), []fleet.MDMProfileUUIDFleetVariables{ + _, err = batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: globalProfiles[0], FleetVariables: []fleet.FleetVarName{fleet.FleetVarHostEndUserIDPUsername, fleet.FleetVarName(string(fleet.FleetVarDigiCertDataPrefix) + "ZZZ")}}, {ProfileUUID: globalProfiles[1], FleetVariables: []fleet.FleetVarName{fleet.FleetVarHostEndUserIDPGroups}}, }, "windows") diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index b9198341f2d..feecaf6a042 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -249,6 +249,7 @@ func mockDatastore(t *testing.T) (sqlmock.Sqlmock, *Datastore) { primary: dbmock, replica: dbmock, logger: slog.New(slog.DiscardHandler), + dialect: mysqlDialect{}, } return mock, ds @@ -1093,7 +1094,7 @@ func TestCompareVersions(t *testing.T) { } func TestDebugs(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) status, err := ds.InnoDBStatus(context.Background()) require.NoError(t, err) @@ -1105,7 +1106,7 @@ func TestDebugs(t *testing.T) { } func TestWantedModesEnabled(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) var sqlMode string err := ds.writer(context.Background()).GetContext(context.Background(), &sqlMode, `SELECT @@SQL_MODE`) diff --git a/server/datastore/mysql/nanomdm_storage_test.go b/server/datastore/mysql/nanomdm_storage_test.go index 4fd28a8f380..751aa12eceb 100644 --- a/server/datastore/mysql/nanomdm_storage_test.go +++ b/server/datastore/mysql/nanomdm_storage_test.go @@ -17,7 +17,7 @@ import ( ) func TestNanoMDMStorage(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string fn func(t *testing.T, ds *Datastore) diff --git a/server/datastore/mysql/operating_system_vulnerabilities_batch_test.go b/server/datastore/mysql/operating_system_vulnerabilities_batch_test.go index 7d596dab6fc..8b8ec5a9ab4 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities_batch_test.go +++ b/server/datastore/mysql/operating_system_vulnerabilities_batch_test.go @@ -13,7 +13,7 @@ import ( ) func TestListVulnsByMultipleOSVersions(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/operating_system_vulnerabilities_test.go b/server/datastore/mysql/operating_system_vulnerabilities_test.go index 18cf3b96a78..35f2f6a8e6b 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities_test.go +++ b/server/datastore/mysql/operating_system_vulnerabilities_test.go @@ -15,7 +15,7 @@ import ( ) func TestOperatingSystemVulnerabilities(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/operating_systems_test.go b/server/datastore/mysql/operating_systems_test.go index d67b7261784..62208fbfa52 100644 --- a/server/datastore/mysql/operating_systems_test.go +++ b/server/datastore/mysql/operating_systems_test.go @@ -17,7 +17,7 @@ import ( func TestListOperatingSystems(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) // no os records list, err := ds.ListOperatingSystems(ctx) @@ -41,7 +41,7 @@ func TestListOperatingSystems(t *testing.T) { func TestListOperatingSystemsForPlatform(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) // no os records list, err := ds.ListOperatingSystemsForPlatform(ctx, "windows") @@ -63,7 +63,7 @@ func TestListOperatingSystemsForPlatform(t *testing.T) { func TestUpdateHostOperatingSystem(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) testHostID := uint(42) testOS := fleet.OperatingSystem{ @@ -145,7 +145,7 @@ func TestUpdateHostOperatingSystem(t *testing.T) { func TestUniqueOS(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) testHostIDs := make([]uint, 50) testOS := fleet.OperatingSystem{ @@ -174,7 +174,7 @@ func TestUniqueOS(t *testing.T) { func TestMaybeNewOperatingSystem(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) seedOperatingSystems(t, ds) list, err := ds.ListOperatingSystems(ctx) @@ -248,7 +248,7 @@ func TestMaybeNewOperatingSystem(t *testing.T) { func TestMaybeUpdateHostOperatingSystem(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) seedOperatingSystems(t, ds) osList, err := ds.ListOperatingSystems(ctx) @@ -261,21 +261,21 @@ func TestMaybeUpdateHostOperatingSystem(t *testing.T) { require.ErrorIs(t, err, sql.ErrNoRows) // insert test host and os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[0].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[0].ID) require.NoError(t, err) osID, err := getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) require.Equal(t, osList[0].ID, osID) // update test host with new os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) osID, err = getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) require.Equal(t, osList[1].ID, osID) // no change - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) osID, err = getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -284,7 +284,7 @@ func TestMaybeUpdateHostOperatingSystem(t *testing.T) { func TestGetHostOperatingSystem(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) seedOperatingSystems(t, ds) osList, err := ds.ListOperatingSystems(ctx) @@ -300,7 +300,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.ErrorIs(t, err, sql.ErrNoRows) // insert test host and os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[0].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[0].ID) require.NoError(t, err) os, err := getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -311,7 +311,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.Equal(t, osList[0], *os) // update test host with new os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) os, err = getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -322,7 +322,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.Equal(t, osList[1], *os) // no change - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) os, err = getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -335,7 +335,7 @@ func TestGetHostOperatingSystem(t *testing.T) { func TestCleanupHostOperatingSystems(t *testing.T) { ctx := context.Background() - ds := CreateMySQLDS(t) + ds := CreateDS(t) seedOperatingSystems(t, ds) testOSs, err := ds.ListOperatingSystems(ctx) @@ -360,7 +360,7 @@ func TestCleanupHostOperatingSystems(t *testing.T) { // insert host operating system record so initially each os is seeded with two hosts hostOS := testOSs[i%len(testOSs)] - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), h.ID, hostOS.ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, h.ID, hostOS.ID) require.NoError(t, err) osByHostID[h.ID] = hostOS } diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 2350e730e7d..0acb4740211 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -15,7 +15,7 @@ import ( ) func TestPacks(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -498,7 +498,7 @@ func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) } } }() @@ -550,7 +550,7 @@ func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) } } }() diff --git a/server/datastore/mysql/password_reset_test.go b/server/datastore/mysql/password_reset_test.go index 2427c325444..6e98d843e6d 100644 --- a/server/datastore/mysql/password_reset_test.go +++ b/server/datastore/mysql/password_reset_test.go @@ -12,7 +12,7 @@ import ( ) func TestPasswordReset(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 7bd17e11d85..a49c19764b0 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -26,7 +26,7 @@ import ( ) func TestPolicies(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -1501,14 +1501,11 @@ func testPolicyQueriesForHost(t *testing.T, ds *Datastore) { assert.Equal(t, "pass", policies[0].Response) // Manually insert a global policy with null resolution. - res, err := ds.writer(context.Background()).ExecContext( - context.Background(), - fmt.Sprintf(`INSERT INTO policies (name, query, description, checksum) VALUES (?, ?, ?, %s)`, policiesChecksumComputedColumn()), - q.Name+"2", q.Query, q.Description+"2", + id, err := ds.insertAndGetID(context.Background(), ds.writer(context.Background()), + `INSERT INTO policies (name, query, description, checksum) VALUES (?, ?, ?, ?)`, + q.Name+"2", q.Query, q.Description+"2", policyChecksum(nil, q.Name+"2"), ) require.NoError(t, err) - id, err := res.LastInsertId() - require.NoError(t, err) require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), host2, map[uint]*bool{uint(id): nil}, //nolint:gosec // dismiss G115 time.Now(), false)) @@ -3053,13 +3050,9 @@ func testPolicyViolationDays(t *testing.T, ds *Datastore) { hosts[i] = h } - createPolStmt := fmt.Sprintf( - `INSERT INTO policies (name, query, description, author_id, platforms, created_at, updated_at, checksum) VALUES (?, ?, '', ?, ?, ?, ?, %s)`, - policiesChecksumComputedColumn(), - ) - res, err := ds.writer(ctx).ExecContext(ctx, createPolStmt, "test_pol", "select 1", user.ID, "", then, then) + createPolStmt := `INSERT INTO policies (name, query, description, author_id, platforms, created_at, updated_at, checksum) VALUES (?, ?, '', ?, ?, ?, ?, ?)` + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), createPolStmt, "test_pol", "select 1", user.ID, "", then, then, policyChecksum(nil, "test_pol")) require.NoError(t, err) - id, _ := res.LastInsertId() pol, err := ds.Policy(ctx, uint(id)) //nolint:gosec // dismiss G115 require.NoError(t, err) @@ -3151,10 +3144,8 @@ func testPolicyCleanupPolicyMembership(t *testing.T, ds *Datastore) { } // create some policies, using direct insert statements to control the timestamps - createPolStmt := fmt.Sprintf( - `INSERT INTO policies (name, query, description, author_id, platforms, created_at, updated_at, checksum) - VALUES (?, ?, '', ?, ?, ?, ?, %s)`, policiesChecksumComputedColumn(), - ) + createPolStmt := `INSERT INTO policies (name, query, description, author_id, platforms, created_at, updated_at, checksum) + VALUES (?, ?, '', ?, ?, ?, ?, ?)` jan2020 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) feb2020 := time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC) @@ -3163,9 +3154,9 @@ func testPolicyCleanupPolicyMembership(t *testing.T, ds *Datastore) { may2020 := time.Date(2020, 5, 1, 0, 0, 0, 0, time.UTC) pols := make([]*fleet.Policy, 3) for i, dt := range []time.Time{jan2020, feb2020, mar2020} { - res, err := ds.writer(ctx).ExecContext(ctx, createPolStmt, "p"+strconv.Itoa(i+1), "select 1", user.ID, "", dt, dt) + name := "p" + strconv.Itoa(i+1) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), createPolStmt, name, "select 1", user.ID, "", dt, dt, policyChecksum(nil, name)) require.NoError(t, err) - id, _ := res.LastInsertId() pol, err := ds.Policy(ctx, uint(id)) //nolint:gosec // dismiss G115 require.NoError(t, err) pols[i] = pol @@ -3331,7 +3322,7 @@ func testDeleteAllPolicyMemberships(t *testing.T, ds *Datastore) { require.NoError(t, ds.writer(ctx).Get(&count, "select COUNT(*) from host_issues WHERE total_issues_count > 0")) assert.Equal(t, 1, count) - err = deleteAllPolicyMemberships(ctx, ds.writer(ctx), host.ID) + err = deleteAllPolicyMemberships(ctx, ds.writer(ctx), ds.dialect, host.ID) require.NoError(t, err) err = ds.writer(ctx).Get(&count, "select COUNT(*) from policy_membership") @@ -3789,7 +3780,7 @@ func testPoliciesNameUnicode(t *testing.T, ds *Datastore) { err = ds.SavePolicy( context.Background(), &fleet.Policy{PolicyData: fleet.PolicyData{ID: policyEmoji.ID, Name: equivalentNames[1]}}, false, false, ) - assert.True(t, IsDuplicate(err), err) + assert.True(t, ds.dialect.IsDuplicate(err), err) // Try to find policy with equivalent name policies, err := ds.ListGlobalPolicies(context.Background(), fleet.ListOptions{MatchQuery: equivalentNames[1]}) @@ -6974,12 +6965,9 @@ func testPolicyModificationResetsAttemptNumber(t *testing.T, ds *Datastore) { // Create script content var scriptContentID int64 ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - res, err := q.ExecContext(ctx, `INSERT INTO script_contents (md5_checksum, contents) VALUES (?, ?)`, + var err error + scriptContentID, err = insertAndGetIDTx(ctx, q, ds.dialect, `INSERT INTO script_contents (md5_checksum, contents) VALUES (?, ?)`, "md5hash", "echo 'test'") - if err != nil { - return err - } - scriptContentID, err = res.LastInsertId() return err }) @@ -6995,24 +6983,18 @@ func testPolicyModificationResetsAttemptNumber(t *testing.T, ds *Datastore) { // Create a software title and installer titleID := int64(0) ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - res, err := q.ExecContext(ctx, `INSERT INTO software_titles (name, source) VALUES (?, ?)`, "Test App", "apps") - if err != nil { - return err - } - titleID, err = res.LastInsertId() + var err error + titleID, err = insertAndGetIDTx(ctx, q, ds.dialect, `INSERT INTO software_titles (name, source) VALUES (?, ?)`, "Test App", "apps") return err }) installerID := int64(0) ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { - res, err := q.ExecContext(ctx, ` + var err error + installerID, err = insertAndGetIDTx(ctx, q, ds.dialect, ` INSERT INTO software_installers (team_id, global_or_team_id, title_id, storage_id, filename, extension, version, install_script_content_id, uninstall_script_content_id, platform, package_ids) VALUES (?, ?, ?, 'storage', 'test.pkg', 'pkg', '1.0', ?, ?, 'darwin', '') `, team.ID, team.ID, titleID, scriptContentID, scriptContentID) - if err != nil { - return err - } - installerID, err = res.LastInsertId() return err }) @@ -7165,7 +7147,7 @@ func testBatchedPolicyMembershipCleanup(t *testing.T, ds *Datastore) { // Run the full cleanup function directly (simulates what ApplyPolicySpecs triggers when a // query changes — shouldRemoveAllPolicyMemberships == true). - err = cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID) + err = cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), ds.dialect, pol.ID) require.NoError(t, err) // All policy_membership rows must be gone. @@ -7237,7 +7219,7 @@ func testBatchedPolicyMembershipCleanupOnPolicyUpdate(t *testing.T, ds *Datastor require.Equal(t, 6, count) // Run the platform-aware cleanup (simulates CleanupPolicyMembership cron). - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform, ds.dialect) require.NoError(t, err) // Only the windows host should remain. @@ -7302,7 +7284,7 @@ func testBatchedPolicyMembershipCleanupOnPolicyUpdate(t *testing.T, ds *Datastor // Run cleanupPolicyMembershipOnPolicyUpdate with no platform restriction so // only the label-based branch fires. - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), lblPol.ID, "" /* no platform filter */) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), lblPol.ID, "" /* no platform filter */, ds.dialect) require.NoError(t, err) // Only the host that belongs to the include label should remain. @@ -7369,11 +7351,11 @@ func testApplyPolicySpecsNeedsFullMembershipCleanupFlag(t *testing.T, ds *Datast {Name: "flag test policy", Query: "select 2;", Platform: "", Type: fleet.PolicyTypeDynamic}, })) - // The flag must be 0 after successful completion (set inside TX, cleared after cleanup). - var flagVal int + // The flag must be false after successful completion (set inside TX, cleared after cleanup). + var flagVal bool require.NoError(t, ds.writer(ctx).Get(&flagVal, `SELECT needs_full_membership_cleanup FROM policies WHERE id = ?`, pol.ID)) - assert.Zero(t, flagVal, "needs_full_membership_cleanup must be cleared after successful cleanup") + assert.False(t, flagVal, "needs_full_membership_cleanup must be cleared after successful cleanup") // All memberships must have been removed. require.NoError(t, ds.writer(ctx).Get(&count, `SELECT COUNT(*) FROM policy_membership WHERE policy_id = ?`, pol.ID)) @@ -7447,7 +7429,7 @@ func testCleanupPolicyMembershipCrashRecovery(t *testing.T, ds *Datastore) { // Simulate: TX committed with the flag set, but cleanup never ran (crash/error). _, err = ds.writer(ctx).ExecContext(ctx, - `UPDATE policies SET needs_full_membership_cleanup = 1 WHERE id = ?`, pol.ID) + `UPDATE policies SET needs_full_membership_cleanup = true WHERE id = ?`, pol.ID) require.NoError(t, err) // Retry GitOps with the same spec. ApplyPolicySpecs must detect the flag and @@ -7457,10 +7439,10 @@ func testCleanupPolicyMembershipCrashRecovery(t *testing.T, ds *Datastore) { })) // Flag must be cleared by the retry. - var flagVal int + var flagVal bool require.NoError(t, ds.writer(ctx).Get(&flagVal, `SELECT needs_full_membership_cleanup FROM policies WHERE id = ?`, pol.ID)) - assert.Zero(t, flagVal, "flag must be cleared by the GitOps retry") + assert.False(t, flagVal, "flag must be cleared by the GitOps retry") // All memberships must be gone. require.NoError(t, ds.writer(ctx).Get(&count, `SELECT COUNT(*) FROM policy_membership WHERE policy_id = ?`, pol.ID)) @@ -7479,17 +7461,17 @@ func testCleanupPolicyMembershipCrashRecovery(t *testing.T, ds *Datastore) { // Simulate interrupted cleanup: set the flag directly, leave membership rows in place. _, err := ds.writer(ctx).ExecContext(ctx, - `UPDATE policies SET needs_full_membership_cleanup = 1 WHERE id = ?`, pol.ID) + `UPDATE policies SET needs_full_membership_cleanup = true WHERE id = ?`, pol.ID) require.NoError(t, err) // CleanupPolicyMembership (cron) should pick up the flag and run the full cleanup. require.NoError(t, ds.CleanupPolicyMembership(ctx, time.Now())) // Flag must be cleared. - var flagVal int + var flagVal bool require.NoError(t, ds.writer(ctx).Get(&flagVal, `SELECT needs_full_membership_cleanup FROM policies WHERE id = ?`, pol.ID)) - assert.Zero(t, flagVal, "flag must be cleared by CleanupPolicyMembership") + assert.False(t, flagVal, "flag must be cleared by CleanupPolicyMembership") // All memberships must be removed. require.NoError(t, ds.writer(ctx).Get(&count, `SELECT COUNT(*) FROM policy_membership WHERE policy_id = ?`, pol.ID)) @@ -7510,17 +7492,17 @@ func testCleanupPolicyMembershipCrashRecovery(t *testing.T, ds *Datastore) { // Set the flag to simulate the crash window between cleanup and flag clear. _, err := ds.writer(ctx).ExecContext(ctx, - `UPDATE policies SET needs_full_membership_cleanup = 1 WHERE id = ?`, pol.ID) + `UPDATE policies SET needs_full_membership_cleanup = true WHERE id = ?`, pol.ID) require.NoError(t, err) // CleanupPolicyMembership (cron) should handle this without errors. require.NoError(t, ds.CleanupPolicyMembership(ctx, time.Now())) // Flag must be cleared. - var flagVal int + var flagVal bool require.NoError(t, ds.writer(ctx).Get(&flagVal, `SELECT needs_full_membership_cleanup FROM policies WHERE id = ?`, pol.ID)) - assert.Zero(t, flagVal, "flag must be cleared even when no membership rows remain") + assert.False(t, flagVal, "flag must be cleared even when no membership rows remain") }) } diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index 4f3fb03ae43..fe62a0b3462 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -19,7 +19,7 @@ import ( ) func TestQueries(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/query_results_test.go b/server/datastore/mysql/query_results_test.go index 7506af69dbd..4030deef9b9 100644 --- a/server/datastore/mysql/query_results_test.go +++ b/server/datastore/mysql/query_results_test.go @@ -15,7 +15,7 @@ import ( ) func TestQueryResults(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/scheduled_queries_test.go b/server/datastore/mysql/scheduled_queries_test.go index d03b2b7d15d..debfd028400 100644 --- a/server/datastore/mysql/scheduled_queries_test.go +++ b/server/datastore/mysql/scheduled_queries_test.go @@ -15,7 +15,7 @@ import ( ) func TestScheduledQueries(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/scim_test.go b/server/datastore/mysql/scim_test.go index 678d42cbfaa..af70c5db0c4 100644 --- a/server/datastore/mysql/scim_test.go +++ b/server/datastore/mysql/scim_test.go @@ -17,7 +17,7 @@ import ( ) func TestScim(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/scripts_test.go b/server/datastore/mysql/scripts_test.go index c464532111b..92c0ddc6622 100644 --- a/server/datastore/mysql/scripts_test.go +++ b/server/datastore/mysql/scripts_test.go @@ -21,7 +21,7 @@ import ( ) func TestScripts(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -1378,16 +1378,14 @@ type scriptContents struct { func testInsertScriptContents(t *testing.T, ds *Datastore) { ctx := context.Background() contents := `echo foobar;` - res, err := insertScriptContents(ctx, ds.writer(ctx), contents) + id, err := insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ := res.LastInsertId() require.Equal(t, int64(1), id) expectedCS := md5ChecksumScriptContent(contents) // insert same contents again, verify that the checksum and ID stayed the same - res, err = insertScriptContents(ctx, ds.writer(ctx), contents) + id, err = insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ = res.LastInsertId() require.Equal(t, int64(1), id) stmt := `SELECT id, HEX(md5_checksum) as md5_checksum FROM script_contents WHERE id = ?` @@ -1523,9 +1521,8 @@ func testCleanupUnusedScriptContents(t *testing.T, ds *Datastore) { func testGetAnyScriptContents(t *testing.T, ds *Datastore) { ctx := context.Background() contents := `echo foobar;` - res, err := insertScriptContents(ctx, ds.writer(ctx), contents) + id, err := insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ := res.LastInsertId() result, err := ds.GetAnyScriptContents(ctx, uint(id)) //nolint:gosec // dismiss G115 require.NoError(t, err) diff --git a/server/datastore/mysql/secret_variables_test.go b/server/datastore/mysql/secret_variables_test.go index 85b024a76af..5da264a72f9 100644 --- a/server/datastore/mysql/secret_variables_test.go +++ b/server/datastore/mysql/secret_variables_test.go @@ -13,7 +13,7 @@ import ( ) func TestSecretVariables(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/sessions_test.go b/server/datastore/mysql/sessions_test.go index db835b59197..59c61f2f04b 100644 --- a/server/datastore/mysql/sessions_test.go +++ b/server/datastore/mysql/sessions_test.go @@ -14,7 +14,7 @@ import ( ) func TestSessions(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/setup_experience_test.go b/server/datastore/mysql/setup_experience_test.go index 027c6529cd1..9b0254c9a7f 100644 --- a/server/datastore/mysql/setup_experience_test.go +++ b/server/datastore/mysql/setup_experience_test.go @@ -17,7 +17,7 @@ import ( ) func TestSetupExperience(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 910f3ceb96c..0c3ef4d643c 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -27,7 +27,7 @@ import ( ) func TestSoftwareInstallers(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 8a70c8f338f..e6b47f13f44 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -33,7 +33,7 @@ import ( ) func TestSoftware(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -3781,7 +3781,7 @@ func testInsertHostSoftwareInstalledPaths(t *testing.T, ds *Datastore) { } func TestCleanupSoftwareTitles(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) ctx := context.Background() host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) @@ -7810,12 +7810,12 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { time.Sleep(time.Second) // assign the label to the software installers - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -7868,7 +7868,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // Update the label to be "include any" - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -7922,7 +7922,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label3, err := ds.NewLabel(ctx, &fleet.Label{Name: "label3" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID2, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID2, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -7985,7 +7985,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label4, err := ds.NewLabel(ctx, &fleet.Label{Name: "label4" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeDynamic}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID3, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID3, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -8029,7 +8029,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { require.True(t, scoped) // Now include hosts with label4. No host has this label, so we shouldn't see installerID3 anymore. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID3, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID3, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -8079,7 +8079,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label5, err := ds.NewLabel(ctx, &fleet.Label{Name: "label5" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID4, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID4, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label5.Name: {LabelName: label5.Name, LabelID: label5.ID}}, }, softwareTypeInstaller) @@ -8100,7 +8100,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { // Scope installer1 to include_all: [label1, label4]. // hostIncludeAll has neither label yet, so installer1 should be out of scope. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAll, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -9013,7 +9013,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { time.Sleep(time.Second) // assign the label to the software installer - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -9093,7 +9093,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { require.True(t, scoped) // Assign the label to the VPP app. Now we should have an empty list - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9175,13 +9175,13 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { opts.OnlyAvailableForInstall = false // Make the label include any. We should have both of them back. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9193,7 +9193,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { // Give the VPP app a different label. Only the installer should show up now, since the host // only has label1. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label2.Name: {LabelName: label2.Name, LabelID: label2.ID}}, }, softwareTypeVPP) @@ -9207,7 +9207,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { require.NoError(t, err) require.False(t, scoped) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9226,7 +9226,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { label3, err := ds.NewLabel(ctx, &fleet.Label{Name: "label3" + t.Name()}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label3.Name: {LabelName: label3.Name, LabelID: label3.ID}}, }, softwareTypeVPP) @@ -9260,7 +9260,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { label4, err := ds.NewLabel(ctx, &fleet.Label{Name: "label4" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeVPP) @@ -9281,7 +9281,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { // Scope the VPP app to include_all: [label5, label6]. // host currently has label1 but not label5 or label6. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAll, ByName: map[string]fleet.LabelIdent{ label5.Name: {LabelName: label5.Name, LabelID: label5.ID}, @@ -9588,13 +9588,13 @@ func testListHostSoftwareSelfServiceWithLabelScopingHostInstalled(t *testing.T, err = ds.UpdateHost(ctx, host) require.NoError(t, err) // label software - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{excludeLabel.Name: {LabelName: excludeLabel.Name, LabelID: excludeLabel.ID}}, }, softwareTypeInstaller) require.NoError(t, err) // label vpp app - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vPPApp.VPPAppTeam.AppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vPPApp.VPPAppTeam.AppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{excludeLabel.Name: {LabelName: excludeLabel.Name, LabelID: excludeLabel.ID}}, }, softwareTypeVPP) @@ -9857,7 +9857,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { } // Dynamic label exclude any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, @@ -9876,7 +9876,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // manual label exclude any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -9919,7 +9919,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // manual label include any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -9952,7 +9952,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Greater(t, label2.CreatedAt, host.LabelUpdatedAt) // Dynamic label include any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{ label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, diff --git a/server/datastore/mysql/software_title_icons_test.go b/server/datastore/mysql/software_title_icons_test.go index 111beb06a25..4ec07a0808f 100644 --- a/server/datastore/mysql/software_title_icons_test.go +++ b/server/datastore/mysql/software_title_icons_test.go @@ -12,7 +12,7 @@ import ( ) func TestSoftwareTitleIcons(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index 564e17da721..e0f56dc960c 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -23,7 +23,7 @@ import ( ) func TestSoftwareTitles(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/software_upgrade_code_test.go b/server/datastore/mysql/software_upgrade_code_test.go index 99bcff1e12c..3e41b937ad3 100644 --- a/server/datastore/mysql/software_upgrade_code_test.go +++ b/server/datastore/mysql/software_upgrade_code_test.go @@ -13,7 +13,7 @@ import ( ) func TestSoftwareUpgradeCode(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index 6c831def968..8b1f8b1368d 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -19,7 +19,7 @@ import ( ) func TestStatistics(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index e042c110650..a858605fdcb 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -16,7 +16,7 @@ import ( ) func TestTargets(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/teams_test.go b/server/datastore/mysql/teams_test.go index 5cfeaaccd0e..3dd56bd1638 100644 --- a/server/datastore/mysql/teams_test.go +++ b/server/datastore/mysql/teams_test.go @@ -22,7 +22,7 @@ import ( ) func TestTeams(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -898,13 +898,13 @@ func testTeamsNameUnicode(t *testing.T, ds *Datastore) { // Try to create team with equivalent name _, err = ds.NewTeam(context.Background(), &fleet.Team{Name: equivalentNames[1]}) - assert.True(t, IsDuplicate(err), err) + assert.True(t, ds.dialect.IsDuplicate(err), err) // Try to update a different team with equivalent name -- not allowed teamEmoji, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "💻"}) require.NoError(t, err) _, err = ds.SaveTeam(context.Background(), &fleet.Team{ID: teamEmoji.ID, Name: equivalentNames[1]}) - assert.True(t, IsDuplicate(err), err) + assert.True(t, ds.dialect.IsDuplicate(err), err) // Try to find team with equivalent name teamFilter := fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}} diff --git a/server/datastore/mysql/unicode_test.go b/server/datastore/mysql/unicode_test.go index 918f4e744ee..972e75edfc3 100644 --- a/server/datastore/mysql/unicode_test.go +++ b/server/datastore/mysql/unicode_test.go @@ -14,7 +14,7 @@ import ( ) func TestUnicode(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) defer ds.Close() l1 := fleet.LabelSpec{ diff --git a/server/datastore/mysql/users_test.go b/server/datastore/mysql/users_test.go index 781b9f9464c..f719f5cda1f 100644 --- a/server/datastore/mysql/users_test.go +++ b/server/datastore/mysql/users_test.go @@ -19,7 +19,7 @@ import ( ) func TestUsers(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string @@ -57,7 +57,7 @@ func testUsersCreate(t *testing.T, ds *Datastore) { {"foobar", "jason3@fleet.co", true, true, false, false, true, nil, nil}, {"foobar", "jason4@fleet.co", true, true, false, false, true, ptr.Uint(1), nil}, // Simulating a race condition where two users accept the same invite - {"foobar", "jason5@fleet.co", true, true, false, false, true, ptr.Uint(1), ptr.String("users.invite_id")}, + {"foobar", "jason5@fleet.co", true, true, false, false, true, ptr.Uint(1), ptr.String("invite_id")}, } for _, tt := range createTests { diff --git a/server/datastore/mysql/vpp_test.go b/server/datastore/mysql/vpp_test.go index 87169e341ad..9dfa9d37bf4 100644 --- a/server/datastore/mysql/vpp_test.go +++ b/server/datastore/mysql/vpp_test.go @@ -21,7 +21,7 @@ import ( ) func TestVPP(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/vulnerabilities_test.go b/server/datastore/mysql/vulnerabilities_test.go index 4f0d81affa1..3f3f2347bb4 100644 --- a/server/datastore/mysql/vulnerabilities_test.go +++ b/server/datastore/mysql/vulnerabilities_test.go @@ -15,7 +15,7 @@ import ( ) func TestVulnerabilities(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/windows_updates_test.go b/server/datastore/mysql/windows_updates_test.go index 3f7b16b9203..303153a4201 100644 --- a/server/datastore/mysql/windows_updates_test.go +++ b/server/datastore/mysql/windows_updates_test.go @@ -11,7 +11,7 @@ import ( ) func TestWindowsUpdates(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) cases := []struct { name string diff --git a/server/datastore/mysql/wstep_test.go b/server/datastore/mysql/wstep_test.go index c6b94d8a895..bd07e3fec15 100644 --- a/server/datastore/mysql/wstep_test.go +++ b/server/datastore/mysql/wstep_test.go @@ -13,7 +13,7 @@ import ( ) func TestWSTEPStore(t *testing.T) { - ds := CreateMySQLDS(t) + ds := CreateDS(t) wantCert, err := cryptoutil.DecodePEMCertificate(testCert) require.NoError(t, err) From 9ee17596fee36e6a6913e8af5fdca16725bc333c Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 25 Mar 2026 16:02:39 -0400 Subject: [PATCH 6/6] refactor(datastore): migrate remaining SQL files to dialect helpers Additional datastore files migrated to use ds.dialect methods: aggregated_stats, android_*, app_configs, carves, campaigns, certificate_authorities, conditional_access, delete, in_house_apps, invites, jobs, locks, maintained_apps, packs, password_reset, scheduled_queries, secret_variables, setup_experience, software_titles*, teams, users, windows_updates, wstep. --- server/datastore/mysql/aggregated_stats.go | 3 +- server/datastore/mysql/android_enterprises.go | 3 +- server/datastore/mysql/android_hosts.go | 6 +-- server/datastore/mysql/android_mysql.go | 4 +- server/datastore/mysql/app_configs.go | 2 +- server/datastore/mysql/ca_config_assets.go | 7 ++- server/datastore/mysql/campaigns.go | 6 +-- server/datastore/mysql/carves.go | 10 ++-- .../mysql/certificate_authorities.go | 26 +++++----- .../mysql/conditional_access_bypass.go | 7 ++- .../mysql/conditional_access_microsoft.go | 5 +- server/datastore/mysql/delete.go | 2 +- server/datastore/mysql/hosts.go | 6 +-- server/datastore/mysql/in_house_apps.go | 45 +++++++--------- server/datastore/mysql/invites.go | 5 +- server/datastore/mysql/jobs.go | 3 +- server/datastore/mysql/locks.go | 2 +- server/datastore/mysql/maintained_apps.go | 11 ++-- .../datastore/mysql/nanomdm_storage_test.go | 7 +-- .../mysql/operating_system_vulnerabilities.go | 52 +++++++++++++------ server/datastore/mysql/packs.go | 20 +++---- server/datastore/mysql/password_reset.go | 6 +-- server/datastore/mysql/postgres_smoke_test.go | 2 +- server/datastore/mysql/scheduled_queries.go | 39 ++++++++------ server/datastore/mysql/secret_variables.go | 5 +- server/datastore/mysql/setup_experience.go | 17 +++--- .../mysql/software_title_display_names.go | 5 +- .../datastore/mysql/software_title_icons.go | 3 +- server/datastore/mysql/software_titles.go | 21 ++++---- server/datastore/mysql/teams.go | 7 +-- server/datastore/mysql/users.go | 8 ++- server/datastore/mysql/windows_updates.go | 2 +- server/datastore/mysql/wstep.go | 8 +-- 33 files changed, 167 insertions(+), 188 deletions(-) diff --git a/server/datastore/mysql/aggregated_stats.go b/server/datastore/mysql/aggregated_stats.go index 21ffb74532d..c6db1849b97 100644 --- a/server/datastore/mysql/aggregated_stats.go +++ b/server/datastore/mysql/aggregated_stats.go @@ -129,8 +129,7 @@ func (ds *Datastore) CalculateAggregatedPerfStatsPercentiles(ctx context.Context ` INSERT INTO aggregated_stats(id, type, global_stats, json_value) VALUES (?, ?, 0, ?) - ON DUPLICATE KEY UPDATE json_value=VALUES(json_value) - `, + `+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value=VALUES(json_value)`), queryID, aggregate, statsJson, ) if err != nil { diff --git a/server/datastore/mysql/android_enterprises.go b/server/datastore/mysql/android_enterprises.go index d26145f555d..857efb9b913 100644 --- a/server/datastore/mysql/android_enterprises.go +++ b/server/datastore/mysql/android_enterprises.go @@ -14,11 +14,10 @@ import ( func (ds *AndroidDatastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { // android_enterprises user_id is only set when the row is created stmt := `INSERT INTO android_enterprises (signup_name, user_id) VALUES ('', ?)` - res, err := ds.Writer(ctx).ExecContext(ctx, stmt, userID) + id, err := insertAndGetIDTx(ctx, ds.Writer(ctx), ds.dialect, stmt, userID) if err != nil { return 0, ctxerr.Wrap(ctx, err, "inserting enterprise") } - id, _ := res.LastInsertId() return uint(id), nil // nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/android_hosts.go b/server/datastore/mysql/android_hosts.go index 7e8998006f8..9486a47c27e 100644 --- a/server/datastore/mysql/android_hosts.go +++ b/server/datastore/mysql/android_hosts.go @@ -66,7 +66,7 @@ func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.De applied_policy_version ) VALUES (?, ?, ?, ?, ?, ?)` - result, err := tx.ExecContext(ctx, stmt, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, device.HostID, device.DeviceID, device.EnterpriseSpecificID, @@ -77,10 +77,6 @@ func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.De if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting device") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting android_devices last insert ID") - } device.ID = uint(id) // nolint:gosec return device, nil } diff --git a/server/datastore/mysql/android_mysql.go b/server/datastore/mysql/android_mysql.go index 3f842777d7e..c393b2b5da7 100644 --- a/server/datastore/mysql/android_mysql.go +++ b/server/datastore/mysql/android_mysql.go @@ -17,14 +17,16 @@ type AndroidDatastore struct { logger *slog.Logger primary *sqlx.DB replica fleet.DBReader // so it cannot be used to perform writes + dialect DialectHelper } // NewAndroidDatastore creates a new Android Datastore -func NewAndroidDatastore(logger *slog.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore { +func NewAndroidDatastore(logger *slog.Logger, primary *sqlx.DB, replica fleet.DBReader, dialect DialectHelper) android.Datastore { return &AndroidDatastore{ logger: logger, primary: primary, replica: replica, + dialect: dialect, } } diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go index 76ffeaede48..771f93cf85c 100644 --- a/server/datastore/mysql/app_configs.go +++ b/server/datastore/mysql/app_configs.go @@ -67,7 +67,7 @@ func (ds *Datastore) SaveAppConfig(ctx context.Context, info *fleet.AppConfig) e } _, err = tx.ExecContext(ctx, - `INSERT INTO app_config_json(json_value) VALUES(?) ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)`, + `INSERT INTO app_config_json(json_value) VALUES(?) `+ds.dialect.OnDuplicateKey("id", `json_value = VALUES(json_value)`), configBytes, ) if err != nil { diff --git a/server/datastore/mysql/ca_config_assets.go b/server/datastore/mysql/ca_config_assets.go index e5d8abbac6b..6ac0139945f 100644 --- a/server/datastore/mysql/ca_config_assets.go +++ b/server/datastore/mysql/ca_config_assets.go @@ -56,10 +56,9 @@ func (ds *Datastore) saveCAConfigAssets(ctx context.Context, tx sqlx.ExtContext, stmt := fmt.Sprintf(` INSERT INTO ca_config_assets (name, type, value) VALUES %s - ON DUPLICATE KEY UPDATE - value = VALUES(value), - type = VALUES(type) - `, strings.TrimSuffix(strings.Repeat("(?,?,?),", len(assets)), ",")) + `+ds.dialect.OnDuplicateKey("id", `value = VALUES(value), + type = VALUES(type)`), + strings.TrimSuffix(strings.Repeat("(?,?,?),", len(assets)), ",")) args := make([]interface{}, 0, len(assets)*3) for _, asset := range assets { diff --git a/server/datastore/mysql/campaigns.go b/server/datastore/mysql/campaigns.go index f3cb58052ae..fb9c9f75235 100644 --- a/server/datastore/mysql/campaigns.go +++ b/server/datastore/mysql/campaigns.go @@ -48,12 +48,11 @@ func (ds *Datastore) NewDistributedQueryCampaign(ctx context.Context, camp *flee ) VALUES(?,?,?%s) `, createdAtField, createdAtPlaceholder) - result, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, args...) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, args...) if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting distributed query campaign") } - id, _ := result.LastInsertId() camp.ID = uint(id) //nolint:gosec // dismiss G115 return camp, nil } @@ -140,12 +139,11 @@ func (ds *Datastore) NewDistributedQueryCampaignTarget(ctx context.Context, targ ) VALUES (?,?,?) ` - result, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert distributed campaign target") } - id, _ := result.LastInsertId() target.ID = uint(id) //nolint:gosec // dismiss G115 return target, nil } diff --git a/server/datastore/mysql/carves.go b/server/datastore/mysql/carves.go index 4fea620a177..0c250f86e28 100644 --- a/server/datastore/mysql/carves.go +++ b/server/datastore/mysql/carves.go @@ -11,7 +11,7 @@ import ( "github.com/jmoiron/sqlx" ) -func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fleet.CarveMetadata) (int64, error) { +func upsertCarveDB(ctx context.Context, writer sqlx.ExtContext, dialect DialectHelper, metadata *fleet.CarveMetadata) (int64, error) { stmt := `INSERT INTO carve_metadata ( host_id, created_at, @@ -36,8 +36,10 @@ func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fle ? )` - result, err := writer.ExecContext( + id, err := insertAndGetIDTx( ctx, + writer, + dialect, stmt, metadata.HostId, metadata.CreatedAt.Format(mySQLTimestampFormat), @@ -53,11 +55,11 @@ func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fle if err != nil { return 0, ctxerr.Wrap(ctx, err, "insert carve metadata") } - return result.LastInsertId() + return id, nil } func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { - id, err := upsertCarveDB(ctx, ds.writer(ctx), metadata) + id, err := upsertCarveDB(ctx, ds.writer(ctx), ds.dialect, metadata) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert carve metadata") } diff --git a/server/datastore/mysql/certificate_authorities.go b/server/datastore/mysql/certificate_authorities.go index ce208316c67..46d9d444950 100644 --- a/server/datastore/mysql/certificate_authorities.go +++ b/server/datastore/mysql/certificate_authorities.go @@ -195,17 +195,13 @@ func (ds *Datastore) NewCertificateAuthority(ctx context.Context, ca *fleet.Cert return nil, err } - result, err := ds.writer(ctx).ExecContext(ctx, fmt.Sprintf(sqlInsertCertificateAuthority, placeholders), args...) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), fmt.Sprintf(sqlInsertCertificateAuthority, placeholders), args...) if err != nil { if strings.Contains(err.Error(), "idx_ca_type_name") { return nil, fleet.ConflictError{Message: "a certificate authority with this name already exists"} } return nil, ctxerr.Wrap(ctx, err, "inserting new certificate authority") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last insert ID for new certificate authority") - } ca.ID = uint(id) //nolint:gosec // dismiss G115 return ca, nil } @@ -230,7 +226,8 @@ const sqlInsertCertificateAuthority = `INSERT INTO certificate_authorities ( client_secret_encrypted ) VALUES %s` -const sqlUpsertCertificateAuthority = sqlInsertCertificateAuthority + ` ON DUPLICATE KEY UPDATE +func sqlUpsertCertificateAuthority(dialect DialectHelper) string { + return sqlInsertCertificateAuthority + ` ` + dialect.OnDuplicateKey("name,type", ` type = VALUES(type), name = VALUES(name), url = VALUES(url), @@ -245,7 +242,8 @@ const sqlUpsertCertificateAuthority = sqlInsertCertificateAuthority + ` ON DUPLI password_encrypted = VALUES(password_encrypted), challenge_encrypted = VALUES(challenge_encrypted), client_id = VALUES(client_id), - client_secret_encrypted = VALUES(client_secret_encrypted)` + client_secret_encrypted = VALUES(client_secret_encrypted)`) +} func sqlGenerateArgsForInsertCertificateAuthority(ctx context.Context, serverPrivateKey string, ca *fleet.CertificateAuthority) ([]interface{}, string, error) { var upns []byte @@ -308,7 +306,7 @@ func sqlGenerateArgsForInsertCertificateAuthority(ctx context.Context, serverPri return args, placeholders, nil } -func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, serverPrivateKey string, certificateAuthorities []*fleet.CertificateAuthority) error { +func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, serverPrivateKey string, certificateAuthorities []*fleet.CertificateAuthority) error { if len(certificateAuthorities) == 0 { return nil } @@ -325,7 +323,7 @@ func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, placeholders.WriteString(fmt.Sprintf("%s,", p)) } - stmt := fmt.Sprintf(sqlUpsertCertificateAuthority, strings.TrimSuffix(placeholders.String(), ",")) + stmt := fmt.Sprintf(sqlUpsertCertificateAuthority(dialect), strings.TrimSuffix(placeholders.String(), ",")) if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "upserting certificate authorities") @@ -334,7 +332,7 @@ func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, return nil } -func batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, certificateAuthorities []*fleet.CertificateAuthority) error { +func (ds *Datastore) batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, certificateAuthorities []*fleet.CertificateAuthority) error { if len(certificateAuthorities) == 0 { return nil } @@ -350,7 +348,7 @@ func batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, _, err := tx.ExecContext(ctx, stmt, args...) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return &fleet.ConflictError{ Message: "Couldn't delete certificate authority. " + fleet.DeleteCAReferencedByTemplatesErrMsg + ". Please remove the certificate templates first.", } @@ -368,10 +366,10 @@ func (ds *Datastore) BatchApplyCertificateAuthorities(ctx context.Context, ops f upserts = append(upserts, ops.Update...) return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - if err := batchUpsertCertificateAuthorities(ctx, tx, ds.serverPrivateKey, upserts); err != nil { + if err := batchUpsertCertificateAuthorities(ctx, tx, ds.dialect, ds.serverPrivateKey, upserts); err != nil { return err } - if err := batchDeleteCertificateAuthorities(ctx, tx, ops.Delete); err != nil { + if err := ds.batchDeleteCertificateAuthorities(ctx, tx, ops.Delete); err != nil { return err } return nil @@ -399,7 +397,7 @@ func (ds *Datastore) DeleteCertificateAuthority(ctx context.Context, certificate stmt = "DELETE FROM certificate_authorities WHERE id = ?" result, err := ds.writer(ctx).ExecContext(ctx, stmt, certificateAuthorityID) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return nil, fleet.ConflictError{ Message: "Couldn't delete certificate authority. " + fleet.DeleteCAReferencedByTemplatesErrMsg + ". Please remove the certificate templates first.", } diff --git a/server/datastore/mysql/conditional_access_bypass.go b/server/datastore/mysql/conditional_access_bypass.go index 12deee4da6b..f48807e94b1 100644 --- a/server/datastore/mysql/conditional_access_bypass.go +++ b/server/datastore/mysql/conditional_access_bypass.go @@ -25,13 +25,12 @@ func (ds *Datastore) ConditionalAccessBypassDevice(ctx context.Context, hostID u AND p.critical = 1 AND pm.passes = 0 ` - const insertStmt = ` + insertStmt := ` INSERT INTO host_conditional_access (host_id, bypassed_at) VALUES - (?, NOW(6)) - ON DUPLICATE KEY UPDATE - bypassed_at = NOW(6)` + (?, NOW()) + ` + ds.dialect.OnDuplicateKey("id", `bypassed_at = NOW()`) var blockCount uint diff --git a/server/datastore/mysql/conditional_access_microsoft.go b/server/datastore/mysql/conditional_access_microsoft.go index a4367f62d95..5cb707f152b 100644 --- a/server/datastore/mysql/conditional_access_microsoft.go +++ b/server/datastore/mysql/conditional_access_microsoft.go @@ -141,11 +141,10 @@ func (ds *Datastore) CreateHostConditionalAccessStatus(ctx context.Context, host `INSERT INTO microsoft_compliance_partner_host_statuses (host_id, device_id, user_principal_name) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - device_id = VALUES(device_id), + `+ds.dialect.OnDuplicateKey("host_id", `device_id = VALUES(device_id), user_principal_name = VALUES(user_principal_name), managed = NULL, - compliant = NULL`, + compliant = NULL`), hostID, deviceID, userPrincipalName, ); err != nil { return ctxerr.Wrap(ctx, err, "create host conditional access status") diff --git a/server/datastore/mysql/delete.go b/server/datastore/mysql/delete.go index 47285401d94..082c7d121f5 100644 --- a/server/datastore/mysql/delete.go +++ b/server/datastore/mysql/delete.go @@ -29,7 +29,7 @@ func (ds *Datastore) deleteEntityByName(ctx context.Context, dbTable entity, nam deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable.name) result, err := ds.writer(ctx).ExecContext(ctx, deleteStmt, name) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey(dbTable.name, name)) } return ctxerr.Wrapf(ctx, err, "delete %s", dbTable) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 094547724f4..9ca6daa8d7a 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -297,7 +297,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper seen[key] = i } if len(seen) < scheduledQueriesQueryCount { - var dedupedArgs []interface{} + var dedupedArgs []any dedupedCount := 0 for i := 0; i < scheduledQueriesQueryCount; i++ { base := i * argsPerRow @@ -332,7 +332,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper seen[key] = i } if len(seen) < userPacksQueryCount { - var dedupedArgs []interface{} + var dedupedArgs []any dedupedCount := 0 for i := 0; i < userPacksQueryCount; i++ { base := i * argsPerRow @@ -357,7 +357,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper // which avoids NOT NULL violations on PG). argsPerRow := 12 // 2 (subquery: teamID, name) + 10 (values) var selectParts []string - var reorderedArgs []interface{} + var reorderedArgs []any for i := 0; i < scheduledQueriesQueryCount; i++ { base := i * argsPerRow selectParts = append(selectParts, diff --git a/server/datastore/mysql/in_house_apps.go b/server/datastore/mysql/in_house_apps.go index 4342f6e09c5..148c8045457 100644 --- a/server/datastore/mysql/in_house_apps.go +++ b/server/datastore/mysql/in_house_apps.go @@ -109,9 +109,9 @@ func (ds *Datastore) insertInHouseAppDB(ctx context.Context, tx sqlx.ExtContext, ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` - res, err := tx.ExecContext(ctx, stmt, args...) + id64, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { return 0, ctxerr.Wrap(ctx, err) @@ -121,13 +121,9 @@ func (ds *Datastore) insertInHouseAppDB(ctx context.Context, tx sqlx.ExtContext, } return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") } - id64, err := res.LastInsertId() installerID := uint(id64) //nolint:gosec // dismiss G115 - if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") - } - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, installerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, installerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") } @@ -286,7 +282,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { return ctxerr.Wrap(ctx, err) @@ -297,7 +293,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if payload.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { return ctxerr.Wrap(ctx, err, "upsert in house app labels") } } @@ -309,7 +305,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if payload.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "update in house app display name") } } @@ -528,7 +524,7 @@ VALUES } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, @@ -540,8 +536,6 @@ VALUES if err != nil { return ctxerr.Wrap(ctx, err, "insert in house app install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertIHAUAStmt, activityID, inHouseAppID, @@ -734,17 +728,17 @@ WHERE } func (ds *Datastore) BatchSetInHouseAppsInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { - const upsertSoftwareTitles = ` + upsertSoftwareTitles := ` INSERT INTO software_titles (name, source, extension_for, bundle_identifier) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` name = VALUES(name), source = VALUES(source), extension_for = VALUES(extension_for), bundle_identifier = VALUES(bundle_identifier) -` +`) const loadSoftwareTitles = ` SELECT @@ -916,7 +910,7 @@ WHERE title_id = ? ` - const insertNewOrEditedInstaller = ` + insertNewOrEditedInstaller := ` INSERT INTO in_house_apps ( title_id, team_id, @@ -931,7 +925,7 @@ INSERT INTO in_house_apps ( ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` filename = VALUES(filename), version = VALUES(version), storage_id = VALUES(storage_id), @@ -939,7 +933,7 @@ ON DUPLICATE KEY UPDATE bundle_identifier = VALUES(bundle_identifier), self_service = VALUES(self_service), url = VALUES(url) -` +`) const loadInHouseInstallerID = ` SELECT @@ -968,7 +962,7 @@ WHERE in_house_app_id = ? ` - const upsertInHouseLabels = ` + upsertInHouseLabels := ` INSERT INTO in_house_app_labels ( in_house_app_id, @@ -978,10 +972,10 @@ INSERT INTO ) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` exclude = VALUES(exclude), require_all = VALUES(require_all) -` +`) const loadExistingInHouseLabels = ` SELECT @@ -1009,8 +1003,7 @@ WHERE software_category_id NOT IN (?) ` - const upsertInHouseCategories = ` -INSERT IGNORE INTO + const upsertInHouseCategoriesSuffix = ` in_house_app_software_categories ( in_house_app_id, software_category_id @@ -1396,7 +1389,7 @@ WHERE upsertCategoriesArgs = append(upsertCategoriesArgs, installerID, catID) } upsertCategoriesValues := strings.TrimSuffix(strings.Repeat("(?,?),", len(installer.CategoryIDs)), ",") - _, err = tx.ExecContext(ctx, fmt.Sprintf(upsertInHouseCategories, upsertCategoriesValues), upsertCategoriesArgs...) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+fmt.Sprintf(upsertInHouseCategoriesSuffix, upsertCategoriesValues)+ds.dialect.OnConflictDoNothing("in_house_app_id,software_category_id"), upsertCategoriesArgs...) if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited categories for in-house with name %q", installer.Filename) } @@ -1405,7 +1398,7 @@ WHERE // update display name for the software title if it needs to be updated or inserted // no deletions will happen, display names will be set to empty if needed if name, ok := displayNameIDMap[titleID]; (ok && name != installer.DisplayName) || (!ok && installer.DisplayName != "") { - if err := updateSoftwareTitleDisplayName(ctx, tx, tmID, titleID, installer.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, tmID, titleID, installer.DisplayName); err != nil { return ctxerr.Wrapf(ctx, err, "update software title display name for in-house app with name %q", installer.Filename) } } diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go index e3305c8a0b8..c11d9fb61b5 100644 --- a/server/datastore/mysql/invites.go +++ b/server/datastore/mysql/invites.go @@ -37,15 +37,14 @@ func (ds *Datastore) NewInvite(ctx context.Context, i *fleet.Invite) (*fleet.Inv VALUES ( ?, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlStmt, i.InvitedBy, i.Email, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStmt, i.InvitedBy, i.Email, i.Name, i.Position, i.Token, i.SSOEnabled, i.MFAEnabled, i.GlobalRole) - if err != nil && IsDuplicate(err) { + if err != nil && ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("Invite", i.Email)) } else if err != nil { return ctxerr.Wrap(ctx, err, "create invite") } - id, _ := result.LastInsertId() i.ID = uint(id) //nolint:gosec // dismiss G115 if len(i.Teams) == 0 { diff --git a/server/datastore/mysql/jobs.go b/server/datastore/mysql/jobs.go index 159b8defa9a..b453912e2dc 100644 --- a/server/datastore/mysql/jobs.go +++ b/server/datastore/mysql/jobs.go @@ -26,12 +26,11 @@ VALUES (?, ?, ?, ?, ?, COALESCE(?, NOW())) if !job.NotBefore.IsZero() { notBefore = &job.NotBefore } - result, err := ds.writer(ctx).ExecContext(ctx, query, job.Name, job.Args, job.State, job.Retries, job.Error, notBefore) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), query, job.Name, job.Args, job.State, job.Retries, job.Error, notBefore) if err != nil { return nil, err } - id, _ := result.LastInsertId() job.ID = uint(id) //nolint:gosec // dismiss G115 return job, nil diff --git a/server/datastore/mysql/locks.go b/server/datastore/mysql/locks.go index 52362e8c5b0..263e7713866 100644 --- a/server/datastore/mysql/locks.go +++ b/server/datastore/mysql/locks.go @@ -37,7 +37,7 @@ func (ds *Datastore) Lock(ctx context.Context, name string, owner string, expira func (ds *Datastore) createLock(ctx context.Context, name string, owner string, expiration time.Duration) (sql.Result, error) { return ds.writer(ctx).ExecContext(ctx, - `INSERT IGNORE INTO locks (name, owner, expires_at) VALUES (?, ?, ?)`, + ds.dialect.InsertIgnoreInto()+` locks (name, owner, expires_at) VALUES (?, ?, ?)`+ds.dialect.OnConflictDoNothing("name"), name, owner, time.Now().Add(expiration), ) } diff --git a/server/datastore/mysql/maintained_apps.go b/server/datastore/mysql/maintained_apps.go index c1754a46729..64a1c51def0 100644 --- a/server/datastore/mysql/maintained_apps.go +++ b/server/datastore/mysql/maintained_apps.go @@ -12,27 +12,24 @@ import ( ) func (ds *Datastore) UpsertMaintainedApp(ctx context.Context, app *fleet.MaintainedApp) (*fleet.MaintainedApp, error) { - const upsertStmt = ` + upsertStmt := ` INSERT INTO fleet_maintained_apps (name, slug, platform, unique_identifier) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - name = VALUES(name), +` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), platform = VALUES(platform), - unique_identifier = VALUES(unique_identifier) -` + unique_identifier = VALUES(unique_identifier)`) var appID uint err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error // upsert the maintained app - res, err := tx.ExecContext(ctx, upsertStmt, app.Name, app.Slug, app.Platform, app.UniqueIdentifier) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, upsertStmt, app.Name, app.Slug, app.Platform, app.UniqueIdentifier) if err != nil { return ctxerr.Wrap(ctx, err, "upsert maintained app") } - id, _ := res.LastInsertId() appID = uint(id) //nolint:gosec // dismiss G115 return nil }) diff --git a/server/datastore/mysql/nanomdm_storage_test.go b/server/datastore/mysql/nanomdm_storage_test.go index 751aa12eceb..4facc42a154 100644 --- a/server/datastore/mysql/nanomdm_storage_test.go +++ b/server/datastore/mysql/nanomdm_storage_test.go @@ -266,9 +266,10 @@ func testEnqueueDeviceLockCommandRaceCondition(t *testing.T, ds *Datastore) { // Create NanoMDMStorage storage := &NanoMDMStorage{ - db: ds.writer(ctx), - logger: slog.New(slog.DiscardHandler), - ds: ds, + db: ds.writer(ctx), + logger: slog.New(slog.DiscardHandler), + ds: ds, + dialect: ds.dialect, } // Number of concurrent lock attempts diff --git a/server/datastore/mysql/operating_system_vulnerabilities.go b/server/datastore/mysql/operating_system_vulnerabilities.go index 832844da61f..c21823f9be7 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities.go +++ b/server/datastore/mysql/operating_system_vulnerabilities.go @@ -57,12 +57,14 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers } // Query with CVSS metadata - baseCTE := ` + gcDistinctResolved := ds.dialect.GroupConcat("DISTINCT v.resolved_in_version", ",") + gcDistinctResolvedOsvv := ds.dialect.GroupConcat("DISTINCT osvv.resolved_in_version", ",") + baseCTE := fmt.Sprintf(` WITH all_vulns AS ( SELECT v.cve, MIN(v.created_at) created_at, - GROUP_CONCAT(DISTINCT v.resolved_in_version SEPARATOR ',') resolved_in_version + %s resolved_in_version FROM operating_system_vulnerabilities v JOIN operating_systems os ON os.id = v.operating_system_id AND os.name = ? AND os.version = ? @@ -73,14 +75,14 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers SELECT DISTINCT osvv.cve, MIN(osvv.created_at) created_at, - GROUP_CONCAT(DISTINCT osvv.resolved_in_version SEPARATOR ',') resolved_in_version + %s resolved_in_version FROM operating_system_version_vulnerabilities osvv JOIN operating_systems os ON os.os_version_id = osvv.os_version_id WHERE os.name = ? AND os.version = ? - ` + linuxTeamFilter + ` + `, gcDistinctResolved, gcDistinctResolvedOsvv) + linuxTeamFilter + ` GROUP BY osvv.cve ) ` @@ -294,11 +296,11 @@ func (ds *Datastore) InsertOSVulnerabilities(ctx context.Context, vulnerabilitie stmt := fmt.Sprintf(` INSERT INTO operating_system_vulnerabilities (operating_system_id, cve, source, resolved_in_version) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = NOW() - `, values) + `), values) var args []any for _, v := range batch { @@ -333,15 +335,27 @@ func (ds *Datastore) InsertOSVulnerability(ctx context.Context, v fleet.OSVulner source, resolved_in_version ) VALUES (?,?,?,?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` operating_system_id = VALUES(operating_system_id), source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = NOW() - ` + `) args = append(args, v.OSID, v.CVE, s, v.ResolvedInVersion) + if ds.dialect.ReturningID() != "" { + // PostgreSQL: use RETURNING id and xmax to distinguish insert from update. + // xmax = 0 means the row was freshly inserted (not updated). + var id int64 + var xmax uint32 + err := ds.writer(ctx).QueryRowContext(ctx, sqlStmt+" RETURNING id, xmax", args...).Scan(&id, &xmax) + if err != nil { + return false, ctxerr.Wrap(ctx, err, "insert operating system vulnerability") + } + return xmax == 0, nil + } + // MySQL path res, err := ds.writer(ctx).ExecContext(ctx, sqlStmt, args...) if err != nil { return false, ctxerr.Wrap(ctx, err, "insert operating system vulnerability") @@ -350,7 +364,11 @@ func (ds *Datastore) InsertOSVulnerability(ctx context.Context, v fleet.OSVulner // inserts affect one row, updates affect 0 or 2; we don't care which because timestamp may not change if we // recently inserted the vuln and changed nothing else; see insertOnDuplicateDidInsertOrUpdate for context affected, _ := res.RowsAffected() - lastID, _ := res.LastInsertId() + lastID, err := res.LastInsertId() + if err != nil { + // PG: no LastInsertId, use RowsAffected == 1 as insert indicator + return affected == 1, nil + } return lastID != 0 && affected == 1, nil } @@ -389,9 +407,11 @@ func (ds *Datastore) DeleteOutOfDateOSVulnerabilities(ctx context.Context, src f func (ds *Datastore) DeleteOrphanedOSVulnerabilities(ctx context.Context) error { if _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE osv FROM operating_system_vulnerabilities osv - LEFT JOIN host_operating_system hos ON hos.os_id = osv.operating_system_id - WHERE hos.host_id IS NULL + DELETE FROM operating_system_vulnerabilities + WHERE NOT EXISTS ( + SELECT 1 FROM host_operating_system hos + WHERE hos.os_id = operating_system_vulnerabilities.operating_system_id + ) `); err != nil { return ctxerr.Wrap(ctx, err, "deleting orphaned OS vulnerabilities") } @@ -603,12 +623,12 @@ func (ds *Datastore) refreshOSVersionVulnerabilities(ctx context.Context) error JOIN software_cve sc ON sc.software_id = khc.software_id WHERE khc.hosts_count > 0 GROUP BY khc.team_id, khc.os_version_id, sc.cve - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), created_at = VALUES(created_at), updated_at = VALUES(updated_at) - `, updatedAt) + `), updatedAt) if err != nil { return ctxerr.Wrap(ctx, err, "refresh per-team OS version vulnerabilities") } @@ -629,12 +649,12 @@ func (ds *Datastore) refreshOSVersionVulnerabilities(ctx context.Context) error JOIN software_cve sc ON sc.software_id = khc.software_id WHERE khc.hosts_count > 0 GROUP BY khc.os_version_id, sc.cve - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), created_at = VALUES(created_at), updated_at = VALUES(updated_at) - `, updatedAt) + `), updatedAt) if err != nil { return ctxerr.Wrap(ctx, err, "refresh all-teams OS version vulnerabilities") } diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index cb1cf9142e8..933a606bbe6 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -14,7 +14,7 @@ import ( func (ds *Datastore) ApplyPackSpecs(ctx context.Context, specs []*fleet.PackSpec) (err error) { err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { for _, spec := range specs { - if err := applyPackSpecDB(ctx, tx, spec); err != nil { + if err := applyPackSpecDB(ctx, tx, ds.dialect, spec); err != nil { return ctxerr.Wrapf(ctx, err, "applying pack '%s'", spec.Name) } } @@ -25,7 +25,7 @@ func (ds *Datastore) ApplyPackSpecs(ctx context.Context, specs []*fleet.PackSpec return err } -func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, spec *fleet.PackSpec) error { +func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, spec *fleet.PackSpec) error { if spec.Name == "" { return ctxerr.New(ctx, "pack name must not be empty") } @@ -34,11 +34,11 @@ func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, spec *fleet.PackSp query := ` INSERT INTO packs (name, description, platform, disabled) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("name", ` name = VALUES(name), description = VALUES(description), platform = VALUES(platform), - disabled = VALUES(disabled) + disabled = VALUES(disabled)`) + ` ` if _, err := tx.ExecContext(ctx, query, spec.Name, spec.Description, spec.Platform, spec.Disabled); err != nil { return ctxerr.Wrap(ctx, err, "insert/update pack") @@ -266,12 +266,11 @@ func (ds *Datastore) NewPack(ctx context.Context, pack *fleet.Pack, opts ...flee (name, description, platform, disabled) VALUES ( ?, ?, ?, ? ) ` - result, err := tx.ExecContext(ctx, query, pack.Name, pack.Description, pack.Platform, pack.Disabled) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, query, pack.Name, pack.Description, pack.Platform, pack.Disabled) if err != nil { return ctxerr.Wrap(ctx, err, "insert pack") } - id, _ := result.LastInsertId() pack.ID = uint(id) //nolint:gosec // dismiss G115 if err := replacePackTargetsDB(ctx, tx, pack); err != nil { @@ -480,13 +479,8 @@ func listPacksForHost(ctx context.Context, db sqlx.QueryerContext, hid uint) ([] SELECT DISTINCT packs.* FROM ( ( SELECT p.* FROM packs p - JOIN pack_targets pt - JOIN label_membership lm - ON ( - p.id = pt.pack_id - AND pt.target_id = lm.label_id - AND pt.type = ? - ) + JOIN pack_targets pt ON p.id = pt.pack_id AND pt.type = ? + JOIN label_membership lm ON pt.target_id = lm.label_id WHERE lm.host_id = ? AND NOT p.disabled AND p.pack_type IS NULL ) UNION ALL diff --git a/server/datastore/mysql/password_reset.go b/server/datastore/mysql/password_reset.go index 4edd3f39514..fa3947a3388 100644 --- a/server/datastore/mysql/password_reset.go +++ b/server/datastore/mysql/password_reset.go @@ -17,14 +17,14 @@ func (ds *Datastore) NewPasswordResetRequest(ctx context.Context, req *fleet.Pas sqlStatement := ` INSERT INTO password_reset_requests ( user_id, token, expires_at) - VALUES (?,?, DATE_ADD(CURRENT_TIMESTAMP, INTERVAL ? MINUTE)) + VALUES (?,?, ?) ` - response, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, req.UserID, req.Token, PasswordResetRequestDuration.Minutes()) + expiresAt := time.Now().Add(PasswordResetRequestDuration) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, req.UserID, req.Token, expiresAt) if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting password reset requests") } - id, _ := response.LastInsertId() req.ID = uint(id) //nolint:gosec // dismiss G115 return req, nil } diff --git a/server/datastore/mysql/postgres_smoke_test.go b/server/datastore/mysql/postgres_smoke_test.go index 8a3651f596e..60fc513bf71 100644 --- a/server/datastore/mysql/postgres_smoke_test.go +++ b/server/datastore/mysql/postgres_smoke_test.go @@ -379,7 +379,7 @@ func TestPostgresDatastoreOperations(t *testing.T) { // --- Host disk encryption key --- t.Run("SetOrUpdateHostDiskEncryptionKey", func(t *testing.T) { - _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", ptr.Bool(false)) + _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", new(bool)) if err != nil { t.Logf("FAIL SetOrUpdateHostDiskEncryptionKey: %v", err) } diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 3c001b1634d..4f334c9e386 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -15,7 +15,7 @@ import ( // ListScheduledQueriesInPackWithStats loads a pack's scheduled queries and its aggregated stats. func (ds *Datastore) ListScheduledQueriesInPackWithStats(ctx context.Context, id uint, opts fleet.ListOptions) ([]*fleet.ScheduledQuery, error) { - query := ` + query := fmt.Sprintf(` SELECT sq.id, sq.pack_id, @@ -31,16 +31,22 @@ func (ds *Datastore) ListScheduledQueriesInPackWithStats(ctx context.Context, id sq.denylist, q.query, q.id AS query_id, - JSON_EXTRACT(ag.json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(ag.json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(ag.json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(ag.json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(ag.json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM scheduled_queries sq JOIN (SELECT * FROM queries WHERE team_id IS NULL) q ON (sq.query_name = q.name) LEFT JOIN aggregated_stats ag ON (ag.id = sq.id AND ag.global_stats = ? AND ag.type = ?) WHERE sq.pack_id = ? - ` + `, + ds.dialect.JSONExtract("ag.json_value", "$.user_time_p50"), + ds.dialect.JSONExtract("ag.json_value", "$.user_time_p95"), + ds.dialect.JSONExtract("ag.json_value", "$.system_time_p50"), + ds.dialect.JSONExtract("ag.json_value", "$.system_time_p95"), + ds.dialect.JSONExtract("ag.json_value", "$.total_executions"), + ) params := []interface{}{false, fleet.AggregatedStatsTypeScheduledQuery, id} query, params = appendListOptionsWithCursorToSQL(query, params, &opts) results := []*fleet.ScheduledQuery{} @@ -83,10 +89,10 @@ func (ds *Datastore) ListScheduledQueriesInPack(ctx context.Context, id uint) (f } func (ds *Datastore) NewScheduledQuery(ctx context.Context, sq *fleet.ScheduledQuery, opts ...fleet.OptionalArg) (*fleet.ScheduledQuery, error) { - return insertScheduledQueryDB(ctx, ds.writer(ctx), sq) + return insertScheduledQueryDB(ctx, ds.writer(ctx), ds.dialect, sq) } -func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { +func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, dialect DialectHelper, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { // This query looks up the query name using the ID (for backwards // compatibility with the UI) query := ` @@ -97,7 +103,7 @@ func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.Sc pack_id, snapshot, removed, - ` + "`interval`" + `, + "interval", platform, version, shard, @@ -107,12 +113,11 @@ func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.Sc FROM queries WHERE id = ? ` - result, err := q.ExecContext(ctx, query, sq.QueryID, sq.Name, sq.PackID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.QueryID) + id, err := insertAndGetIDTx(ctx, q, dialect, query, sq.QueryID, sq.Name, sq.PackID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.QueryID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert scheduled query") } - id, _ := result.LastInsertId() sq.ID = uint(id) //nolint:gosec // dismiss G115 query = `SELECT query, name FROM queries WHERE id = ? LIMIT 1` @@ -145,7 +150,7 @@ func (ds *Datastore) SaveScheduledQuery(ctx context.Context, sq *fleet.Scheduled func saveScheduledQueryDB(ctx context.Context, exec sqlx.ExecerContext, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { query := ` UPDATE scheduled_queries - SET pack_id = ?, query_id = ?, ` + "`interval`" + ` = ?, snapshot = ?, removed = ?, platform = ?, version = ?, shard = ?, denylist = ? + SET pack_id = ?, query_id = ?, "interval" = ?, snapshot = ?, removed = ?, platform = ?, version = ?, shard = ?, denylist = ? WHERE id = ? ` result, err := exec.ExecContext(ctx, query, sq.PackID, sq.QueryID, sq.Interval, sq.Snapshot, sq.Removed, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.ID) @@ -276,8 +281,8 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, // in SaveHostPackStats (in hosts.go) - that is, the behaviour per host must // be the same. - stmt := ` - INSERT IGNORE INTO scheduled_query_stats ( + stmt := ds.dialect.InsertIgnoreInto() + ` + scheduled_query_stats ( scheduled_query_id, host_id, average_memory, @@ -290,7 +295,7 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, user_time, wall_time ) - VALUES %s ON DUPLICATE KEY UPDATE + VALUES %s ` + ds.dialect.OnDuplicateKey("scheduled_query_id,host_id", ` scheduled_query_id = VALUES(scheduled_query_id), host_id = VALUES(host_id), average_memory = VALUES(average_memory), @@ -301,7 +306,7 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, output_size = VALUES(output_size), system_time = VALUES(system_time), user_time = VALUES(user_time), - wall_time = VALUES(wall_time); + wall_time = VALUES(wall_time)`) + `; ` var countExecs int diff --git a/server/datastore/mysql/secret_variables.go b/server/datastore/mysql/secret_variables.go index 0dcf041d4af..ea8aa77afa4 100644 --- a/server/datastore/mysql/secret_variables.go +++ b/server/datastore/mysql/secret_variables.go @@ -103,17 +103,16 @@ func (ds *Datastore) CreateSecretVariable(ctx context.Context, name string, valu if err != nil { return 0, ctxerr.Wrap(ctx, err, "encrypt secret value for insert with server private key") } - res, err := ds.writer(ctx).ExecContext(ctx, + id_, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO secret_variables (name, value) VALUES (?, ?)`, name, valueEncrypted, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return 0, ctxerr.Wrap(ctx, alreadyExists("name", name), "found duplicate") } return 0, ctxerr.Wrap(ctx, err, "insert secret variable") } - id_, _ := res.LastInsertId() return uint(id_), nil //nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/setup_experience.go b/server/datastore/mysql/setup_experience.go index 8804699f097..ee1d446e15a 100644 --- a/server/datastore/mysql/setup_experience.go +++ b/server/datastore/mysql/setup_experience.go @@ -249,7 +249,7 @@ WHERE global_or_team_id = ?` // Set setup experience on Apple hosts only if they have something configured. if fleetPlatform == "darwin" || fleetPlatform == "ios" || fleetPlatform == "ipados" { if totalInsertions > 0 { - if err := setHostAwaitingConfiguration(ctx, tx, hostUUID, true); err != nil { + if err := setHostAwaitingConfiguration(ctx, tx, ds.dialect, hostUUID, true); err != nil { return ctxerr.Wrap(ctx, err, "setting host awaiting configuration to true") } } @@ -733,14 +733,11 @@ WHERE func (ds *Datastore) SetSetupExperienceScript(ctx context.Context, script *fleet.Script) error { err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - var err error - // first insert script contents - scRes, err := insertScriptContents(ctx, tx, script.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, script.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() // This clause allows for PUT semantics. The basic idea is: // - no existing setup script -> go through the usual insert logic @@ -824,17 +821,15 @@ func (ds *Datastore) deleteSetupExperienceScript(ctx context.Context, tx sqlx.Ex func (ds *Datastore) SetHostAwaitingConfiguration(ctx context.Context, hostUUID string, awaitingConfiguration bool) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return setHostAwaitingConfiguration(ctx, tx, hostUUID, awaitingConfiguration) + return setHostAwaitingConfiguration(ctx, tx, ds.dialect, hostUUID, awaitingConfiguration) }) } -func setHostAwaitingConfiguration(ctx context.Context, tx sqlx.ExtContext, hostUUID string, awaitingConfiguration bool) error { - const stmt = ` +func setHostAwaitingConfiguration(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostUUID string, awaitingConfiguration bool) error { + stmt := ` INSERT INTO host_mdm_apple_awaiting_configuration (host_uuid, awaiting_configuration) VALUES (?, ?) -ON DUPLICATE KEY UPDATE - awaiting_configuration = VALUES(awaiting_configuration) - ` +` + dialect.OnDuplicateKey("host_uuid", "awaiting_configuration = VALUES(awaiting_configuration)") _, err := tx.ExecContext(ctx, stmt, hostUUID, awaitingConfiguration) if err != nil { diff --git a/server/datastore/mysql/software_title_display_names.go b/server/datastore/mysql/software_title_display_names.go index 4e5f7d13d53..33311f3e4ae 100644 --- a/server/datastore/mysql/software_title_display_names.go +++ b/server/datastore/mysql/software_title_display_names.go @@ -8,7 +8,7 @@ import ( "github.com/jmoiron/sqlx" ) -func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, teamID *uint, titleID uint, displayName string) error { +func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, teamID *uint, titleID uint, displayName string) error { var tmID uint if teamID != nil { tmID = *teamID @@ -17,8 +17,7 @@ func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, tea INSERT INTO software_title_display_names (team_id, software_title_id, display_name) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - display_name = VALUES(display_name)`, tmID, titleID, displayName) + `+dialect.OnDuplicateKey("title_id", "display_name = VALUES(display_name)"), tmID, titleID, displayName) if err != nil { return err } diff --git a/server/datastore/mysql/software_title_icons.go b/server/datastore/mysql/software_title_icons.go index d0ae9c1e1d9..1b53c9af359 100644 --- a/server/datastore/mysql/software_title_icons.go +++ b/server/datastore/mysql/software_title_icons.go @@ -15,8 +15,7 @@ func (ds *Datastore) CreateOrUpdateSoftwareTitleIcon(ctx context.Context, payloa var args []any query = ` INSERT INTO software_title_icons (team_id, software_title_id, storage_id, filename) - VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE - storage_id = VALUES(storage_id), filename = VALUES(filename) + VALUES (?, ?, ?, ?) ` + ds.dialect.OnDuplicateKey("id", `storage_id = VALUES(storage_id), filename = VALUES(filename)`) + ` ` args = []any{payload.TeamID, payload.TitleID, payload.StorageID, payload.Filename} diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index 9f88d2fd317..f23c5a5108f 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -1081,7 +1081,7 @@ SELECT s.title_id, s.id, s.version, %s -- placeholder for optional host_counts - CONCAT('[', GROUP_CONCAT(JSON_QUOTE(scve.cve) SEPARATOR ','), ']') as vulnerabilities + CONCAT('[', ` + ds.dialect.GroupConcat("JSON_QUOTE(scve.cve)", ",") + `, ']') as vulnerabilities FROM software s LEFT JOIN software_host_counts shc ON shc.software_id = s.id AND %s LEFT JOIN software_cve scve ON shc.software_id = scve.software_id @@ -1158,17 +1158,16 @@ func (ds *Datastore) SyncHostsSoftwareTitles(ctx context.Context, updatedAt time WHERE h.team_id IS NULL AND hs.software_id > 0 GROUP BY st.id` - insertStmt = ` + valuesPart = `(?, ?, ?, ?, ?),` + ) + + insertStmt := ` INSERT INTO ` + swapTable + ` (software_title_id, hosts_count, team_id, global_stats, updated_at) VALUES %s - ON DUPLICATE KEY UPDATE - hosts_count = VALUES(hosts_count), - updated_at = VALUES(updated_at)` - - valuesPart = `(?, ?, ?, ?, ?),` - ) + ` + ds.dialect.OnDuplicateKey("software_id,team_id", `hosts_count = VALUES(hosts_count), + updated_at = VALUES(updated_at)`) // Create a fresh swap table to populate with new counts. If a previous run left a partial swap table, drop it first. w := ds.writer(ctx) @@ -1279,11 +1278,9 @@ func (ds *Datastore) UpdateSoftwareTitleAutoUpdateConfig(ctx context.Context, ti INSERT INTO software_update_schedules (title_id, team_id, enabled, start_time, end_time) VALUES (?, ?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - enabled = VALUES(enabled), +` + ds.dialect.OnDuplicateKey("id", `enabled = VALUES(enabled), start_time = IF(VALUES(start_time) = '', start_time, VALUES(start_time)), - end_time = IF(VALUES(end_time) = '', end_time, VALUES(end_time)) -` + end_time = IF(VALUES(end_time) = '', end_time, VALUES(end_time))`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, titleID, teamID, config.AutoUpdateEnabled, startTime, endTime) if err != nil { return ctxerr.Wrap(ctx, err, "updating software title auto update config") diff --git a/server/datastore/mysql/teams.go b/server/datastore/mysql/teams.go index a641bbc3f72..76a760c5c8d 100644 --- a/server/datastore/mysql/teams.go +++ b/server/datastore/mysql/teams.go @@ -33,9 +33,7 @@ func (ds *Datastore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team config ) VALUES (?, ?, ?, ?) ` - result, err := tx.ExecContext( - ctx, - query, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, query, team.Name, team.Filename, team.Description, @@ -45,7 +43,6 @@ func (ds *Datastore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team return ctxerr.Wrap(ctx, err, "insert team") } - id, _ := result.LastInsertId() team.ID = uint(id) //nolint:gosec // dismiss G115 team.CreatedAt = time.Now().UTC().Truncate(time.Second) @@ -618,7 +615,7 @@ func (ds *Datastore) SaveDefaultTeamConfig(ctx context.Context, config *fleet.Te _, err = ds.writer(ctx).ExecContext(ctx, `INSERT INTO default_team_config_json(id, json_value) VALUES(1, ?) - ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)`, + `+ds.dialect.OnDuplicateKey("id", `json_value = VALUES(json_value)`), configBytes, ) return ctxerr.Wrap(ctx, err, "save default team config") diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go index 4e4b4938528..5b27b433b78 100644 --- a/server/datastore/mysql/users.go +++ b/server/datastore/mysql/users.go @@ -53,7 +53,7 @@ func (ds *Datastore) NewUser(ctx context.Context, user *fleet.User) (*fleet.User invite_id ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ` - result, err := tx.ExecContext(ctx, sqlStatement, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, user.Password, user.Salt, user.Name, @@ -76,7 +76,6 @@ func (ds *Datastore) NewUser(ctx context.Context, user *fleet.User) (*fleet.User return ctxerr.Wrap(ctx, err, "create new user") } - id, _ := result.LastInsertId() user.ID = uint(id) //nolint:gosec // dismiss G115 if err := saveTeamsForUserDB(ctx, tx, user); err != nil { @@ -385,9 +384,8 @@ func (ds *Datastore) DeleteUser(ctx context.Context, id uint) error { SELECT u.id, u.name, u.email FROM users AS u WHERE u.id = ? - ON DUPLICATE KEY UPDATE - name = u.name, - email = u.email` + ` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), + email = VALUES(email)`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, id) if err != nil { return ctxerr.Wrap(ctx, err, "populate users_deleted entry") diff --git a/server/datastore/mysql/windows_updates.go b/server/datastore/mysql/windows_updates.go index d364f48a70d..0f17bf5edd5 100644 --- a/server/datastore/mysql/windows_updates.go +++ b/server/datastore/mysql/windows_updates.go @@ -60,7 +60,7 @@ func (ds *Datastore) InsertWindowsUpdates(ctx context.Context, hostID uint, upda if len(args) > 0 { smt := fmt.Sprintf( - `INSERT IGNORE INTO windows_updates (host_id, date_epoch, kb_id) VALUES %s`, + ds.dialect.InsertIgnoreInto()+` windows_updates (host_id, date_epoch, kb_id) VALUES %s`+ds.dialect.OnConflictDoNothing("host_id,date_epoch,kb_id"), strings.Join(placeholders, ","), ) diff --git a/server/datastore/mysql/wstep.go b/server/datastore/mysql/wstep.go index ebcd43468eb..bc1393c45a0 100644 --- a/server/datastore/mysql/wstep.go +++ b/server/datastore/mysql/wstep.go @@ -45,15 +45,11 @@ VALUES // WSTEPNewSerial allocates and returns a new (increasing) serial number. func (ds *Datastore) WSTEPNewSerial(ctx context.Context) (*big.Int, error) { - result, err := ds.writer(ctx).ExecContext(ctx, `INSERT INTO wstep_serials () VALUES ();`) + lid, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO wstep_serials () VALUES ();`) if err != nil { return nil, err } - lid, err := result.LastInsertId() // TODO: ok if sequential and not random? - if err != nil { - return nil, err - } - // TODO: check maxSerialNumber? + // TODO: check maxSerialNumber? ok if sequential and not random? return big.NewInt(lid), nil }