From b44e955e26a47501040769c07a5aebb0f69601d1 Mon Sep 17 00:00:00 2001 From: Gautier DI FOLCO Date: Tue, 4 Aug 2026 21:04:49 +0200 Subject: [PATCH] WPB-22972: bootstrap Spar Postgres infrastructure Wire a Hasql Postgres pool, schema-migration run, and shared storage-location config into spar, and add a cassandraSpar client to the background-worker (with full cassandraSpar TLS plumbing mirroring cassandraBrig). This is the shared infrastructure prerequisite for the Spar Cassandra->Postgres store migrations (WPB-22972/22973/22974); no store interpreters, SQL migrations, or migration workers are included yet. --- changelog.d/5-internal/WPB-22972-0 | 1 + charts/wire-server/templates/_helpers.tpl | 8 +++++++ .../background-worker/configmap.yaml | 14 +++++++++++ .../background-worker/deployment.yaml | 9 ++++++++ .../wire-server/templates/spar/configmap.yaml | 7 ++++++ charts/wire-server/values.yaml | 22 ++++++++++++++++++ .../src/developer/reference/config-options.md | 7 +++++- hack/helm_vars/wire-server/values.yaml.gotmpl | 23 +++++++++++++++++++ .../background-worker.integration.yaml | 6 +++++ .../src/Wire/BackgroundWorker/Env.hs | 3 +++ .../src/Wire/BackgroundWorker/Options.hs | 1 + .../Wire/BackendNotificationPusherSpec.hs | 2 ++ .../background-worker/test/Test/Wire/Util.hs | 1 + services/spar/spar.integration.yaml | 19 +++++++++++++++ services/spar/src/Spar/App.hs | 4 +++- .../spar/src/Spar/CanonicalInterpreter.hs | 3 +++ services/spar/src/Spar/Options.hs | 13 ++++++++++- services/spar/src/Spar/Run.hs | 4 ++++ services/spar/test-integration/Util/Core.hs | 1 + services/spar/test/Test/Spar/Saml/IdPSpec.hs | 17 +++++++++++++- 20 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 changelog.d/5-internal/WPB-22972-0 diff --git a/changelog.d/5-internal/WPB-22972-0 b/changelog.d/5-internal/WPB-22972-0 new file mode 100644 index 00000000000..e51a6f26290 --- /dev/null +++ b/changelog.d/5-internal/WPB-22972-0 @@ -0,0 +1 @@ +Bootstrap Spar Postgres infrastructure (connection pool, schema migrations, storage-location config) for the Cassandra->Postgres store migration. diff --git a/charts/wire-server/templates/_helpers.tpl b/charts/wire-server/templates/_helpers.tpl index 5edb0251456..7374573f167 100644 --- a/charts/wire-server/templates/_helpers.tpl +++ b/charts/wire-server/templates/_helpers.tpl @@ -39,6 +39,14 @@ {{- end -}} {{- end -}} +{{- define "sparTlsSecretRef" -}} +{{- if .cassandraSpar.tlsCaSecretRef -}} +{{ .cassandraSpar.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "background-worker-cassandra-spar" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + {{/* BRIG */}} {{- define "brig.tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} diff --git a/charts/wire-server/templates/background-worker/configmap.yaml b/charts/wire-server/templates/background-worker/configmap.yaml index d4fe2a63202..e6675b15b44 100644 --- a/charts/wire-server/templates/background-worker/configmap.yaml +++ b/charts/wire-server/templates/background-worker/configmap.yaml @@ -64,6 +64,20 @@ data: {{- end }} {{- end }} + {{- if .cassandraSpar }} + cassandraSpar: + endpoint: + host: {{ .cassandraSpar.host }} + port: 9042 + keyspace: spar + {{- if hasKey .cassandraSpar "filterNodesByDatacentre" }} + filterNodesByDatacentre: {{ .cassandraSpar.filterNodesByDatacentre }} + {{- end }} + {{- if eq (include "useCassandraTLS" .cassandraSpar) "true" }} + tlsCa: /etc/wire/background-worker/cassandra-spar/{{- (include "sparTlsSecretRef" . | fromYaml).key }} + {{- end }} + {{- end }} + {{- with .rabbitmq }} rabbitmq: host: {{ .host }} diff --git a/charts/wire-server/templates/background-worker/deployment.yaml b/charts/wire-server/templates/background-worker/deployment.yaml index f91ccb5ee17..c11f35313b0 100644 --- a/charts/wire-server/templates/background-worker/deployment.yaml +++ b/charts/wire-server/templates/background-worker/deployment.yaml @@ -59,6 +59,11 @@ spec: secret: secretName: {{ (include "brigTlsSecretRef" $backgroundWorker.config | fromYaml).name }} {{- end }} + {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }} + - name: "background-worker-cassandra-spar" + secret: + secretName: {{ (include "sparTlsSecretRef" $backgroundWorker.config | fromYaml).name }} + {{- end }} {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }} - name: "galley-cassandra" secret: @@ -97,6 +102,10 @@ spec: - name: "background-worker-cassandra-brig" mountPath: "/etc/wire/background-worker/cassandra-brig" {{- end }} + {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }} + - name: "background-worker-cassandra-spar" + mountPath: "/etc/wire/background-worker/cassandra-spar" + {{- end }} {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" diff --git a/charts/wire-server/templates/spar/configmap.yaml b/charts/wire-server/templates/spar/configmap.yaml index 6dfd671442a..f9c42620cf0 100644 --- a/charts/wire-server/templates/spar/configmap.yaml +++ b/charts/wire-server/templates/spar/configmap.yaml @@ -29,6 +29,13 @@ data: tlsCa: /etc/wire/spar/cassandra/{{- (include "spar.tlsSecretRef" . | fromYaml).key }} {{- end }} + postgresql: {{ toYaml .postgresql | nindent 6 }} + postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }} + {{- if .postgresqlPassword }} + postgresqlPassword: {{ .postgresqlPassword | quote }} + {{- end }} + postgresMigration: {{- toYaml .postgresMigration | nindent 6 }} + maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 89e688a2ef0..1250ed13e70 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -857,6 +857,26 @@ spar: # tlsCaSecretRef: # name: # key: + + # Postgres connection (libpq key/value map). See: + # https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS + postgresql: + host: postgresql + port: "5432" + user: wire-server + dbname: wire-server + postgresqlPool: + size: 100 + acquisitionTimeout: 10s + idlenessTimeout: 10m + # Per-store storage-location flags (shared type). Spar-relevant stores are + # added by their migration tasks; defaults stay on Cassandra. + postgresMigration: + conversation: cassandra + conversationCodes: cassandra + teamFeatures: cassandra + domainRegistration: cassandra + user: cassandra richInfoLimit: 5000 maxScimTokens: 0 @@ -997,6 +1017,8 @@ background-worker: host: aws-cassandra cassandraBrig: host: aws-cassandra + cassandraSpar: + host: aws-cassandra postgresqlPool: size: 5 diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 066637dbb07..b6ba338bd70 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -2029,7 +2029,7 @@ server, verification can be turned off by settings `insecureSkipVerifyTls` to ## Configure PostgreSQL -`brig`, `galley`, and `background-worker` require a PostgreSQL database. The configured user needs to +`brig`, `galley`, `spar`, and `background-worker` require a PostgreSQL database. The configured user needs to be able to write data and change the schema (e.g. create and alter tables.) The internal configuration YAML file format and the Helm charts for `brig` and @@ -2114,6 +2114,11 @@ background-worker: migrateDomainRegistration: false ``` +`spar` additionally exposes its own `spar.config.postgresMigration` block, using the +same shared storage-location type. Spar-owned stores (e.g. SAML users, SCIM tokens, +external ids) are added to it by their respective migration tasks; until then the +block mirrors the shared defaults and stays on Cassandra. + #### Migration for existing installations Existing installations should migrate Cassandra-backed data to PostgreSQL over diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index bd6cdadf2a8..11b5d4aea00 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -583,6 +583,21 @@ spar: name: "cassandra-jks-keystore" key: "ca.crt" {{- end }} + postgresql: + host: postgresql + port: "5432" + user: wire-server + dbname: wire-server + postgresqlPool: + size: 100 + acquisitionTimeout: 10s + idlenessTimeout: 10m + postgresMigration: + conversation: {{ .Values.conversationStore }} + conversationCodes: {{ .Values.conversationCodesStore }} + teamFeatures: {{ .Values.teamFeaturesStore }} + domainRegistration: {{ .Values.domainRegistration }} + user: {{ .Values.userStore }} logLevel: Debug domain: zinfra.io appUri: http://spar:8080/ @@ -689,6 +704,14 @@ background-worker: name: "cassandra-jks-keystore" key: "ca.crt" {{- end }} + cassandraSpar: + host: {{ .Values.cassandraHost }} + replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} rabbitmq: port: 5671 adminPort: 15671 diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml index e264ce14016..76992bfe075 100644 --- a/services/background-worker/background-worker.integration.yaml +++ b/services/background-worker/background-worker.integration.yaml @@ -30,6 +30,12 @@ cassandraBrig: port: 9042 keyspace: brig_test +cassandraSpar: + endpoint: + host: 127.0.0.1 + port: 9042 + keyspace: spar_test + postgresqlPool: size: 5 acquisitionTimeout: 10s diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs index ed784a33db9..e6696f1ed7c 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs @@ -90,6 +90,7 @@ data Env = Env cassandra :: ClientState, cassandraGalley :: ClientState, cassandraBrig :: ClientState, + cassandraSpar :: ClientState, hasqlPool :: Hasql.Pool, -- May contain the PostgreSQL password. Do not unwrap outside the Arbiter boundary. arbiterConnStr :: SecretText, @@ -156,6 +157,8 @@ mkEnv opts galleyOpts = do cassandraGalley <- defInitCassandra galleyOpts._cassandra =<< setLoggerName "cassandra-galley" logger Log.info logger $ Log.msg @Text "Connecting to Cassandra (brig)..." cassandraBrig <- defInitCassandra opts.cassandraBrig =<< setLoggerName "cassandra-brig" logger + Log.info logger $ Log.msg @Text "Connecting to Cassandra (spar)..." + cassandraSpar <- defInitCassandra opts.cassandraSpar =<< setLoggerName "cassandra-spar" logger Log.info logger $ Log.msg @Text "Cassandra connections established" http2Manager <- initHttp2Manager httpManager <- newManager defaultManagerSettings diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs index 61df5d5d14f..5cae03ae28f 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs @@ -48,6 +48,7 @@ data Opts = Opts backendNotificationPusher :: BackendNotificationsConfig, cassandra :: CassandraOpts, cassandraBrig :: CassandraOpts, + cassandraSpar :: CassandraOpts, postgresqlPool :: !PoolConfig, postgresMigration :: !PostgresMigrationOpts, migrateConversations :: !Bool, diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs index 7222120d93a..74691fb7c23 100644 --- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs +++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs @@ -348,6 +348,7 @@ spec = do let cassandra = undefined cassandraGalley = undefined cassandraBrig = undefined + cassandraSpar = undefined federatorInternal = Endpoint "localhost" 8097 http2Manager = undefined statuses = undefined @@ -413,6 +414,7 @@ spec = do httpManager <- newManager defaultManagerSettings let federatorInternal = Endpoint "localhost" 8097 cassandraBrig = undefined + cassandraSpar = undefined http2Manager = undefined statuses = undefined rabbitmqAdminClient = Just $ mockRabbitMqAdminClient mockAdmin diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs index 5d89532bfec..bd6c055c848 100644 --- a/services/background-worker/test/Test/Wire/Util.hs +++ b/services/background-worker/test/Test/Wire/Util.hs @@ -44,6 +44,7 @@ testEnv = do let cassandra = undefined cassandraGalley = undefined cassandraBrig = undefined + cassandraSpar = undefined postgresMigration = PostgresMigrationOpts { conversation = CassandraStorage, diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index a50e065c263..dcd10f6bcf8 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -31,6 +31,25 @@ cassandra: keyspace: spar_test filterNodesByDatacentre: datacenter1 +postgresql: + host: 127.0.0.1 + port: "5432" + user: wire-server + dbname: backendA + password: posty-the-gres + +postgresqlPool: + size: 20 + acquisitionTimeout: 10s + idlenessTimeout: 10m + +postgresMigration: + conversation: cassandra + conversationCodes: cassandra + teamFeatures: cassandra + domainRegistration: cassandra + user: cassandra + # Wire/AWS specific, optional # discoUrl: "https://" diff --git a/services/spar/src/Spar/App.hs b/services/spar/src/Spar/App.hs index dc1c950fd4e..1d14ab1583d 100644 --- a/services/spar/src/Spar/App.hs +++ b/services/spar/src/Spar/App.hs @@ -38,6 +38,7 @@ where import Bilge import qualified Cassandra as Cas +import qualified Hasql.Pool.Extended as HasqlPoolExt import Control.Exception (assert) import Control.Lens hiding ((.=)) import Control.Monad.Trans.Maybe (MaybeT (..), runMaybeT) @@ -127,7 +128,8 @@ data Env = Env sparCtxHttpBrig :: Bilge.Request, sparCtxHttpGalley :: Bilge.Request, sparCtxRequestId :: RequestId, - sparCtxScimSubsystemConfig :: ScimSubsystemConfig + sparCtxScimSubsystemConfig :: ScimSubsystemConfig, + sparCtxHasqlPool :: HasqlPoolExt.Pool } -- | Get a user by UserRef, no matter what the team. diff --git a/services/spar/src/Spar/CanonicalInterpreter.hs b/services/spar/src/Spar/CanonicalInterpreter.hs index b2a4405b850..f8158a8ec1e 100644 --- a/services/spar/src/Spar/CanonicalInterpreter.hs +++ b/services/spar/src/Spar/CanonicalInterpreter.hs @@ -25,6 +25,7 @@ module Spar.CanonicalInterpreter where import qualified Cassandra as Cas +import qualified Hasql.Pool.Extended as HasqlPoolExt import Control.Monad.Except hiding (mapError) import Imports import Polysemy @@ -120,6 +121,7 @@ type LowerLevelCanonicalEffs = Logger (TinyLog.Msg -> TinyLog.Msg), Input Opts, Input TinyLog.Logger, + Input HasqlPoolExt.Pool, Random, Now, Embed IO, @@ -132,6 +134,7 @@ runSparToIO ctx = . embedToFinal @IO . nowToIO . randomToIO + . runInputConst (sparCtxHasqlPool ctx) . runInputConst (sparCtxLogger ctx) . runInputConst (sparCtxOpts ctx) . loggerToTinyLog (sparCtxLogger ctx) diff --git a/services/spar/src/Spar/Options.hs b/services/spar/src/Spar/Options.hs index 887933a0a9f..73d763584aa 100644 --- a/services/spar/src/Spar/Options.hs +++ b/services/spar/src/Spar/Options.hs @@ -35,6 +35,7 @@ import qualified Data.Set as Set import Data.Time import Data.X509.Extended (Fingerprint, parseFingerprintHex) import qualified Data.Yaml as Yaml +import Hasql.Pool.Extended (PoolConfig) import Imports import Options.Applicative import SAML2.WebSSO @@ -45,6 +46,7 @@ import Util.Options import Wire.API.Routes.Version import Wire.API.User.Orphans () import Wire.API.User.Saml +import Wire.PostgresMigrationOpts (PostgresMigrationOpts) data Opts = Opts { saml :: !SAML.Config, @@ -70,7 +72,16 @@ data Opts = Opts -- disables the check. When set, every cert in 'edCertAuthnResponse' -- (on create/update and on AuthnResponse) must be listed or the request -- is rejected with 'SparIdPCertNotAllowed' (HTTP 403). - idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist) + idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist), + -- | Postgres connection parameters (libpq key/value map). + postgresql :: !(Map Text Text), + -- | Postgres connection-pool sizing. + postgresqlPool :: !PoolConfig, + -- | Per-store Cassandra/Postgres storage-location flags (shared type; + -- Spar-relevant stores are added by their migration tasks). + postgresMigration :: !PostgresMigrationOpts, + -- | Optional path to a Postgres password secrets file (prod only). + postgresqlPassword :: !(Maybe FilePathSecrets) } deriving (Show, Generic) diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 36482a10abf..441766c2ab0 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -31,6 +31,7 @@ where import qualified Bilge import Cassandra as Cas import Cassandra.Util (initCassandraForService) +import Hasql.Pool.Extended (initPostgresPool, rawPool) import Control.Exception (ErrorCall (ErrorCall), throwIO) import Control.Lens (to, (^.)) import qualified Data.ByteString.UTF8 as UTF8 @@ -60,6 +61,7 @@ import qualified Web.Scim.Schema.Common as Scim import Wire.API.Routes.Version (expandVersionExp) import Wire.API.Routes.Version.Wai import Wire.ScimSubsystem.Interpreter +import Wire.PostgresMigrations (runAllMigrations) ---------------------------------------------------------------------- -- cassandra @@ -82,6 +84,7 @@ runServer sparCtxOpts = do sport :: Int = sparCtxOpts ^. to saml . SAML.cfgSPPort (wrappedApp, ctxOpts) <- mkApp sparCtxOpts let logger = sparCtxLogger ctxOpts + runAllMigrations ctxOpts.sparCtxHasqlPool.rawPool logger let settings = newSettings $ defaultServer shost (fromIntegral sport) logger WU.runSettingsWithShutdown settings wrappedApp Nothing @@ -90,6 +93,7 @@ mkApp sparCtxOpts = do let logLevel = saml sparCtxOpts ^. SAML.cfgLogLevel sparCtxLogger <- Log.mkLogger logLevel (logNetStrings sparCtxOpts) (logFormat sparCtxOpts) sparCtxCas <- initCassandra sparCtxOpts sparCtxLogger + sparCtxHasqlPool <- initPostgresPool (Opt.postgresqlPool sparCtxOpts) (Opt.postgresql sparCtxOpts) (Opt.postgresqlPassword sparCtxOpts) sparCtxHttpManager <- Bilge.newManager Bilge.defaultManagerSettings let sparCtxHttpBrig = Bilge.host (sparCtxOpts ^. to brig . to host . to encodeUtf8) diff --git a/services/spar/test-integration/Util/Core.hs b/services/spar/test-integration/Util/Core.hs index 4fe1e23e92f..991ad7975f6 100644 --- a/services/spar/test-integration/Util/Core.hs +++ b/services/spar/test-integration/Util/Core.hs @@ -277,6 +277,7 @@ mkEnv tstOpts opts = do sparCtxHttpGalley = galley empty sparCtxRequestId = RequestId "" sparCtxScimSubsystemConfig = error "mkEnv: implement sparCtxScimSubsystemConfig when needed" + sparCtxHasqlPool = error "mkEnv: sparCtxHasqlPool is not available in integration tests" pure $ TestEnv mgr diff --git a/services/spar/test/Test/Spar/Saml/IdPSpec.hs b/services/spar/test/Test/Spar/Saml/IdPSpec.hs index 93f591a750c..4babacc306e 100644 --- a/services/spar/test/Test/Spar/Saml/IdPSpec.hs +++ b/services/spar/test/Test/Spar/Saml/IdPSpec.hs @@ -18,6 +18,9 @@ import Data.X509 (SignedCertificate) import Data.X509.Extended (renderFingerprintHex) import qualified Data.X509.Extended as X509E import Imports +import Data.Misc (unsafeParseDuration) +import Hasql.Pool.Extended (PoolConfig (..)) +import Wire.PostgresMigrationOpts (PostgresMigrationOpts (..), StorageLocation (..)) import Polysemy import qualified Polysemy.Error import Polysemy.Input (Input, runInputConst) @@ -1016,7 +1019,19 @@ defaultTestOpts = disabledAPIVersions = mempty, scimBaseUri = [uri|http://localhost:8088/scim/v2|], enableIdPByEmailDiscovery = False, - idpCertFingerprintAllowlist = Nothing + idpCertFingerprintAllowlist = Nothing, + -- Postgres fields are placeholders; only the allowlist is read here. + postgresql = mempty, + postgresqlPool = PoolConfig 1 (unsafeParseDuration "10s") (unsafeParseDuration "10m"), + postgresMigration = + PostgresMigrationOpts + { conversation = CassandraStorage, + conversationCodes = CassandraStorage, + teamFeatures = CassandraStorage, + domainRegistration = CassandraStorage, + user = CassandraStorage + }, + postgresqlPassword = Nothing } galleyAccessMock :: Sem (GalleyAPIAccess ': r) a -> Sem r a