WPB-22972: bootstrap Spar Postgres infrastructure

changelog.d/5-internal/WPB-22972-0

@@ -0,0 +1 @@
+Bootstrap Spar Postgres infrastructure (connection pool, schema migrations, storage-location config) for the Cassandra->Postgres store migration.

charts/wire-server/templates/_helpers.tpl

@@ -39,6 +39,14 @@
 {{- end -}}
 {{- end -}}
 
+{{- define "sparTlsSecretRef" -}}
+{{- if .cassandraSpar.tlsCaSecretRef -}}
+{{ .cassandraSpar.tlsCaSecretRef | toYaml }}
+{{- else }}
+{{- dict "name" "background-worker-cassandra-spar" "key" "ca.pem" | toYaml -}}
+{{- end -}}
+{{- end -}}
+
 {{/* BRIG */}}
 {{- define "brig.tlsSecretRef" -}}
 {{- if .cassandra.tlsCaSecretRef -}}

charts/wire-server/templates/background-worker/configmap.yaml

@@ -64,6 +64,20 @@ data:
       {{- end }}
     {{- end }}
 
+    {{- if .cassandraSpar }}
+    cassandraSpar:
+      endpoint:
+        host: {{ .cassandraSpar.host }}
+        port: 9042
+      keyspace: spar
+      {{- if hasKey .cassandraSpar "filterNodesByDatacentre" }}
+      filterNodesByDatacentre: {{ .cassandraSpar.filterNodesByDatacentre }}
+      {{- end }}
+      {{- if eq (include "useCassandraTLS" .cassandraSpar) "true" }}
+      tlsCa: /etc/wire/background-worker/cassandra-spar/{{- (include "sparTlsSecretRef" . | fromYaml).key }}
+      {{- end }}
+    {{- end }}
+
     {{- with .rabbitmq }}
     rabbitmq:
       host: {{ .host }}

charts/wire-server/templates/background-worker/deployment.yaml

@@ -59,6 +59,11 @@ spec:
           secret:
             secretName: {{ (include "brigTlsSecretRef" $backgroundWorker.config | fromYaml).name }}
         {{- end }}
+        {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }}
+        - name: "background-worker-cassandra-spar"
+          secret:
+            secretName: {{ (include "sparTlsSecretRef" $backgroundWorker.config | fromYaml).name }}
+        {{- end }}
         {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }}
         - name: "galley-cassandra"
           secret:
@@ -97,6 +102,10 @@ spec:
           - name: "background-worker-cassandra-brig"
             mountPath: "/etc/wire/background-worker/cassandra-brig"
           {{- end }}
+          {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }}
+          - name: "background-worker-cassandra-spar"
+            mountPath: "/etc/wire/background-worker/cassandra-spar"
+          {{- end }}
           {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }}
           - name: "galley-cassandra"
             mountPath: "/etc/wire/galley/cassandra"

charts/wire-server/templates/spar/configmap.yaml

@@ -29,6 +29,13 @@ data:
       tlsCa: /etc/wire/spar/cassandra/{{- (include "spar.tlsSecretRef" . | fromYaml).key }}
       {{- end }}
 
+    postgresql: {{ toYaml .postgresql | nindent 6 }}
+    postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }}
+    {{- if .postgresqlPassword }}
+    postgresqlPassword: {{ .postgresqlPassword | quote }}
+    {{- end }}
+    postgresMigration: {{- toYaml .postgresMigration | nindent 6 }}
+
     maxttlAuthreq: {{ .maxttlAuthreq }}
     maxttlAuthresp: {{ .maxttlAuthresp }}
 

charts/wire-server/values.yaml

@@ -857,6 +857,26 @@ spar:
       # tlsCaSecretRef:
       #   name: <secret-name>
       #   key: <ca-attribute>
+
+    # Postgres connection (libpq key/value map). See:
+    # https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+    postgresql:
+      host: postgresql
+      port: "5432"
+      user: wire-server
+      dbname: wire-server
+    postgresqlPool:
+      size: 100
+      acquisitionTimeout: 10s
+      idlenessTimeout: 10m
+    # Per-store storage-location flags (shared type). Spar-relevant stores are
+    # added by their migration tasks; defaults stay on Cassandra.
+    postgresMigration:
+      conversation: cassandra
+      conversationCodes: cassandra
+      teamFeatures: cassandra
+      domainRegistration: cassandra
+      user: cassandra
     richInfoLimit: 5000
     maxScimTokens: 0
 
@@ -997,6 +1017,8 @@ background-worker:
       host: aws-cassandra
     cassandraBrig:
       host: aws-cassandra
+    cassandraSpar:
+      host: aws-cassandra
 
     postgresqlPool:
       size: 5

docs/src/developer/reference/config-options.md

@@ -2029,7 +2029,7 @@ server, verification can be turned off by settings `insecureSkipVerifyTls` to
 
 ## Configure PostgreSQL
 
-`brig`, `galley`, and `background-worker` require a PostgreSQL database. The configured user needs to
+`brig`, `galley`, `spar`, and `background-worker` require a PostgreSQL database. The configured user needs to
 be able to write data and change the schema (e.g. create and alter tables.)
 
 The internal configuration YAML file format and the Helm charts for `brig` and
@@ -2114,6 +2114,11 @@ background-worker:
     migrateDomainRegistration: false
 ```
 
+`spar` additionally exposes its own `spar.config.postgresMigration` block, using the
+same shared storage-location type. Spar-owned stores (e.g. SAML users, SCIM tokens,
+external ids) are added to it by their respective migration tasks; until then the
+block mirrors the shared defaults and stays on Cassandra.
+
 #### Migration for existing installations
 
 Existing installations should migrate Cassandra-backed data to PostgreSQL over

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -583,6 +583,21 @@ spar:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
     {{- end }}
+    postgresql:
+      host: postgresql
+      port: "5432"
+      user: wire-server
+      dbname: wire-server
+    postgresqlPool:
+      size: 100
+      acquisitionTimeout: 10s
+      idlenessTimeout: 10m
+    postgresMigration:
+      conversation: {{ .Values.conversationStore }}
+      conversationCodes: {{ .Values.conversationCodesStore }}
+      teamFeatures: {{ .Values.teamFeaturesStore }}
+      domainRegistration: {{ .Values.domainRegistration }}
+      user: {{ .Values.userStore }}
     logLevel: Debug
     domain: zinfra.io
     appUri: http://spar:8080/
@@ -689,6 +704,14 @@ background-worker:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
       {{- end }}
+    cassandraSpar:
+      host: {{ .Values.cassandraHost }}
+      replicaCount: 1
+      {{- if .Values.useK8ssandraSSL.enabled }}
+      tlsCaSecretRef:
+          name: "cassandra-jks-keystore"
+          key: "ca.crt"
+      {{- end }}
     rabbitmq:
       port: 5671
       adminPort: 15671

services/background-worker/background-worker.integration.yaml

@@ -30,6 +30,12 @@ cassandraBrig:
     port: 9042
   keyspace: brig_test
 
+cassandraSpar:
+  endpoint:
+    host: 127.0.0.1
+    port: 9042
+  keyspace: spar_test
+
 postgresqlPool:
   size: 5
   acquisitionTimeout: 10s

services/background-worker/src/Wire/BackgroundWorker/Env.hs

@@ -90,6 +90,7 @@ data Env = Env
     cassandra :: ClientState,
     cassandraGalley :: ClientState,
     cassandraBrig :: ClientState,
+    cassandraSpar :: ClientState,
     hasqlPool :: Hasql.Pool,
     -- May contain the PostgreSQL password. Do not unwrap outside the Arbiter boundary.
     arbiterConnStr :: SecretText,
@@ -156,6 +157,8 @@ mkEnv opts galleyOpts = do
   cassandraGalley <- defInitCassandra galleyOpts._cassandra =<< setLoggerName "cassandra-galley" logger
   Log.info logger $ Log.msg @Text "Connecting to Cassandra (brig)..."
   cassandraBrig <- defInitCassandra opts.cassandraBrig =<< setLoggerName "cassandra-brig" logger
+  Log.info logger $ Log.msg @Text "Connecting to Cassandra (spar)..."
+  cassandraSpar <- defInitCassandra opts.cassandraSpar =<< setLoggerName "cassandra-spar" logger
   Log.info logger $ Log.msg @Text "Cassandra connections established"
   http2Manager <- initHttp2Manager
   httpManager <- newManager defaultManagerSettings

services/background-worker/src/Wire/BackgroundWorker/Options.hs

@@ -48,6 +48,7 @@ data Opts = Opts
     backendNotificationPusher :: BackendNotificationsConfig,
     cassandra :: CassandraOpts,
     cassandraBrig :: CassandraOpts,
+    cassandraSpar :: CassandraOpts,
     postgresqlPool :: !PoolConfig,
     postgresMigration :: !PostgresMigrationOpts,
     migrateConversations :: !Bool,

services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs

@@ -348,6 +348,7 @@ spec = do
       let cassandra = undefined
           cassandraGalley = undefined
           cassandraBrig = undefined
+          cassandraSpar = undefined
           federatorInternal = Endpoint "localhost" 8097
           http2Manager = undefined
           statuses = undefined
@@ -413,6 +414,7 @@ spec = do
       httpManager <- newManager defaultManagerSettings
       let federatorInternal = Endpoint "localhost" 8097
           cassandraBrig = undefined
+          cassandraSpar = undefined
           http2Manager = undefined
           statuses = undefined
           rabbitmqAdminClient = Just $ mockRabbitMqAdminClient mockAdmin

services/background-worker/test/Test/Wire/Util.hs

@@ -44,6 +44,7 @@ testEnv = do
   let cassandra = undefined
       cassandraGalley = undefined
       cassandraBrig = undefined
+      cassandraSpar = undefined
       postgresMigration =
         PostgresMigrationOpts
           { conversation = CassandraStorage,

services/spar/spar.integration.yaml

@@ -31,6 +31,25 @@ cassandra:
   keyspace: spar_test
   filterNodesByDatacentre: datacenter1
 
+postgresql:
+  host: 127.0.0.1
+  port: "5432"
+  user: wire-server
+  dbname: backendA
+  password: posty-the-gres
+
+postgresqlPool:
+  size: 20
+  acquisitionTimeout: 10s
+  idlenessTimeout: 10m
+
+postgresMigration:
+  conversation: cassandra
+  conversationCodes: cassandra
+  teamFeatures: cassandra
+  domainRegistration: cassandra
+  user: cassandra
+
 # Wire/AWS specific, optional
 # discoUrl: "https://"
 

services/spar/src/Spar/App.hs

@@ -38,6 +38,7 @@ where
 
 import Bilge
 import qualified Cassandra as Cas
+import qualified Hasql.Pool.Extended as HasqlPoolExt
 import Control.Exception (assert)
 import Control.Lens hiding ((.=))
 import Control.Monad.Trans.Maybe (MaybeT (..), runMaybeT)
@@ -127,7 +128,8 @@ data Env = Env
     sparCtxHttpBrig :: Bilge.Request,
     sparCtxHttpGalley :: Bilge.Request,
     sparCtxRequestId :: RequestId,
-    sparCtxScimSubsystemConfig :: ScimSubsystemConfig
+    sparCtxScimSubsystemConfig :: ScimSubsystemConfig,
+    sparCtxHasqlPool :: HasqlPoolExt.Pool
   }
 
 -- | Get a user by UserRef, no matter what the team.

services/spar/src/Spar/CanonicalInterpreter.hs

@@ -25,6 +25,7 @@ module Spar.CanonicalInterpreter
 where
 
 import qualified Cassandra as Cas
+import qualified Hasql.Pool.Extended as HasqlPoolExt
 import Control.Monad.Except hiding (mapError)
 import Imports
 import Polysemy
@@ -120,6 +121,7 @@ type LowerLevelCanonicalEffs =
      Logger (TinyLog.Msg -> TinyLog.Msg),
      Input Opts,
      Input TinyLog.Logger,
+    Input HasqlPoolExt.Pool,
      Random,
      Now,
      Embed IO,
@@ -132,6 +134,7 @@ runSparToIO ctx =
     . embedToFinal @IO
     . nowToIO
     . randomToIO
+    . runInputConst (sparCtxHasqlPool ctx)
     . runInputConst (sparCtxLogger ctx)
     . runInputConst (sparCtxOpts ctx)
     . loggerToTinyLog (sparCtxLogger ctx)

services/spar/src/Spar/Options.hs

@@ -35,6 +35,7 @@ import qualified Data.Set as Set
 import Data.Time
 import Data.X509.Extended (Fingerprint, parseFingerprintHex)
 import qualified Data.Yaml as Yaml
+import Hasql.Pool.Extended (PoolConfig)
 import Imports
 import Options.Applicative
 import SAML2.WebSSO
@@ -45,6 +46,7 @@ import Util.Options
 import Wire.API.Routes.Version
 import Wire.API.User.Orphans ()
 import Wire.API.User.Saml
+import Wire.PostgresMigrationOpts (PostgresMigrationOpts)
 
 data Opts = Opts
   { saml :: !SAML.Config,
@@ -70,7 +72,16 @@ data Opts = Opts
     -- disables the check.  When set, every cert in 'edCertAuthnResponse'
     -- (on create/update and on AuthnResponse) must be listed or the request
     -- is rejected with 'SparIdPCertNotAllowed' (HTTP 403).
-    idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist)
+    idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist),
+    -- | Postgres connection parameters (libpq key/value map).
+    postgresql :: !(Map Text Text),
+    -- | Postgres connection-pool sizing.
+    postgresqlPool :: !PoolConfig,
+    -- | Per-store Cassandra/Postgres storage-location flags (shared type;
+    --   Spar-relevant stores are added by their migration tasks).
+    postgresMigration :: !PostgresMigrationOpts,
+    -- | Optional path to a Postgres password secrets file (prod only).
+    postgresqlPassword :: !(Maybe FilePathSecrets)
   }
   deriving (Show, Generic)
 

services/spar/src/Spar/Run.hs

@@ -31,6 +31,7 @@ where
 import qualified Bilge
 import Cassandra as Cas
 import Cassandra.Util (initCassandraForService)
+import Hasql.Pool.Extended (initPostgresPool, rawPool)
 import Control.Exception (ErrorCall (ErrorCall), throwIO)
 import Control.Lens (to, (^.))
 import qualified Data.ByteString.UTF8 as UTF8
@@ -60,6 +61,7 @@ import qualified Web.Scim.Schema.Common as Scim
 import Wire.API.Routes.Version (expandVersionExp)
 import Wire.API.Routes.Version.Wai
 import Wire.ScimSubsystem.Interpreter
+import Wire.PostgresMigrations (runAllMigrations)
 
 ----------------------------------------------------------------------
 -- cassandra
@@ -82,6 +84,7 @@ runServer sparCtxOpts = do
       sport :: Int = sparCtxOpts ^. to saml . SAML.cfgSPPort
   (wrappedApp, ctxOpts) <- mkApp sparCtxOpts
   let logger = sparCtxLogger ctxOpts
+  runAllMigrations ctxOpts.sparCtxHasqlPool.rawPool logger
   let settings = newSettings $ defaultServer shost (fromIntegral sport) logger
   WU.runSettingsWithShutdown settings wrappedApp Nothing
 
@@ -90,6 +93,7 @@ mkApp sparCtxOpts = do
   let logLevel = saml sparCtxOpts ^. SAML.cfgLogLevel
   sparCtxLogger <- Log.mkLogger logLevel (logNetStrings sparCtxOpts) (logFormat sparCtxOpts)
   sparCtxCas <- initCassandra sparCtxOpts sparCtxLogger
+  sparCtxHasqlPool <- initPostgresPool (Opt.postgresqlPool sparCtxOpts) (Opt.postgresql sparCtxOpts) (Opt.postgresqlPassword sparCtxOpts)
   sparCtxHttpManager <- Bilge.newManager Bilge.defaultManagerSettings
   let sparCtxHttpBrig =
         Bilge.host (sparCtxOpts ^. to brig . to host . to encodeUtf8)

services/spar/test-integration/Util/Core.hs

@@ -277,6 +277,7 @@ mkEnv tstOpts opts = do
       sparCtxHttpGalley = galley empty
       sparCtxRequestId = RequestId "<fake request id>"
       sparCtxScimSubsystemConfig = error "mkEnv: implement sparCtxScimSubsystemConfig when needed"
+      sparCtxHasqlPool = error "mkEnv: sparCtxHasqlPool is not available in integration tests"
   pure $
     TestEnv
       mgr