Cleanup old team feature storage

cassandra-schema.cql

@@ -1207,6 +1207,7 @@ CREATE TABLE galley_test.team_features (
     guest_links_status int,
     legalhold_status int,
     limited_event_fanout_status int,
+    migration_state int,
     mls_allowed_ciphersuites set<int>,
     mls_default_ciphersuite int,
     mls_default_protocol int,
@@ -1516,6 +1517,29 @@ CREATE TABLE galley_test.legalhold_service (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE galley_test.team_features_dyn (
+    team uuid,
+    feature text,
+    config text,
+    lock_status int,
+    status int,
+    PRIMARY KEY (team, feature)
+) WITH CLUSTERING ORDER BY (feature ASC)
+    AND bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE galley_test.conversation_codes (
     key ascii,
     scope int,

changelog.d/0-release-notes/simplify-feature-table

@@ -0,0 +1 @@
+This release introduces a new data storage format for team features and a corresponding migration. While the migration is running, team features are going to operate in read-only mode for the team that is currently being migrated. After migration, the new storage is going to be used. No special action should be required on the part of instance operators.

changelog.d/5-internal/cleanup-old-features

@@ -0,0 +1 @@
+Remove legacy team feature storage support

changelog.d/5-internal/simplify-feature-table

@@ -0,0 +1 @@
+Introduce a new feature table in Cassandra: `team_features_dyn`. This table has a fixed number of fields, as opposed to the ever-growing collection of all the fields of all the features that we were using before.

integration/test/API/GalleyInternal.hs

@@ -1,7 +1,9 @@
 module API.GalleyInternal where
 
+import qualified Data.Aeson as A
 import qualified Data.Aeson as Aeson
 import Data.String.Conversions (cs)
+import qualified Data.Text as T
 import qualified Data.Vector as Vector
 import GHC.Stack
 import Testlib.Prelude
@@ -136,3 +138,8 @@ getTeam :: (HasCallStack, MakesValue domain) => domain -> String -> App Response
 getTeam domain tid = do
   req <- baseRequest domain Galley Unversioned $ joinHttpPath ["i", "teams", tid]
   submit "GET" $ req
+
+setTeamFeatureMigrationState :: (HasCallStack, MakesValue domain) => domain -> String -> String -> App Response
+setTeamFeatureMigrationState domain tid state = do
+  req <- baseRequest domain Galley Unversioned $ joinHttpPath ["i", "teams", tid, "feature-migration-state"]
+  submit "PUT" $ req & addJSON (A.String (T.pack state))

libs/schema-profunctor/src/Data/Schema.hs

@@ -36,6 +36,7 @@ module Data.Schema
     schemaIn,
     schemaOut,
     HasDoc (..),
+    HasOpt (..),
     doc',
     HasSchemaRef (..),
     HasObject (..),
@@ -52,15 +53,12 @@ module Data.Schema
     objectOver,
     jsonObject,
     jsonValue,
-    FieldFunctor (..),
     field,
     fieldWithDocModifier,
-    fieldOver,
     optField,
+    optField',
     optFieldWithDocModifier,
-    fieldF,
-    fieldOverF,
-    fieldWithDocModifierF,
+    fieldOver,
     array,
     set,
     nonEmptyArray,
@@ -283,26 +281,14 @@ schemaIn (SchemaP _ (SchemaIn i) _) = i
 schemaOut :: SchemaP ss v m a b -> a -> Maybe m
 schemaOut (SchemaP _ _ (SchemaOut o)) = o
 
-class (Functor f) => FieldFunctor doc f where
-  parseFieldF :: (A.Value -> A.Parser a) -> A.Object -> Text -> A.Parser (f a)
-  mkDocF :: doc -> doc
-
-instance FieldFunctor doc Identity where
-  parseFieldF f obj key = Identity <$> A.explicitParseField f obj (Key.fromText key)
-  mkDocF = id
-
-instance (HasOpt doc) => FieldFunctor doc Maybe where
-  parseFieldF f obj key = A.explicitParseFieldMaybe f obj (Key.fromText key)
-  mkDocF = mkOpt
-
 -- | A schema for a one-field JSON object.
 field ::
   forall doc' doc a b.
   (HasField doc' doc) =>
   Text ->
   SchemaP doc' A.Value A.Value a b ->
   SchemaP doc A.Object [A.Pair] a b
-field = fieldOver id
+field name = fieldOver id name
 
 -- | A schema for a JSON object with a single optional field.
 optField ::
@@ -311,16 +297,24 @@ optField ::
   Text ->
   SchemaP doc' A.Value A.Value a b ->
   SchemaP doc A.Object [A.Pair] a (Maybe b)
-optField = fieldF
-
--- | Generalization of 'optField' with 'FieldFunctor'.
-fieldF ::
-  forall doc' doc f a b.
-  (HasField doc' doc, FieldFunctor doc f) =>
+optField =
+  fieldOverF
+    (\f obj key -> A.explicitParseFieldMaybe f obj (Key.fromText key))
+    mkOpt
+    id
+
+-- | Same as 'optField', but null values are parsed as normal values rather than 'Nothing'.
+optField' ::
+  forall doc doc' a b.
+  (HasOpt doc, HasField doc' doc) =>
   Text ->
   SchemaP doc' A.Value A.Value a b ->
-  SchemaP doc A.Object [A.Pair] a (f b)
-fieldF = fieldOverF id
+  SchemaP doc A.Object [A.Pair] a (Maybe b)
+optField' =
+  fieldOverF
+    (\f obj key -> A.explicitParseFieldMaybe' f obj (Key.fromText key))
+    mkOpt
+    id
 
 newtype Positive x y a = Positive {runPositive :: (a -> x) -> y}
   deriving (Functor)
@@ -334,15 +328,17 @@ newtype Positive x y a = Positive {runPositive :: (a -> x) -> y}
 -- See 'bind' for use cases.
 fieldOverF ::
   forall f doc' doc v v' a b.
-  (HasField doc' doc, FieldFunctor doc f) =>
+  (HasField doc' doc) =>
+  (forall x. (A.Value -> A.Parser x) -> A.Object -> Text -> A.Parser (f x)) ->
+  (doc -> doc) ->
   Lens v v' A.Object A.Value ->
   Text ->
   SchemaP doc' v' A.Value a b ->
   SchemaP doc v [A.Pair] a (f b)
-fieldOverF l name sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w)
+fieldOverF pf mkDoc l name sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w)
   where
     parseField :: A.Object -> Positive (A.Parser b) (A.Parser (f b)) A.Value
-    parseField obj = Positive $ \k -> parseFieldF @doc k obj name
+    parseField obj = Positive $ \k -> pf k obj name
 
     r :: v -> A.Parser (f b)
     r obj = runPositive (l parseField obj) (schemaIn sch)
@@ -351,7 +347,7 @@ fieldOverF l name sch = SchemaP (SchemaDoc s) (SchemaIn r) (SchemaOut w)
       v <- schemaOut sch x
       pure [Key.fromText name A..= v]
 
-    s = mkDocF @doc @f (mkField name (schemaDoc sch))
+    s = mkDoc (mkField name (schemaDoc sch))
 
 -- | Like 'fieldOverF', but specialised to the identity functor.
 fieldOver ::
@@ -361,7 +357,13 @@ fieldOver ::
   Text ->
   SchemaP doc' v' A.Value a b ->
   SchemaP doc v [A.Pair] a b
-fieldOver l name = fmap runIdentity . fieldOverF l name
+fieldOver l name =
+  fmap runIdentity
+    . fieldOverF
+      (\f obj key -> Identity <$> A.explicitParseField f obj (Key.fromText key))
+      id
+      l
+      name
 
 -- | Like 'field', but apply an arbitrary function to the
 -- documentation of the field.
@@ -385,17 +387,6 @@ optFieldWithDocModifier ::
   SchemaP doc A.Object [A.Pair] a (Maybe b)
 optFieldWithDocModifier name modify sch = optField @doc @doc' name (over doc modify sch)
 
--- | Like 'fieldF', but apply an arbitrary function to the
--- documentation of the field.
-fieldWithDocModifierF ::
-  forall doc' doc f a b.
-  (HasField doc' doc, FieldFunctor doc f) =>
-  Text ->
-  (doc' -> doc') ->
-  SchemaP doc' A.Value A.Value a b ->
-  SchemaP doc A.Object [A.Pair] a (f b)
-fieldWithDocModifierF name modify sch = fieldF @doc' @doc name (over doc modify sch)
-
 -- | Change the input type of a schema.
 (.=) :: (Profunctor p) => (a -> a') -> p a' b -> p a b
 (.=) = lmap

libs/wire-api/default.nix

@@ -10,6 +10,7 @@
 , asn1-encoding
 , async
 , attoparsec
+, barbies
 , base
 , base64-bytestring
 , binary
@@ -23,6 +24,7 @@
 , cereal
 , comonad
 , conduit
+, constraints
 , containers
 , cookie
 , crypton
@@ -68,6 +70,7 @@
 , pem
 , polysemy
 , process
+, profunctors
 , proto-lens
 , protobuf
 , QuickCheck
@@ -123,6 +126,7 @@ mkDerivation {
     aeson
     asn1-encoding
     attoparsec
+    barbies
     base
     base64-bytestring
     binary
@@ -135,6 +139,7 @@ mkDerivation {
     cereal
     comonad
     conduit
+    constraints
     containers
     cookie
     crypton
@@ -174,6 +179,7 @@ mkDerivation {
     openapi3
     pem
     polysemy
+    profunctors
     proto-lens
     protobuf
     QuickCheck