diff --git a/changes/.gitkeep b/changes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/changes/394.changed b/changes/394.changed new file mode 100644 index 00000000..bc016a18 --- /dev/null +++ b/changes/394.changed @@ -0,0 +1 @@ +- BREAKING: remove `type` from `datamanager` table ([https://github.com/Substra/orchestrator/pull/394](https://github.com/Substra/orchestrator/pull/394)) \ No newline at end of file diff --git a/docs/schemas/standalone-database.svg b/docs/schemas/standalone-database.svg index c5d16f88..3eed7ca5 100644 --- a/docs/schemas/standalone-database.svg +++ b/docs/schemas/standalone-database.svg @@ -4,11 +4,11 @@ - + postgres - + public.schema_migrations @@ -155,7 +155,7 @@ FOREIGN KEY (description) REFERENCES addressables(storage_address) - + public.function_statuses @@ -176,65 +176,62 @@ public.datamanagers - - -public.datamanagers -     -[BASE TABLE] - -key -[uuid] - -channel -[varchar(100)] - -name -[varchar(100)] - -owner -[varchar(100)] - -permissions -[jsonb] - -description -[varchar(200)] - -opener -[varchar(200)] - -type -[varchar(30)] - -creation_date -[timestamp with time zone] - -logs_permission -[jsonb] - -metadata -[jsonb] + + +public.datamanagers +     +[BASE TABLE] + +key +[uuid] + +channel +[varchar(100)] + +name +[varchar(100)] + +owner +[varchar(100)] + +permissions +[jsonb] + +description +[varchar(200)] + +opener +[varchar(200)] + +creation_date +[timestamp with time zone] + +logs_permission +[jsonb] + +metadata +[jsonb] public.datamanagers:owner->public.organizations:id - - -FOREIGN KEY (owner, channel) REFERENCES organizations(id, channel) + + +FOREIGN KEY (owner, channel) REFERENCES organizations(id, channel) public.datamanagers:description->public.addressables:storage_address - - -FOREIGN KEY (description) REFERENCES addressables(storage_address) + + +FOREIGN KEY (description) REFERENCES addressables(storage_address) public.datamanagers:opener->public.addressables:storage_address - - -FOREIGN KEY (opener) REFERENCES addressables(storage_address) + + +FOREIGN KEY (opener) REFERENCES addressables(storage_address) @@ -444,7 +441,7 @@ [varchar(100)] - + public.compute_task_outputs @@ -508,7 +505,7 @@ [bigint] - + public.asset_kinds @@ -527,7 +524,7 @@ FOREIGN KEY (asset_kind) REFERENCES asset_kinds(kind) - + public.event_kinds @@ -609,7 +606,7 @@ FOREIGN KEY (error_type) REFERENCES error_types(error_type) - + public.failed_asset_kinds @@ -659,56 +656,8 @@ FOREIGN KEY (parent_task_key) REFERENCES compute_tasks(key) - - -public.expanded_datamanagers - - -public.expanded_datamanagers -     -[VIEW] - -key -[uuid] - -name -[varchar(100)] - -owner -[varchar(100)] - -channel -[varchar(100)] - -permissions -[jsonb] - -description_address -[varchar(200)] - -description_checksum -[varchar(64)] - -opener_address -[varchar(200)] - -opener_checksum -[varchar(64)] - -type -[varchar(30)] - -creation_date -[timestamp with time zone] - -logs_permission -[jsonb] - -metadata -[jsonb] - - + public.datasample_datamanagers @@ -732,12 +681,12 @@ public.datasample_datamanagers:datamanager_key->public.datamanagers:key - - + + FOREIGN KEY (datamanager_key) REFERENCES datamanagers(key) - + public.function_inputs @@ -775,7 +724,7 @@ FOREIGN KEY (kind) REFERENCES asset_kinds(kind) - + public.function_outputs @@ -810,7 +759,7 @@ FOREIGN KEY (kind) REFERENCES asset_kinds(kind) - + public.compute_task_inputs @@ -858,7 +807,7 @@ FOREIGN KEY (compute_task_key) REFERENCES compute_tasks(key) - + public.compute_task_output_assets @@ -884,8 +833,8 @@ public.compute_task_output_assets:asset_kind->public.asset_kinds:kind - - + + FOREIGN KEY (asset_kind) REFERENCES asset_kinds(kind) @@ -896,148 +845,193 @@ FOREIGN KEY (compute_task_key, compute_task_output_identifier) REFERENCES compute_task_outputs(compute_task_key, identifier) - + public.expanded_models - - -public.expanded_models -     -[VIEW] - -key -[uuid] - -compute_task_key -[uuid] - -address -[varchar(200)] - -checksum -[varchar(64)] - -permissions -[jsonb] - -owner -[varchar(100)] - -channel -[varchar(100)] - -creation_date -[timestamp with time zone] + + +public.expanded_models +     +[VIEW] + +key +[uuid] + +compute_task_key +[uuid] + +address +[varchar(200)] + +checksum +[varchar(64)] + +permissions +[jsonb] + +owner +[varchar(100)] + +channel +[varchar(100)] + +creation_date +[timestamp with time zone] - + public.expanded_datasamples - - -public.expanded_datasamples -     -[VIEW] - -key -[uuid] - -owner -[varchar(100)] - -channel -[varchar(100)] - -checksum -[varchar(64)] - -creation_date -[timestamp with time zone] - -datamanager_keys -[jsonb] + + +public.expanded_datasamples +     +[VIEW] + +key +[uuid] + +owner +[varchar(100)] + +channel +[varchar(100)] + +checksum +[varchar(64)] + +creation_date +[timestamp with time zone] + +datamanager_keys +[jsonb] - + public.expanded_failure_reports - - -public.expanded_failure_reports -     -[VIEW] - -asset_key -[uuid] - -asset_type -[varchar(100)] - -error_type -[varchar(50)] - -logs_address -[varchar(200)] - -logs_checksum -[varchar(64)] - + + +public.expanded_failure_reports +     +[VIEW] + +asset_key +[uuid] + +asset_type +[varchar(100)] + +error_type +[varchar(50)] + +logs_address +[varchar(200)] + +logs_checksum +[varchar(64)] + +creation_date +[timestamp with time zone] + +owner +[varchar(100)] + +channel +[varchar(100)] + + + +public.expanded_functions + + +public.expanded_functions +     +[VIEW] + +key +[uuid] + +name +[varchar(100)] + +description_address +[varchar(200)] + +description_checksum +[varchar(64)] + +archive_address +[varchar(200)] + +archive_checksum +[varchar(64)] + +permissions +[jsonb] + +owner +[varchar(100)] + creation_date [timestamp with time zone] - -owner -[varchar(100)] - + +metadata +[jsonb] + channel [varchar(100)] + +status +[varchar(100)] + +image_address +[varchar(200)] + +image_checksum +[varchar(64)] - + -public.expanded_functions - - -public.expanded_functions -     -[VIEW] - -key -[uuid] - -name -[varchar(100)] - -description_address -[varchar(200)] - -description_checksum -[varchar(64)] - -archive_address -[varchar(200)] - -archive_checksum -[varchar(64)] - -permissions -[jsonb] - -owner -[varchar(100)] - -creation_date -[timestamp with time zone] - -metadata -[jsonb] - -channel -[varchar(100)] - -status -[varchar(100)] - -image_address -[varchar(200)] - -image_checksum -[varchar(64)] +public.expanded_datamanagers + + +public.expanded_datamanagers +     +[VIEW] + +key +[uuid] + +name +[varchar(100)] + +owner +[varchar(100)] + +channel +[varchar(100)] + +permissions +[jsonb] + +description_address +[varchar(200)] + +description_checksum +[varchar(64)] + +opener_address +[varchar(200)] + +opener_checksum +[varchar(64)] + +creation_date +[timestamp with time zone] + +logs_permission +[jsonb] + +metadata +[jsonb] diff --git a/e2e/client/client.go b/e2e/client/client.go index 73fb757f..07d8bcfa 100644 --- a/e2e/client/client.go +++ b/e2e/client/client.go @@ -217,7 +217,6 @@ func (c *TestClient) RegisterDataManager(o *DataManagerOptions) *asset.DataManag Checksum: "1d55e9c55fa7ad6b6a49ad79da897d58be7ce8b76f92ced4c20f361ba3a0af6e", StorageAddress: "http://somewhere.local/opener" + uuid.NewString(), }, - Type: "test", LogsPermission: o.LogsPermission, } c.logger.Debug().Interface("datamanager", newDm).Msg("registering datamanager") diff --git a/lib/asset/datamanager_validation.go b/lib/asset/datamanager_validation.go index cdb6f780..e0a0905d 100644 --- a/lib/asset/datamanager_validation.go +++ b/lib/asset/datamanager_validation.go @@ -15,7 +15,6 @@ func (d *NewDataManager) Validate() error { validation.Field(&d.Description, validation.Required), validation.Field(&d.Opener, validation.Required), validation.Field(&d.Metadata, validation.By(validateMetadata)), - validation.Field(&d.Type, validation.Required, validation.Length(1, 30)), validation.Field(&d.LogsPermission, validation.Required), ) } diff --git a/lib/asset/datamanager_validation_test.go b/lib/asset/datamanager_validation_test.go index cf8b683c..dde569ea 100644 --- a/lib/asset/datamanager_validation_test.go +++ b/lib/asset/datamanager_validation_test.go @@ -35,7 +35,6 @@ func TestDataManagerValidate(t *testing.T) { NewPermissions: validPermissions, Description: validAddressable, Opener: validAddressable, - Type: "test", LogsPermission: validPermissions, }, false}, "valid": {&NewDataManager{ @@ -44,7 +43,6 @@ func TestDataManagerValidate(t *testing.T) { NewPermissions: validPermissions, Description: validAddressable, Opener: validAddressable, - Type: "test", LogsPermission: validPermissions, }, true}, } diff --git a/lib/service/datamanager.go b/lib/service/datamanager.go index 9f6ad9e3..3c828540 100644 --- a/lib/service/datamanager.go +++ b/lib/service/datamanager.go @@ -74,7 +74,6 @@ func (s *DataManagerService) RegisterDataManager(d *asset.NewDataManager, owner Description: d.GetDescription(), Opener: d.GetOpener(), Metadata: d.GetMetadata(), - Type: d.GetType(), CreationDate: timestamppb.New(s.GetTimeService().GetTransactionTime()), } diff --git a/lib/service/datamanager_test.go b/lib/service/datamanager_test.go index 12acfea8..1f7133b4 100644 --- a/lib/service/datamanager_test.go +++ b/lib/service/datamanager_test.go @@ -50,7 +50,6 @@ func TestRegisterDataManager(t *testing.T) { Name: "test datamanager", Description: description, Opener: opener, - Type: "test dm", NewPermissions: newPerms, LogsPermission: newLogsPerm, } @@ -62,7 +61,6 @@ func TestRegisterDataManager(t *testing.T) { Permissions: perms, Description: description, Opener: opener, - Type: "test dm", CreationDate: timestamppb.New(time.Unix(1337, 0)), LogsPermission: logsPerm, } diff --git a/server/standalone/dbal/datamanager.go b/server/standalone/dbal/datamanager.go index bdb2e8a1..b26ff76f 100644 --- a/server/standalone/dbal/datamanager.go +++ b/server/standalone/dbal/datamanager.go @@ -20,7 +20,6 @@ type sqlDataManager struct { Permissions asset.Permissions Description asset.Addressable Opener asset.Addressable - Type string CreationDate time.Time LogsPermission asset.Permission Metadata map[string]string @@ -34,7 +33,6 @@ func (dm *sqlDataManager) toDataManager() *asset.DataManager { Permissions: &dm.Permissions, Description: &dm.Description, Opener: &dm.Opener, - Type: dm.Type, CreationDate: timestamppb.New(dm.CreationDate), LogsPermission: &dm.LogsPermission, Metadata: dm.Metadata, @@ -55,8 +53,8 @@ func (d *DBAL) AddDataManager(datamanager *asset.DataManager) error { stmt := getStatementBuilder(). Insert("datamanagers"). - Columns("key", "channel", "name", "owner", "permissions", "description", "opener", "type", "creation_date", "logs_permission", "metadata"). - Values(datamanager.Key, d.channel, datamanager.Name, datamanager.Owner, datamanager.Permissions, datamanager.Description.StorageAddress, datamanager.Opener.StorageAddress, datamanager.Type, datamanager.CreationDate.AsTime(), datamanager.LogsPermission, datamanager.Metadata) + Columns("key", "channel", "name", "owner", "permissions", "description", "opener", "creation_date", "logs_permission", "metadata"). + Values(datamanager.Key, d.channel, datamanager.Name, datamanager.Owner, datamanager.Permissions, datamanager.Description.StorageAddress, datamanager.Opener.StorageAddress, datamanager.CreationDate.AsTime(), datamanager.LogsPermission, datamanager.Metadata) return d.exec(stmt) } @@ -82,7 +80,7 @@ func (d *DBAL) DataManagerExists(key string) (bool, error) { // GetDataManager implements persistence.DataManagerDBAL func (d *DBAL) GetDataManager(key string) (*asset.DataManager, error) { stmt := getStatementBuilder(). - Select("key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "type", "creation_date", "logs_permission", "metadata"). + Select("key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "creation_date", "logs_permission", "metadata"). From("expanded_datamanagers"). Where(sq.Eq{"channel": d.channel, "key": key}) @@ -92,7 +90,7 @@ func (d *DBAL) GetDataManager(key string) (*asset.DataManager, error) { } dm := new(sqlDataManager) - err = row.Scan(&dm.Key, &dm.Name, &dm.Owner, &dm.Permissions, &dm.Description.StorageAddress, &dm.Description.Checksum, &dm.Opener.StorageAddress, &dm.Opener.Checksum, &dm.Type, &dm.CreationDate, &dm.LogsPermission, &dm.Metadata) + err = row.Scan(&dm.Key, &dm.Name, &dm.Owner, &dm.Permissions, &dm.Description.StorageAddress, &dm.Description.Checksum, &dm.Opener.StorageAddress, &dm.Opener.Checksum, &dm.CreationDate, &dm.LogsPermission, &dm.Metadata) if err != nil { if errors.Is(err, pgx.ErrNoRows) { @@ -112,7 +110,7 @@ func (d *DBAL) QueryDataManagers(p *common.Pagination) ([]*asset.DataManager, co } stmt := getStatementBuilder(). - Select("key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "type", "creation_date", "logs_permission", "metadata"). + Select("key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "creation_date", "logs_permission", "metadata"). From("expanded_datamanagers"). Where(sq.Eq{"channel": d.channel}). OrderByClause("creation_date ASC, key"). @@ -132,7 +130,7 @@ func (d *DBAL) QueryDataManagers(p *common.Pagination) ([]*asset.DataManager, co for rows.Next() { dm := new(sqlDataManager) - err = rows.Scan(&dm.Key, &dm.Name, &dm.Owner, &dm.Permissions, &dm.Description.StorageAddress, &dm.Description.Checksum, &dm.Opener.StorageAddress, &dm.Opener.Checksum, &dm.Type, &dm.CreationDate, &dm.LogsPermission, &dm.Metadata) + err = rows.Scan(&dm.Key, &dm.Name, &dm.Owner, &dm.Permissions, &dm.Description.StorageAddress, &dm.Description.Checksum, &dm.Opener.StorageAddress, &dm.Opener.Checksum, &dm.CreationDate, &dm.LogsPermission, &dm.Metadata) if err != nil { return nil, "", err } diff --git a/server/standalone/dbal/datamanager_test.go b/server/standalone/dbal/datamanager_test.go index 98a49e7f..84b0e9eb 100644 --- a/server/standalone/dbal/datamanager_test.go +++ b/server/standalone/dbal/datamanager_test.go @@ -12,9 +12,9 @@ import ( ) func makeDataManagerRows() *pgxmock.Rows { - return pgxmock.NewRows([]string{"key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "type", "creation_date", "logs_permission", "metadata"}). - AddRow("key1", "name", "owner", []byte("{}"), "https://example.com/desc", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "https://example.com/chksm", "993b6d90e0ed15d80e7e39c6fb298855d9544420be07faec52935649780e8f19", "", time.Unix(12, 0), []byte("{}"), map[string]string{}). - AddRow("key2", "name", "owner", []byte("{}"), "https://example.com/desc", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "https://example.com/chksm", "993b6d90e0ed15d80e7e39c6fb298855d9544420be07faec52935649780e8f19", "", time.Unix(12, 0), []byte("{}"), map[string]string{}) + return pgxmock.NewRows([]string{"key", "name", "owner", "permissions", "description_address", "description_checksum", "opener_address", "opener_checksum", "creation_date", "logs_permission", "metadata"}). + AddRow("key1", "name", "owner", []byte("{}"), "https://example.com/desc", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "https://example.com/chksm", "993b6d90e0ed15d80e7e39c6fb298855d9544420be07faec52935649780e8f19", time.Unix(12, 0), []byte("{}"), map[string]string{}). + AddRow("key2", "name", "owner", []byte("{}"), "https://example.com/desc", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "https://example.com/chksm", "993b6d90e0ed15d80e7e39c6fb298855d9544420be07faec52935649780e8f19", time.Unix(12, 0), []byte("{}"), map[string]string{}) } func TestQueryDataManagers(t *testing.T) { diff --git a/server/standalone/migration/000063_delete_type_from_datamanager.up.sql b/server/standalone/migration/000063_delete_type_from_datamanager.up.sql new file mode 100644 index 00000000..8a7b1094 --- /dev/null +++ b/server/standalone/migration/000063_delete_type_from_datamanager.up.sql @@ -0,0 +1,26 @@ +SELECT execute($$ + DROP VIEW IF EXISTS expanded_datamanagers; + CREATE VIEW expanded_datamanagers AS + SELECT key, + name, + owner, + channel, + permissions, + description AS description_address, + desc_add.checksum AS description_checksum, + opener AS opener_address, + opener_add.checksum AS opener_checksum, + creation_date, + logs_permission, + metadata + FROM datamanagers + JOIN addressables desc_add ON datamanagers.description = desc_add.storage_address + JOIN addressables opener_add ON datamanagers.opener = opener_add.storage_address; + + ALTER TABLE datamanagers + DROP COLUMN type; + + UPDATE events SET asset = asset - 'type' + WHERE asset_kind = 'ASSET_DATA_MANAGER'; + +$$) WHERE column_exists('public', 'datamanagers', 'type'); \ No newline at end of file