Merge test to stable (#639) (#640)

* CSCFAIRMETA-647: [FIX] bug with almost simultaneous requests

- LightFileserializer made modifications to class variable
  so when two requests came almost at the same time the latter
  was using modified version of the variable. Fixed with deepcopy
- Changed app to use local Elasticsearch with tests since it is now
  working

* CSCFAIRMETA-710: [FIX] _mapping endpoint and ES in testcases

- Correctly call elasticsearch when request comes to _mapping endpoint
- Tests can be run against local elasticsearch which reduces the
  population time
- Follow naming convention in refdata loader with non-internal
  functions

* CSCFAIRMETA-710: [FIX] remove unnecessary variable

* CSCFAIRADM-337: [ADD] Update domain references (#631)

- These mostly concern test data, but should be updated still

* CSCFAIRMETA-704 [FIX] merge-draft-does-not-save-changes
- In merge_draft take preservation_state from draft cr

* CSCFAIRMETA-704-merge-draft-does-not-save-changes
- Change metax-demo to metax.demo

* CSCFAIRMETA-716: [FIX] create-draft-internal-error-when-draft-already-exists
- Add a check for next_draft

* CSCFAIRMETA-712-Issues-with-emails
-Add migration to change email and organization name

* CSCFAIRMETA-712-Issues-with-emails
- Add check whether metadata_owner_org or metadata_provider_org exists

* CSCFAIRMETA-636-return-if-files-are-part-of-some-dataset
- Add ?keys=datasets|files and ?keysonly=bool
- Remove parameter ?detailed as it is the same as ?keys=files

* CSCFAIRMETA-636-return-if-files-are-part-of-some-dataset
- Add test for parameter ?keysonly

* CSCFAIRMETA-636-return-if-files-are-part-of-some-dataset
- Add support for leaving argument ?detailed

* CSCFAIRMETA-636-return-if-files-are-part-of-some-dataset
Add ?keysonly return input list except standalone files or datasets without files

* CSCFAIRMETA-636-return-if-files-are-part-of-some-dataset
- Refactor sql's
- Fix parameters 'request' and 'preferred_identifier'

Co-authored-by: Tommi Pulli <[email protected]>
Co-authored-by: Tommi Pulli <[email protected]>
Co-authored-by: Mattias Levlin <[email protected]>

Co-authored-by: Tommi Pulli <[email protected]>
Co-authored-by: Tommi Pulli <[email protected]>
Co-authored-by: Mattias Levlin <[email protected]>

.travis-deploy.sh

@@ -13,11 +13,11 @@ fi
 # if [[ "$TRAVIS_BRANCH" == "test" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
 #     echo "Deploying to test.."
 #     ansible-galaxy -r requirements.yml install --roles-path=roles
-#     ansible-playbook -vv -i inventories/test/hosts site_deploy.yml --extra-vars "ssh_user=metax-deploy-user server_domain_name=metax-test.csc.fi"
+#     ansible-playbook -vv -i inventories/test/hosts site_deploy.yml --extra-vars "ssh_user=metax-deploy-user server_domain_name=metax.fd-test.csc.fi"
 # elif [[ "$TRAVIS_BRANCH" == "stable" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
 #     echo "Deploying to stable.."
 #     ansible-galaxy -r requirements.yml install --roles-path=roles
-#     ansible-playbook -vv -i inventories/stable/hosts site_deploy.yml --extra-vars "ssh_user=metax-deploy-user server_domain_name=metax-stable.csc.fi"
+#     ansible-playbook -vv -i inventories/stable/hosts site_deploy.yml --extra-vars "ssh_user=metax-deploy-user server_domain_name=metax.fd-stable.csc.fi"
 # fi
 
 # # Make sure the last command to run before this part is the ansible-playbook command

src/metax_api/api/rest/base/schemas/att_dataset_schema.json

@@ -614,7 +614,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/base/schemas/harvester_dataset_schema.json

@@ -718,7 +718,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/base/schemas/ida_dataset_schema.json

@@ -762,7 +762,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/base/serializers/common_serializer.py

@@ -5,9 +5,10 @@
 # :author: CSC - IT Center for Science Ltd., Espoo Finland <[email protected]>
 # :license: MIT
 
-import logging
+from copy import deepcopy
 from collections import OrderedDict
 from datetime import datetime
+import logging
 
 from django.db import transaction
 from django.db.models.query import QuerySet
@@ -315,7 +316,8 @@ def ls_field_list(cls, received_field_list=[]):
 
         else:
             # get all fields
-            field_list = cls.allowed_fields
+            # deepcopy prevents inheriting classes from modifying the allowed_fields -set which should be static
+            field_list = deepcopy(cls.allowed_fields)
 
         return field_list
 

src/metax_api/api/rest/base/views/file_view.py

@@ -166,18 +166,36 @@ def partial_update_bulk(self, request, *args, **kwargs):
     @action(detail=False, methods=['post'], url_path="datasets")
     def datasets(self, request):
         """
-        Find out which datasets a list of files belongs to, and return their
+        keys=files: Find out which datasets a list of files belongs to, and return their
         metadata_version_identifiers as a list.
 
+        keys=datasets: Find out which files belong to a list of datasets, and return their
+        identifiers as a list.
+
+        keysonly: return the same list as input, but only with files that belong to some dataset /
+        only with datasets that have files
+
         The method is invoked using POST, because there are limits to length of query
         parameters in GET. Also, some clients forcibly shove parameters in body in GET
         requests to query parameters, so using POST instead is more guaranteed to work.
         """
 
-        if CommonService.get_boolean_query_param(request, 'detailed'):
-            return FileService.get_detailed_datasets_where_file_belongs_to(request.data)
+        keysonly = CommonService.get_boolean_query_param(request, 'keysonly')
+        detailed = CommonService.get_boolean_query_param(request, 'detailed')
+
+        params = request.query_params
+
+        if not params.keys():
+            return FileService.get_identifiers(request.data, 'noparams', True)
+
+        if 'keys' in params.keys():
+            if params['keys'] in ['files', 'datasets']:
+                return FileService.get_identifiers(request.data, params['keys'], keysonly)
+
+        if detailed: # This can be removed as soon as front can listen to ?keys=files which returns the same
+            return FileService.get_identifiers(request.data, 'files', False)
 
-        return FileService.get_datasets_where_file_belongs_to(request.data)
+        raise Http403({ 'detail': [ 'Invalid parameters' ]})
 
     @action(detail=False, methods=['post'], url_path="restore")
     def restore_files(self, request):

src/metax_api/api/rest/v2/api_schemas/ida_dataset_schema.json

@@ -761,7 +761,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/v2/schemas/att_dataset_schema.json

@@ -561,7 +561,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/v2/schemas/harvester_dataset_schema.json

@@ -665,7 +665,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rest/v2/schemas/ida_dataset_schema.json

@@ -709,7 +709,7 @@
                 "place_uri":{
                     "@id":"http://purl.org/dc/terms/subject",
                     "title":"Place uri",
-                    "description":"URI for the place. Use reference data from: https://metax-test.csc.fi/es/reference_data/location/_search?pretty=true",
+                    "description":"URI for the place. Use reference data from: https://metax.fd-test.csc.fi/es/reference_data/location/_search?pretty=true",
                     "@type":"@id",
                     "type":"object",
                     "$ref":"#/definitions/Concept"

src/metax_api/api/rpc/base/views/elasticsearch_rpc.py

@@ -27,12 +27,21 @@ def map_refdata(self, request):
         if not isinstance(django_settings, dict):
             settings = django_settings.ELASTICSEARCH
 
-        connection_params = RDL._get_connection_parameters(settings)
-        esclient, scan = RDL._get_es_imports(settings['HOSTS'], connection_params)
+        connection_params = RDL.get_connection_parameters(settings)
+        # returns scan object as well but that is not needed here
+        esclient = RDL.get_es_imports(settings['HOSTS'], connection_params)[0]
 
         if not self.request.query_params:
             return Response(status=status.HTTP_200_OK)
 
+        elif '_mapping' in self.request.query_params:
+            try:
+                res = esclient.indices.get_mapping()
+            except Exception as e:
+                raise Http400(f'Error when accessing elasticsearch. {e}')
+
+            return Response(data=res, status=status.HTTP_200_OK)
+
         params = {}
         for k, v in self.request.query_params.items():
             # python dict.items() keeps order so the url is always the first one

src/metax_api/migrations/0026_auto_20200917_1615.py

@@ -0,0 +1,88 @@
+# Generated by Django 3.0.7 on 2020-09-17 12:04
+
+from django.db import migrations
+import logging
+
+_logger = logging.getLogger(__name__)
+
+def change_emails(apps, schema_editor):
+    import json
+    CatalogRecord = apps.get_model('metax_api', 'CatalogRecord')
+
+    for cr in CatalogRecord.objects.all():
+        owner = json.loads(json.dumps(cr.metadata_owner_org))
+        provider = json.loads(json.dumps(cr.metadata_provider_org))
+
+        cr_changed = False
+
+        owner_data = ''
+        prov_data = ''
+        if owner:
+            if 'tut.fi' in owner:
+                owner_data = owner.replace("tut.fi", "tuni.fi")
+                cr_changed = True
+            elif 'uta.fi' in owner:
+                owner_data = owner.replace("uta.fi", "tuni.fi")
+                cr_changed = True
+        if provider:
+            if 'tut.fi' in provider:
+                prov_data = provider.replace("tut.fi", "tuni.fi")
+                cr_changed = True
+            elif 'uta.fi' in provider:
+                prov_data = provider.replace("uta.fi", "tuni.fi")
+                cr_changed = True
+
+        if cr_changed:
+            cr.metadata_owner_org = owner_data
+            cr.metadata_provider_org = prov_data
+            cr.save()
+
+        # Change actor emails in research dataset
+        research_dataset_changed = False
+
+        dataset = json.loads(json.dumps(cr.research_dataset))
+
+        roles = ['curator', 'creator', 'contributor', 'publisher', 'rights_holder']
+
+        for role in roles:
+            if role in dataset:
+                for actor in dataset[role]:
+                    email = ''
+                    if isinstance(actor, str):
+                        if 'email' in dataset[role].keys():
+                            email = dataset[role]['email']
+                    if isinstance(actor, dict):
+                        if 'email' in actor.keys():
+                            email = actor['email']
+
+                    changed = False
+                    if 'tut.fi' in email:
+                        email = email.replace("tut.fi", "tuni.fi")
+                        changed = True
+                    elif 'uta.fi' in email:
+                        email = email.replace("uta.fi", "tuni.fi")
+                        changed = True
+
+                    if changed:
+                        research_dataset_changed = True
+                        if isinstance(actor, str):
+                            dataset[role]['email'] = email
+                        else:
+                            actor['email'] = email
+        if research_dataset_changed:
+            cr.research_dataset = dataset
+            cr.save()
+
+
+def revert(apps, schema_editor): 
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('metax_api', '0025_auto_20200811_1050'),
+    ]
+
+    operations = [
+        migrations.RunPython(change_emails, revert),
+    ]

src/metax_api/models/catalog_record_v2.py

@@ -303,6 +303,7 @@ def merge_draft(self):
         origin_cr = CatalogRecordV2.objects.get(next_draft_id=self.id)
         draft_cr = self
 
+        origin_cr.preservation_state = draft_cr.preservation_state
         origin_cr.date_modified = get_tz_aware_now_without_micros()
         origin_cr.user_modified = draft_cr.user_modified
 
@@ -1135,6 +1136,11 @@ def create_draft(self):
             # a new dataset in draft state
             raise Http400('Dataset is already draft.')
 
+        if self.next_draft:
+            raise Http400(
+                'The dataset already has an existing unmerged draft: {}'.format(self.next_draft.preferred_identifier)
+            )
+
         origin_cr = self
 
         draft_cr = origin_cr._create_new_dataset_version_template()