diff --git a/app/migration/import/authorization_request_events.rb b/app/migration/import/authorization_request_events.rb index 4da32b557..eab0bbac4 100644 --- a/app/migration/import/authorization_request_events.rb +++ b/app/migration/import/authorization_request_events.rb @@ -3,7 +3,7 @@ class Import::AuthorizationRequestEvents < Import::Base include UserLegacyUtils def extract(event_row) - authorization_request = AuthorizationRequest.find(event_row['enrollment_id']) + authorization_request = options[:authorization_request] || AuthorizationRequest.find(event_row['enrollment_id']) case event_row['name'] when 'create' @@ -50,6 +50,8 @@ def extract(event_row) authorization_request.update!(last_submitted_at: event_created_at) end when 'approve', 'validate' + return if options[:create_from_authorization_request_import].blank? && authorization_request.type == 'AuthorizationRequest::FranceConnect' + authorization_request.update( last_validated_at: event_row['created_at'], ) @@ -156,8 +158,11 @@ def create_authorization(event_row, authorization_request) end def find_closest_authorization(event_row, authorization_request) + event_created_at_as_datetime = DateTime.parse(event_row['created_at']) + Authorization.where( - request_id: authorization_request.id + request_id: authorization_request.id, + created_at: (event_created_at_as_datetime-10.seconds..event_created_at_as_datetime+10.seconds), ).order(created_at: :desc).first end diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 6fbeec0c5..2cea65c52 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -16,10 +16,12 @@ def extract(enrollment_row) authorization_request.applicant = user authorization_request.organization_id = fetch_organization(user, enrollment_row).try(:id) - authorization_request.form_uid = fetch_form(authorization_request).id + authorization_request.form_uid ||= fetch_form(authorization_request).try(:id) authorization_request.state = enrollment_row['status'] authorization_request.external_provider_id = enrollment_row['external_provider_id'] - authorization_request.copied_from_request = AuthorizationRequest.find(enrollment_row['copied_from_enrollment_id']) if enrollment_row['copied_from_enrollment_id'] && AuthorizationRequest.exists?(enrollment_row['copied_from_enrollment_id']) + authorization_request.last_validated_at = enrollment_row['last_validated_at'] + # FIXME to delete, will use raw data anyway + # authorization_request.copied_from_request = AuthorizationRequest.find(enrollment_row['copied_from_enrollment_id']) if enrollment_row['copied_from_enrollment_id'] && AuthorizationRequest.exists?(enrollment_row['copied_from_enrollment_id']) if authorization_request.state != 'draft' authorization_request.assign_attributes( @@ -28,23 +30,35 @@ def extract(enrollment_row) ) end - handle_authorization_request_type_specific_fields(authorization_request, enrollment_row) + authorization_request = handle_authorization_request_type_specific_fields(authorization_request, enrollment_row) begin authorization_request.save! + after_save_authorization_request(authorization_request, enrollment_row) @models << authorization_request rescue ActiveRecord::RecordInvalid => e - log("DataPass: https://datapass.api.gouv.fr/#{enrollment_row['target_api'].gsub('_', '-')}/#{enrollment_row['id']}") + log("DataPass: https://datapass.api.gouv.fr/#{enrollment_row['target_api'].gsub('_', '-')}/#{enrollment_row['id']} (status: #{enrollment_row['status']})") log("Errors: #{authorization_request.errors.full_messages.join("\n")}") byebug rescue ActiveStorage::IntegrityError => e byebug + ensure + ($dummy_files || {}).each do |key, value| + value.close + end + $dummy_files = nil end end private + def after_save_authorization_request(authorization_request, enrollment_row) + return unless @authorization_request_type_specific_field_builder.respond_to?(:after_save) + + @authorization_request_type_specific_field_builder.after_save(authorization_request, enrollment_row) + end + def sql_tables_to_save super.concat( %w[ @@ -86,21 +100,37 @@ def fetch_organization(user, enrollment_row) end new_potential_siret = { - # PM => DINUM - '11000101300017' => '13002526500013', - # Agence de la recherche fermée - '13000250400020' => '13000250400038', - # DRJSCS => DREETS - '13001252900017' => '13002921800018', - # Recia - '18450311800020' => '12002503600035', - # Port de strasbourg - '77564141800014' => '77564141800089', + # # PM => DINUM + # '11000101300017' => '13002526500013', + # # Agence de la recherche fermée + # '13000250400020' => '13000250400038', + # # DRJSCS => DREETS + # '13001252900017' => '13002921800018', + # # Recia + # '18450311800020' => '12002503600035', + # # Port de strasbourg + # '77564141800014' => '77564141800089', }[enrollment_row['siret']] - return if new_potential_siret.blank? + if new_potential_siret.present? + user.organizations.find_by(siret: new_potential_siret) + else + organization = build_organization(enrollment_row['siret']) + + begin + organization.save! + rescue ActiveRecord::RecordInvalid => e + if e.record.errors.include?(:siret) + raise Import::AuthorizationRequests::Base::SkipRow.new(:invalid_siret_for_unknown_user_and_organization, id: enrollment_row['id'], target_api: enrollment_row['target_api'], authorization_request: e.record) + else + raise + end + end + + user.organizations << organization - user.organizations.find_by(siret: new_potential_siret) + organization + end end def authorization_ids_where_user_belongs_to_organization @@ -138,16 +168,13 @@ def try_to_create_user!(enrollment_row, demandeur) ) ) - organization = Organization.find_or_initialize_by(siret: enrollment_row['siret']) - organization.assign_attributes( - mon_compte_pro_payload: { siret: enrollment_row['siret'], manual_creation: true }, - last_mon_compte_pro_updated_at: DateTime.now, - ) + organization = build_organization(enrollment_row['siret']) + begin organization.save! rescue ActiveRecord::RecordInvalid => e if e.record.errors.include?(:siret) - raise Import::AuthorizationRequests::Base::SkipRow.new(:invalid_siret_for_unknown_user_and_organization, id: enrollment_row['id'], target_api: enrollment_row['target_api']) + raise Import::AuthorizationRequests::Base::SkipRow.new(:invalid_siret_for_unknown_user_and_organization, id: enrollment_row['id'], target_api: enrollment_row['target_api'], authorization_request: e.record) else raise end @@ -161,6 +188,19 @@ def try_to_create_user!(enrollment_row, demandeur) user end + def build_organization(siret) + organization = Organization.find_or_initialize_by(siret: siret) + + return organization if organization.persisted? + + organization.assign_attributes( + mon_compte_pro_payload: { siret:, manual_creation: true }, + last_mon_compte_pro_updated_at: DateTime.now, + ) + + organization + end + def fetch_form(authorization_request) if authorization_request.definition.available_forms.one? authorization_request.definition.available_forms.first @@ -172,7 +212,7 @@ def fetch_form(authorization_request) end def handle_authorization_request_type_specific_fields(authorization_request, enrollment_row) - Kernel.const_get( + @authorization_request_type_specific_field_builder = Kernel.const_get( "Import::AuthorizationRequests::#{authorization_request.type.split('::')[-1]}Attributes" ).new( authorization_request, @@ -180,7 +220,8 @@ def handle_authorization_request_type_specific_fields(authorization_request, enr fetch_team_members(enrollment_row['id']), options[:warned], @models, - ).perform + ) + @authorization_request_type_specific_field_builder.perform end def import?(enrollment_row) @@ -188,16 +229,23 @@ def import?(enrollment_row) ( !old_unused?(enrollment_row) && !ignore?(enrollment_row['id']) && + !more_recent_validated_production?(enrollment_row) && from_target_api_to_type(enrollment_row).present? ) end + def more_recent_validated_production?(enrollment_row) + enrollment_row['target_api'] =~ /_production$/ && + database.execute("select id from enrollments where status = 'validated' and copied_from_enrollment_id = ?", enrollment_row['id']).any? + end + def whitelisted_enrollment?(enrollment_row) (options[:authorization_request_ids] || []).include?(enrollment_row['id'].to_i) end def old_unused?(enrollment_row) %w[refused revoked changes_requested draft archived].include?(enrollment_row['status']) && + enrollment_row['last_validated_at'].blank? && DateTime.parse(enrollment_row['created_at']) < DateTime.new(2022, 1, 1) end @@ -225,6 +273,35 @@ def from_target_api_to_type(enrollment) 'hubee_portail_dila' => 'hubee_dila', 'api_entreprise' => 'api_entreprise', 'api_particulier' => 'api_particulier', + 'api_impot_particulier_sandbox' => 'api_impot_particulier_sandbox', + 'api_impot_particulier_fc_sandbox' => 'api_impot_particulier_sandbox', + 'api_impot_particulier_production' => 'api_impot_particulier', + 'api_impot_particulier_fc_production' => 'api_impot_particulier', + 'franceconnect' => 'france_connect', + 'api_rial_sandbox' => 'api_rial_sandbox', + 'api_rial_production' => 'api_rial', + 'api_cpr_pro_sandbox' => 'api_cpr_pro_adelie_sandbox', + 'api_cpr_pro_production' => 'api_cpr_pro_adelie', + 'api_e_contacts_sandbox' => 'api_e_contacts_sandbox', + 'api_e_contacts_production' => 'api_e_contacts', + 'api_e_pro_sandbox' => 'api_e_pro_sandbox', + 'api_e_pro_production' => 'api_e_pro', + 'api_ensu_documents_sandbox' => 'api_ensu_documents_sandbox', + 'api_ensu_documents_production' => 'api_ensu_documents', + 'api_hermes_sandbox' => 'api_hermes_sandbox', + 'api_hermes_production' => 'api_hermes', + 'api_imprimfip_sandbox' => 'api_imprimfip_sandbox', + 'api_imprimfip_production' => 'api_imprimfip', + 'api_mire_sandbox' => 'api_mire_sandbox', + 'api_mire_production' => 'api_mire', + 'api_ocfi_sandbox' => 'api_ocfi_sandbox', + 'api_ocfi_production' => 'api_ocfi', + 'api_opale_sandbox' => 'api_opale_sandbox', + 'api_opale_production' => 'api_opale', + 'api_robf_sandbox' => 'api_robf_sandbox', + 'api_robf_production' => 'api_robf', + 'api_satelit_sandbox' => 'api_satelit_sandbox', + 'api_satelit_production' => 'api_satelit', }[enrollment['target_api']].try(:classify) end diff --git a/app/migration/import/authorization_requests/api_cpr_pro_adelie_attributes.rb b/app/migration/import/authorization_requests/api_cpr_pro_adelie_attributes.rb new file mode 100644 index 000000000..c2432bbc3 --- /dev/null +++ b/app/migration/import/authorization_requests/api_cpr_pro_adelie_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APICprProAdelieAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_cpr_pro_adelie_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_cpr_pro_adelie_sandbox_attributes.rb new file mode 100644 index 000000000..f5b0e6dd8 --- /dev/null +++ b/app/migration/import/authorization_requests/api_cpr_pro_adelie_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APICprProAdelieSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_e_contacts_attributes.rb b/app/migration/import/authorization_requests/api_e_contacts_attributes.rb new file mode 100644 index 000000000..f72b0085f --- /dev/null +++ b/app/migration/import/authorization_requests/api_e_contacts_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIEContactsAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_e_contacts_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_e_contacts_sandbox_attributes.rb new file mode 100644 index 000000000..a4ac69a53 --- /dev/null +++ b/app/migration/import/authorization_requests/api_e_contacts_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIEContactsSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_e_pro_attributes.rb b/app/migration/import/authorization_requests/api_e_pro_attributes.rb new file mode 100644 index 000000000..3e89f5094 --- /dev/null +++ b/app/migration/import/authorization_requests/api_e_pro_attributes.rb @@ -0,0 +1,3 @@ +class Import::AuthorizationRequests::APIEProAttributes < Import::AuthorizationRequests::APIRialAttributes +end + diff --git a/app/migration/import/authorization_requests/api_e_pro_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_e_pro_sandbox_attributes.rb new file mode 100644 index 000000000..16e4de5ac --- /dev/null +++ b/app/migration/import/authorization_requests/api_e_pro_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIEProSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_ensu_documents_attributes.rb b/app/migration/import/authorization_requests/api_ensu_documents_attributes.rb new file mode 100644 index 000000000..6067242ee --- /dev/null +++ b/app/migration/import/authorization_requests/api_ensu_documents_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIENSUDocumentsAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_ensu_documents_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_ensu_documents_sandbox_attributes.rb new file mode 100644 index 000000000..051b8b71e --- /dev/null +++ b/app/migration/import/authorization_requests/api_ensu_documents_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIENSUDocumentsSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_entreprise_attributes.rb b/app/migration/import/authorization_requests/api_entreprise_attributes.rb index 3ad6f6d28..f0e6c635a 100644 --- a/app/migration/import/authorization_requests/api_entreprise_attributes.rb +++ b/app/migration/import/authorization_requests/api_entreprise_attributes.rb @@ -29,14 +29,6 @@ def affect_potential_legal_document end end - def affect_form_uid - form_uid = demarche_to_form_uid - - return if form_uid.blank? - - authorization_request.form_uid = form_uid - end - def demarche_to_form_uid case enrollment_row['demarche'] when 'marches_publics' @@ -74,12 +66,6 @@ def affect_not_specified_to_duree_conservation authorization_request.duree_conservation_donnees_caractere_personnel = 'Non renseigné' end - def recent_validated_enrollment_exists? - bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? - database.close - bool - end - def attributes_mapping { "intitule" => "intitule", diff --git a/app/migration/import/authorization_requests/api_hermes_attributes.rb b/app/migration/import/authorization_requests/api_hermes_attributes.rb new file mode 100644 index 000000000..95a5851cc --- /dev/null +++ b/app/migration/import/authorization_requests/api_hermes_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIHermesAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_hermes_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_hermes_sandbox_attributes.rb new file mode 100644 index 000000000..cf18da632 --- /dev/null +++ b/app/migration/import/authorization_requests/api_hermes_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIHermesSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb new file mode 100644 index 000000000..cf6a07e01 --- /dev/null +++ b/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb @@ -0,0 +1,34 @@ +class Import::AuthorizationRequests::APIImpotParticulierAttributes < Import::AuthorizationRequests::APIImpotParticulierSandboxAttributes + def affect_data + migrate_from_sandbox_to_production! + + affect_form_uid + + affect_operational_acceptance + affect_safety_certification + affect_volumetrie + + handle_franceconnect + + authorization_request.state = enrollment_row['status'] + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + end + + private + + def handle_franceconnect + return unless enrollment_row['target_api'] == 'api_impot_particulier_fc_production' + + authorization_request.modalities ||= [] + authorization_request.modalities = authorization_request.modalities.concat(['with_france_connect']) + authorization_request.modalities = authorization_request.modalities.uniq + end + + # FIXME + def affect_form_uid + authorization_request.form_uid = authorization_request.definition.available_forms.first.id + end +end diff --git a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb new file mode 100644 index 000000000..8cbe75929 --- /dev/null +++ b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb @@ -0,0 +1,106 @@ +class Import::AuthorizationRequests::APIImpotParticulierSandboxAttributes < Import::AuthorizationRequests::DGFIPSandboxAttributes + def affect_data + affect_scopes + affect_attributes + affect_contacts + affect_potential_legal_document + affect_potential_specific_requirements + affect_modalities + # affect_form_uid + handle_incompatible_scopes_error + + affect_duree_conservation_donnees_caractere_personnel_justification + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + end + + def handle_incompatible_scopes_error + return if authorization_request.valid? + return unless authorization_request.errors[:scopes].any? + + skip_row!("invalid_scopes_in_status_#{authorization_request.state}") + end + + def affect_modalities + authorization_request.modalities ||= [] + + { + 'acces_spi' => 'with_spi', + 'acces_etat_civil' => 'with_etat_civil' + }.each do |from, to| + next unless additional_content[from] + + authorization_request.modalities = authorization_request.modalities.concat([to]) + end + + affect_franceconnect_data if with_franceconnect? + + authorization_request.modalities = authorization_request.modalities.uniq + + return if authorization_request.modalities.present? + return if %w[refused validated].exclude?(authorization_request.state) + + skip_row!("no_modalities_in_status_#{authorization_request.state}") + end + + def affect_contacts + { + 'responsable_technique' => 'contact_technique', + 'responsable_traitement' => 'responsable_traitement', + 'delegue_protection_donnees' => 'delegue_protection_donnees', + }.each do |from_contact, to_contact| + affect_contact(from_contact, to_contact) + end + end + + # FIXME check https://metabase.entreprise.api.gouv.fr/dashboard/50-datapass-exploration?nom_de_l%27api_(target_api)=api_impot_particulier_sandbox + def demarche_to_form_uid + case enrollment_row['demarche'] + when 'marches_publics' + 'api-entreprise-marches-publics' + when 'aides_publiques' + 'api-entreprise-aides-publiques' + when 'subventions_associations' + 'api-entreprise-subventions-associations' + when 'portail_gru' + 'api-entreprise-portail-gru-preremplissage' + when 'portail_gru_instruction' + 'api-entreprise-portail-gru-instruction' + when 'detection_fraude' + 'api-entreprise-detection-fraude' + when 'e_attestations' + 'api-entreprise-e-attestations' + when 'provigis' + 'api-entreprise-provigis' + when 'achat_solution' + 'api-entreprise-achat-solution' + when 'atexo' + 'api-entreprise-atexo' + when 'mgdis' + 'api-entreprise-mgdis' + when 'setec' + 'api-entreprise-setec-atexo' + when 'editeur' + 'api-entreprise-editeur' + end + end + + def attributes_mapping + { + "intitule" => "intitule", + "description" => "description", + "fondement_juridique_title" => "cadre_juridique_nature", + "fondement_juridique_url" => "cadre_juridique_url", + "date_mise_en_production" => "date_prevue_mise_en_production", + "data_recipients" => "destinataire_donnees_caractere_personnel", + "data_retention_period" => "duree_conservation_donnees_caractere_personnel", + "data_retention_comment" => "duree_conservation_donnees_caractere_personnel_justification", + } + end + + def attributes_with_possible_null_values + ['destinataire_donnees_caractere_personnel'] + end +end diff --git a/app/migration/import/authorization_requests/api_imprimfip_attributes.rb b/app/migration/import/authorization_requests/api_imprimfip_attributes.rb new file mode 100644 index 000000000..88c683691 --- /dev/null +++ b/app/migration/import/authorization_requests/api_imprimfip_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIImprimfipAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_imprimfip_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_imprimfip_sandbox_attributes.rb new file mode 100644 index 000000000..7fcdf1189 --- /dev/null +++ b/app/migration/import/authorization_requests/api_imprimfip_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIImprimfipSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_mire_attributes.rb b/app/migration/import/authorization_requests/api_mire_attributes.rb new file mode 100644 index 000000000..4c8e86ff3 --- /dev/null +++ b/app/migration/import/authorization_requests/api_mire_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIMireAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_mire_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_mire_sandbox_attributes.rb new file mode 100644 index 000000000..d7ddb8188 --- /dev/null +++ b/app/migration/import/authorization_requests/api_mire_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIMireSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_ocfi_attributes.rb b/app/migration/import/authorization_requests/api_ocfi_attributes.rb new file mode 100644 index 000000000..a5fe498bd --- /dev/null +++ b/app/migration/import/authorization_requests/api_ocfi_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIOcfiAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_ocfi_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_ocfi_sandbox_attributes.rb new file mode 100644 index 000000000..270303d9e --- /dev/null +++ b/app/migration/import/authorization_requests/api_ocfi_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIOcfiSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_opale_attributes.rb b/app/migration/import/authorization_requests/api_opale_attributes.rb new file mode 100644 index 000000000..ae18716d8 --- /dev/null +++ b/app/migration/import/authorization_requests/api_opale_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIOpaleAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_opale_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_opale_sandbox_attributes.rb new file mode 100644 index 000000000..c75e02374 --- /dev/null +++ b/app/migration/import/authorization_requests/api_opale_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIOpaleSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_particulier_attributes.rb b/app/migration/import/authorization_requests/api_particulier_attributes.rb index 40c296492..3c5f868cf 100644 --- a/app/migration/import/authorization_requests/api_particulier_attributes.rb +++ b/app/migration/import/authorization_requests/api_particulier_attributes.rb @@ -50,16 +50,6 @@ def affect_potentiel_maquette_projet_document attach_file('maquette_projet', row) end - def affect_duree_conservation_donnees_caractere_personnel_justification - return unless authorization_request.duree_conservation_donnees_caractere_personnel > 36 && authorization_request.duree_conservation_donnees_caractere_personnel_justification.blank? - - authorization_request.duree_conservation_donnees_caractere_personnel_justification = 'Non renseigné' - end - - def affect_form_uid - authorization_request.form_uid = demarche_to_form_uid - end - def demarche_to_form_uid case enrollment_row['demarche'] when 'arpege-concerto' diff --git a/app/migration/import/authorization_requests/api_rial_attributes.rb b/app/migration/import/authorization_requests/api_rial_attributes.rb new file mode 100644 index 000000000..4a421f1ae --- /dev/null +++ b/app/migration/import/authorization_requests/api_rial_attributes.rb @@ -0,0 +1,15 @@ +class Import::AuthorizationRequests::APIRialAttributes < Import::AuthorizationRequests::DGFIPProductionAttributes + def affect_data + migrate_from_sandbox_to_production! + + affect_operational_acceptance + affect_safety_certification + affect_volumetrie + + authorization_request.state = enrollment_row['status'] + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + end +end diff --git a/app/migration/import/authorization_requests/api_rial_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_rial_sandbox_attributes.rb new file mode 100644 index 000000000..19847894c --- /dev/null +++ b/app/migration/import/authorization_requests/api_rial_sandbox_attributes.rb @@ -0,0 +1,40 @@ +class Import::AuthorizationRequests::APIRialSandboxAttributes < Import::AuthorizationRequests::DGFIPSandboxAttributes + def affect_data + affect_attributes + affect_contacts + affect_potential_legal_document + affect_potential_specific_requirements + affect_duree_conservation_donnees_caractere_personnel_justification + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + end + + def affect_contacts + { + 'responsable_technique' => 'contact_technique', + 'responsable_traitement' => 'responsable_traitement', + 'delegue_protection_donnees' => 'delegue_protection_donnees', + }.each do |from_contact, to_contact| + affect_contact(from_contact, to_contact) + end + end + + def attributes_mapping + { + "intitule" => "intitule", + "description" => "description", + "fondement_juridique_title" => "cadre_juridique_nature", + "fondement_juridique_url" => "cadre_juridique_url", + "date_mise_en_production" => "date_prevue_mise_en_production", + "data_recipients" => "destinataire_donnees_caractere_personnel", + "data_retention_period" => "duree_conservation_donnees_caractere_personnel", + "data_retention_comment" => "duree_conservation_donnees_caractere_personnel_justification", + } + end + + def attributes_with_possible_null_values + ['destinataire_donnees_caractere_personnel'] + end +end diff --git a/app/migration/import/authorization_requests/api_robf_attributes.rb b/app/migration/import/authorization_requests/api_robf_attributes.rb new file mode 100644 index 000000000..e1d22e8b2 --- /dev/null +++ b/app/migration/import/authorization_requests/api_robf_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIRobfAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_robf_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_robf_sandbox_attributes.rb new file mode 100644 index 000000000..ae4282837 --- /dev/null +++ b/app/migration/import/authorization_requests/api_robf_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APIRobfSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/api_satelit_attributes.rb b/app/migration/import/authorization_requests/api_satelit_attributes.rb new file mode 100644 index 000000000..5297e5a01 --- /dev/null +++ b/app/migration/import/authorization_requests/api_satelit_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APISatelitAttributes < Import::AuthorizationRequests::APIRialAttributes +end diff --git a/app/migration/import/authorization_requests/api_satelit_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_satelit_sandbox_attributes.rb new file mode 100644 index 000000000..67eda012e --- /dev/null +++ b/app/migration/import/authorization_requests/api_satelit_sandbox_attributes.rb @@ -0,0 +1,2 @@ +class Import::AuthorizationRequests::APISatelitSandboxAttributes < Import::AuthorizationRequests::APIRialSandboxAttributes +end diff --git a/app/migration/import/authorization_requests/base.rb b/app/migration/import/authorization_requests/base.rb index 439077a5d..0ca71a0c9 100644 --- a/app/migration/import/authorization_requests/base.rb +++ b/app/migration/import/authorization_requests/base.rb @@ -6,12 +6,13 @@ class Import::AuthorizationRequests::Base include LocalDatabaseUtils class AbstractRow < StandardError - attr_reader :kind, :id, :target_api + attr_reader :kind, :id, :target_api, :authorization_request - def initialize(kind = nil, id:, target_api:) + def initialize(kind = nil, id:, target_api:, authorization_request:) @kind = kind @id = id @target_api = target_api + @authorization_request = authorization_request end end @@ -29,7 +30,11 @@ def initialize(authorization_request, enrollment_row, team_members, warned, all_ end def perform - affect_data + begin + affect_data + ensure + database.close + end authorization_request end @@ -59,6 +64,29 @@ def affect_attributes end end + def affect_potential_legal_document + return if authorization_request.cadre_juridique_url.present? + + affect_potential_document('Document::LegalBasis', 'cadre_juridique_document') + end + + def affect_potential_document(kind, field) + row = csv('documents').find { |row| row['attachable_id'] == enrollment_row['id'] && row['type'] == kind } + + return false unless row + + attach_file(field, row) + true + end + + def affect_form_uid + form_uid = demarche_to_form_uid + + return if form_uid.blank? + + authorization_request.form_uid = form_uid + end + def find_team_member_by_type(type) team_member = team_members.find { |team_member| team_member['type'] == type } @@ -145,7 +173,7 @@ def affect_contact(from_contact, to_contact) end end - byebug + # byebug if recent_validated_enrollment_exists? skip_row!('incomplete_contact_data_with_new_enrollments') @@ -181,25 +209,43 @@ def attributes_with_possible_null_values end def skip_row!(kind) - raise SkipRow.new(kind.to_s, id: enrollment_row['id'], target_api: enrollment_row['target_api']) + raise SkipRow.new(kind.to_s, id: enrollment_row['id'], target_api: enrollment_row['target_api'], authorization_request:) end def warn_row!(kind) - @warned << WarnRow.new(kind.to_s, id: enrollment_row['id'], target_api: enrollment_row['target_api']) + @warned << WarnRow.new(kind.to_s, id: enrollment_row['id'], target_api: enrollment_row['target_api'], authorization_request:) end def attach_file(kind, row_data) - filename, io = extract_attachable(row_data) + filename, io = extract_attachable(kind, row_data) authorization_request.public_send("#{kind}").attach(io:, filename:) end - def extract_attachable(row_data) + def additional_content + @additional_content ||= enrollment_row['additional_content'].is_a?(String) ? JSON.parse(enrollment_row['additional_content']) : enrollment_row['additional_content'] + end + + def affect_duree_conservation_donnees_caractere_personnel_justification + return if authorization_request.duree_conservation_donnees_caractere_personnel.blank? + return unless authorization_request.duree_conservation_donnees_caractere_personnel > 36 && authorization_request.duree_conservation_donnees_caractere_personnel_justification.blank? + + authorization_request.duree_conservation_donnees_caractere_personnel_justification = 'Non renseigné' + end + + def extract_attachable(kind, row_data) if ENV['LOCAL'] == 'true' - [ - 'dummy.pdf', - dummy_pdf_as_io, - ] + if kind == 'specific_requirements_document' + [ + 'dummy.xlsx', + dummy_file_as_io('xlsx'), + ] + else + [ + 'dummy.pdf', + dummy_file_as_io('pdf'), + ] + end else [ row_data['attachment'], @@ -208,8 +254,17 @@ def extract_attachable(row_data) end end - def dummy_pdf_as_io - Rails.root.join('spec', 'fixtures', 'dummy.pdf').open + def dummy_file_as_io(extension) + $dummy_files ||= {} + $dummy_files[extension] ||= Rails.root.join('spec', 'fixtures', "dummy.#{extension}").open + $dummy_files[extension].rewind + $dummy_files[extension] + end + + def recent_validated_enrollment_exists? + bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? + database.close + bool end def extract_io(row_data) diff --git a/app/migration/import/authorization_requests/dgfip_production_attributes.rb b/app/migration/import/authorization_requests/dgfip_production_attributes.rb new file mode 100644 index 000000000..f6ce57a7f --- /dev/null +++ b/app/migration/import/authorization_requests/dgfip_production_attributes.rb @@ -0,0 +1,48 @@ +class Import::AuthorizationRequests::DGFIPProductionAttributes < Import::AuthorizationRequests::Base + def migrate_from_sandbox_to_production! + @authorization_request = AuthorizationRequest.find(enrollment_row['previous_enrollment_id']) + + authorization_request.update!(type: "AuthorizationRequest::#{self.class.to_s.split('::')[-1].sub('Attributes', '')}", state: 'draft', id: enrollment_row['id']) + + ActiveStorage::Attachment.where(record_type: 'AuthorizationRequest', record_id: enrollment_row['previous_enrollment_id']).update_all(record_id: enrollment_row['id']) + + @authorization_request = AuthorizationRequest.find(enrollment_row['id']) + rescue ActiveRecord::RecordNotFound + skip_row!(:sandbox_missing) + end + + def affect_operational_acceptance + authorization_request.operational_acceptance_done = additional_content['recette_fonctionnelle'] ? '1' : nil + end + + def affect_safety_certification + { + 'autorite_homologation_nom' => 'safety_certification_authority_name', + 'autorite_homologation_fonction' => 'safety_certification_authority_function', + 'date_homologation' => 'safety_certification_begin_date', + 'date_fin_homologation' => 'safety_certification_end_date' + }.each do |from, to| + authorization_request.public_send("#{to}=", additional_content[from]) + end + + affect_potential_document('Document::DecisionHomologation', 'safety_certification_document') + + return if authorization_request.safety_certification_begin_date.blank? || authorization_request.safety_certification_end_date.blank? + + begin + return unless Date.parse(authorization_request.safety_certification_begin_date) == Date.parse(authorization_request.safety_certification_end_date) + rescue TypeError + return + end + + authorization_request.safety_certification_end_date = (Date.parse(authorization_request.safety_certification_begin_date) + 1.day).to_s + end + + def affect_volumetrie + authorization_request.volumetrie_appels_par_minute = additional_content['volumetrie_appels_par_minute'].try(:to_i) + + return if authorization_request.volumetrie_appels_par_minute == 50 + + authorization_request.volumetrie_justification = 'Non renseignée' + end +end diff --git a/app/migration/import/authorization_requests/dgfip_sandbox_attributes.rb b/app/migration/import/authorization_requests/dgfip_sandbox_attributes.rb new file mode 100644 index 000000000..396f38f8f --- /dev/null +++ b/app/migration/import/authorization_requests/dgfip_sandbox_attributes.rb @@ -0,0 +1,19 @@ +class Import::AuthorizationRequests::DGFIPSandboxAttributes < Import::AuthorizationRequests::Base + def affect_potential_specific_requirements + return unless affect_potential_document('Document::ExpressionBesoinSpecifique', 'specific_requirements_document') + + authorization_request.specific_requirements = '1' + end + + def with_franceconnect? + enrollment_row['previous_enrollment_id'].present? + end + + def affect_franceconnect_data + authorization_request.modalities = authorization_request.modalities.concat(['with_france_connect']) + + france_connect_authorization = AuthorizationRequest::FranceConnect.find_by(id: enrollment_row['previous_enrollment_id']) + + authorization_request.france_connect_authorization_id = france_connect_authorization.latest_authorization.id.to_s + end +end diff --git a/app/migration/import/authorization_requests/france_connect_attributes.rb b/app/migration/import/authorization_requests/france_connect_attributes.rb new file mode 100644 index 000000000..b8ce34a74 --- /dev/null +++ b/app/migration/import/authorization_requests/france_connect_attributes.rb @@ -0,0 +1,97 @@ +class Import::AuthorizationRequests::FranceConnectAttributes < Import::AuthorizationRequests::Base + def affect_data + affect_scopes + affect_attributes + affect_contacts + affect_potential_legal_document + # affect_not_specified_to_duree_conservation + affect_form_uid + authorization_request.france_connect_eidas = extract_eidas + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + skip_row!(:no_scopes) if authorization_request.scopes.blank? + skip_row!(:no_destinataire_donnees_caractere_personnel) if authorization_request.destinataire_donnees_caractere_personnel.blank? + end + + def affect_contacts + { + 'responsable_technique' => 'contact_technique', + 'responsable_traitement' => 'responsable_traitement', + 'delegue_protection_donnees' => 'delegue_protection_donnees', + }.each do |from_contact, to_contact| + affect_contact(from_contact, to_contact) + end + end + + def affect_potential_legal_document + return if authorization_request.cadre_juridique_url.present? + + row = csv('documents').find { |row| row['attachable_id'] == enrollment_row['id'] && row['type'] == 'Document::LegalBasis' } + + if row + attach_file('cadre_juridique_document', row) + end + end + + def after_save(authorization_request, row) + @authorization_request = authorization_request + + create_approved_events + end + + def create_approved_events + @database = SQLite3::Database.new(db_path) + + event_rows = database.execute('select * from events where authorization_request_id = ?', enrollment_row['id']).to_a.map do |row| + JSON.parse(row[-1]).to_h + end.select { |e| e['name'] == 'validate' } + + event_rows.map do |event_row| + Import::AuthorizationRequestEvents.new(authorization_request:, create_from_authorization_request_import: true).send(:extract, event_row) + end + + @database.close + end + + def extract_eidas + case additional_content['eidas_level'] + when '2' + 'eidas_2' + else + 'eidas_1' + end + end + + def demarche_to_form_uid + case enrollment_row['demarche'] + when 'collectivite' + 'france-connect-collectivite-administration' + when 'epermis' + 'france-connect-collectivite-epermis' + when 'sns' + 'france-connect-sante' + else + 'france-connect' + end + end + + def attributes_mapping + { + "intitule" => "intitule", + "description" => "description", + "fondement_juridique_title" => "cadre_juridique_nature", + "fondement_juridique_url" => "cadre_juridique_url", + "date_mise_en_production" => "date_prevue_mise_en_production", + "volumetrie_approximative" => "volumetrie_approximative", + "data_recipients" => "destinataire_donnees_caractere_personnel", + "data_retention_period" => "duree_conservation_donnees_caractere_personnel", + "data_retention_comment" => "duree_conservation_donnees_caractere_personnel_justification", + } + end + + def attributes_with_possible_null_values + ['duree_conservation_donnees_caractere_personnel_justification'] + end +end diff --git a/app/migration/import/base.rb b/app/migration/import/base.rb index ac0ba5897..b30af5b40 100644 --- a/app/migration/import/base.rb +++ b/app/migration/import/base.rb @@ -77,7 +77,10 @@ def load_from_csv! extract(row) print '.' rescue Import::AuthorizationRequests::Base::SkipRow => e + print 's' options[:skipped] << e + + e.authorization_request.save(validate: false) if ENV['SKIP_VALIDATION'] == 'true' rescue => e log(" ERROR: #{e.message}") log(e.backtrace.join("\n")) diff --git a/app/migration/local_reset.sh b/app/migration/local_reset.sh index e9184ad3b..8c60d9b29 100755 --- a/app/migration/local_reset.sh +++ b/app/migration/local_reset.sh @@ -6,4 +6,4 @@ rm -f app/migration/dumps/*.json rm -f app/migration/dumps/data.db ./app/migration/export_v1.sh ./app/migration/export_v2.sh -./app/migration/export_hubee.sh +# ./app/migration/export_hubee.sh diff --git a/app/migration/local_run.sh b/app/migration/local_run.sh index a4d91ad89..85181ddd9 100755 --- a/app/migration/local_run.sh +++ b/app/migration/local_run.sh @@ -7,8 +7,8 @@ LOCAL=true bundle exec rails runner "ImportDataInLocalDb.new.perform(delete_db_f if [ $# -eq 1 ]; then echo "Run for $1 only" - DUMP=false LOCAL=true bundle exec rails runner "MainImport.new(authorization_request_ids: [$1]).perform" + SKIP_VALIDATION=true DUMP=true LOCAL=true bundle exec rails runner "MainImport.new(authorization_request_ids: [$1]).perform" else echo "Run for all" - DUMP=false LOCAL=true bundle exec rails runner "MainImport.new.perform" + SKIP_VALIDATION=true DUMP=true LOCAL=true bundle exec rails runner "MainImport.new.perform" fi diff --git a/app/migration/main_import.rb b/app/migration/main_import.rb index 7519fc549..7be0fc25b 100644 --- a/app/migration/main_import.rb +++ b/app/migration/main_import.rb @@ -12,9 +12,12 @@ def initialize(authorization_request_ids: []) end def perform - organizations = import(:organizations, { load_from_sql: ENV['DUMP'] == 'true' }) - import(:users, { load_from_sql: ENV['DUMP'] == 'true' }) - authorization_requests = import(:authorization_requests, { load_from_sql: ENV['DUMP'] == 'true' }) + import(:organizations, { load_from_sql: true }) + import(:users, { load_from_sql: true }) + + # import_extra_authorization_requests_sql_data + + authorization_requests = import(:authorization_requests, { load_from_sql: false, dump_sql: true }) if types_to_import.any? valid_authorization_request_ids = AuthorizationRequest.where(id: authorization_requests.pluck(:id), type: types_to_import).pluck(:id) @@ -22,7 +25,7 @@ def perform valid_authorization_request_ids = authorization_requests.pluck(:id) end - import(:authorization_request_events, { dump_sql: ENV['DUMP'] == 'true', valid_authorization_request_ids: }) + # import(:authorization_request_events, { dump_sql: ENV['DUMP'] == 'true', valid_authorization_request_ids: }) %i[warned skipped].each do |kind| export(kind) @@ -32,10 +35,16 @@ def perform private + def import_extra_authorization_requests_sql_data + Dir[Rails.root.join('app/migration/dumps/authorization_requests_sql_to_load/*.sql')].sort.each do |file| + log("# Importing file #{File.basename(file)}") + + `psql -d #{ActiveRecord::Base.connection.current_database} -f #{file}` + end + end + def types_to_import %w[ - AuthorizationRequest::HubEEDila - AuthorizationRequest::HubEECertDC ] end @@ -56,10 +65,10 @@ def export(kind) data = public_send(kind) CSV.open(export_path(kind), 'w') do |csv| - csv << %w[id target_api kind] + csv << %w[id target_api kind url] data.each do |datum| - csv << [datum.id, datum.target_api, datum.kind] + csv << [datum.id, datum.target_api, datum.kind, "https://datapass.api.gouv.fr/#{datum.target_api.gsub('_', '-')}/#{datum.id}"] end end end @@ -87,17 +96,31 @@ def export_path(kind) def global_options { authorization_requests_filter: ->(enrollment_row) do - %w[ - 5 - 26 - 129 - 25590 - 54115 - ].exclude?(enrollment_row['id']) + true end, - authorization_requests_sql_where: 'target_api in (\'hubee_portail\', \'hubee_portail_dila\') order by id desc', + # authorization_requests_sql_where: 'target_api in (\'franceconnect\', \'api_impot_particulier_fc_sandbox\', \'api_impot_particulier_fc_production\') order by case when target_api = \'franceconnect\' then 1 when target_api like \'%_sandbox\' then 2 else 3 end', + authorization_requests_sql_where: "target_api in (#{target_apis_to_import}) order by case when target_api = 'franceconnect' then 1 when target_api like '%_sandbox' then 2 else 3 end", skipped: @skipped, warned: @warned, } end + + def target_apis_to_import + %w[ + api_rial + api_cpr_pro + api_e_contacts + api_e_pro + api_ensu_documents + api_hermes + api_imprimfip + api_mire + api_ocfi + api_opale + api_robf + api_satelit + ].map do |name| + ["'#{name}_sandbox'", "'#{name}_production'"] + end.flatten.join(', ') + end end diff --git a/app/models/authorization.rb b/app/models/authorization.rb index 1bc59f7eb..24f5228c5 100644 --- a/app/models/authorization.rb +++ b/app/models/authorization.rb @@ -36,7 +36,7 @@ class Authorization < ApplicationRecord through: :approve_authorization_request_event, source: :user - scope :validated, -> { joins(:request).where(authorization_requests: { state: 'validated' }) } + scope :validated, -> { where(revoked: false) } delegate :name, :kind, to: :request diff --git a/app/models/authorization_request/api_impot_particulier.rb b/app/models/authorization_request/api_impot_particulier.rb index 2557d1035..e2c38b2fa 100644 --- a/app/models/authorization_request/api_impot_particulier.rb +++ b/app/models/authorization_request/api_impot_particulier.rb @@ -23,9 +23,7 @@ class AuthorizationRequest::APIImpotParticulier < AuthorizationRequest add_attribute :contact_technique_extra_email, format: { with: URI::MailTo::EMAIL_REGEXP } add_attribute :extra_organization_contact_name - add_scopes(validation: { - presence: true, if: -> { need_complete_validation?(:scopes) && !specific_requirements? } - }) + add_scopes contact :contact_technique, validation_condition: ->(record) { record.need_complete_validation?(:contacts) } end diff --git a/app/models/authorization_request/api_impot_particulier_sandbox.rb b/app/models/authorization_request/api_impot_particulier_sandbox.rb index 0a218bafc..10a0a0fbc 100644 --- a/app/models/authorization_request/api_impot_particulier_sandbox.rb +++ b/app/models/authorization_request/api_impot_particulier_sandbox.rb @@ -13,9 +13,7 @@ class AuthorizationRequest::APIImpotParticulierSandbox < AuthorizationRequest add_attribute :contact_technique_extra_email, format: { with: URI::MailTo::EMAIL_REGEXP } add_attribute :extra_organization_contact_name - add_scopes(validation: { - presence: true, if: -> { need_complete_validation?(:scopes) } - }) + add_scopes contact :contact_technique, validation_condition: ->(record) { record.need_complete_validation?(:contacts) } diff --git a/app/views/reopen_authorizations/new.html.erb b/app/views/reopen_authorizations/new.html.erb index 1e333bd51..c7151a009 100644 --- a/app/views/reopen_authorizations/new.html.erb +++ b/app/views/reopen_authorizations/new.html.erb @@ -10,6 +10,12 @@
<%= t('.disclaimer').html_safe %>
+ + <% unless @authorization.request.valid? %> ++ <%= t('.invalid_request') %> +
+ <% end %>