From 3fec9f2f4ec4c4e94c28dcba04bc60149a7dbacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Tue, 7 Jan 2025 12:10:58 +0100 Subject: [PATCH 01/40] AuthorizationRequest::APIImpotParticulier does not need presence validation on scopes Scopes presence validation is not required when specific_requirements document is present. Moreover with custom validators, it covers all cases. --- app/models/authorization_request/api_impot_particulier.rb | 4 +--- .../authorization_request/api_impot_particulier_sandbox.rb | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/authorization_request/api_impot_particulier.rb b/app/models/authorization_request/api_impot_particulier.rb index 2557d1035..e2c38b2fa 100644 --- a/app/models/authorization_request/api_impot_particulier.rb +++ b/app/models/authorization_request/api_impot_particulier.rb @@ -23,9 +23,7 @@ class AuthorizationRequest::APIImpotParticulier < AuthorizationRequest add_attribute :contact_technique_extra_email, format: { with: URI::MailTo::EMAIL_REGEXP } add_attribute :extra_organization_contact_name - add_scopes(validation: { - presence: true, if: -> { need_complete_validation?(:scopes) && !specific_requirements? } - }) + add_scopes contact :contact_technique, validation_condition: ->(record) { record.need_complete_validation?(:contacts) } end diff --git a/app/models/authorization_request/api_impot_particulier_sandbox.rb b/app/models/authorization_request/api_impot_particulier_sandbox.rb index 0a218bafc..10a0a0fbc 100644 --- a/app/models/authorization_request/api_impot_particulier_sandbox.rb +++ b/app/models/authorization_request/api_impot_particulier_sandbox.rb @@ -13,9 +13,7 @@ class AuthorizationRequest::APIImpotParticulierSandbox < AuthorizationRequest add_attribute :contact_technique_extra_email, format: { with: URI::MailTo::EMAIL_REGEXP } add_attribute :extra_organization_contact_name - add_scopes(validation: { - presence: true, if: -> { need_complete_validation?(:scopes) } - }) + add_scopes contact :contact_technique, validation_condition: ->(record) { record.need_complete_validation?(:contacts) } From 716fba7725b882047c21386e7931e1624507d7be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Wed, 22 Jan 2025 17:58:30 +0100 Subject: [PATCH 02/40] Authorization.validated relies on revoked boolean Previous version excluded reopened authorization requests, which is not valid. --- app/models/authorization.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/authorization.rb b/app/models/authorization.rb index 1bc59f7eb..24f5228c5 100644 --- a/app/models/authorization.rb +++ b/app/models/authorization.rb @@ -36,7 +36,7 @@ class Authorization < ApplicationRecord through: :approve_authorization_request_event, source: :user - scope :validated, -> { joins(:request).where(authorization_requests: { state: 'validated' }) } + scope :validated, -> { where(revoked: false) } delegate :name, :kind, to: :request From 807475e7c49a9d571cf5c249258d367f8a36ffe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Tue, 4 Feb 2025 15:24:35 +0100 Subject: [PATCH 03/40] Harden revoked trait --- spec/factories/authorization_requests.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/factories/authorization_requests.rb b/spec/factories/authorization_requests.rb index b1e5beb37..7eb6524b7 100644 --- a/spec/factories/authorization_requests.rb +++ b/spec/factories/authorization_requests.rb @@ -139,6 +139,10 @@ after(:build) do |authorization_request| authorization_request.revocations << build(:revocation_of_authorization, authorization_request:) end + + after(:create) do |authorization_request| + authorization_request.authorizations.update_all(revoked: true) # rubocop:disable Rails/SkipsModelValidations + end end trait :validated do From 671b3251bf0f53b2da6194b7c19d966bb2b9f067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Wed, 29 Jan 2025 08:35:39 +0100 Subject: [PATCH 04/40] Migration: old unused exclude reopenings --- app/migration/import/authorization_requests.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 6fbeec0c5..35f37f56e 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -198,6 +198,7 @@ def whitelisted_enrollment?(enrollment_row) def old_unused?(enrollment_row) %w[refused revoked changes_requested draft archived].include?(enrollment_row['status']) && + enrollment_row['last_validated_at'].blank? && DateTime.parse(enrollment_row['created_at']) < DateTime.new(2022, 1, 1) end From dc821366fa86c6c8739c40306411ab729294fb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Tue, 7 Jan 2025 15:21:05 +0100 Subject: [PATCH 05/40] Migration: Enhance Import::AuthorizationRequests classes methods Refactor some methods, prepare for DGFIP --- .../api_entreprise_attributes.rb | 8 ---- .../api_particulier_attributes.rb | 4 -- .../import/authorization_requests/base.rb | 46 +++++++++++++++---- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/app/migration/import/authorization_requests/api_entreprise_attributes.rb b/app/migration/import/authorization_requests/api_entreprise_attributes.rb index 3ad6f6d28..7c9013b13 100644 --- a/app/migration/import/authorization_requests/api_entreprise_attributes.rb +++ b/app/migration/import/authorization_requests/api_entreprise_attributes.rb @@ -29,14 +29,6 @@ def affect_potential_legal_document end end - def affect_form_uid - form_uid = demarche_to_form_uid - - return if form_uid.blank? - - authorization_request.form_uid = form_uid - end - def demarche_to_form_uid case enrollment_row['demarche'] when 'marches_publics' diff --git a/app/migration/import/authorization_requests/api_particulier_attributes.rb b/app/migration/import/authorization_requests/api_particulier_attributes.rb index 40c296492..ff5f42de0 100644 --- a/app/migration/import/authorization_requests/api_particulier_attributes.rb +++ b/app/migration/import/authorization_requests/api_particulier_attributes.rb @@ -56,10 +56,6 @@ def affect_duree_conservation_donnees_caractere_personnel_justification authorization_request.duree_conservation_donnees_caractere_personnel_justification = 'Non renseigné' end - def affect_form_uid - authorization_request.form_uid = demarche_to_form_uid - end - def demarche_to_form_uid case enrollment_row['demarche'] when 'arpege-concerto' diff --git a/app/migration/import/authorization_requests/base.rb b/app/migration/import/authorization_requests/base.rb index 439077a5d..ae5f3edc9 100644 --- a/app/migration/import/authorization_requests/base.rb +++ b/app/migration/import/authorization_requests/base.rb @@ -59,6 +59,29 @@ def affect_attributes end end + def affect_potential_legal_document + return if authorization_request.cadre_juridique_url.present? + + affect_potential_document('Document::LegalBasis', 'cadre_juridique_document') + end + + def affect_potential_document(kind, field) + row = csv('documents').find { |row| row['attachable_id'] == enrollment_row['id'] && row['type'] == kind } + + return false unless row + + attach_file(field, row) + true + end + + def affect_form_uid + form_uid = demarche_to_form_uid + + return if form_uid.blank? + + authorization_request.form_uid = form_uid + end + def find_team_member_by_type(type) team_member = team_members.find { |team_member| team_member['type'] == type } @@ -189,17 +212,24 @@ def warn_row!(kind) end def attach_file(kind, row_data) - filename, io = extract_attachable(row_data) + filename, io = extract_attachable(kind, row_data) authorization_request.public_send("#{kind}").attach(io:, filename:) end - def extract_attachable(row_data) + def extract_attachable(kind, row_data) if ENV['LOCAL'] == 'true' - [ - 'dummy.pdf', - dummy_pdf_as_io, - ] + if kind == 'specific_requirements_document' + [ + 'dummy.xlsx', + dummy_file_as_io('xlsx'), + ] + else + [ + 'dummy.pdf', + dummy_file_as_io('pdf'), + ] + end else [ row_data['attachment'], @@ -208,8 +238,8 @@ def extract_attachable(row_data) end end - def dummy_pdf_as_io - Rails.root.join('spec', 'fixtures', 'dummy.pdf').open + def dummy_file_as_io(extension) + Rails.root.join('spec', 'fixtures', "dummy.#{extension}").open end def extract_io(row_data) From 0006e7bc89de8e8996299c0adb53d3ed900655de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Tue, 7 Jan 2025 15:39:35 +0100 Subject: [PATCH 06/40] Migration: enhance debugging by putting enrollment status --- app/migration/import/authorization_requests.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 35f37f56e..3fecc0d78 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -34,7 +34,7 @@ def extract(enrollment_row) authorization_request.save! @models << authorization_request rescue ActiveRecord::RecordInvalid => e - log("DataPass: https://datapass.api.gouv.fr/#{enrollment_row['target_api'].gsub('_', '-')}/#{enrollment_row['id']}") + log("DataPass: https://datapass.api.gouv.fr/#{enrollment_row['target_api'].gsub('_', '-')}/#{enrollment_row['id']} (status: #{enrollment_row['status']})") log("Errors: #{authorization_request.errors.full_messages.join("\n")}") byebug From a2b380a1eacfca8a28406d631f79b22872c87885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Tue, 7 Jan 2025 15:40:03 +0100 Subject: [PATCH 07/40] Migration: Introduce Import::AuthorizationRequests::APIImpotParticulierSandboxAttributes --- ...pi_impot_particulier_sandbox_attributes.rb | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb diff --git a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb new file mode 100644 index 000000000..032cd2d6d --- /dev/null +++ b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb @@ -0,0 +1,110 @@ +class Import::AuthorizationRequests::APIImpotParticulierSandboxAttributes < Import::AuthorizationRequests::Base + def affect_data + affect_scopes + affect_attributes + affect_contacts + affect_potential_legal_document + affect_potential_specific_requirements + affect_modalities + # affect_form_uid + handle_incompatible_scopes_error + end + + def handle_incompatible_scopes_error + return if authorization_request.valid? + return unless authorization_request.errors[:scopes].any? + + skip_row!("invalid_scopes_in_status_#{authorization_request.state}") + end + + def affect_potential_specific_requirements + return unless affect_potential_document('Document::ExpressionBesoinSpecifique', 'specific_requirements_document') + + authorization_request.specific_requirements = '1' + end + + def affect_modalities + additional_content = JSON.parse(enrollment_row['additional_content']) || {} + authorization_request.modalities ||= [] + + { + 'acces_spi' => 'with_spi', + 'acces_etat_civil' => 'with_etat_civil' + }.each do |from, to| + next unless additional_content[from] + + authorization_request.modalities = authorization_request.modalities.concat([to]) + end + + authorization_request.modalities = authorization_request.modalities.uniq + + return if authorization_request.modalities.present? + return if %w[refused validated].exclude?(authorization_request.state) + + skip_row!("no_modalities_in_status_#{authorization_request.state}") + end + + def affect_contacts + { + 'responsable_technique' => 'contact_technique', + 'responsable_traitement' => 'responsable_traitement', + 'delegue_protection_donnees' => 'delegue_protection_donnees', + }.each do |from_contact, to_contact| + affect_contact(from_contact, to_contact) + end + end + + def demarche_to_form_uid + case enrollment_row['demarche'] + when 'marches_publics' + 'api-entreprise-marches-publics' + when 'aides_publiques' + 'api-entreprise-aides-publiques' + when 'subventions_associations' + 'api-entreprise-subventions-associations' + when 'portail_gru' + 'api-entreprise-portail-gru-preremplissage' + when 'portail_gru_instruction' + 'api-entreprise-portail-gru-instruction' + when 'detection_fraude' + 'api-entreprise-detection-fraude' + when 'e_attestations' + 'api-entreprise-e-attestations' + when 'provigis' + 'api-entreprise-provigis' + when 'achat_solution' + 'api-entreprise-achat-solution' + when 'atexo' + 'api-entreprise-atexo' + when 'mgdis' + 'api-entreprise-mgdis' + when 'setec' + 'api-entreprise-setec-atexo' + when 'editeur' + 'api-entreprise-editeur' + end + end + + def recent_validated_enrollment_exists? + bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? + database.close + bool + end + + def attributes_mapping + { + "intitule" => "intitule", + "description" => "description", + "fondement_juridique_title" => "cadre_juridique_nature", + "fondement_juridique_url" => "cadre_juridique_url", + "date_mise_en_production" => "date_prevue_mise_en_production", + "data_recipients" => "destinataire_donnees_caractere_personnel", + "data_retention_period" => "duree_conservation_donnees_caractere_personnel", + "data_retention_comment" => "duree_conservation_donnees_caractere_personnel_justification", + } + end + + def attributes_with_possible_null_values + ['destinataire_donnees_caractere_personnel'] + end +end From 97ac46056c407e717d444fbddb16da05965131ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Tue, 7 Jan 2025 15:40:45 +0100 Subject: [PATCH 08/40] Migration: wip main scripts --- app/migration/import/authorization_requests.rb | 1 + app/migration/local_reset.sh | 2 +- app/migration/local_run.sh | 4 ++-- app/migration/main_import.rb | 16 +++++----------- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 3fecc0d78..4366ed871 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -226,6 +226,7 @@ def from_target_api_to_type(enrollment) 'hubee_portail_dila' => 'hubee_dila', 'api_entreprise' => 'api_entreprise', 'api_particulier' => 'api_particulier', + 'api_impot_particulier_sandbox' => 'api_impot_particulier_sandbox', }[enrollment['target_api']].try(:classify) end diff --git a/app/migration/local_reset.sh b/app/migration/local_reset.sh index e9184ad3b..8c60d9b29 100755 --- a/app/migration/local_reset.sh +++ b/app/migration/local_reset.sh @@ -6,4 +6,4 @@ rm -f app/migration/dumps/*.json rm -f app/migration/dumps/data.db ./app/migration/export_v1.sh ./app/migration/export_v2.sh -./app/migration/export_hubee.sh +# ./app/migration/export_hubee.sh diff --git a/app/migration/local_run.sh b/app/migration/local_run.sh index a4d91ad89..5e081c0bb 100755 --- a/app/migration/local_run.sh +++ b/app/migration/local_run.sh @@ -7,8 +7,8 @@ LOCAL=true bundle exec rails runner "ImportDataInLocalDb.new.perform(delete_db_f if [ $# -eq 1 ]; then echo "Run for $1 only" - DUMP=false LOCAL=true bundle exec rails runner "MainImport.new(authorization_request_ids: [$1]).perform" + DUMP=true LOCAL=true bundle exec rails runner "MainImport.new(authorization_request_ids: [$1]).perform" else echo "Run for all" - DUMP=false LOCAL=true bundle exec rails runner "MainImport.new.perform" + DUMP=true LOCAL=true bundle exec rails runner "MainImport.new.perform" fi diff --git a/app/migration/main_import.rb b/app/migration/main_import.rb index 7519fc549..d4d5da386 100644 --- a/app/migration/main_import.rb +++ b/app/migration/main_import.rb @@ -12,9 +12,9 @@ def initialize(authorization_request_ids: []) end def perform - organizations = import(:organizations, { load_from_sql: ENV['DUMP'] == 'true' }) - import(:users, { load_from_sql: ENV['DUMP'] == 'true' }) - authorization_requests = import(:authorization_requests, { load_from_sql: ENV['DUMP'] == 'true' }) + import(:organizations, { load_from_sql: true }) + import(:users, { load_from_sql: true }) + authorization_requests = import(:authorization_requests, { load_from_sql: false }) if types_to_import.any? valid_authorization_request_ids = AuthorizationRequest.where(id: authorization_requests.pluck(:id), type: types_to_import).pluck(:id) @@ -34,8 +34,6 @@ def perform def types_to_import %w[ - AuthorizationRequest::HubEEDila - AuthorizationRequest::HubEECertDC ] end @@ -88,14 +86,10 @@ def global_options { authorization_requests_filter: ->(enrollment_row) do %w[ - 5 - 26 - 129 - 25590 - 54115 ].exclude?(enrollment_row['id']) end, - authorization_requests_sql_where: 'target_api in (\'hubee_portail\', \'hubee_portail_dila\') order by id desc', + authorization_requests_sql_where: 'target_api in (\'api_impot_particulier_sandbox\') order by id desc', + # authorization_requests_sql_where: 'target_api not in (\'hubee_portail\', \'hubee_portail_dila\', \'api_entreprise\', \'api_particulier\') order by id desc', skipped: @skipped, warned: @warned, } From a22cc5775ba3ebb0cd3a8c12035a209e5d29c599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Wed, 8 Jan 2025 13:31:09 +0100 Subject: [PATCH 09/40] Migration: Introduce Import::AuthorizationRequests::APIImpotParticulierAttributes --- .../import/authorization_requests.rb | 5 +- .../api_impot_particulier_attributes.rb | 67 +++++++++++++++++++ ...pi_impot_particulier_sandbox_attributes.rb | 5 +- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 app/migration/import/authorization_requests/api_impot_particulier_attributes.rb diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 4366ed871..b1dfe3bec 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -16,7 +16,7 @@ def extract(enrollment_row) authorization_request.applicant = user authorization_request.organization_id = fetch_organization(user, enrollment_row).try(:id) - authorization_request.form_uid = fetch_form(authorization_request).id + authorization_request.form_uid = fetch_form(authorization_request).try(:id) authorization_request.state = enrollment_row['status'] authorization_request.external_provider_id = enrollment_row['external_provider_id'] authorization_request.copied_from_request = AuthorizationRequest.find(enrollment_row['copied_from_enrollment_id']) if enrollment_row['copied_from_enrollment_id'] && AuthorizationRequest.exists?(enrollment_row['copied_from_enrollment_id']) @@ -28,7 +28,7 @@ def extract(enrollment_row) ) end - handle_authorization_request_type_specific_fields(authorization_request, enrollment_row) + authorization_request = handle_authorization_request_type_specific_fields(authorization_request, enrollment_row) begin authorization_request.save! @@ -227,6 +227,7 @@ def from_target_api_to_type(enrollment) 'api_entreprise' => 'api_entreprise', 'api_particulier' => 'api_particulier', 'api_impot_particulier_sandbox' => 'api_impot_particulier_sandbox', + 'api_impot_particulier_production' => 'api_impot_particulier', }[enrollment['target_api']].try(:classify) end diff --git a/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb new file mode 100644 index 000000000..8bf0db286 --- /dev/null +++ b/app/migration/import/authorization_requests/api_impot_particulier_attributes.rb @@ -0,0 +1,67 @@ +class Import::AuthorizationRequests::APIImpotParticulierAttributes < Import::AuthorizationRequests::APIImpotParticulierSandboxAttributes + def affect_data + migrate_from_sandbox_to_production! + + affect_form_uid + + affect_operational_acceptance + affect_safety_certification + affect_volumetrie + + authorization_request.state = enrollment_row['status'] + end + + private + + def migrate_from_sandbox_to_production! + @authorization_request = AuthorizationRequest.find(enrollment_row['previous_enrollment_id']) + + authorization_request.update!(type: 'AuthorizationRequest::APIImpotParticulier', state: 'draft', id: enrollment_row['id']) + + ActiveStorage::Attachment.where(record_type: 'AuthorizationRequest', record_id: enrollment_row['previous_enrollment_id']).update_all(record_id: enrollment_row['id']) + + @authorization_request = AuthorizationRequest.find(enrollment_row['id']) + rescue ActiveRecord::RecordNotFound + skip_row!(:sandbox_missing) + end + + # FIXME + def affect_form_uid + authorization_request.form_uid = authorization_request.definition.available_forms.first.id + end + + def affect_operational_acceptance + authorization_request.operational_acceptance_done = additional_content['recette_fonctionnelle'] ? '1' : nil + end + + def affect_safety_certification + { + 'autorite_homologation_nom' => 'safety_certification_authority_name', + 'autorite_homologation_fonction' => 'safety_certification_authority_function', + 'date_homologation' => 'safety_certification_begin_date', + 'date_fin_homologation' => 'safety_certification_end_date' + }.each do |from, to| + authorization_request.public_send("#{to}=", additional_content[from]) + end + + affect_potential_document('Document::DecisionHomologation', 'safety_certification_document') + + return if authorization_request.safety_certification_begin_date.blank? || authorization_request.safety_certification_end_date.blank? + + begin + return unless Date.parse(authorization_request.safety_certification_begin_date) == Date.parse(authorization_request.safety_certification_end_date) + rescue TypeError + return + end + + authorization_request.safety_certification_end_date = (Date.parse(authorization_request.safety_certification_begin_date) + 1.day).to_s + end + + def affect_volumetrie + authorization_request.volumetrie_appels_par_minute = additional_content['volumetrie_appels_par_minute'].try(:to_i) + + return if authorization_request.volumetrie_appels_par_minute == 50 + + authorization_request.volumetrie_justification = 'Non renseignée' + end +end diff --git a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb index 032cd2d6d..56f3c2658 100644 --- a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb +++ b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb @@ -24,7 +24,6 @@ def affect_potential_specific_requirements end def affect_modalities - additional_content = JSON.parse(enrollment_row['additional_content']) || {} authorization_request.modalities ||= [] { @@ -107,4 +106,8 @@ def attributes_mapping def attributes_with_possible_null_values ['destinataire_donnees_caractere_personnel'] end + + def additional_content + @additional_content ||= JSON.parse(enrollment_row['additional_content']) + end end From 697f83849d52d56cd0cba4679d7f82a69731d2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Wed, 15 Jan 2025 10:30:25 +0100 Subject: [PATCH 10/40] Migration: print skipped while loading --- app/migration/import/base.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/migration/import/base.rb b/app/migration/import/base.rb index ac0ba5897..b8956c51c 100644 --- a/app/migration/import/base.rb +++ b/app/migration/import/base.rb @@ -77,6 +77,7 @@ def load_from_csv! extract(row) print '.' rescue Import::AuthorizationRequests::Base::SkipRow => e + print 's' options[:skipped] << e rescue => e log(" ERROR: #{e.message}") From ab7c8874e8755da7b9f9e1b619ce1d5e1d237577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Wed, 15 Jan 2025 10:32:07 +0100 Subject: [PATCH 11/40] Migration: Harden Import::AuthorizationRequests classes --- .../import/authorization_requests.rb | 8 ++++++- .../api_entreprise_attributes.rb | 6 ----- ...pi_impot_particulier_sandbox_attributes.rb | 10 --------- .../import/authorization_requests/base.rb | 22 ++++++++++++++++--- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index b1dfe3bec..8c1a96752 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -16,7 +16,7 @@ def extract(enrollment_row) authorization_request.applicant = user authorization_request.organization_id = fetch_organization(user, enrollment_row).try(:id) - authorization_request.form_uid = fetch_form(authorization_request).try(:id) + authorization_request.form_uid ||= fetch_form(authorization_request).try(:id) authorization_request.state = enrollment_row['status'] authorization_request.external_provider_id = enrollment_row['external_provider_id'] authorization_request.copied_from_request = AuthorizationRequest.find(enrollment_row['copied_from_enrollment_id']) if enrollment_row['copied_from_enrollment_id'] && AuthorizationRequest.exists?(enrollment_row['copied_from_enrollment_id']) @@ -40,6 +40,11 @@ def extract(enrollment_row) byebug rescue ActiveStorage::IntegrityError => e byebug + ensure + ($dummy_files || {}).each do |key, value| + value.close + end + $dummy_files = nil end end @@ -228,6 +233,7 @@ def from_target_api_to_type(enrollment) 'api_particulier' => 'api_particulier', 'api_impot_particulier_sandbox' => 'api_impot_particulier_sandbox', 'api_impot_particulier_production' => 'api_impot_particulier', + 'franceconnect' => 'france_connect', }[enrollment['target_api']].try(:classify) end diff --git a/app/migration/import/authorization_requests/api_entreprise_attributes.rb b/app/migration/import/authorization_requests/api_entreprise_attributes.rb index 7c9013b13..f0e6c635a 100644 --- a/app/migration/import/authorization_requests/api_entreprise_attributes.rb +++ b/app/migration/import/authorization_requests/api_entreprise_attributes.rb @@ -66,12 +66,6 @@ def affect_not_specified_to_duree_conservation authorization_request.duree_conservation_donnees_caractere_personnel = 'Non renseigné' end - def recent_validated_enrollment_exists? - bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? - database.close - bool - end - def attributes_mapping { "intitule" => "intitule", diff --git a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb index 56f3c2658..9a1606f58 100644 --- a/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb +++ b/app/migration/import/authorization_requests/api_impot_particulier_sandbox_attributes.rb @@ -84,12 +84,6 @@ def demarche_to_form_uid end end - def recent_validated_enrollment_exists? - bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? - database.close - bool - end - def attributes_mapping { "intitule" => "intitule", @@ -106,8 +100,4 @@ def attributes_mapping def attributes_with_possible_null_values ['destinataire_donnees_caractere_personnel'] end - - def additional_content - @additional_content ||= JSON.parse(enrollment_row['additional_content']) - end end diff --git a/app/migration/import/authorization_requests/base.rb b/app/migration/import/authorization_requests/base.rb index ae5f3edc9..513c4c6cc 100644 --- a/app/migration/import/authorization_requests/base.rb +++ b/app/migration/import/authorization_requests/base.rb @@ -29,7 +29,11 @@ def initialize(authorization_request, enrollment_row, team_members, warned, all_ end def perform - affect_data + begin + affect_data + ensure + database.close + end authorization_request end @@ -168,7 +172,7 @@ def affect_contact(from_contact, to_contact) end end - byebug + # byebug if recent_validated_enrollment_exists? skip_row!('incomplete_contact_data_with_new_enrollments') @@ -217,6 +221,10 @@ def attach_file(kind, row_data) authorization_request.public_send("#{kind}").attach(io:, filename:) end + def additional_content + @additional_content ||= JSON.parse(enrollment_row['additional_content']) + end + def extract_attachable(kind, row_data) if ENV['LOCAL'] == 'true' if kind == 'specific_requirements_document' @@ -239,7 +247,15 @@ def extract_attachable(kind, row_data) end def dummy_file_as_io(extension) - Rails.root.join('spec', 'fixtures', "dummy.#{extension}").open + $dummy_files ||= {} + $dummy_files[extension] ||= Rails.root.join('spec', 'fixtures', "dummy.#{extension}").open + $dummy_files[extension] + end + + def recent_validated_enrollment_exists? + bool = database.execute('select id from enrollments where copied_from_enrollment_id = ? and status = "validated" limit 1;', enrollment_row['id']).any? + database.close + bool end def extract_io(row_data) From 2486f85968948fcc1273906baf001ed1306f16b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loi=CC=88c=20Delmaire?= Date: Wed, 15 Jan 2025 10:32:11 +0100 Subject: [PATCH 12/40] Migration: Introduce Import::AuthorizationRequests::FranceConnectAttributes --- .../france_connect_attributes.rb | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 app/migration/import/authorization_requests/france_connect_attributes.rb diff --git a/app/migration/import/authorization_requests/france_connect_attributes.rb b/app/migration/import/authorization_requests/france_connect_attributes.rb new file mode 100644 index 000000000..77d3806ab --- /dev/null +++ b/app/migration/import/authorization_requests/france_connect_attributes.rb @@ -0,0 +1,77 @@ +class Import::AuthorizationRequests::FranceConnectAttributes < Import::AuthorizationRequests::Base + def affect_data + affect_scopes + affect_attributes + affect_contacts + affect_potential_legal_document + # affect_not_specified_to_duree_conservation + affect_form_uid + authorization_request.france_connect_eidas = extract_eidas + + return if authorization_request.valid? + + skip_row!(:invalid_cadre_juridique) if authorization_request.errors[:cadre_juridique_url].any? + end + + def affect_contacts + { + 'responsable_technique' => 'contact_technique', + 'responsable_traitement' => 'responsable_traitement', + 'delegue_protection_donnees' => 'delegue_protection_donnees', + }.each do |from_contact, to_contact| + affect_contact(from_contact, to_contact) + end + end + + def affect_potential_legal_document + return if authorization_request.cadre_juridique_url.present? + + row = csv('documents').find { |row| row['attachable_id'] == enrollment_row['id'] && row['type'] == 'Document::LegalBasis' } + + if row + attach_file('cadre_juridique_document', row) + end + end + + def extract_eidas + case additional_content['eidas_level'] + when '1' + 'eidas_1' + when '2' + 'eidas_2' + else + skip_row!(:eidas_missing) + end + end + + def demarche_to_form_uid + case enrollment_row['demarche'] + when 'collectivite' + 'france-connect-collectivite-administration' + when 'epermis' + 'france-connect-collectivite-epermis' + when 'sns' + 'france-connect-sante' + else + 'france-connect' + end + end + + def attributes_mapping + { + "intitule" => "intitule", + "description" => "description", + "fondement_juridique_title" => "cadre_juridique_nature", + "fondement_juridique_url" => "cadre_juridique_url", + "date_mise_en_production" => "date_prevue_mise_en_production", + "volumetrie_approximative" => "volumetrie_approximative", + "data_recipients" => "destinataire_donnees_caractere_personnel", + "data_retention_period" => "duree_conservation_donnees_caractere_personnel", + "data_retention_comment" => "duree_conservation_donnees_caractere_personnel_justification", + } + end + + def attributes_with_possible_null_values + ['duree_conservation_donnees_caractere_personnel_justification'] + end +end From cc12619a4bdb23229b1d2f587da01e53743744b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Tue, 21 Jan 2025 11:54:48 +0100 Subject: [PATCH 13/40] Migration: FC no eidas => eidas 1 --- .../authorization_requests/france_connect_attributes.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/migration/import/authorization_requests/france_connect_attributes.rb b/app/migration/import/authorization_requests/france_connect_attributes.rb index 77d3806ab..dbb55bc4e 100644 --- a/app/migration/import/authorization_requests/france_connect_attributes.rb +++ b/app/migration/import/authorization_requests/france_connect_attributes.rb @@ -35,12 +35,10 @@ def affect_potential_legal_document def extract_eidas case additional_content['eidas_level'] - when '1' - 'eidas_1' when '2' 'eidas_2' else - skip_row!(:eidas_missing) + 'eidas_1' end end From dad4a0a9ea304cd5cb72a36a9e8c3db963a456a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Tue, 21 Jan 2025 10:07:14 +0100 Subject: [PATCH 14/40] Migration: Harden Import::AuthorizationRequests organization build --- .../import/authorization_requests.rb | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/app/migration/import/authorization_requests.rb b/app/migration/import/authorization_requests.rb index 8c1a96752..3a51ee646 100644 --- a/app/migration/import/authorization_requests.rb +++ b/app/migration/import/authorization_requests.rb @@ -103,9 +103,25 @@ def fetch_organization(user, enrollment_row) '77564141800014' => '77564141800089', }[enrollment_row['siret']] - return if new_potential_siret.blank? + if new_potential_siret.present? + user.organizations.find_by(siret: new_potential_siret) + else + organization = build_organization(enrollment_row['siret']) + + begin + organization.save! + rescue ActiveRecord::RecordInvalid => e + if e.record.errors.include?(:siret) + raise Import::AuthorizationRequests::Base::SkipRow.new(:invalid_siret_for_unknown_user_and_organization, id: enrollment_row['id'], target_api: enrollment_row['target_api']) + else + raise + end + end - user.organizations.find_by(siret: new_potential_siret) + user.organizations << organization + + organization + end end def authorization_ids_where_user_belongs_to_organization @@ -143,11 +159,8 @@ def try_to_create_user!(enrollment_row, demandeur) ) ) - organization = Organization.find_or_initialize_by(siret: enrollment_row['siret']) - organization.assign_attributes( - mon_compte_pro_payload: { siret: enrollment_row['siret'], manual_creation: true }, - last_mon_compte_pro_updated_at: DateTime.now, - ) + organization = build_organization(enrollment_row['siret']) + begin organization.save! rescue ActiveRecord::RecordInvalid => e @@ -166,6 +179,19 @@ def try_to_create_user!(enrollment_row, demandeur) user end + def build_organization(siret) + organization = Organization.find_or_initialize_by(siret: siret) + + return organization if organization.persisted? + + organization.assign_attributes( + mon_compte_pro_payload: { siret:, manual_creation: true }, + last_mon_compte_pro_updated_at: DateTime.now, + ) + + organization + end + def fetch_form(authorization_request) if authorization_request.definition.available_forms.one? authorization_request.definition.available_forms.first From 000f375a3f1bb676655df9187f52c904420f7301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Delmaire=20Lo=C3=AFc?= Date: Tue, 21 Jan 2025 19:37:27 +0100 Subject: [PATCH 15/40] Migration: add disclaimer on reopening authorizations for invalid request from migration --- app/views/reopen_authorizations/new.html.erb | 6 ++++++ config/locales/fr.yml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/app/views/reopen_authorizations/new.html.erb b/app/views/reopen_authorizations/new.html.erb index 1e333bd51..c7151a009 100644 --- a/app/views/reopen_authorizations/new.html.erb +++ b/app/views/reopen_authorizations/new.html.erb @@ -10,6 +10,12 @@

<%= t('.disclaimer').html_safe %>

+ + <% unless @authorization.request.valid? %> +

+ <%= t('.invalid_request') %> +

+ <% end %>