Add pre-creation of body CSV files.

mysociety · Nov 16, 2023 · ef56818 · ef56818
1 parent 5cb01fb
commit ef56818
Show file tree

Hide file tree

Showing 38 changed files with 838 additions and 67 deletions.
diff --git a/bin/csv-export b/bin/csv-export
@@ -71,9 +71,13 @@ my $reporting = FixMyStreet::Reporting->new(
     $opts->start_date ? (start_date => $opts->start_date) : (),
     end_date => $opts->end_date,
 );
-$reporting->construct_rs_filter;
-$reporting->csv_parameters;
-$reporting->generate_csv($fh);
+if ($reporting->premade_csv_exists) {
+    $reporting->filter_premade_csv($fh);
+} else {
+    $reporting->construct_rs_filter;
+    $reporting->csv_parameters;
+    $reporting->generate_csv($fh);
+}
 unless ($use_stdout) {
     $file->move($opts->out);
 }
diff --git a/bin/fixmystreet.com/all-csv-export b/bin/fixmystreet.com/all-csv-export
@@ -0,0 +1,31 @@
+#!/usr/bin/env perl
+
+# all-csv-export
+# Pre-generates all body CSV files for quicker dashboard export
+# Uses DBI directly
+
+use v5.14;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../../setenv.pl";
+}
+
+use Getopt::Long::Descriptive;
+use CronFns;
+use FixMyStreet::Script::CSVExport;
+
+my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
+CronFns::language($site);
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['verbose|v', 'more verbose output'],
+    ['help|h', "print usage message and exit" ],
+);
+$usage->die if $opts->help;
+
+FixMyStreet::Script::CSVExport::process(verbose => $opts->verbose);
diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm
@@ -158,7 +158,11 @@ sub index : Path : Args(0) {
         $reporting->csv_parameters;
         if ($export == 1) {
             # Existing method, generate and serve
-            $reporting->generate_csv_http($c);
+            if ($reporting->premade_csv_exists) {
+                $reporting->filter_premade_csv_http($c);
+            } else {
+                $reporting->generate_csv_http($c);
+            }
         } elsif ($export == 2) {
             # New offline method
             $reporting->kick_off_process;

diff --git a/perllib/FixMyStreet/Cobrand/BathNES.pm b/perllib/FixMyStreet/Cobrand/BathNES.pm
@@ -210,7 +210,8 @@ sub dashboard_export_updates_add_columns {
 sub dashboard_export_problems_add_columns {
     my ($self, $csv) = @_;
 
-    return unless $csv->user->has_body_permission_to('export_extra_columns');
+    # If running via DBI export, we need the extra columns saved
+    return unless $csv->dbi || $csv->user->has_body_permission_to('export_extra_columns');
 
     $csv->add_csv_columns(
         user_email => 'User Email',
@@ -227,14 +228,14 @@ sub dashboard_export_problems_add_columns {
 
     $csv->csv_extra_data(sub {
         my $report = shift;
-
-        my $staff_user = $self->csv_staff_user_lookup($report->get_extra_metadata('contributed_by'), $user_lookup);
-        my $attribute_data = join "; ", map { $_->{name} . " = " . $_->{value} } @{ $report->get_extra_fields };
+        my $attribute_data = join "; ", map { $_->{name} . " = " . $_->{value} } @{ $csv->_extra_field($report) };
         return {
-            user_email => $report->user->email || '',
-            user_phone => $report->user->phone || '',
-            staff_user => $staff_user,
             attribute_data => $attribute_data,
+            $csv->dbi ? () : (
+                user_email => $report->user->email || '',
+                user_phone => $report->user->phone || '',
+                staff_user => $self->csv_staff_user_lookup($report->get_extra_metadata('contributed_by'), $user_lookup),
+            ),
         };
     });
 }

diff --git a/perllib/FixMyStreet/Cobrand/Brent.pm b/perllib/FixMyStreet/Cobrand/Brent.pm
@@ -227,8 +227,8 @@ sub dashboard_export_problems_add_columns {
         (
             street_name => 'Street Name',
             location_name => 'Location Name',
-            created_by => 'Created By',
-            email => 'Email',
+            name => 'Created By',
+            user_email => 'Email',
             usrn => 'USRN',
             uprn => 'UPRN',
             external_id => 'External ID',
@@ -260,35 +260,38 @@ sub dashboard_export_problems_add_columns {
 
     my $flytipping_lookup = sub {
         my ($report, $field) = @_;
-
-        my $v = $report->get_extra_field_value($field) // return '';
+        my $v = $csv->_extra_field($report, $field) // return '';
         return $values->{$field}{$v} || '';
     };
 
     $csv->csv_extra_data(sub {
         my $report = shift;
 
         my $data = {
-            street_name => $report->nearest_address_parts->{street},
-            location_name => $report->get_extra_field_value('location_name') || '',
-            created_by => $report->name || '',
-            email => $report->user->email || '',
-            usrn => $report->get_extra_field_value('usrn') || '',
-            uprn => $report->get_extra_field_value('uprn') || '',
-            external_id => $report->external_id || '',
-            image_included => $report->photo ? 'Y' : 'N',
+            location_name => $csv->_extra_field($report, 'location_name'),
+            $csv->dbi ? (
+                street_name => FixMyStreet::Geocode::Address->new($report->{geocode})->parts->{street},
+                image_included => $report->{photo} ? 'Y' : 'N',
+            ) : (
+                street_name => $report->nearest_address_parts->{street},
+                name => $report->name || '',
+                user_email => $report->user->email || '',
+                image_included => $report->photo ? 'Y' : 'N',
+                external_id => $report->external_id || '',
+            ),
+            usrn => $csv->_extra_field($report, 'usrn'),
+            uprn => $csv->_extra_field($report, 'uprn'),
             flytipping_did_you_see => $flytipping_lookup->($report, 'Did_you_see_the_Flytip_take_place?_'),
             flytipping_statement => $flytipping_lookup->($report, 'Are_you_willing_to_be_a_WItness?_'),
             flytipping_quantity => $flytipping_lookup->($report, 'Flytip_Size'),
             flytipping_type => $flytipping_lookup->($report, 'Flytip_Type'),
-            container_req_action => $report->get_extra_field_value('Container_Request_Action') || '',
-            container_req_type => $report->get_extra_field_value('Container_Request_Container_Type') || '',
-            container_req_reason => $report->get_extra_field_value('Container_Request_Reason') || '',
-            missed_collection_id => $report->get_extra_field_value('service_id') || '',
+            container_req_action => $csv->_extra_field($report, 'Container_Request_Action'),
+            container_req_type => $csv->_extra_field($report, 'Container_Request_Container_Type'),
+            container_req_reason => $csv->_extra_field($report, 'Container_Request_Reason'),
+            missed_collection_id => $csv->_extra_field($report, 'service_id'),
         };
 
-        my $extra = $report->get_extra_metadata;
-
+        my $extra = $csv->_extra_metadata($report);
         %$data = (%$data, map {$_ => $extra->{$_} || ''} grep { $_ =~ /^(item_\d+)$/ } keys %$extra);
 
         return $data;

diff --git a/perllib/FixMyStreet/Cobrand/Bromley.pm b/perllib/FixMyStreet/Cobrand/Bromley.pm
@@ -971,6 +971,8 @@ sub dashboard_export_problems_add_columns {
         staff_role => 'Staff Role',
     );
 
+    return if $csv->dbi; # All covered already
+
     my $user_lookup = $self->csv_staff_users;
     my $userroles = $self->csv_staff_roles($user_lookup);
 

diff --git a/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm b/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm
@@ -333,6 +333,8 @@ sub _dashboard_export_add_columns {
 
     $csv->add_csv_columns( staff_user => 'Staff User' );
 
+    return if $csv->dbi; # staff_user included by default
+
     my $user_lookup = $self->csv_staff_users;
 
     $csv->csv_extra_data(sub {

diff --git a/perllib/FixMyStreet/Cobrand/Camden.pm b/perllib/FixMyStreet/Cobrand/Camden.pm
@@ -143,7 +143,7 @@ sub dashboard_export_problems_add_columns {
     my ($self, $csv) = @_;
 
     $csv->add_csv_columns(
-        user_name => 'User Name',
+        name => 'User Name',
         user_email => 'User Email',
     );
 
@@ -152,11 +152,13 @@ sub dashboard_export_problems_add_columns {
         join => 'user',
     });
 
+    return if $csv->dbi; # Already covered
+
     $csv->csv_extra_data(sub {
         my $report = shift;
 
         return {
-            user_name => $report->name || '',
+            name => $report->name || '',
             user_email => $report->user->email || '',
         };
     });

diff --git a/perllib/FixMyStreet/Cobrand/CentralBedfordshire.pm b/perllib/FixMyStreet/Cobrand/CentralBedfordshire.pm
@@ -281,6 +281,8 @@ sub dashboard_export_problems_add_columns {
         external_id => 'CRNo',
     );
 
+    return if $csv->dbi; # Already covered
+
     $csv->csv_extra_data(sub {
         my $report = shift;
 

diff --git a/perllib/FixMyStreet/Cobrand/Hackney.pm b/perllib/FixMyStreet/Cobrand/Hackney.pm
@@ -349,16 +349,23 @@ sub dashboard_export_problems_add_columns {
 
         my $address = '';
         my $postcode = '';
-
-        if ( $report->geocode ) {
-            $address = $report->nearest_address;
-            $postcode = $report->nearest_address_parts->{postcode};
+        if ($csv->dbi) {
+            if ( $report->{geocode} ) {
+                my $addr = FixMyStreet::Geocode::Address->new($report->{geocode});
+                $address = $addr->summary;
+                $postcode = $addr->parts->{postcode};
+            }
+        } else {
+            if ( $report->geocode ) {
+                $address = $report->nearest_address;
+                $postcode = $report->nearest_address_parts->{postcode};
+            }
         }
 
         return {
             nearest_address => $address,
             nearest_address_postcode => $postcode,
-            extra_details => $report->get_extra_metadata('detailed_information') || '',
+            extra_details => $csv->_extra_metadata($report, 'detailed_information') || '',
         };
     });
 }

diff --git a/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm b/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm
@@ -330,15 +330,40 @@ sub dashboard_export_problems_add_columns {
         );
     }
 
+    if ($csv->dbi) {
+        my $JSON = JSON::MaybeXS->new->allow_nonref;
+        $csv->csv_extra_data(sub {
+            my $report = shift;
+
+            my $fields = {
+                user_name_display => $report->{name},
+                area_name => $csv->_extra_field($report, 'area_name'),
+                where_hear => $csv->_extra_metadata($report, 'where_hear'),
+            };
+
+            my $i = $report->{comment_rn};
+            if ($report->{comment_id} && $i <= 5) {
+                $fields->{"update_text_$i"} = $report->{comment_text};
+                $fields->{"update_date_$i"} = $report->{comment_confirmed};
+                my $extra = $JSON->decode($report->{comment_extra} || '{}');
+                my $staff = $extra->{contributed_by} || $extra->{is_body_user} || $extra->{is_superuser};
+                $fields->{"update_name_$i"} = $staff ? $report->{comment_name} : 'public';
+            }
+
+            return $fields;
+        });
+        return;
+    }
+
     $csv->csv_extra_data(sub {
         my $report = shift;
 
         my $fields = {
             user_name_display => $report->name,
             user_email => $report->user->email || '',
             user_phone => $report->user->phone || '',
-            area_name => $report->get_extra_field_value('area_name'),
-            where_hear => $report->get_extra_metadata('where_hear'),
+            area_name => $csv->_extra_field($report, 'area_name'),
+            where_hear => $csv->_extra_metadata($report, 'where_hear'),
         };
 
         my $i = 1;

diff --git a/perllib/FixMyStreet/Cobrand/Northamptonshire.pm b/perllib/FixMyStreet/Cobrand/Northamptonshire.pm
@@ -165,6 +165,8 @@ sub dashboard_export_problems_add_columns {
         external_id => 'External ID',
     );
 
+    return if $csv->dbi;
+
     $csv->csv_extra_data(sub {
         my $report = shift;
 

diff --git a/perllib/FixMyStreet/Cobrand/Northumberland.pm b/perllib/FixMyStreet/Cobrand/Northumberland.pm
@@ -86,6 +86,16 @@ sub dashboard_export_problems_add_columns {
         assigned_to => 'Assigned To',
     );
 
+    if ($csv->dbi) {
+        $csv->csv_extra_data(sub {
+            my $report = shift;
+            return {
+                user_name_display => $report->{name},
+            };
+        });
+        return; # Rest already covered
+    }
+
     my $user_lookup = $self->csv_staff_users;
     my $userroles = $self->csv_staff_roles($user_lookup);
     my $problems_to_user = $self->csv_active_planned_reports;

diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm
@@ -346,14 +346,18 @@ sub dashboard_export_problems_add_columns {
 
     $csv->csv_extra_data(sub {
         my $report = shift;
-        my $usrn = $report->get_extra_field_value('usrn') || '';
+        my $usrn = $csv->_extra_field($report, 'usrn') || '';
         # Try and get a HIAMS reference first of all
-        my $ref = $report->get_extra_metadata('customer_reference');
+        my $ref = $csv->_extra_metadata($report, 'customer_reference');
         unless ($ref) {
             # No HIAMS ref which means it's either an older Exor report
             # or a HIAMS report which hasn't had its reference set yet.
             # We detect the latter case by the id and external_id being the same.
-            $ref = $report->external_id if $report->id ne ( $report->external_id || '' );
+            if ($csv->dbi) {
+                $ref = $report->{external_id} if $report->{id} ne ( $report->{external_id} || '' );
+            } else {
+                $ref = $report->external_id if $report->id ne ( $report->external_id || '' );
+            }
         }
         return {
             external_ref => ( $ref || '' ),

diff --git a/perllib/FixMyStreet/Cobrand/Peterborough.pm b/perllib/FixMyStreet/Cobrand/Peterborough.pm
@@ -395,14 +395,39 @@ sub dashboard_export_problems_add_columns {
         @extra_columns,
     );
 
+    if ($csv->dbi) {
+        $csv->csv_extra_data(sub {
+            my $report = shift;
+
+            my $addr = FixMyStreet::Geocode::Address->new($report->{geocode});
+            my $address = $addr->summary;
+            my $ext_code = $csv->_extra_metadata($report, 'external_status_code');
+            my $state = FixMyStreet::DB->resultset("State")->display($report->{state});
+            my $extra = {
+                nearest_address => $address,
+                external_status_code => $ext_code,
+                state => $state,
+                db_state => $report->{state},
+            };
+
+            foreach (@{$csv->_extra_field($report)}) {
+                $extra->{usrn} = $_->{value} if $_->{name} eq 'site_code';
+                $extra->{"extra.$_->{name}"} = $_->{value} if $_->{name} =~ /^PCC-/i;
+            }
+
+            return $extra;
+        });
+        return;
+    }
+
     my $user_lookup = $self->csv_staff_users;
 
     $csv->csv_extra_data(sub {
         my $report = shift;
 
         my $address = $report->nearest_address(1);
         my $staff_user = $self->csv_staff_user_lookup($report->get_extra_metadata('contributed_by'), $user_lookup);
-        my $ext_code = $report->get_extra_metadata('external_status_code');
+        my $ext_code = $csv->_extra_metadata($report, 'external_status_code');
         my $state = FixMyStreet::DB->resultset("State")->display($report->state);
         my $extra = {
             nearest_address => $address,
@@ -412,7 +437,7 @@ sub dashboard_export_problems_add_columns {
             state => $state,
         };
 
-        foreach (@{$report->get_extra_fields}) {
+        foreach (@{$csv->_extra_field($report)}) {
             $extra->{usrn} = $_->{value} if $_->{name} eq 'site_code';
             $extra->{"extra.$_->{name}"} = $_->{value} if $_->{name} =~ /^PCC-/i;
         }

diff --git a/perllib/FixMyStreet/Cobrand/Shropshire.pm b/perllib/FixMyStreet/Cobrand/Shropshire.pm
@@ -192,8 +192,9 @@ sub dashboard_export_problems_add_columns {
     $csv->csv_extra_data(sub {
         my $report = shift;
 
+        my $non_public = $csv->dbi ? $report->{non_public} : $report->non_public;
         return {
-            private_report => $report->non_public ? 'Yes' : 'No'
+            private_report => $non_public ? 'Yes' : 'No'
         };
     });
 }