Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@nikhiltri
nikhiltri committed Jun 20, 2018
2 parents c66714b + 9f487c6 commit 6d17874
Show file tree
Hide file tree
Showing 29 changed files with 959 additions and 599 deletions.
26 changes: 26 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,32 @@
Data Aggregator Changelog
=============================

### 1.0-beta2 – Minor enhancements, code cleanup and reorganization

* Refactor deletes from LAKE so they can run every five minutes instead of hourly
* Move titles into all listing endpoints, where previously they were only indexed in search
* Add `alt_text` to `thumbnail` block of all resources


ARTWORKS

* The following modifications have been made to the API schema:
- `has_multimedia_resources` - Added, whether this artwork has any associated microsites, digital publications, or documents tagged as multimedia
- `has_educational_resources` - Added, whether this artwork has any documents tagged as educational
- `theme_titles` - Added, the names of all thematic publish categories related to this artwork


EVENTS

* Add fields to support sales.artic.edu
* The following modifications have been made to the API schema:
- `is_admission_required` - Added, whether admission to the museum is required to attend this event
- `ticketed_event_id` - Added, unique identifier of the event in the ticketing system this website event is tied to
- `survey_url` - Added, URL to the survey associated with this event
- `email_series` - Added, the email series associated with this event
- `door_time` - Added, the time the doors open for this event


### 1.0-beta1 – Final cleanup before beta and bug fixes

* Upgrade to Laravel 5.6
Expand Down
102 changes: 57 additions & 45 deletions app/Console/Commands/DeleteCollectionsRecords.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@
namespace App\Console\Commands;

use Illuminate\Console\Command;
use Carbon\Carbon;

class DeleteCollectionsRecords extends Command
class DeleteCollectionsRecords extends AbstractImportCommand
{
/**
* The name and signature of the console command.
Expand All @@ -20,7 +21,22 @@ class DeleteCollectionsRecords extends Command
*/
protected $description = 'Delete records that have been removed from the source system';

protected $size = 100;
protected $models = [
\App\Models\Collections\ArtworkPlaceQualifier::class,
\App\Models\Collections\ArtworkDateQualifier::class,
\App\Models\Collections\AgentRole::class,
\App\Models\Collections\ArtworkType::class,
\App\Models\Collections\AgentType::class,
\App\Models\Collections\Agent::class,
\App\Models\Collections\Category::class,
\App\Models\Collections\Term::class,
\App\Models\Collections\Place::class,
\App\Models\Collections\Gallery::class,
\App\Models\Collections\Catalogue::class,
\App\Models\Collections\Asset::class,
\App\Models\Collections\Artwork::class,
\App\Models\Collections\Exhibition::class,
];

/**
* Create a new command instance.
Expand All @@ -29,7 +45,10 @@ class DeleteCollectionsRecords extends Command
*/
public function __construct()
{

$this->api = env('COLLECTIONS_DATA_SERVICE_URL');
parent::__construct();

}

/**
Expand All @@ -40,63 +59,56 @@ public function __construct()
public function handle()
{

$this->deleteUnpublished(\App\Models\Collections\ArtworkDateQualifier::class);
$this->deleteUnpublished(\App\Models\Collections\ArtworkType::class, 'object-types');
$this->deleteUnpublished(\App\Models\Collections\AgentType::class);
$this->deleteUnpublished(\App\Models\Collections\AgentPlace::class);
$this->deleteUnpublished(\App\Models\Collections\Agent::class);
$this->deleteUnpublished(\App\Models\Collections\Category::class);
$this->deleteUnpublished(\App\Models\Collections\Term::class);
$this->deleteUnpublished(\App\Models\Collections\Place::class);
$this->deleteUnpublished(\App\Models\Collections\Gallery::class);
$this->deleteUnpublished(\App\Models\Collections\ArtworkCatalogue::class);
$this->deleteUnpublished(\App\Models\Collections\Catalogue::class);
$this->deleteUnpublished(\App\Models\Collections\Artwork::class);
$this->deleteUnpublished(\App\Models\Collections\Video::class);
$this->deleteUnpublished(\App\Models\Collections\Text::class);
$this->deleteUnpublished(\App\Models\Collections\Sound::class);
$this->deleteUnpublished(\App\Models\Collections\Image::class);
$this->deleteUnpublished(\App\Models\Collections\AgentExhibition::class);
$this->deleteUnpublished(\App\Models\Collections\Exhibition::class);
$current = 1;

}
$json = $this->query('deletes', $current);
$pages = $json->pagination->total_pages;

private function deleteUnpublished($model = \App\Models\Collections\Artwork::class, $endpoint = '')
{
$this->warn( 'Found ' . $pages . ' pages' );

if (!$endpoint)
while( $current <= $pages )
{

$endpoint = app('Resources')->getEndpointForModel($model);

}
$this->warn( 'Deleteing ' . $current . ' of ' . $pages);

$model::chunk($this->size, function ($resources) use ($model, $endpoint) {
$this->info( 'Working on ' . $endpoint . ' starting at ' .$resources->pluck($model::instance()->getKeyName())->first());
// Assumes the dataservice wraps its results in a `data` field
foreach( $json->data as $datum )
{

// Get a list of CITI IDs of works in the Data Aggregator
$daIds = $resources->pluck($model::instance()->getKeyName());
// Break if we're past the last time we checked
$sourceTime = new Carbon( $datum->indexed_at );
$sourceTime->timezone = config('app.timezone');

// Use those IDs to retrieve a list from the Collections Data Service
$json = json_decode(file_get_contents(env('COLLECTIONS_DATA_SERVICE_URL')
.'/'
.$endpoint
.'?flo=id&limit=' .$this->size .'&ids='
.implode(',',$daIds->all())));
if( $this->command->last_success_at->gt( $sourceTime ) )
{
break 2;
}

$cdsIds = collect($json->data)->pluck('id');
// Now execute an actual delete
// Loop through all model types
foreach ( $this->models as $model)
{

// Compare those two lists, and delete anything in the Data Hub that's not in the CSD
$diff = $daIds->diff($cdsIds)->all();
if ($diff)
{
// Check if a resource with a matching lake_guid exists
if ($m = $model::where('lake_guid', '=', $datum->lake_guid)->first())
{
// If it does, destroy the model and break
$this->warn( 'Deleteing ' . $model . ' ' . $datum->lake_guid);
$m->delete();
break;
}

$this->warn( 'Deleting ' . implode(',', $diff) );
$model::destroy($diff);
}

}

});
$current++;

// TODO: This structure causes an extra query to be run, when it might not need to be
$json = $this->query( 'deletes', $current );

}

}

}
123 changes: 98 additions & 25 deletions app/Console/Commands/ImportWebFull.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ class ImportWebFull extends AbstractImportCommand
{

protected $signature = 'import:web-full
{--y|yes : Answer "yes" to all prompts}';
{endpoint? : Endpoint on dataservice to query, e.g. `events`}
{--y|yes : Answer "yes" to all prompts}
{page? : Page to begin importing from}';

protected $description = "Import all Web CMS data";

Expand All @@ -33,14 +35,25 @@ public function handle()

$this->api = env('WEB_CMS_DATA_SERVICE_URL');

if( !$this->reset() )
{
return false;
}
$endpoint = $this->argument('endpoint');

if ($endpoint) {

$page = $this->argument('page') ?: 1;

$this->importEndpoints();
$this->importFromWeb($endpoint, $page);
$this->info("Imported ${endpoint} web CMS content!");

$this->info("Imported all web CMS content!");
} else {

if( !$this->reset() )
{
return false;
}
$this->importEndpoints();
$this->info("Imported all web CMS content!");

}

}

Expand Down Expand Up @@ -89,30 +102,90 @@ protected function reset()
protected function importEndpoints()
{

$this->importFromWeb(Article::class, 'articles');
$this->importFromWeb('articles');
// Incorrect integer value: 'Also known as' for column 'also_known_as'
// $this->importFromWeb(Artist::class, 'artists');
$this->importFromWeb(Closure::class, 'closures');
$this->importFromWeb(Event::class, 'events');
$this->importFromWeb(Exhibition::class, 'exhibitions');
$this->importFromWeb(Hour::class, 'hours');
$this->importFromWeb(Location::class, 'locations');
$this->importFromWeb(Selection::class, 'selections');
$this->importFromWeb(Tag::class, 'tags');

$this->importFromWeb(GenericPage::class, 'genericpages');
$this->importFromWeb(PressRelease::class, 'pressreleases');
$this->importFromWeb(ResearchGuide::class, 'researchguides');
$this->importFromWeb(EducatorResource::class, 'educatorresources');
$this->importFromWeb(DigitalCatalog::class, 'digitalcatalogs');
$this->importFromWeb(PrintedCatalog::class, 'printedcatalogs');
// $this->importFromWeb('artists');
$this->importFromWeb('closures');
$this->importFromWeb('events');
$this->importFromWeb('exhibitions');
$this->importFromWeb('hours');
$this->importFromWeb('locations');
$this->importFromWeb('selections');
$this->importFromWeb('tags');

$this->importFromWeb('genericpages');
$this->importFromWeb('pressreleases');
$this->importFromWeb('researchguides');
$this->importFromWeb('educatorresources');
$this->importFromWeb('digitalcatalogs');
$this->importFromWeb('printedcatalogs');

}

protected function getModelForEndpoint($endpoint)
{

switch( $endpoint )
{
case 'articles':
return Article::class;
break;
case 'artists':
return Artist::class;
break;
case 'closures':
return Closure::class;
break;
case 'events':
return Event::class;
break;
case 'exhibitions':
return Exhibition::class;
break;
case 'hours':
return Hour::class;
break;
case 'locations':
return Location::class;
break;
case 'selections':
return Selection::class;
break;
case 'tags':
return Tag::class;
break;
case 'genericpages':
return GenericPage::class;
break;
case 'pressreleases':
return PressRelease::class;
break;
case 'researchguides':
return ResearchGuide::class;
break;
case 'educatorresources':
return EducatorResource::class;
break;
case 'digitalcatalogs':
return DigitalCatalog::class;
break;
case 'printedcatalogs':
return PrintedCatalog::class;
break;
default:
// TODO: This gets endpoints as outbound from our API
// Endpoints in the dataservice might be different!
return app('Resources')->getModelForEndpoint($endpoint);
break;
}

}

protected function importFromWeb($model, $endpoint)
protected function importFromWeb($endpoint, $page = 1)
{

return $this->import( 'Web', $model, $endpoint );
$model = $this->getModelForEndpoint($endpoint);
return $this->import( 'Web', $model, $endpoint, $page );

}

Expand Down
7 changes: 7 additions & 0 deletions app/Console/Commands/Update/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
This directory contains commands that are migration-esque, but for one reason or another, weren't a great fit for migrations.

1. Commands that seem like re-usable utils, rather than one-offs.
2. Commands that cause tests to fail, because SQLite complains.
3. Commands that update the search index.

We are still not quite comfortable putting the latter into migrations.
Loading

0 comments on commit 6d17874

Please sign in to comment.