Merge branch 'develop'

.env.example

@@ -116,3 +116,6 @@ MAIL_PORT=2525
 MAIL_USERNAME=null
 MAIL_PASSWORD=null
 MAIL_ENCRYPTION=null
+
+GTM_ENABLED=false
+GTM_ID=

.env.testing.example

@@ -1,16 +1,33 @@
-APP_NAME=Laravel
-APP_ENV=testing
+APP_NAME='Data Aggregator'
 APP_KEY=
 APP_DEBUG=true
-APP_LOG_LEVEL=debug
-APP_URL=http://localhost
+APP_LOG_LEVEL='debug'
+APP_URL='http://data-aggregator.test'
 
-DB_CONNECTION=sqlite
-DB_DATABASE=:memory:
+# Also defined in phpunit.xml?
+APP_ENV='testing'
+BCRYPT_ROUNDS=4
+CACHE_DRIVER='array'
+MAIL_DRIVER='array'
+QUEUE_CONNECTION='sync'
+SESSION_DRIVER='array'
+
+# WEB-347: Use MySQL instead of SQLite for testing
+DB_CONNECTION='sqlite'
+DB_DATABASE=':memory:'
+
+# DB_CONNECTION='mysql'
+# DB_HOST='127.0.0.1'
+# DB_PORT=3306
+# DB_DATABASE='data_aggregator_test'
+# DB_USERNAME='homestead'
+# DB_PASSWORD='secret'
+
+DEBUGBAR_ENABLED=false
 
 SCOUT_DRIVER=null
 
-PRIMO_API_SOURCE=FOOBAR
+PRIMO_API_SOURCE='FOOBAR'
 
 # WEB-1189: Write tests for authentication behavior
 APP_RESTRICTED=false

.gitignore

@@ -12,6 +12,7 @@ npm-debug.log
 !.env.*example
 /storage/clockwork
 Envoy.blade.php
+.phpunit.result.cache
 .php_cs.cache
 .php_cs
 

app/Console/Commands/BaseCommand.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Aic\Hub\Foundation\AbstractCommand;
+
+abstract class BaseCommand extends AbstractCommand
+{
+
+    /**
+     * Keys are namespaces under `\App\Models`. Values are human-readable headings for the docs.
+     * Here, order determines order of sections in the docs. Resources under these sections will
+     * be ordered according to their definitions in `config/resources.php`.
+     *
+     * @TODO: Consider moving this to `resources.sources`?
+     *
+     * @return array
+     */
+    protected function getCategories()
+    {
+        return [
+            'Collections' => 'Collections',
+            'Shop' => 'Shop',
+            //'Membership' => 'Membership',
+            'Mobile' => 'Mobile',
+            'Dsc' => 'Digital Scholarly Catalogs',
+            'StaticArchive' => 'Static Archive',
+            //'Archive' => 'Archive',
+            //'Library' => 'Library',
+            'Web' => 'Website',
+        ];
+    }
+
+    protected function getModelsForNamespace($desiredNamespace)
+    {
+        return $this->getModels()
+            ->filter(function ($modelNamespace, $model) use ($desiredNamespace) {
+                return $modelNamespace === $desiredNamespace;
+            })
+            ->keys();
+    }
+
+    protected function getModels()
+    {
+        if (isset($this->models)) {
+            return $this->models;
+        }
+
+        return $this->models = collect(config('resources.outbound.base'))
+            ->filter(function ($value, $key) {
+                return array_key_exists('endpoint', $value) && (!array_key_exists('is_restricted', $value) || $value['is_restricted'] === false);
+            })
+            ->pluck('model')
+            ->unique()
+            ->filter()
+            ->values()
+            ->map(function ($model) {
+                $segments = explode('\\', $model);
+                return [$model => $segments[count($segments) - 2]];
+            })
+            ->collapse();
+    }
+
+}

app/Console/Commands/Docs/AbstractDocCommand.php

@@ -2,65 +2,11 @@
 
 namespace App\Console\Commands\Docs;
 
-use Aic\Hub\Foundation\AbstractCommand as BaseCommand;
+use App\Console\Commands\BaseCommand;
 
 abstract class AbstractDocCommand extends BaseCommand
 {
 
     private $models;
 
-    /**
-     * Keys are namespaces under `\App\Models`. Values are human-readable headings for the docs.
-     * Here, order determines order of sections in the docs. Resources under these sections will
-     * be ordered according to their definitions in `config/resources.php`.
-     *
-     * @TODO: Consider moving this to `resources.sources`?
-     *
-     * @return array
-     */
-    protected function getCategories()
-    {
-        return [
-            'Collections' => 'Collections',
-            'Shop' => 'Shop',
-            //'Membership' => 'Membership',
-            'Mobile' => 'Mobile',
-            'Dsc' => 'Digital Scholarly Catalogs',
-            'StaticArchive' => 'Static Archive',
-            //'Archive' => 'Archive',
-            //'Library' => 'Library',
-            'Web' => 'Website',
-        ];
-    }
-
-    protected function getModelsForNamespace($desiredNamespace)
-    {
-        return $this->getModels()
-            ->filter(function ($modelNamespace, $model) use ($desiredNamespace) {
-                return $modelNamespace === $desiredNamespace;
-            })
-            ->keys();
-    }
-
-    private function getModels()
-    {
-        if (isset($this->models)) {
-            return $this->models;
-        }
-
-        return $this->models = collect(config('resources.outbound.base'))
-            ->filter(function ($value, $key) {
-                return array_key_exists('endpoint', $value) && (!array_key_exists('is_restricted', $value) || $value['is_restricted'] === false);
-            })
-            ->pluck('model')
-            ->unique()
-            ->filter()
-            ->values()
-            ->map(function ($model) {
-                $segments = explode('\\', $model);
-                return [$model => $segments[count($segments) - 2]];
-            })
-            ->collapse();
-    }
-
 }

app/Console/Commands/Docs/DownloadCssJs.php

@@ -37,7 +37,7 @@ public function handle()
 
         foreach ($files as $vanityName => $fileName) {
             $contents = file_get_contents('https://www.artic.edu/dist/' .$fileName);
-            Storage::disk('local')->put($vanityName, $contents);
+            Storage::disk('public')->put($vanityName, $contents);
         }
     }
 }

app/Console/Commands/Dump/AbstractDumpCommand.php

@@ -6,7 +6,7 @@
 use App\Library\Shell;
 use Exception;
 
-use Aic\Hub\Foundation\AbstractCommand as BaseCommand;
+use App\Console\Commands\BaseCommand;
 
 abstract class AbstractDumpCommand extends BaseCommand
 {

app/Console/Commands/Dump/DumpJsonExport.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace App\Console\Commands\Dump;
+
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\File;
+use Exception;
+use Throwable;
+use App\Scopes\PublishedScope;
+
+class DumpJsonExport extends AbstractDumpCommand
+{
+
+    protected $signature = 'dump:json-export
+                            {--path= : Directory where to save dump, with `tables` subdir }';
+
+    protected $description = 'Create JSON dumps of all pubic endpoints';
+
+    public function handle()
+    {
+        // Get all models for export
+        $models = $this->getModels();
+
+        foreach ($models as $model => $category) {
+            // Remove any old CSVs in this dump
+            $dumpPath = $this->getDumpPath('local/json/' .app('Resources')->getEndpointForModel($model));
+            array_map('unlink', glob($dumpPath . '/*.json') ?: []);
+
+            // Create transformer used for generating JSON output
+            $transformer = app('Resources')->getTransformerForModel($model);
+            $transformer = new $transformer;
+
+            $model::addGlobalScope(new PublishedScope);
+
+            // Give feedback to the user
+            $this->info(app('Resources')->getEndpointForModel($model));
+            $bar = $this->output->createProgressBar($model::count());
+
+            // Loop through each record and dump its contents into a file
+            $model::chunk(100, function ($items) use ($transformer, $model, $bar) {
+                foreach ($items as $key => $item) {
+                    $filename = 'local/json/' .app('Resources')->getEndpointForModel($model) .'/' .$item->{$item->getKeyName()}. '.json';
+                    Storage::disk('dumps')->put($filename, json_encode($transformer->transform($item), JSON_PRETTY_PRINT));
+                }
+                $bar->advance($items->count());
+            });
+            $bar->finish();
+            $this->output->newLine(1);
+        }
+    }
+}

app/Console/Commands/Dump/DumpUpload.php

@@ -2,6 +2,7 @@
 
 namespace App\Console\Commands\Dump;
 
+use Illuminate\Support\Facades\File;
 use Exception;
 
 class DumpUpload extends AbstractDumpCommand
@@ -24,10 +25,15 @@ public function handle()
         // If you change these, you'll need to clean up the repo manually
         $tablesSrcPath = $this->getDumpPath('local/tables');
         $tablesDestPath = $repoPath . '/tables';
+        $jsonsSrcPath = $this->getDumpPath('local/json');
+        $jsonsDestPath = $repoPath . '/json';
 
         if (count(glob($tablesSrcPath . '/*.csv') ?: []) < 1) {
             throw new Exception('No CSV files found in ' . $tablesSrcPath);
         }
+        if (count(glob($jsonsSrcPath . '/*/*.json') ?: []) < 1) {
+            throw new Exception('No JSON files found in ' . $jsonsSrcPath);
+        }
 
         if ($this->option('remove') && file_exists($repoPath)) {
             $this->shell->passthru('rm -rf %s', $repoPath);
@@ -49,15 +55,20 @@ public function handle()
             $this->shell->passthru('git -C %s reset --hard %s', $repoPath, $commit);
         }
 
-        // Remove all existing CSVs from the repo
+        // Remove all existing CSVs and JSONs from the repo
         // This should take care of any tables that were removed or renamed
         if (file_exists($tablesDestPath)) {
             $this->shell->passthru('find %s -name *.csv | xargs rm', $tablesDestPath);
         } else {
             mkdir($tablesDestPath);
         }
+        if (file_exists($jsonsDestPath)) {
+            $this->shell->passthru('find %s -name *.json | xargs rm', $jsonsDestPath);
+        } else {
+            mkdir($jsonsDestPath);
+        }
 
-        // Copy dumps of whitelisted tables into the repo
+        // Copy dumps of whitelisted tables and endpoints into the repo
         foreach ($this->whitelistedTables as $tableName) {
             $csvPaths = $this->shell->exec('find %s -name %s', $tablesSrcPath, $tableName . '*.csv')['output'];
 
@@ -71,6 +82,18 @@ public function handle()
                 $this->shell->passthru('cp %s %s', $tablesSrcPath . $csvSubPath, $tablesDestPath . $csvSubPath);
             }
         }
+        foreach ($this->getModels() as $model => $category) {
+            if(!File::exists($jsonsSrcPath .'/' .app('Resources')->getEndpointForModel($model))) {
+                continue;
+            }
+            $jsonPaths = $this->shell->exec('find %s -name %s', $jsonsSrcPath .'/' .app('Resources')->getEndpointForModel($model), '*.json')['output'];
+
+            $this->shell->passthru('mkdir -p %s', $jsonsDestPath . '/' .app('Resources')->getEndpointForModel($model));
+            foreach ($jsonPaths as $jsonPath) {
+                $jsonSubPath = '/' . app('Resources')->getEndpointForModel($model) .'/' . basename($jsonPath);
+                $this->shell->passthru('cp %s %s', $jsonsSrcPath . $jsonSubPath, $jsonsDestPath . $jsonSubPath);
+            }
+        }
 
         // Add VERSION file with current commit
         $this->shell->passthru('git -C %s rev-parse HEAD > %s', base_path(), $repoPath . '/VERSION');
@@ -87,7 +110,7 @@ public function handle()
         );
 
         // TODO: Fix how this works without --reset?
-        $this->shell->passthru('git -C %s push %s', $repoPath, ($this->option('reset') ? '--force' : ''));
+        $this->shell->passthru('git -C %s push %s', $repoPath, ($this->option('reset') ? '--force' : '--no-force-with-lease'));
     }
 
 }

app/Console/Kernel.php

@@ -76,14 +76,17 @@ protected function schedule(Schedule $schedule)
             ->withoutOverlapping()
             ->sendOutputTo(storage_path('logs/import-collections-last-run.log'));
 
-        $schedule->command('dump:export')
-            ->dailyAt('22:45')
-            ->withoutOverlapping()
+        $schedule->command('dump:json-export')
+            ->before(function () {
+                $this->call('dump:export');
+            })
             ->after(function () {
                 $this->call('dump:upload', [
                     '--reset' => 'default',
                 ]);
-            });
+            })
+            ->dailyAt('22:45')
+            ->withoutOverlapping();
     }
 
     /**