From 258a26ec5b40585b505f8bab01ea77f06208789c Mon Sep 17 00:00:00 2001 From: fmizzell Date: Fri, 19 Mar 2021 09:51:13 -0500 Subject: [PATCH] Refactoring (#22) * refactoring * More stuff * Tests stuff * code style --- .circleci/config.yml | 15 +- .ddev/config.yaml | 171 +++++++++++++++++++++ .ddev/docker-compose.environment.yml | 5 + composer.json | 6 +- src/PhpFunctionsBridge.php | 24 +++ src/PhpFunctionsBridgeTrait.php | 21 +++ src/Processor/AbstractChunkedProcessor.php | 106 +++++++++++++ src/Processor/LastResort.php | 68 ++++++-- src/Processor/Local.php | 40 +++-- src/Processor/Remote.php | 107 ++----------- src/TemporaryFilePathFromUrl.php | 18 +-- test/FileFetcherTest.php | 158 +++++++------------ test/Mock/FakeRemote.php | 24 +++ test/Processor/LastResortTest.php | 92 +++++++++++ test/Processor/RemoteTest.php | 87 +++++++++++ 15 files changed, 694 insertions(+), 248 deletions(-) create mode 100644 .ddev/config.yaml create mode 100644 .ddev/docker-compose.environment.yml create mode 100644 src/PhpFunctionsBridge.php create mode 100644 src/PhpFunctionsBridgeTrait.php create mode 100644 src/Processor/AbstractChunkedProcessor.php create mode 100644 test/Mock/FakeRemote.php create mode 100644 test/Processor/LastResortTest.php create mode 100644 test/Processor/RemoteTest.php diff --git a/.circleci/config.yml b/.circleci/config.yml index e4452e0e..c8782982 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,18 +1,17 @@ version: 2.0 jobs: build: + machine: + image: ubuntu-2004:202010-01 environment: CC_TEST_REPORTER_ID: 05b97154cabfafb769fb0afb99744dca1b5b1bdc6436657c0ac32887cfa599da - docker: - - image: circleci/php:7-cli-node-browsers-legacy working_directory: ~/repo steps: - checkout - run: - name: Setup dependencies + name: Setup DDEV command: | - sudo composer self-update - composer install -n --prefer-dist + curl -LO https://raw.githubusercontent.com/drud/ddev/master/scripts/install_ddev.sh && bash install_ddev.sh - run: name: Setup Code Climate test-reporter command: | @@ -21,6 +20,10 @@ jobs: - run: name: Run tests command: | + ddev start + ddev xdebug + ddev composer install ./cc-test-reporter before-build - vendor/bin/phpunit --testsuite all --coverage-clover clover.xml + ddev exec ./vendor/bin/phpunit --testsuite all --coverage-clover clover.xml + sed -i 's+/var/www/html/+/home/circleci/repo/+g' clover.xml ./cc-test-reporter after-build --coverage-input-type clover --exit-code $? diff --git a/.ddev/config.yaml b/.ddev/config.yaml new file mode 100644 index 00000000..537fd0d1 --- /dev/null +++ b/.ddev/config.yaml @@ -0,0 +1,171 @@ +name: file-fetcher +type: php +docroot: "" +php_version: "7.3" +webserver_type: nginx-fpm +router_http_port: "80" +router_https_port: "443" +xdebug_enabled: false +additional_hostnames: [] +additional_fqdns: [] +mariadb_version: "10.2" +mysql_version: "" +provider: default +use_dns_when_possible: true +composer_version: "" + + +# This config.yaml was created with ddev version v1.16.5 +# webimage: drud/ddev-webserver:v1.16.3 +# dbimage: drud/ddev-dbserver-mariadb-10.2:v1.16.0 +# dbaimage: phpmyadmin:5 +# However we do not recommend explicitly wiring these images into the +# config.yaml as they may break future versions of ddev. +# You can update this config.yaml using 'ddev config'. + +# Key features of ddev's config.yaml: + +# name: # Name of the project, automatically provides +# http://projectname.ddev.site and https://projectname.ddev.site + +# type: # drupal6/7/8, backdrop, typo3, wordpress, php + +# docroot: # Relative path to the directory containing index.php. + +# php_version: "7.3" # PHP version to use, "5.6", "7.0", "7.1", "7.2", "7.3", "7.4" "8.0" + +# You can explicitly specify the webimage, dbimage, dbaimage lines but this +# is not recommended, as the images are often closely tied to ddev's' behavior, +# so this can break upgrades. + +# webimage: # nginx/php docker image. +# dbimage: # mariadb docker image. +# dbaimage: + +# mariadb_version and mysql_version +# ddev can use many versions of mariadb and mysql +# However these directives are mutually exclusive +# mariadb_version: 10.2 +# mysql_version: 8.0 + +# router_http_port: # Port to be used for http (defaults to port 80) +# router_https_port: # Port for https (defaults to 443) + +# xdebug_enabled: false # Set to true to enable xdebug and "ddev start" or "ddev restart" +# Note that for most people the commands +# "ddev xdebug" to enable xdebug and "ddev xdebug off" to disable it work better, +# as leaving xdebug enabled all the time is a big performance hit. + +# webserver_type: nginx-fpm # or apache-fpm + +# timezone: Europe/Berlin +# This is the timezone used in the containers and by PHP; +# it can be set to any valid timezone, +# see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# For example Europe/Dublin or MST7MDT + +# composer_version: "2" +# if composer_version:"" it will use the current ddev default composer release. +# It can also be set to "1", to get most recent composer v1 +# or "2" for most recent composer v2. +# It can be set to any existing specific composer version. +# After first project 'ddev start' this will not be updated until it changes + +# additional_hostnames: +# - somename +# - someothername +# would provide http and https URLs for "somename.ddev.site" +# and "someothername.ddev.site". + +# additional_fqdns: +# - example.com +# - sub1.example.com +# would provide http and https URLs for "example.com" and "sub1.example.com" +# Please take care with this because it can cause great confusion. + +# upload_dir: custom/upload/dir +# would set the destination path for ddev import-files to custom/upload/dir. + +# working_dir: +# web: /var/www/html +# db: /home +# would set the default working directory for the web and db services. +# These values specify the destination directory for ddev ssh and the +# directory in which commands passed into ddev exec are run. + +# omit_containers: [db, dba, ddev-ssh-agent] +# Currently only these containers are supported. Some containers can also be +# omitted globally in the ~/.ddev/global_config.yaml. Note that if you omit +# the "db" container, several standard features of ddev that access the +# database container will be unusable. + +# nfs_mount_enabled: false +# Great performance improvement but requires host configuration first. +# See https://ddev.readthedocs.io/en/stable/users/performance/#using-nfs-to-mount-the-project-into-the-container + +# host_https_port: "59002" +# The host port binding for https can be explicitly specified. It is +# dynamic unless otherwise specified. +# This is not used by most people, most people use the *router* instead +# of the localhost port. + +# host_webserver_port: "59001" +# The host port binding for the ddev-webserver can be explicitly specified. It is +# dynamic unless otherwise specified. +# This is not used by most people, most people use the *router* instead +# of the localhost port. + +# host_db_port: "59002" +# The host port binding for the ddev-dbserver can be explicitly specified. It is dynamic +# unless explicitly specified. + +# phpmyadmin_port: "8036" +# phpmyadmin_https_port: "8037" +# The PHPMyAdmin ports can be changed from the default 8036 and 8037 + +# mailhog_port: "8025" +# mailhog_https_port: "8026" +# The MailHog ports can be changed from the default 8025 and 8026 + +# webimage_extra_packages: [php7.3-tidy, php-bcmath] +# Extra Debian packages that are needed in the webimage can be added here + +# dbimage_extra_packages: [telnet,netcat] +# Extra Debian packages that are needed in the dbimage can be added here + +# use_dns_when_possible: true +# If the host has internet access and the domain configured can +# successfully be looked up, DNS will be used for hostname resolution +# instead of editing /etc/hosts +# Defaults to true + +# project_tld: ddev.site +# The top-level domain used for project URLs +# The default "ddev.site" allows DNS lookup via a wildcard +# If you prefer you can change this to "ddev.local" to preserve +# pre-v1.9 behavior. + +# ngrok_args: --subdomain mysite --auth username:pass +# Provide extra flags to the "ngrok http" command, see +# https://ngrok.com/docs#http or run "ngrok http -h" + +# disable_settings_management: false +# If true, ddev will not create CMS-specific settings files like +# Drupal's settings.php/settings.ddev.php or TYPO3's AdditionalSettings.php +# In this case the user must provide all such settings. + +# no_project_mount: false +# (Experimental) If true, ddev will not mount the project into the web container; +# the user is responsible for mounting it manually or via a script. +# This is to enable experimentation with alternate file mounting strategies. +# For advanced users only! + +# provider: default # Currently either "default" or "pantheon" +# +# Many ddev commands can be extended to run tasks before or after the +# ddev command is executed, for example "post-start", "post-import-db", +# "pre-composer", "post-composer" +# See https://ddev.readthedocs.io/en/stable/users/extending-commands/ for more +# information on the commands that can be extended and the tasks you can define +# for them. Example: +#hooks: diff --git a/.ddev/docker-compose.environment.yml b/.ddev/docker-compose.environment.yml new file mode 100644 index 00000000..5f892ef1 --- /dev/null +++ b/.ddev/docker-compose.environment.yml @@ -0,0 +1,5 @@ +services: + web: + environment: + XDEBUG_MODE: 'coverage' + PHP_IDE_CONFIG: 'serverName=myproject.ddev.local' diff --git a/composer.json b/composer.json index 1831792b..abe333d9 100644 --- a/composer.json +++ b/composer.json @@ -4,7 +4,9 @@ "license": "GPL-3.0-only", "minimum-stability": "dev", "require-dev": { - "phpunit/phpunit": "^8.2" + "phpunit/phpunit": "^8.2", + "badoo/soft-mocks": "dev-master", + "getdkan/mock-chain": "dev-master" }, "authors": [ { @@ -15,7 +17,7 @@ "autoload": { "psr-4": { "FileFetcher\\": "src/", - "FileFetcherTest\\": "test/" + "FileFetcherTests\\": "test/" } }, "require": { diff --git a/src/PhpFunctionsBridge.php b/src/PhpFunctionsBridge.php new file mode 100644 index 00000000..23395c6e --- /dev/null +++ b/src/PhpFunctionsBridge.php @@ -0,0 +1,24 @@ +php = new PhpFunctionsBridge(); + } + + public function setPhpFunctionsBridge(PhpFunctionsBridge $bridge) + { + $this->php = $bridge; + } +} diff --git a/src/Processor/AbstractChunkedProcessor.php b/src/Processor/AbstractChunkedProcessor.php new file mode 100644 index 00000000..b1b9bacd --- /dev/null +++ b/src/Processor/AbstractChunkedProcessor.php @@ -0,0 +1,106 @@ +initializePhpFunctionsBridge(); + } + + + public function setupState(array $state): array + { + $state['destination'] = $this->getTemporaryFilePath($state); + $state['temporary'] = true; + $state['total_bytes'] = $this->getFileSize($state['source']); + + if (file_exists($state['destination'])) { + clearstatcache(); + $state['total_bytes_copied'] = filesize($state['destination']); + } + + return $state; + } + + public function isTimeLimitIncompatible(): bool + { + return false; + } + + public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX): array + { + $destinationFile = $state['destination']; + $total = $state['total_bytes_copied']; + + $expiration = time() + $timeLimit; + + while ($chunk = $this->getTheChunk($state)) { + $bytesWritten = $this->createOrAppend($destinationFile, $chunk); + + if ($bytesWritten !== strlen($chunk)) { + throw new \RuntimeException( + "Unable to fetch {$state['source']}. " . + " Reason: Failed to write to destination " . $destinationFile, + 0 + ); + } + + $total += $bytesWritten; + $state['total_bytes_copied'] = $total; + + $currentTime = time(); + if ($currentTime > $expiration) { + $result->setStatus(Result::STOPPED); + return ['state' => $state, 'result' => $result]; + } + } + + $result->setStatus(Result::DONE); + return ['state' => $state, 'result' => $result]; + } + + private function createOrAppend($filePath, $chunk) + { + if (!file_exists($filePath)) { + $bytesWritten = file_put_contents($filePath, $chunk); + } else { + $bytesWritten = file_put_contents($filePath, $chunk, FILE_APPEND); + } + return $bytesWritten; + } + + private function getTheChunk(array $state) + { + // 10 MB. + $bytesToRead = 10 * 1000 * 1000; + + $filePath = $state['source']; + $start = $state['total_bytes_copied']; + $end = $start + $bytesToRead; + + if ($end > $state['total_bytes']) { + $end = $state['total_bytes']; + } + + if ($start == $end) { + return false; + } + + return $this->getChunk($filePath, $start, $end); + } +} diff --git a/src/Processor/LastResort.php b/src/Processor/LastResort.php index 46d59598..beddfa61 100644 --- a/src/Processor/LastResort.php +++ b/src/Processor/LastResort.php @@ -3,12 +3,30 @@ namespace FileFetcher\Processor; use FileFetcher\LastResortException; +use FileFetcher\PhpFunctionsBridgeTrait; use FileFetcher\TemporaryFilePathFromUrl; use Procrastinator\Result; +/** + * Class LastResort + * + * The "last resort" processor does a regular copy of a file if non of the safer options were possible. This + * processor will attempt at getting all of the data in one shot and placing it in a file. + * + * @package FileFetcher\Processor + */ class LastResort implements ProcessorInterface { use TemporaryFilePathFromUrl; + use PhpFunctionsBridgeTrait; + + /** + * LastResort constructor. + */ + public function __construct() + { + $this->initializePhpFunctionsBridge(); + } public function isServerCompatible(array $state): bool { @@ -31,24 +49,21 @@ public function isTimeLimitIncompatible(): bool public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX): array { - // 1 MB. - $bytesToRead = 1024 * 1000; + list($from, $to) = $this->validateAndGetInfoFromState($state); + + $bytesToRead = 10 * 1000 * 1000; $bytesCopied = 0; - $from = $state['source']; - $to = $state['destination']; + $fin = $this->ensureExistsForReading($from); $fout = $this->ensureCreatingForWriting($to); while (!feof($fin)) { - $bytesRead = fread($fin, $bytesToRead); - if ($bytesRead === false) { - throw new LastResortException("reading from", $from); - } - $bytesWritten = fwrite($fout, $bytesRead); - if ($bytesWritten === false) { - throw new LastResortException("writing to", $to); - } - $bytesCopied += $bytesWritten; + $bytesCopied += $this->readAndWrite( + $fin, + $fout, + $bytesToRead, + $state + ); } $result->setStatus(Result::DONE); @@ -60,6 +75,29 @@ public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX) return ['state' => $state, 'result' => $result]; } + private function readAndWrite($fin, $fout, $bytesToRead, $state): int + { + list($from, $to) = $this->validateAndGetInfoFromState($state); + + $bytesRead = $this->php->fread($fin, $bytesToRead); + if ($bytesRead === false) { + throw new LastResortException("reading from", $from); + } + $bytesWritten = fwrite($fout, $bytesRead); + if ($bytesWritten === false) { + throw new LastResortException("writing to", $to); + } + return $bytesWritten; + } + + private function validateAndGetInfoFromState($state) + { + if (!isset($state['source']) && !isset($state['destination'])) { + throw new \Exception("Incorrect state missing source, destination, or both."); + } + return [$state['source'], $state['destination']]; + } + /** * Ensure the target file can be read from. * @@ -71,7 +109,7 @@ public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX) */ private function ensureExistsForReading(string $from) { - $fin = fopen($from, "rb"); + $fin = @$this->php->fopen($from, "rb"); if ($fin === false) { throw new LastResortException("opening", $from); } @@ -91,7 +129,7 @@ private function ensureCreatingForWriting(string $to) { // Delete destination first to avoid appending if existing. $this->deleteFile($to); - $fout = fopen($to, "w"); + $fout = $this->php->fopen($to, "w"); if ($fout === false) { throw new LastResortException("creating", $to); } diff --git a/src/Processor/Local.php b/src/Processor/Local.php index c0735fe8..5f4d2837 100644 --- a/src/Processor/Local.php +++ b/src/Processor/Local.php @@ -2,36 +2,32 @@ namespace FileFetcher\Processor; -use Procrastinator\Result; - -class Local implements ProcessorInterface +class Local extends AbstractChunkedProcessor { - public function isServerCompatible(array $state): bool - { - try { - $file = new \SplFileObject($state['source']); - return $file->isFile(); - } catch (\Exception $e) { - return false; - } - } - public function setupState(array $state): array + protected function getFileSize(string $filePath): int { - $size = filesize($state['source']); - $state['total_bytes'] = $size; - $state['total_bytes_copied'] = $size; - return $state; + return $this->php->filesize($filePath); } - public function isTimeLimitIncompatible(): bool + public function isServerCompatible(array $state): bool { - return true; + $path = $state['source']; + + if ($this->php->file_exists($path) && !$this->php->is_dir($path)) { + return true; + } + + return false; } - public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX): array + protected function getChunk(string $filePath, int $start, int $end) { - $result->setStatus(Result::DONE); - return ['state' => $state, 'result' => $result]; + $fp = fopen($filePath, 'r'); + fseek($fp, $start); + $bytesToCopy = $end - $start; + $data = fread($fp, $bytesToCopy); + fclose($fp); + return $data; } } diff --git a/src/Processor/Remote.php b/src/Processor/Remote.php index 7426cd7d..d9b423cd 100644 --- a/src/Processor/Remote.php +++ b/src/Processor/Remote.php @@ -2,12 +2,16 @@ namespace FileFetcher\Processor; -use FileFetcher\TemporaryFilePathFromUrl; use Procrastinator\Result; -class Remote implements ProcessorInterface +class Remote extends AbstractChunkedProcessor { - use TemporaryFilePathFromUrl; + + protected function getFileSize(string $filePath): int + { + $headers = $this->getHeaders($filePath); + return $headers['Content-Length']; + } public function isServerCompatible(array $state): bool { @@ -20,94 +24,19 @@ public function isServerCompatible(array $state): bool return false; } - public function setupState(array $state): array - { - $state['destination'] = $this->getTemporaryFilePath($state); - $state['temporary'] = true; - $state['total_bytes'] = $this->getRemoteFileSize($state['source']); - - if (file_exists($state['destination'])) { - $state['total_bytes_copied'] = filesize($state['destination']); - } - - return $state; - } - - public function isTimeLimitIncompatible(): bool - { - return false; - } - - public function copy(array $state, Result $result, int $timeLimit = PHP_INT_MAX): array - { - $destinationFile = $state['destination']; - $total = $state['total_bytes_copied']; - - $expiration = time() + $timeLimit; - - while ($chunk = $this->getChunk($state)) { - $bytesWritten = $this->createOrAppend($destinationFile, $chunk); - - if ($bytesWritten !== strlen($chunk)) { - throw new \RuntimeException( - "Unable to fetch {$state['source']}. " . - " Reason: Failed to write to destination " . $destinationFile, - 0 - ); - } - - $total += $bytesWritten; - $state['total_bytes_copied'] = $total; - - $currentTime = time(); - if ($currentTime > $expiration) { - $result->setStatus(Result::STOPPED); - return ['state' => $state, 'result' => $result]; - } - } - - $result->setStatus(Result::DONE); - return ['state' => $state, 'result' => $result]; - } - - private function createOrAppend($filePath, $chunk) - { - if (!file_exists($filePath)) { - $bytesWritten = file_put_contents($filePath, $chunk); - } else { - $bytesWritten = file_put_contents($filePath, $chunk, FILE_APPEND); - } - return $bytesWritten; - } - - private function getChunk(array $state) + protected function getChunk(string $filePath, int $start, int $end) { - // 1 MB. - $bytesToRead = 1024 * 1000; - - $url = $state['source']; - $start = $state['total_bytes_copied']; - $end = $start + $bytesToRead; - - if ($end > $state['total_bytes']) { - $end = $state['total_bytes']; - } - - if ($start == $end) { - return false; - } - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_URL, $filePath); curl_setopt($ch, CURLOPT_RANGE, "{$start}-{$end}"); curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - $result = curl_exec($ch); + $result = $this->php->curl_exec($ch); curl_close($ch); return $result; } - private function getHeaders($url) + protected function getHeaders($url) { $ch = curl_init(); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); @@ -116,24 +45,24 @@ private function getHeaders($url) curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_NOBODY, true); - $headers = $this->parseHeaders(curl_exec($ch)); + $headers = $this->parseHeaders($this->php->curl_exec($ch)); curl_close($ch); return $headers; } - private function parseHeaders($string) + public static function parseHeaders($string) { $headers = []; $lines = explode(PHP_EOL, $string); foreach ($lines as $line) { $line = trim($line); - $keyvalue = $this->getKeyValueFromLine($line); + $keyvalue = self::getKeyValueFromLine($line); $headers[$keyvalue['key']] = $keyvalue['value']; } return $headers; } - private function getKeyValueFromLine($line): array + private static function getKeyValueFromLine($line): array { $key = null; $value = null; @@ -148,10 +77,4 @@ private function getKeyValueFromLine($line): array return ['key' => $key, 'value' => $value]; } - - private function getRemoteFileSize($url) - { - $headers = $this->getHeaders($url); - return $headers['Content-Length']; - } } diff --git a/src/TemporaryFilePathFromUrl.php b/src/TemporaryFilePathFromUrl.php index cf8ddf1b..bb0f84a1 100644 --- a/src/TemporaryFilePathFromUrl.php +++ b/src/TemporaryFilePathFromUrl.php @@ -5,15 +5,15 @@ trait TemporaryFilePathFromUrl { - /** - * Get temporary file path, depending on flag keep_original_filename value. - * - * @param array $state - * State. - * - * @return string - * Temporary file path. - */ + /** + * Get temporary file path, depending on flag keep_original_filename value. + * + * @param array $state + * State. + * + * @return string + * Temporary file path. + */ private function getTemporaryFilePath(array $state): string { if ($state['keep_original_filename']) { diff --git a/test/FileFetcherTest.php b/test/FileFetcherTest.php index c0d0feaf..2ce55104 100644 --- a/test/FileFetcherTest.php +++ b/test/FileFetcherTest.php @@ -1,6 +1,6 @@ "http://samplecsvs.s3.amazonaws.com/Sacramentorealestatetransactions.csv", - "processors" => [Local::class] + "filePath" => __DIR__ . '/files/tiny.csv' ] ); - $result = $fetcher->run(); + // How much time do we want to spend copying the file (In seconds). + $fetcher->setTimeLimit(1); + + $fetcher->run(); // [Basic Usage] - $data = json_decode($result->getData()); - $filepath = "/tmp/samplecsvs_s3_amazonaws_com_sacramentorealestatetransactions.csv"; - $this->assertEquals($filepath, $data->destination); - $this->assertTrue($data->temporary); + $state = $fetcher->getState(); + + $this->assertEquals( + file_get_contents($state['source']), + file_get_contents($state['destination']) + ); + + unlink($state['destination']); } public function testKeepOriginalFilename() @@ -43,132 +48,81 @@ public function testKeepOriginalFilename() "2", new Memory(), [ - "filePath" => "http://samplecsvs.s3.amazonaws.com/Sacramentorealestatetransactions.csv", - "processors" => [Remote::class], + "filePath" => __DIR__ . '/files/tiny.csv', "keep_original_filename" => true, + "processors" => [Local::class], ] ); - $result = $fetcher->run(); - - $data = json_decode($result->getData()); - $filepath = "/tmp/Sacramentorealestatetransactions.csv"; - $this->assertEquals($filepath, $data->destination); - $this->assertTrue($data->temporary); - } - - public function testLocal() - { - $local_file = __DIR__ . "/files/tiny.csv"; - - $config = [ - "filePath" => $local_file, - "processors" => [TestCase::class] - ]; + $fetcher->run(); + $state = $fetcher->getState(); - $fetcher = FileFetcher::get( - "1", - new Memory(), - $config + $this->assertEquals( + basename($state['source']), + basename($state['destination']) ); - $fetcher->setTimeLimit(1); - $result = $fetcher->run(); - $data = json_decode($result->getData()); - $this->assertEquals($local_file, $data->destination); - $this->assertFalse($fetcher->getStateProperty('keep_original_filename')); - $this->assertFalse($data->temporary); + unlink($state['destination']); } - public function testMissingConfigFilePath() + public function testConfigValidationErrorConfigurationMissing() { - $this->expectExceptionMessage("Constructor missing expected config filePath."); - $fetcher = FileFetcher::get( - "1", + $this->expectExceptionMessage('Constructor missing expected config filePath.'); + FileFetcher::get( + "2", new Memory() ); } - public function testTimeOut() + public function testConfigValidationErrorMissingFilePath() { - $store = new Memory(); - $config = [ - "filePath" => "https://dkan-default-content-files.s3.amazonaws.com/files/do_not_delete.csv", - "processors" => "Bad" - ]; - - $fetcher = FileFetcher::get("1", $store, $config); - - $fetcher->setTimeLimit(1); - $fetcher->run(); - $file_size = $fetcher->getStateProperty('total_bytes'); - $this->assertLessThanOrEqual($file_size, $fetcher->getStateProperty('total_bytes_copied')); - $this->assertGreaterThan(0, $fetcher->getStateProperty('total_bytes_copied')); - $this->assertEquals($fetcher->getResult()->getStatus(), \Procrastinator\Result::STOPPED); - - $fetcher2 = FileFetcher::get("1", $store, $config); - - $fetcher2->setTimeLimit(PHP_INT_MAX); - $fetcher2->run(); - $this->assertEquals($file_size, $fetcher2->getStateProperty('total_bytes_copied')); - - clearstatcache(); - $actualFileSize = filesize( - "/tmp/dkan_default_content_files_s3_amazonaws_com_files_do_not_delete.csv" + $this->expectExceptionMessage('Constructor missing expected config filePath.'); + FileFetcher::get( + "2", + new Memory(), + [] ); - - $this->assertEquals($actualFileSize, $fetcher2->getStateProperty('total_bytes_copied')); - - $this->assertEquals($fetcher2->getResult()->getStatus(), \Procrastinator\Result::DONE); } - public function testIncompatibleServer() + public function testCustomProcessorsValidationIsNotAnArray() { - $url = "https://data.medicare.gov/api/views/wue8-3vwe/rows.csv?accessType=DOWNLOAD&sorting=true"; $fetcher = FileFetcher::get( - "1", + "2", new Memory(), [ - "filePath" => $url, - "processors" => ["Bad"] + "filePath" => __DIR__ . '/files/tiny.csv', + "processors" => "hello" ] ); - $fetcher->setTimeLimit(1); - $result = $fetcher->run(); - $this->assertEquals(Result::DONE, $result->getStatus()); - $this->assertGreaterThan(0, json_decode($result->getData())->total_bytes_copied); + // Not sure what to assert. + $this->assertTrue(true); } - public function testLastResortErrorOpening() + public function testCustomProcessorsValidationNotAClass() { $fetcher = FileFetcher::get( - "1", + "2", new Memory(), [ - "filePath" => __DIR__ . "/files/non-existent.csv", - "processors" => [LastResort::class], + "filePath" => __DIR__ . '/files/tiny.csv', + "processors" => ["hello"] ] ); - $fetcher->setTimeLimit(1); - $result = $fetcher->run(); - $this->assertEquals(Result::ERROR, $result->getStatus()); + // Not sure what to assert. + $this->assertTrue(true); } - public function tearDown(): void + public function testCustomProcessorsValidationImproperClass() { - parent::tearDown(); - $files = [ - "/tmp/samplecsvs_s3_amazonaws_com_sacramentorealestatetransactions.csv", - "/tmp/Sacramentorealestatetransactions.csv", - "/tmp/dkan_default_content_files_s3_amazonaws_com_{$this->sampleCsvSize}_mb_sample.csv", - "/tmp/data_medicare_gov_api_views_42wc_33ci_rows.csv", - "/tmp/dkan_default_content_files_s3_amazonaws_com_files_do_not_delete.csv" - ]; - - foreach ($files as $file) { - if (file_exists($file)) { - unlink($file); - } - } + $fetcher = FileFetcher::get( + "2", + new Memory(), + [ + "filePath" => __DIR__ . '/files/tiny.csv', + "processors" => [\SplFileInfo::class] + ] + ); + // Not sure what to assert. + $this->assertTrue(true); } } diff --git a/test/Mock/FakeRemote.php b/test/Mock/FakeRemote.php new file mode 100644 index 00000000..4f35100c --- /dev/null +++ b/test/Mock/FakeRemote.php @@ -0,0 +1,24 @@ + __DIR__ . '/../files/tiny.csv', + "processors" => [LastResort::class] + ] + ); + + // Last resort does not support time limits. + $this->assertFalse($fetcher->setTimeLimit(1)); + + $fetcher->run(); + + $state = $fetcher->getState(); + + $this->assertEquals( + file_get_contents($state['source']), + file_get_contents($state['destination']) + ); + + unlink($state['destination']); + } + + public function testStateIsValid() + { + $this->expectExceptionMessage('Incorrect state missing source, destination, or both.'); + $processor = new LastResort(); + $processor->copy([], new Result()); + } + + public function testOpeningSourceException() + { + $this->expectExceptionMessage('Error opening file: hello.'); + $state = ['source' => 'hello', 'destination' => 'goodbye']; + + $options = (new Options()) + ->add('fopen', false) + ->index(0); + + $phpFunctionsBridge = (new Chain($this)) + ->add(PhpFunctionsBridge::class, '__call', $options) + ->getMock(); + + $processor = new LastResort(); + $processor->setPhpFunctionsBridge($phpFunctionsBridge); + + $processor->copy($state, new Result()); + } + + public function testOpeningDestinationException() + { + $this->expectExceptionMessage('Error creating file: goodbye.'); + $state = ['source' => 'hello', 'destination' => 'goodbye']; + + $sequence = (new Sequence()) + ->add(1) + ->add(false); + + $options = (new Options()) + ->add('fopen', $sequence) + ->index(0); + + $phpFunctionsBridge = (new Chain($this)) + ->add(PhpFunctionsBridge::class, '__call', $options) + ->getMock(); + + $processor = new LastResort(); + $processor->setPhpFunctionsBridge($phpFunctionsBridge); + + $processor->copy($state, new Result()); + } +} diff --git a/test/Processor/RemoteTest.php b/test/Processor/RemoteTest.php new file mode 100644 index 00000000..25c9f096 --- /dev/null +++ b/test/Processor/RemoteTest.php @@ -0,0 +1,87 @@ + 'http://notreal.blah/notacsv.csv', + "processors" => [\FileFetcherTests\Mock\FakeRemote::class] + ] + ); + + $fetcher->setTimeLimit(1); + + $counter = 0; + do { + $result = $fetcher->run(); + $counter++; + } while ($result->getStatus() == Result::STOPPED); + + $state = $fetcher->getState(); + + $this->assertTrue(true); + + + unlink($state['destination']); + } + + public function testCurlCopy() + { + $options = (new Options()) + ->add('curl_exec', "") + ->index(0); + + $bridge = (new Chain($this)) + ->add(PhpFunctionsBridge::class, '__call', $options) + ->getMock(); + + $processor = new Remote(); + $processor->setPhpFunctionsBridge($bridge); + $processor->copy([ + 'source' => 'hello', + 'destination' => 'goodbye', + 'total_bytes_copied' => 1, + 'total_bytes' => 10, + ], new Result()); + $this->assertTrue(true); + } + + public function testCurlHeaders() + { + $options = (new Options()) + ->add('curl_exec', "Accept-Ranges:TRUE\nContent-Length:10") + ->index(0); + + $bridge = (new Chain($this)) + ->add(PhpFunctionsBridge::class, '__call', $options) + ->getMock(); + + $processor = new Remote(); + $processor->setPhpFunctionsBridge($bridge); + $this->assertTrue( + $processor->isServerCompatible([ + 'source' => 'hello', + 'destination' => 'goodbye', + 'total_bytes_copied' => 1, + 'total_bytes' => 10, + ]) + ); + } +}