From 5038033cc66a4207ea54d2eb93f83eea690c1452 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Thu, 27 Jun 2024 12:02:22 -0400 Subject: [PATCH 01/22] Switch storage driver to S3 --- .env | 25 ++++- composer.json | 16 ++- composer.lock | 14 +-- docker/config/php/materia.php.ini | 2 +- docker/docker-compose.yml | 10 +- fuel/app/classes/controller/media.php | 10 +- fuel/app/classes/materia/widget/asset.php | 2 +- .../classes/materia/widget/asset/manager.php | 4 +- .../materia/widget/asset/storage/s3.php | 106 ++++++++++++++---- fuel/app/config/development/materia.php | 26 ++++- fuel/app/config/materia.php | 9 +- src/components/media-importer.jsx | 2 +- 12 files changed, 169 insertions(+), 57 deletions(-) diff --git a/.env b/.env index 8f36e198c..9f359cdd5 100755 --- a/.env +++ b/.env @@ -34,12 +34,25 @@ BOOL_SEND_EMAILS=false #URLS_STATIC= #URLS_ENGINES= #BOOL_ADMIN_UPLOADER_ENABLE=true -#ASSET_STORAGE_DRIVER=file -#ASSET_STORAGE_S3_REGION=us-east-1 -#ASSET_STORAGE_S3_BUCKET= -#ASSET_STORAGE_S3_BASEPATH= -#ASSET_STORAGE_S3_KEY= -#ASSET_STORAGE_S3_SECRET= + +# AWS S3 =================== + +ASSET_STORAGE_DRIVER=s3 +ASSET_STORAGE_S3_REGION=us-east-1 +ASSET_STORAGE_S3_BASEPATH=media + +# AWS S3 DEVELOPMENT =================== + +# ASSET_STORAGE_S3_BUCKET=fake_bucket +# ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001 +# AWS_SESSION_TOKEN= // STS token for s3 development + +# AWS S3 PRODUCTION =================== + +# ASSET_STORAGE_S3_BUCKET= +# ASSET_STORAGE_S3_ENDPOINT= +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= # SESSION & CACHE =================== diff --git a/composer.json b/composer.json index 01d97360d..5c17293bc 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "monolog/monolog": "1.18.*", "phpseclib/phpseclib": "2.0.31", "eher/oauth": "1.0.7", - "aws/aws-sdk-php": "3.288.1", + "aws/aws-sdk-php": "^3.314", "symfony/dotenv": "^5.1", "ucfopen/materia-theme-ucf": "2.0.2" }, @@ -66,7 +66,8 @@ "vendor-dir": "fuel/vendor", "optimize-autoloader": true, "allow-plugins": { - "composer/installers": true + "composer/installers": true, + "php-http/discovery": true } }, "extra": { @@ -112,5 +113,14 @@ } ], "minimum-stability": "dev", - "prefer-stable": true + "prefer-stable": true, + "autoload": { + "psr-0": { + "": "*" + }, + "psr-4": { + "S3\\": "../s3/", + "AwsUtilities\\": "../aws_utilities/" + } + } } diff --git a/composer.lock b/composer.lock index a5ebe9d68..95af327b4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d68f3eabd3a95508cee1cdfc097109ca", + "content-hash": "1b20e7f4c2040fd429fc45f53d37cb8e", "packages": [ { "name": "aws/aws-crt-php", @@ -62,16 +62,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.288.1", + "version": "3.314.7", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "a1dfa12c7165de0b731ae8074c4ba1f3ae733f89" + "reference": "3a7ea3e49ae50eaaa969dc15d69b2dd46b4d7284" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/a1dfa12c7165de0b731ae8074c4ba1f3ae733f89", - "reference": "a1dfa12c7165de0b731ae8074c4ba1f3ae733f89", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/3a7ea3e49ae50eaaa969dc15d69b2dd46b4d7284", + "reference": "3a7ea3e49ae50eaaa969dc15d69b2dd46b4d7284", "shasum": "" }, "require": { @@ -151,9 +151,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.288.1" + "source": "https://github.com/aws/aws-sdk-php/tree/3.314.7" }, - "time": "2023-11-22T19:35:38+00:00" + "time": "2024-06-24T21:03:27+00:00" }, { "name": "composer/installers", diff --git a/docker/config/php/materia.php.ini b/docker/config/php/materia.php.ini index 9fdebe247..1e07d5bb6 100644 --- a/docker/config/php/materia.php.ini +++ b/docker/config/php/materia.php.ini @@ -2,7 +2,7 @@ short_open_tag = Off expose_php = off max_execution_time = 100 -memory_limit = 250M +memory_limit = 256M track_errors = Off html_errors = On variables_order = "EGPCS" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 25c26347b..b9341a438 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,11 +22,11 @@ services: dockerfile: materia-app.Dockerfile environment: # View Materia README for env settings - - ASSET_STORAGE_DRIVER=file - - ASSET_STORAGE_S3_BUCKET=fake_bucket - - ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001 - - ASSET_STORAGE_S3_KEY=KEY - - ASSET_STORAGE_S3_SECRET=SECRET + # - ASSET_STORAGE_DRIVER=${ASSET_STORAGE_DRIVER} + # - ASSET_STORAGE_S3_BUCKET=${ASSET_STORAGE_S3_BUCKET} + # - ASSET_STORAGE_S3_ENDPOINT=${ASSET_STORAGE_S3_ENDPOINT} + # - ASSET_STORAGE_S3_KEY=${AWS_ACCESS_KEY_ID} + # - ASSET_STORAGE_S3_SECRET=${AWS_SECRET_ACCESS_KEY} - AUTH_DRIVERS=Materiaauth - AUTH_SALT=${DEV_ONLY_AUTH_SALT} - AUTH_SIMPLEAUTH_SALT=${DEV_ONLY_AUTH_SIMPLEAUTH_SALT} diff --git a/fuel/app/classes/controller/media.php b/fuel/app/classes/controller/media.php index b4a4beec7..0e0666f02 100644 --- a/fuel/app/classes/controller/media.php +++ b/fuel/app/classes/controller/media.php @@ -104,7 +104,15 @@ public function action_upload() ]; $name = Input::post('name', 'New Asset'); - $asset = Widget_Asset_Manager::new_asset_from_file($name, $file_info); + + try { + $asset = Widget_Asset_Manager::new_asset_from_file($name, $file_info); + } + catch (\Exception $e) { + $res->body('{"error":{"code":"15","message":"'.$e->getMessage().'"}}'); + $res->set_status(400); + return $res; + } if ( ! isset($asset->id)) { diff --git a/fuel/app/classes/materia/widget/asset.php b/fuel/app/classes/materia/widget/asset.php index e831f25dc..ed5df1d55 100644 --- a/fuel/app/classes/materia/widget/asset.php +++ b/fuel/app/classes/materia/widget/asset.php @@ -125,7 +125,7 @@ public function db_update() 'file_size' => $this->file_size, 'created_at' => time(), 'is_deleted' => $this->is_deleted - + ]) ->where('id','=',$this->id) ->execute(); diff --git a/fuel/app/classes/materia/widget/asset/manager.php b/fuel/app/classes/materia/widget/asset/manager.php index f6676b73f..0036d5e44 100644 --- a/fuel/app/classes/materia/widget/asset/manager.php +++ b/fuel/app/classes/materia/widget/asset/manager.php @@ -58,13 +58,15 @@ static public function new_asset_from_file($name, $file_info) Perm_Manager::set_user_object_perms($asset->id, Perm::ASSET, \Model_User::find_current_id(), [Perm::FULL => Perm::ENABLE]); return $asset; } - catch (\OutsideAreaException | InvalidPathException | \FileAccessException $e) + catch (\OutsideAreaException | InvalidPathException | \FileAccessException | \Exception $e) { trace($e); } // failed, remove the asset $asset->db_remove(); + + throw new \Exception('Failed to store asset data'); } return $asset; diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index adc3d403c..3f949f59c 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -56,10 +56,21 @@ public function delete(string $id, string $size = '*'): void } else { - $s3->deleteObject([ - 'Bucket' => static::$_config['bucket'], - 'Key' => $this->get_key_name($id, $size), - ]); + try { + $s3->deleteObject([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size), + ]); + } catch (\Exception $e) { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to delete asset {$id} {$size}. {$error_code} {$source}"); + } } } @@ -73,7 +84,22 @@ public function exists(string $id, string $size): bool { $s3 = $this->get_s3_client(); - return $s3->doesObjectExist(static::$_config['bucket'], $this->get_key_name($id, $size)); + try { + return $s3->doesObjectExistV2( + static::$_config['bucket'], + $this->get_key_name($id, $size) + ); + } catch (\Exception $e) { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to check if asset {$id} {$size} exists. {$error_code} {$source}"); + return false; + } } /** @@ -98,8 +124,17 @@ public function retrieve(string $id, string $size, string $target_file_path): vo ]); } catch (\Exception $e) { - throw new \Exception("Missing asset data for asset: {$id} {$size}"); + $source = ''; + $error_code = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to retrieve asset {$key}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to retrieve asset {$key}. {$error_code} {$source}"); } + } /** @@ -115,16 +150,33 @@ public function store(Widget_Asset $asset, string $image_path, string $size): vo // Force all uploads in development to have the same bucket sub-directory $key = $this->get_key_name($asset->id, $size); - $s3 = $this->get_s3_client(); - - $result = $s3->putObject([ - 'ACL' => 'public-read', - 'Metadata' => ['Content-Type' => $asset->get_mime_type()], - 'Bucket' => static::$_config['bucket'], - 'Key' => $key, - 'SourceFile' => $image_path, - // 'Body' => $image_data, // use instead of SourceFile to send data - ]); + \Log::info("Storing asset data in s3: {$key} ({$asset->get_mime_type()})"); + \Log::info("Asset data path: {$image_path}"); + \Log::info("Size: {$size}"); + \Log::info('Bucket: '.static::$_config['bucket']); + \Log::info("Asset file_size: {$asset->file_size}"); + + try { + $s3 = $this->get_s3_client(); + + $result = $s3->putObject([ + 'Metadata' => ['Content-Type' => $asset->get_mime_type()], + 'Bucket' => static::$_config['bucket'], + 'Key' => $key, + 'SourceFile' => $image_path, + // 'Body' => $image_data, // use instead of SourceFile to send data + ]); + } catch (\Exception $e) { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to store asset {$key}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to store asset {$key}. {$error_code} {$source}"); + } } /** @@ -149,22 +201,30 @@ protected function get_s3_client(): \Aws\S3\S3Client if (static::$_s3_client) return static::$_s3_client; $config = [ - 'endpoint' => '', + 'endpoint' => static::$_config['endpoint'] ?? '', 'region' => static::$_config['region'], + 'force_path_style' => static::$_config['force_path_style'] ?? false, 'version' => 'latest', 'credentials' => [ 'key' => static::$_config['key'], 'secret' => static::$_config['secret_key'], + 'token' => static::$_config['token'] ?? null, ] ]; - // should we use a mock endpoint for testing? - if (static::$_config['endpoint'] !== false) - { - $config['endpoint'] = static::$_config['endpoint']; + try { + static::$_s3_client = new \Aws\S3\S3Client($config); + } catch (\Exception $e) { + $source = ''; + $error_code = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to create S3 client. {$error_code} {$source}"); + throw new \Exception("S3: Failed to create S3 client. {$error_code} {$source}"); } - - static::$_s3_client = new \Aws\S3\S3Client($config); return static::$_s3_client; } diff --git a/fuel/app/config/development/materia.php b/fuel/app/config/development/materia.php index cec4350ad..eaf14868d 100644 --- a/fuel/app/config/development/materia.php +++ b/fuel/app/config/development/materia.php @@ -23,12 +23,30 @@ // Storage driver can be overridden from env here // s3 uses fakes3 on dev - 'asset_storage_driver' => 'file', + 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 's3', 'asset_storage' => [ - 's3' => [ - 'endpoint' => 'http://fakes3:10001', - 'bucket' => 'fake_bucket', // bucket to store original user uploads + 'file' => [ + 'driver_class' => '\Materia\Widget_Asset_Storage_File', + 'media_dir' => APPPATH.'media'.DS, ], + 'db' => [ + 'driver_class' => '\Materia\Widget_Asset_Storage_Db' + ], + 's3' => ( + (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') + ? [ + 'driver_class' => '\Materia\Widget_Asset_Storage_S3', + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? 'http://fakes3:10001', // set to url for testing endpoint + 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? 'fake_bucket', // bucket to store original user uploads + 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token + 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 + ] + : null + ), ] ]; \ No newline at end of file diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 0d8204309..ae3522100 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -96,7 +96,7 @@ 'google_tracking_id' => $_ENV['GOOGLE_ANALYTICS_ID'] ?? false, // Asset storage configuration - 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 'file', + 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 's3', 'asset_storage' => [ 'file' => [ @@ -110,12 +110,13 @@ (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', - 'endpoint' =>$_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? false, // set to url for testing endpoint + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'], // bucket to store original user uploads 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets - 'secret_key' => $_ENV['ASSET_STORAGE_S3_SECRET'], // aws api secret key - 'key' => $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY' // aws api key + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? null, // aws session token ] : null ), diff --git a/src/components/media-importer.jsx b/src/components/media-importer.jsx index 11c5d9379..e0db90caf 100644 --- a/src/components/media-importer.jsx +++ b/src/components/media-importer.jsx @@ -212,7 +212,7 @@ const MediaImporter = () => { setSelectedAsset(res.id) } else { setErrorState('Something went wrong with uploading your file.') - _onCancel() + // _onCancel() // uncomment to close the modal on error return } } From 9541ded3839f55d8c44c98905acbce10aa2b8885 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Thu, 27 Jun 2024 12:27:01 -0400 Subject: [PATCH 02/22] Remove unnecessary plugin --- composer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 5c17293bc..57108a8be 100644 --- a/composer.json +++ b/composer.json @@ -66,8 +66,7 @@ "vendor-dir": "fuel/vendor", "optimize-autoloader": true, "allow-plugins": { - "composer/installers": true, - "php-http/discovery": true + "composer/installers": true } }, "extra": { From 90b33cd539d2d05f311508e99122f6fcd0aa6397 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Thu, 27 Jun 2024 12:40:46 -0400 Subject: [PATCH 03/22] Fix undefined array key error without an .env.local --- fuel/app/config/materia.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index ae3522100..47ac53ef9 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -112,7 +112,7 @@ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket - 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'], // bucket to store original user uploads + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key From aa8683a935f52bd4db4f74e60d5ca0c14e001e24 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Thu, 27 Jun 2024 13:34:58 -0400 Subject: [PATCH 04/22] Start a different fakes3 container for running tests --- docker/docker-compose.override.test.yml | 15 ++++++++++++++- docker/docker-compose.override.yml | 4 ++++ docker/docker-compose.yml | 4 ---- docker/run_tests.sh | 9 +++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.override.test.yml b/docker/docker-compose.override.test.yml index 2115b1a11..bb06ceba3 100644 --- a/docker/docker-compose.override.test.yml +++ b/docker/docker-compose.override.test.yml @@ -24,6 +24,10 @@ services: - uploaded_media_test:/var/www/html/fuel/packages/materia/media - ./config/php/materia.test.php.ini:/usr/local/etc/php/conf.d/test.ini - ./dockerfiles/wait-for-it.sh:/wait-for-it.sh + depends_on: + - fakes3_test + - mysql + - memcached mysql: environment: @@ -36,9 +40,18 @@ services: # tmpfs: # - /var/lib/mysql - fakes3: + fakes3_test: + image: ucfopen/materia:fake-s3-dev + build: + context: ../ + dockerfile: materia-fake-s3.Dockerfile + ports: + - "10002:10001" volumes: - uploaded_media_test:/s3mnt/fakes3_root/fakes3_uploads/media/ + networks: + - frontend + - backend volumes: # static_files: {} # compiled js/css and uploaded widgets diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index a705ea366..0a15c5212 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -19,6 +19,10 @@ services: - ..:/var/www/html/ - uploaded_widgets:/var/www/html/public/widget/ - ./dockerfiles/wait-for-it.sh:/wait-for-it.sh + depends_on: + - fakes3 + - mysql + - memcached mysql: environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b9341a438..caf9e8f9f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -54,10 +54,6 @@ services: networks: - frontend - backend - depends_on: - - mysql - - memcached - - fakes3 mysql: image: mysql:5.7.34 diff --git a/docker/run_tests.sh b/docker/run_tests.sh index 3e6660317..2cba06678 100755 --- a/docker/run_tests.sh +++ b/docker/run_tests.sh @@ -16,3 +16,12 @@ set -e set -o xtrace $DCTEST run -T --rm app /wait-for-it.sh mysql:3306 -t 20 -- composer run testci -- "$@" + +# Remove fakes3_test container +CONTAINER_ID=$(docker-compose -f docker-compose.yml -f docker-compose.override.test.yml ps -q fakes3_test) +if [ -z "$CONTAINER_ID" ]; then + echo "fakes3_test container not found" +else + docker stop $CONTAINER_ID + docker rm $CONTAINER_ID +fi \ No newline at end of file From 7525512e8a17f46743751908bac0451333a26c88 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Fri, 28 Jun 2024 10:59:28 -0400 Subject: [PATCH 05/22] Add object locking --- fuel/app/classes/materia/widget/asset.php | 23 ++- .../materia/widget/asset/storage/s3.php | 131 +++++++++++++++++- 2 files changed, 146 insertions(+), 8 deletions(-) diff --git a/fuel/app/classes/materia/widget/asset.php b/fuel/app/classes/materia/widget/asset.php index ed5df1d55..dc2de80f8 100644 --- a/fuel/app/classes/materia/widget/asset.php +++ b/fuel/app/classes/materia/widget/asset.php @@ -343,8 +343,14 @@ protected function build_size(string $size): string break; } - $this->_storage_driver->lock_for_processing($this->id, $size); - + try { + // lock the original asset so we can process it + $this->_storage_driver->lock_for_processing($this->id, 'original'); + } catch (\Throwable $e) + { + \LOG::error($e); + throw($e); + } // get the original file $original_asset_path = $this->copy_asset_to_temp_file($this->id, 'original'); @@ -378,10 +384,17 @@ protected function build_size(string $size): string throw($e); } - $this->_storage_driver->store($this, $resized_file_path, $size); + try { + // store the resized asset in s3 or wherever + $this->_storage_driver->store($this, $resized_file_path, $size); - // update asset_data - $this->_storage_driver->unlock_for_processing($this->id, $size); + // unlock original asset + $this->_storage_driver->unlock_for_processing($this->id, 'original'); + } catch (\Throwable $e) + { + \LOG::error($e); + throw($e); + } // close the file handles and delete temp files unlink($original_asset_path); diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index 3f949f59c..d9bf8eec3 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -20,14 +20,117 @@ public static function instance(array $config): Widget_Asset_Storage_Driver } /** - * Create a lock on a specific size of an asset. + * Create a lock on a specific size of an asset for a period of time + * Used to prevent multiple requests from using excessive resources. + * For object locking to work, the bucket must have versioning enabled + * @param string $id Asset Id to lock + * @param string $size Size of asset data to lock + */ + public function lock_for_period(string $id, string $size): void + { + $s3 = $this->get_s3_client(); + + try { + $s3->putObjectRetention([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size), + 'Retention' => [ + 'Mode' => 'GOVERNANCE', + 'RetainUntilDate' => new \DateTime('+1 hour'), + ], + // 'VersionId' => '', + // 'BypassGovernanceRetention' => true || false, + // 'ChecksumAlgorithm' => 'CRC32|CRC32C|SHA1|SHA256', + // 'ContentMD5' => '', + // 'ExpectedBucketOwner' => '', + // 'RequestPayer' => 'requester', + ]); + } + catch (\Exception $e) + { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to lock asset for period {$id} {$size}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to lock asset for period {$id} {$size}. {$error_code} {$source}"); + } + } + + /** + * Get the lock status of a specific size of an asset + * @param string $id Asset Id to lock + * @param string $size Size of asset data to lock + * @return bool True if locked + */ + public function get_lock(string $id, string $size): bool + { + $s3 = $this->get_s3_client(); + + try { + $result = $s3->getObjectRetention([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size), + // 'VersionId' => '', + // 'RequestPayer' => 'requester', + ]); + // Result syntax: + // [ + // 'Retention' => [ + // 'Mode' => 'GOVERNANCE|COMPLIANCE', + // 'RetainUntilDate' => , + // ], + // ] + return $result['Retention']['Mode'] === 'GOVERNANCE'; // if it's not governance, it's not locked + } + catch (\Exception $e) + { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to get lock asset {$id} {$size}. {$error_code} {$source}"); + return false; + } + } + + /** + * Lock a specific size of an asset * Used to prevent multiple requests from using excessive resources. * @param string $id Asset Id to lock * @param string $size Size of asset data to lock */ public function lock_for_processing(string $id, string $size): void { - // @TODO + $s3 = $this->get_s3_client(); + + try { + $s3->putObjectLegalHold([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size), + 'LegalHold' => [ + 'Status' => 'ON', + ] + ]); + } + catch (\Exception $e) + { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to lock asset for processing {$id} {$size}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to lock asset for processing {$id} {$size}. {$error_code} {$source}"); + } } /** @@ -38,7 +141,28 @@ public function lock_for_processing(string $id, string $size): void */ public function unlock_for_processing(string $id, string $size): void { - // @TODO + $s3 = $this->get_s3_client(); + + try { + // Remove object lock + $s3->putObjectLegalHold([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size), + 'LegalHold' => [ + 'Status' => 'OFF', + ] + ]); + } catch (\Exception $e) { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to unlock asset {$id} {$size}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to unlock asset {$id} {$size}. {$error_code} {$source}"); + } } /** @@ -70,6 +194,7 @@ public function delete(string $id, string $size = '*'): void $source = $e->getAwsErrorMessage(); } \Log::error("S3: Failed to delete asset {$id} {$size}. {$error_code} {$source}"); + throw new \Exception("S3: Failed to delete asset {$id} {$size}. {$error_code} {$source}"); } } } From 8bd3c95f99369cca000ab5502f49b6dc36bd0401 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Mon, 1 Jul 2024 13:47:33 -0400 Subject: [PATCH 06/22] Disable object locking on fakes3 and fix fakes3 volume mounts --- .env | 11 ++---- README.md | 34 +++++++++++------- docker/docker-compose.override.test.yml | 2 +- docker/docker-compose.override.yml | 2 +- fuel/app/classes/materia/widget/asset.php | 22 ++++++++---- .../materia/widget/asset/storage/s3.php | 36 +++++++++++++++++-- fuel/app/config/development/materia.php | 17 ++++----- fuel/app/config/materia.php | 15 ++++---- 8 files changed, 91 insertions(+), 48 deletions(-) diff --git a/.env b/.env index 9f359cdd5..7a3ffe892 100755 --- a/.env +++ b/.env @@ -40,19 +40,12 @@ BOOL_SEND_EMAILS=false ASSET_STORAGE_DRIVER=s3 ASSET_STORAGE_S3_REGION=us-east-1 ASSET_STORAGE_S3_BASEPATH=media - -# AWS S3 DEVELOPMENT =================== - -# ASSET_STORAGE_S3_BUCKET=fake_bucket -# ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001 -# AWS_SESSION_TOKEN= // STS token for s3 development - -# AWS S3 PRODUCTION =================== - # ASSET_STORAGE_S3_BUCKET= # ASSET_STORAGE_S3_ENDPOINT= # AWS_ACCESS_KEY_ID= # AWS_SECRET_ACCESS_KEY= +# AWS_SESSION_TOKEN= # STS token for s3 development +# FAKES3_DISABLED=false # set to true if using real S3 on development # SESSION & CACHE =================== diff --git a/README.md b/README.md index e1402e526..b65854bee 100644 --- a/README.md +++ b/README.md @@ -8,21 +8,21 @@ View the [Materia Docs](http://ucfopen.github.io/Materia-Docs/) for info on inst [Join UCF Open Slack Discussions](https://dl.ucf.edu/join-ucfopen/) [![Join UCF Open Slack Discussions](https://badgen.net/badge/icon/ucfopen?icon=slack&label=slack&color=e01563)](https://dl.ucf.edu/join-ucfopen/) -# Installation +## Installation Materia is configured to use Docker containers in production environments, orchestrated through docker compose, though other orchestration frameworks could potentially be used instead. While it may be possible to deploy Materia without Docker, we **do not recommend doing so**. -## Docker Deployment +### Docker Deployment We publish production ready docker and nginx containers in the [Materia Docker repository](https://github.com/orgs/ucfopen/packages/container/package/materia). For more info on using Docker in Production, read the [Materia Docker Readme](docker/README.md) -## Configuration +### Configuration Visit the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for information about configuration through environment variables. -# Development +## Development -## Local Dev with Docker +### Local Dev with Docker Get started with a local dev server: @@ -38,21 +38,21 @@ The `run_first.sh` script only has to be run once for initial setup. Afterwards, Use `docker-compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default. -In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets. +In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets. Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for both `127.0.0.1:443` and `127.0.0.1:8008`. More info about Materia Docker can be found in the [Materia Docker Readme](docker/README.md) -## Creating additional users +### Creating additional users See the wiki page for [Creating a local user](https://github.com/ucfopen/Materia/wiki#creating-a-local-user). -## Running Tests +### Running Tests Tests run in the docker environment to maintain consistency. View the `run_tests_*.sh` scripts in the docker directory for options. -### Running A Single Test Group +#### Running A Single Test Group Inspect the actual test command in `/.run_tests.sh` for guidance, but as of the time of writing this, you can run a subset of the tests in the docker environment to save time. @@ -62,17 +62,25 @@ The following command will run just the **Oauth** tests rather quickly: ./run_tests.sh --group=Oauth ``` -## Git Hooks +### Git Hooks There is a pre-commit hook available to ensure your code follows our linting standards. Check out the comments contained inside the hook files (in the githooks directory) to install it, you may need a few dependencies installed to get linting working. -# Authentication +## Authentication Materia supports two forms of authentication: - Direct authentication through direct logins. Note that Materia does not provide an out-of-the-box tool for user generation. If your goal is to connect to an external identity management platform or service, you will need to author an authentication module to support this. Review FuelPHP's [Auth package and Login driver](https://fuelphp.com/docs/packages/auth/types/login.html) documentation, as well as the `ltiauth` and `materiaauth` packages located in `fuel/packages` to get started. - Authentication over LTI. This is the more out-of-the-box solution for user management and authentication. In fact, you can disable direct authentication altogether through the `BOOL_LTI_RESTRICT_LOGINS_TO_LAUNCHES` environment variable, making LTI authentication the only way to access Materia. Visit our [LTI Integration Overview](https://ucfopen.github.io/Materia-Docs/develop/lti-integrations.html) page on the docs site for more information. -# Asset Storage +## Asset Storage -Materia enables users to upload media assets for their widgets, including images and audio. There are two asset storage drivers available out of the box: `file` and `db`. `file` is the default asset storage driver, which can be explicitly set via the `ASSET_STORAGE_DRIVER` environment variable. \ No newline at end of file +Materia enables users to upload media assets for their widgets, including images and audio. There are three asset storage drivers available out of the box: `s3`, `file` and `db`. `s3` is the default asset storage driver, which can be explicitly set via the `ASSET_STORAGE_DRIVER` environment variable. + +### Local Asset Storage + +By default, a fake `s3` server will be spun up. To test Materia with AWS S3, set the following variables in `.env.local`: +1. Set `FAKES3_DISABLED` environment variable to `true` +2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name +3. Set `ASSET_STORAGE_S3_ENDPOINT` to your endpoint +4. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these) diff --git a/docker/docker-compose.override.test.yml b/docker/docker-compose.override.test.yml index bb06ceba3..7cceb634e 100644 --- a/docker/docker-compose.override.test.yml +++ b/docker/docker-compose.override.test.yml @@ -48,7 +48,7 @@ services: ports: - "10002:10001" volumes: - - uploaded_media_test:/s3mnt/fakes3_root/fakes3_uploads/media/ + - uploaded_media_test:/s3mnt/fakes3_root/fake_bucket/media/ networks: - frontend - backend diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 0a15c5212..37c1ca0fe 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -33,7 +33,7 @@ services: fakes3: volumes: - - uploaded_media:/s3mnt/fakes3_root/fakes3_uploads/media/ + - uploaded_media:/s3mnt/fakes3_root/fake_bucket/media/ volumes: # static_files: {} # compiled js/css and uploaded widgets diff --git a/fuel/app/classes/materia/widget/asset.php b/fuel/app/classes/materia/widget/asset.php index dc2de80f8..9d2e285a4 100644 --- a/fuel/app/classes/materia/widget/asset.php +++ b/fuel/app/classes/materia/widget/asset.php @@ -343,14 +343,19 @@ protected function build_size(string $size): string break; } - try { - // lock the original asset so we can process it - $this->_storage_driver->lock_for_processing($this->id, 'original'); - } catch (\Throwable $e) + // if we're using fakes3, can't lock the original asset + if ( ! \Config::get('materia.asset_storage.s3.fakes3_enabled')) { - \LOG::error($e); - throw($e); + try { + // lock the original asset so we can process it + $this->_storage_driver->lock_for_processing($this->id, 'original'); + } catch (\Throwable $e) + { + \LOG::error($e); + throw($e); + } } + // get the original file $original_asset_path = $this->copy_asset_to_temp_file($this->id, 'original'); @@ -389,7 +394,10 @@ protected function build_size(string $size): string $this->_storage_driver->store($this, $resized_file_path, $size); // unlock original asset - $this->_storage_driver->unlock_for_processing($this->id, 'original'); + if ( ! \Config::get('materia.asset_storage.s3.fakes3_enabled')) + { + $this->_storage_driver->unlock_for_processing($this->id, 'original'); + } } catch (\Throwable $e) { \LOG::error($e); diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index d9bf8eec3..aa7258c3c 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -66,7 +66,7 @@ public function lock_for_period(string $id, string $size): void * @param string $size Size of asset data to lock * @return bool True if locked */ - public function get_lock(string $id, string $size): bool + public function get_lock_retention(string $id, string $size): bool { $s3 = $this->get_s3_client(); @@ -95,7 +95,7 @@ public function get_lock(string $id, string $size): bool $error_code = $e->getAwsErrorCode(); $source = $e->getAwsErrorMessage(); } - \Log::error("S3: Failed to get lock asset {$id} {$size}. {$error_code} {$source}"); + \Log::error("S3: Failed to get lock retention status for asset {$id} {$size}. {$error_code} {$source}"); return false; } } @@ -165,6 +165,38 @@ public function unlock_for_processing(string $id, string $size): void } } + /** + * Get the lock status of a specific size of an asset + * @param string $id Asset Id to lock + * @param string $size Size of asset data to lock + * @return bool True if locked + */ + public function get_lock(string $id, string $size): bool + { + $s3 = $this->get_s3_client(); + + try { + $result = $s3->getObjectLegalHold([ + 'Bucket' => static::$_config['bucket'], + 'Key' => $this->get_key_name($id, $size) + ]); + return $result['LegalHold']['Status'] === 'ON'; + } + catch (\Exception $e) + { + $error_code = ''; + $source = ''; + if (get_class($e) == 'Aws\S3\Exception\S3Exception') + { + $error_code = $e->getAwsErrorCode(); + $source = $e->getAwsErrorMessage(); + } + \Log::error("S3: Failed to get lock statusfor asset {$id} {$size}. {$error_code} {$source}"); + return false; + } + } + + /** * Delete asset data. Set size to '*' to delete all. * @param string $id Asset Id of asset data to delete diff --git a/fuel/app/config/development/materia.php b/fuel/app/config/development/materia.php index eaf14868d..0107ebd28 100644 --- a/fuel/app/config/development/materia.php +++ b/fuel/app/config/development/materia.php @@ -37,14 +37,15 @@ (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', - 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? 'http://fakes3:10001', // set to url for testing endpoint - 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket - 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? 'fake_bucket', // bucket to store original user uploads - 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets - 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key - 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key - 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token - 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? 'http://fakes3:10001', // set to url for testing endpoint + 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? 'fake_bucket', // bucket to store original user uploads + 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token + 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 + 'fakes3_enabled' => $_ENV['FAKES3_DISABLED'] ?? true, // using fakes3 unless explicitly disabled ] : null ), diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 47ac53ef9..4c11107cf 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -110,13 +110,14 @@ (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', - 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint - 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket - 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads - 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets - 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key - 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key - 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? null, // aws session token + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint + 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads + 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? null, // aws session token + 'fakes3_enabled' => false, // using fakes3 ] : null ), From 97132432a0aa0698a4968280800454c6728f4c3e Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Mon, 1 Jul 2024 15:24:16 -0400 Subject: [PATCH 07/22] Test delete functionality; add commented section for deletion from S3 --- README.md | 3 ++- fuel/app/classes/materia/widget/asset.php | 8 ++++++++ fuel/app/classes/materia/widget/asset/manager.php | 7 +++++++ fuel/app/classes/materia/widget/asset/storage/s3.php | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b65854bee..7b3d72497 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ View the [Materia Docs](http://ucfopen.github.io/Materia-Docs/) for info on inst [Join UCF Open Slack Discussions](https://dl.ucf.edu/join-ucfopen/) [![Join UCF Open Slack Discussions](https://badgen.net/badge/icon/ucfopen?icon=slack&label=slack&color=e01563)](https://dl.ucf.edu/join-ucfopen/) -## Installation +## Installation Materia is configured to use Docker containers in production environments, orchestrated through docker compose, though other orchestration frameworks could potentially be used instead. While it may be possible to deploy Materia without Docker, we **do not recommend doing so**. @@ -80,6 +80,7 @@ Materia enables users to upload media assets for their widgets, including images ### Local Asset Storage By default, a fake `s3` server will be spun up. To test Materia with AWS S3, set the following variables in `.env.local`: + 1. Set `FAKES3_DISABLED` environment variable to `true` 2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name 3. Set `ASSET_STORAGE_S3_ENDPOINT` to your endpoint diff --git a/fuel/app/classes/materia/widget/asset.php b/fuel/app/classes/materia/widget/asset.php index 0d2183306..fda3fc039 100644 --- a/fuel/app/classes/materia/widget/asset.php +++ b/fuel/app/classes/materia/widget/asset.php @@ -423,6 +423,14 @@ public function upload_asset_data(string $source_asset_path): void $this->_storage_driver->store($this, $source_asset_path, 'original'); } + /** + * Delete an asset of a specific size + */ + public function delete_asset_data(string $size): void + { + $this->_storage_driver->delete($this->id, $size); + } + /** * Copy the binary of an asset of a specific size to a temp file * @param string $id Asset Id diff --git a/fuel/app/classes/materia/widget/asset/manager.php b/fuel/app/classes/materia/widget/asset/manager.php index f74acfbeb..48f085130 100644 --- a/fuel/app/classes/materia/widget/asset/manager.php +++ b/fuel/app/classes/materia/widget/asset/manager.php @@ -123,6 +123,13 @@ static public function delete_asset($id) if (Widget_Asset_Manager::can_asset_be_deleted($id)) { $asset = Widget_Asset::fetch_by_id($id); + // Uncomment to delete from S3 + // try { + // $asset->delete_asset_data('original'); + // } catch (\Exception $e) { + // trace($e); + // return false; + // } return $asset->db_remove(); } return false; diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index aa7258c3c..9c3e48aaa 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -191,7 +191,7 @@ public function get_lock(string $id, string $size): bool $error_code = $e->getAwsErrorCode(); $source = $e->getAwsErrorMessage(); } - \Log::error("S3: Failed to get lock statusfor asset {$id} {$size}. {$error_code} {$source}"); + \Log::error("S3: Failed to get lock status for asset {$id} {$size}. {$error_code} {$source}"); return false; } } From ecc2269e63b52f9ef3cb18a2935ce37b0d51b75e Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 19 Jul 2024 13:42:02 -0400 Subject: [PATCH 08/22] Added retry: false to react queries with error states --- src/components/header.jsx | 3 +- .../my-widgets-collaborate-dialog.jsx | 1 + src/components/my-widgets-page.jsx | 2 + .../my-widgets-score-semester-individual.jsx | 1 + .../my-widgets-score-semester-storage.jsx | 1 + src/components/my-widgets-scores.jsx | 1 + .../my-widgets-selected-instance.jsx | 1 + src/components/my-widgets-settings-dialog.jsx | 1 + src/components/notifications.jsx | 471 +++++++++--------- src/components/profile-page.jsx | 1 + src/components/question-history.jsx | 1 + src/components/scores.jsx | 3 + src/components/settings-page.jsx | 1 + src/components/support-page.jsx | 2 + src/components/support-selected-instance.jsx | 1 + src/components/user-admin-page.jsx | 2 + src/components/user-admin-selected.jsx | 1 + src/components/widget-admin-page.jsx | 1 + src/components/widget-creator.jsx | 6 + src/components/widget-player.jsx | 3 + 20 files changed, 268 insertions(+), 236 deletions(-) diff --git a/src/components/header.jsx b/src/components/header.jsx index a76bef1bb..4bd8bf072 100644 --- a/src/components/header.jsx +++ b/src/components/header.jsx @@ -12,7 +12,8 @@ const Header = ({ const { data: verified} = useQuery({ queryKey: 'isLoggedIn', queryFn: apiAuthorVerify, - staleTime: Infinity + staleTime: Infinity, + retry: false }) const { data: user, isLoading: userLoading} = useQuery({ queryKey: 'user', diff --git a/src/components/my-widgets-collaborate-dialog.jsx b/src/components/my-widgets-collaborate-dialog.jsx index 7362a2cd6..37e9ab00c 100644 --- a/src/components/my-widgets-collaborate-dialog.jsx +++ b/src/components/my-widgets-collaborate-dialog.jsx @@ -36,6 +36,7 @@ const MyWidgetsCollaborateDialog = ({onClose, inst, myPerms, otherUserPerms, set queryFn: () => apiGetUsers(Array.from(otherUserPerms.keys())), staleTime: Infinity, placeholderData: {}, + retry: false, onSuccess: (data) => { setCollabUsers({...collabUsers, ...data}) }, diff --git a/src/components/my-widgets-page.jsx b/src/components/my-widgets-page.jsx index ddeeb91c0..1fe6bc3b1 100644 --- a/src/components/my-widgets-page.jsx +++ b/src/components/my-widgets-page.jsx @@ -58,6 +58,7 @@ const MyWidgetsPage = () => { queryKey: 'user', queryFn: apiGetUser, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { @@ -80,6 +81,7 @@ const MyWidgetsPage = () => { enabled: !!state.selectedInst && !!state.selectedInst.id && state.selectedInst?.id !== undefined, placeholderData: null, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setInvalidLogin(true) diff --git a/src/components/my-widgets-score-semester-individual.jsx b/src/components/my-widgets-score-semester-individual.jsx index 84dade5b2..15ae8fb30 100644 --- a/src/components/my-widgets-score-semester-individual.jsx +++ b/src/components/my-widgets-score-semester-individual.jsx @@ -38,6 +38,7 @@ const MyWidgetScoreSemesterIndividual = ({ semester, instId, setInvalidLogin }) enabled: !!instId && !!semester && !!semester.term && !!semester.year, placeholderData: [], refetchOnWindowFocus: false, + retry: false, onSuccess: (result) => { if (page <= result?.total_num_pages) setPage(page + 1) if (result && result.pagination) { diff --git a/src/components/my-widgets-score-semester-storage.jsx b/src/components/my-widgets-score-semester-storage.jsx index 1315fa600..8169325b9 100644 --- a/src/components/my-widgets-score-semester-storage.jsx +++ b/src/components/my-widgets-score-semester-storage.jsx @@ -36,6 +36,7 @@ const MyWidgetScoreSemesterStorage = ({semester, instId, setInvalidLogin}) => { enabled: !!instId, staleTime: Infinity, placeholderData: {}, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setInvalidLogin(true); diff --git a/src/components/my-widgets-scores.jsx b/src/components/my-widgets-scores.jsx index e1badb875..1a69baab8 100644 --- a/src/components/my-widgets-scores.jsx +++ b/src/components/my-widgets-scores.jsx @@ -20,6 +20,7 @@ const MyWidgetsScores = ({inst, beardMode, setInvalidLogin}) => { enabled: !!inst && !!inst.id, staleTime: Infinity, placeholderData: [], + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setInvalidLogin(true); diff --git a/src/components/my-widgets-selected-instance.jsx b/src/components/my-widgets-selected-instance.jsx index f843542d7..3d170e2f0 100644 --- a/src/components/my-widgets-selected-instance.jsx +++ b/src/components/my-widgets-selected-instance.jsx @@ -80,6 +80,7 @@ const MyWidgetSelectedInstance = ({ placeholderData: null, enabled: !!inst.id, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { diff --git a/src/components/my-widgets-settings-dialog.jsx b/src/components/my-widgets-settings-dialog.jsx index a30471b1f..e65ecee6f 100644 --- a/src/components/my-widgets-settings-dialog.jsx +++ b/src/components/my-widgets-settings-dialog.jsx @@ -84,6 +84,7 @@ const MyWidgetsSettingsDialog = ({ onClose, inst, currentUser, otherUserPerms, o placeholderData: {}, enabled: !!otherUserPerms && Array.from(otherUserPerms.keys())?.length > 0, staleTime: Infinity, + retry: false, onError: (err) => { console.error(`Error: ${err.message}`); if (err.message == "Invalid Login") { diff --git a/src/components/notifications.jsx b/src/components/notifications.jsx index d66b3cd2f..250bc35f5 100644 --- a/src/components/notifications.jsx +++ b/src/components/notifications.jsx @@ -5,17 +5,17 @@ import useDeleteNotification from './hooks/useDeleteNotification' import setUserInstancePerms from './hooks/useSetUserInstancePerms' const Notifications = (user) => { - const [navOpen, setNavOpen] = useState(false); - const [showDeleteBtn, setShowDeleteBtn] = useState(-1); - const deleteNotification = useDeleteNotification() - const queryClient = useQueryClient() - const setUserPerms = setUserInstancePerms() - const numNotifications = useRef(0); - const [errorMsg, setErrorMsg] = useState({ - notif_id: '', - msg: '' - }); - let modalRef = useRef(); + const [navOpen, setNavOpen] = useState(false); + const [showDeleteBtn, setShowDeleteBtn] = useState(-1); + const deleteNotification = useDeleteNotification() + const queryClient = useQueryClient() + const setUserPerms = setUserInstancePerms() + const numNotifications = useRef(0); + const [errorMsg, setErrorMsg] = useState({ + notif_id: '', + msg: '' + }); + let modalRef = useRef(); const { data: notifications} = useQuery({ queryKey: 'notifications', @@ -24,230 +24,231 @@ const Notifications = (user) => { refetchOnMount: false, refetchOnWindowFocus: true, queryFn: apiGetNotifications, - staleTime: Infinity, - onSuccess: (data) => { - numNotifications.current = 0; - if (data && data.length > 0) data.forEach(element => { - if (!element.remove) numNotifications.current++; - }); - }, - onError: (err) => { - if (err.message == "Invalid Login") { - window.location.href = '/users/login' - } else { - console.error(err) - } - } - }) - - // Close notification modal if user clicks outside of it - useEffect(() => { - if (navOpen) - { - const checkIfClickedOutsideModal = e => { - if (modalRef.current && !modalRef.current.contains(e.target) && !e.target.className.includes("noticeClose")) - { - setNavOpen(false); - } - } - document.addEventListener("click", checkIfClickedOutsideModal); - - return () => { - document.removeEventListener("click", checkIfClickedOutsideModal); - } - } - }, [navOpen]) - - const toggleNavOpen = () => - { - setNavOpen(!navOpen); - } - // Sets the index of the hovered notification - // Shows delete button on hover - const showDeleteButton = (index) => - { - setShowDeleteBtn(index); - } - const hideDeleteButton = () => - { - setShowDeleteBtn(-1); - } - - const removeNotification = (index, id = null) => { - let notif = null; - if (index >= 0) notif = notifications[index]; - if (id == null) id = notif.id; - - deleteNotification.mutate({ - notifId: id, - deleteAll: false, - successFunc: () => { - Object.keys(notifications).forEach((key, index) => { - if (notifications[key].id == id) - { - notifications[key].remove = true; - numNotifications.current--; - return; - } - }) - }, - errorFunc: (err) => { - setErrorMsg({notif_id: id, msg: 'Action failed.'}); - } - }); - } - - const removeAllNotifications = () => { - deleteNotification.mutate({ - notifId: '', - deleteAll: true, - successFunc: () => {}, - errorFunc: (err) => {} - }); - } - - const onChangeAccessLevel = (notif, access) => { - if (access != "") - { - document.getElementById(notif.id + '_action_button').className = "action_button notification_action enabled"; - } - } - - const onClickGrantAccess = (notif) => { - let accessLevel = document.getElementById(notif.id + '-access-level').value; - - if (accessLevel == "") - { - return; - } - - const expireTime = null; - - const userPerms = [{ - user_id: notif.from_id, - expiration: expireTime, - perms: { - [accessLevel]: true - }, - }] - setUserPerms.mutate({ - instId: notif.item_id, - permsObj: userPerms, - successFunc: (data) => { - // Redirect to widget - if (!window.location.pathname.includes('my-widgets')) - { - // No idea why this works - // But setting hash after setting pathname would set the hash first and then the pathname in URL - window.location.hash = notif.item_id + '-collab'; - window.location.pathname = '/my-widgets' - } - else - { - queryClient.invalidateQueries(['user-perms', notif.item_id]) - window.location.hash = notif.item_id + '-collab'; - } - - setErrorMsg({notif_id: notif.id, msg: ''}); - - removeNotification(-1, notif.id); - - // Close notifications - setNavOpen(false) - }, - errorFunc: (err) => { - setErrorMsg({notif_id: notif.id, msg: 'Action failed.'}) - } - }) - - - } - - let render = null; - let notificationElements = null; - let notificationIcon = null; - - if (notifications?.length > 0) { - notificationElements = [] - for (let index = notifications.length - 1; index >= 0; index--) - { - const notification = notifications[index]; - // If notification was deleted don't show - if (notification.remove) continue; - - let actionButton = null; - let grantAccessDropdown = null; - if (notification.action == "access_request") - { - grantAccessDropdown =
-

Grant Access

- -
- actionButton = - } - let createdAt = new Date(0); - createdAt.setUTCSeconds(notification.created_at) - let notifRow =
showDeleteButton(index)} - onMouseLeave={hideDeleteButton} - > - -
-
${notification.subject}

`}}>
- { grantAccessDropdown } - { actionButton } -

Sent on {createdAt.toLocaleString()}

-
- {removeNotification(index)}} - /> -

{errorMsg.notif_id == notification.id ? errorMsg.msg : ''}

-
- - notificationIcon = - - - notificationElements.push(notifRow) - } - - // In the case that some notifications were removed, we don't want to render the empty notificationElements - if (notificationElements.length > 0) - { - render = ( -
- { notificationIcon } - { navOpen ? -
-

Messages:

- { notificationElements } - removeAllNotifications()}>Remove all Notifications -
- : <> } -
- ) - } - } - else - { - render = null; - - // Keeping this here in case the empty notification icon gets used - notificationElements =

You have no messages!

- - notificationIcon = - - } - - return render; + staleTime: Infinity, + retry: false, + onSuccess: (data) => { + numNotifications.current = 0; + if (data && data.length > 0) data.forEach(element => { + if (!element.remove) numNotifications.current++; + }); + }, + onError: (err) => { + if (err.message == "Invalid Login") { + window.location.href = '/users/login' + } else { + console.error(err) + } + } + }) + + // Close notification modal if user clicks outside of it + useEffect(() => { + if (navOpen) + { + const checkIfClickedOutsideModal = e => { + if (modalRef.current && !modalRef.current.contains(e.target) && !e.target.className.includes("noticeClose")) + { + setNavOpen(false); + } + } + document.addEventListener("click", checkIfClickedOutsideModal); + + return () => { + document.removeEventListener("click", checkIfClickedOutsideModal); + } + } + }, [navOpen]) + + const toggleNavOpen = () => + { + setNavOpen(!navOpen); + } + // Sets the index of the hovered notification + // Shows delete button on hover + const showDeleteButton = (index) => + { + setShowDeleteBtn(index); + } + const hideDeleteButton = () => + { + setShowDeleteBtn(-1); + } + + const removeNotification = (index, id = null) => { + let notif = null; + if (index >= 0) notif = notifications[index]; + if (id == null) id = notif.id; + + deleteNotification.mutate({ + notifId: id, + deleteAll: false, + successFunc: () => { + Object.keys(notifications).forEach((key, index) => { + if (notifications[key].id == id) + { + notifications[key].remove = true; + numNotifications.current--; + return; + } + }) + }, + errorFunc: (err) => { + setErrorMsg({notif_id: id, msg: 'Action failed.'}); + } + }); + } + + const removeAllNotifications = () => { + deleteNotification.mutate({ + notifId: '', + deleteAll: true, + successFunc: () => {}, + errorFunc: (err) => {} + }); + } + + const onChangeAccessLevel = (notif, access) => { + if (access != "") + { + document.getElementById(notif.id + '_action_button').className = "action_button notification_action enabled"; + } + } + + const onClickGrantAccess = (notif) => { + let accessLevel = document.getElementById(notif.id + '-access-level').value; + + if (accessLevel == "") + { + return; + } + + const expireTime = null; + + const userPerms = [{ + user_id: notif.from_id, + expiration: expireTime, + perms: { + [accessLevel]: true + }, + }] + setUserPerms.mutate({ + instId: notif.item_id, + permsObj: userPerms, + successFunc: (data) => { + // Redirect to widget + if (!window.location.pathname.includes('my-widgets')) + { + // No idea why this works + // But setting hash after setting pathname would set the hash first and then the pathname in URL + window.location.hash = notif.item_id + '-collab'; + window.location.pathname = '/my-widgets' + } + else + { + queryClient.invalidateQueries(['user-perms', notif.item_id]) + window.location.hash = notif.item_id + '-collab'; + } + + setErrorMsg({notif_id: notif.id, msg: ''}); + + removeNotification(-1, notif.id); + + // Close notifications + setNavOpen(false) + }, + errorFunc: (err) => { + setErrorMsg({notif_id: notif.id, msg: 'Action failed.'}) + } + }) + + + } + + let render = null; + let notificationElements = null; + let notificationIcon = null; + + if (notifications?.length > 0) { + notificationElements = [] + for (let index = notifications.length - 1; index >= 0; index--) + { + const notification = notifications[index]; + // If notification was deleted don't show + if (notification.remove) continue; + + let actionButton = null; + let grantAccessDropdown = null; + if (notification.action == "access_request") + { + grantAccessDropdown =
+

Grant Access

+ +
+ actionButton = + } + let createdAt = new Date(0); + createdAt.setUTCSeconds(notification.created_at) + let notifRow =
showDeleteButton(index)} + onMouseLeave={hideDeleteButton} + > + +
+
${notification.subject}

`}}>
+ { grantAccessDropdown } + { actionButton } +

Sent on {createdAt.toLocaleString()}

+
+ {removeNotification(index)}} + /> +

{errorMsg.notif_id == notification.id ? errorMsg.msg : ''}

+
+ + notificationIcon = + + + notificationElements.push(notifRow) + } + + // In the case that some notifications were removed, we don't want to render the empty notificationElements + if (notificationElements.length > 0) + { + render = ( +
+ { notificationIcon } + { navOpen ? +
+

Messages:

+ { notificationElements } + removeAllNotifications()}>Remove all Notifications +
+ : <> } +
+ ) + } + } + else + { + render = null; + + // Keeping this here in case the empty notification icon gets used + notificationElements =

You have no messages!

+ + notificationIcon = + + } + + return render; } export default Notifications diff --git a/src/components/profile-page.jsx b/src/components/profile-page.jsx index aac336ccd..2ba78bf94 100644 --- a/src/components/profile-page.jsx +++ b/src/components/profile-page.jsx @@ -21,6 +21,7 @@ const ProfilePage = () => { queryKey: 'user', queryFn: apiGetUser, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setAlertDialog({ diff --git a/src/components/question-history.jsx b/src/components/question-history.jsx index 91fe737d9..3aafddf3f 100644 --- a/src/components/question-history.jsx +++ b/src/components/question-history.jsx @@ -20,6 +20,7 @@ const QuestionHistory = () => { queryFn: () => apiGetQuestionSetHistory(instId), enabled: !!instId, staleTime: Infinity, + retry: false, onError: (err) => { setError("Error fetching question set history.") console.error(err.cause) diff --git a/src/components/scores.jsx b/src/components/scores.jsx index c06672a92..edc0803b4 100644 --- a/src/components/scores.jsx +++ b/src/components/scores.jsx @@ -76,6 +76,7 @@ const Scores = ({ inst_id, play_id, single_id, send_token, isEmbedded, isPreview enabled: false, // enabled is set to false so the query can be manually called with the refetch function staleTime: Infinity, refetchOnWindowFocus: false, + retry: false, onSuccess: (result) => { _populateScores(result.scores) setAttemptsLeft(result.attempts_left) @@ -99,6 +100,7 @@ const Scores = ({ inst_id, play_id, single_id, send_token, isEmbedded, isPreview enabled: false, // enabled is set to false so the query can be manually called with the refetch function staleTime: Infinity, refetchOnWindowFocus: false, + retry: false, onSuccess: (result) => { _populateScores(result) }, @@ -138,6 +140,7 @@ const Scores = ({ inst_id, play_id, single_id, send_token, isEmbedded, isPreview queryFn: () => apiGetScoreDistribution(inst_id), enabled: false, staleTime: Infinity, + retry: false, onSuccess: (data) => { _sendToWidget('scoreDistribution', [data]) }, diff --git a/src/components/settings-page.jsx b/src/components/settings-page.jsx index 729b544cd..ce0c1ebd7 100644 --- a/src/components/settings-page.jsx +++ b/src/components/settings-page.jsx @@ -21,6 +21,7 @@ const SettingsPage = () => { queryKey: 'user', queryFn: apiGetUser, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setAlertDialog({ diff --git a/src/components/support-page.jsx b/src/components/support-page.jsx index e26e6dc92..c8db863f8 100644 --- a/src/components/support-page.jsx +++ b/src/components/support-page.jsx @@ -15,6 +15,7 @@ const SupportPage = () => { queryKey: 'user', queryFn: apiGetUser, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { window.location.href = '/login' @@ -29,6 +30,7 @@ const SupportPage = () => { queryFn: () => apiSearchInstances(widgetHash), enabled: widgetHash != undefined && widgetHash != selectedInstance?.id, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { window.location.href = '/login' diff --git a/src/components/support-selected-instance.jsx b/src/components/support-selected-instance.jsx index 3ffb7b886..f7b9d2232 100644 --- a/src/components/support-selected-instance.jsx +++ b/src/components/support-selected-instance.jsx @@ -76,6 +76,7 @@ const SupportSelectedInstance = ({inst, currentUser, onCopySuccess, embed = fals enabled: !!inst && inst.id !== undefined, placeholderData: null, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setInvalidLogin(true) diff --git a/src/components/user-admin-page.jsx b/src/components/user-admin-page.jsx index 66198f748..de72cdec5 100644 --- a/src/components/user-admin-page.jsx +++ b/src/components/user-admin-page.jsx @@ -14,6 +14,7 @@ const UserAdminPage = () => { queryKey: 'user', queryFn: apiGetUser, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { window.location.href = '/login' @@ -29,6 +30,7 @@ const UserAdminPage = () => { enabled: userHash != undefined && userHash != selectedUser?.id, placeholderData: null, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { window.location.href = '/login' diff --git a/src/components/user-admin-selected.jsx b/src/components/user-admin-selected.jsx index f59ea5479..93e90d4b5 100644 --- a/src/components/user-admin-selected.jsx +++ b/src/components/user-admin-selected.jsx @@ -14,6 +14,7 @@ const UserAdminSelected = ({selectedUser, currentUser, onReturn}) => { queryFn: () => apiGetInstancesForUser(updatedUser.id), placeholderData: null, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { window.location.href = '/login' diff --git a/src/components/widget-admin-page.jsx b/src/components/widget-admin-page.jsx index 0ef723997..5fc596629 100644 --- a/src/components/widget-admin-page.jsx +++ b/src/components/widget-admin-page.jsx @@ -14,6 +14,7 @@ const WidgetAdminPage = () => { queryKey: ['widgets'], queryFn: apiGetWidgetsAdmin, staleTime: Infinity, + retry: false, onSuccess: (widgetData) => { widgetData.forEach((w) => { w.icon = iconUrl('/widget/', w.dir, 60) diff --git a/src/components/widget-creator.jsx b/src/components/widget-creator.jsx index 79194916d..e6bb9a669 100644 --- a/src/components/widget-creator.jsx +++ b/src/components/widget-creator.jsx @@ -71,6 +71,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { queryFn: () => apiGetWidget(widgetId), enabled: !!widgetId, staleTime: Infinity, + retry: false, onSuccess: (info) => { if (info) { setInstance({ ...instance, widget: info }) @@ -88,6 +89,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { queryFn: () => apiGetWidgetInstance(instId), enabled: !!instId, staleTime: Infinity, + retry: false, onSuccess: (data) => { // this value will include a qset that's always empty // it will override the instance's qset property even if it's already set @@ -108,6 +110,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { staleTime: Infinity, placeholderData: null, enabled: !!instIdRef.current, // requires instance state object to be prepopulated + retry: false, onSuccess: (data) => { if (data) { setCreatorState({...creatorState, invalid: false}) @@ -126,6 +129,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { queryFn: () => apiCanBePublishedByCurrentUser(instance.widget?.id), enabled: instance?.widget !== null, staleTime: Infinity, + retry: false, onSuccess: (success) => { if (!success && !instance.is_draft) { onInitFail('Widget type can not be edited by students after publishing.') @@ -142,6 +146,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { staleTime: 30000, refetchInterval: 30000, enabled: creatorState.heartbeatEnabled, + retry: 1, onError: (error) => { onInitFail(error) }, @@ -160,6 +165,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { queryFn: () => apiGetWidgetLock(instance.id), enabled: !!instance.id, staleTime: Infinity, + retry: false, onSuccess: (success) => { if (!success) { onInitFail('Someone else is editing this widget, you will be able to edit after they finish.') diff --git a/src/components/widget-player.jsx b/src/components/widget-player.jsx index 7dc679c40..d599b9433 100644 --- a/src/components/widget-player.jsx +++ b/src/components/widget-player.jsx @@ -123,6 +123,7 @@ const WidgetPlayer = ({instanceId, playId, minHeight='', minWidth='',showFooter= queryFn: () => apiGetWidgetInstance(instanceId), enabled: instanceId !== null, staleTime: Infinity, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setAlert({ @@ -148,6 +149,7 @@ const WidgetPlayer = ({instanceId, playId, minHeight='', minWidth='',showFooter= queryFn: () => apiGetQuestionSet(instanceId, playId), staleTime: Infinity, placeholderData: null, + retry: false, onError: (err) => { if (err.message == "Invalid Login") { setAlert({ @@ -174,6 +176,7 @@ const WidgetPlayer = ({instanceId, playId, minHeight='', minWidth='',showFooter= staleTime: Infinity, refetchInterval: HEARTBEAT_INTERVAL, enabled: !!playId && heartbeatActive, + retry: 1, onError: (err) => { if (err.message == "Invalid Login") { setAlert({ From bcb16ae505804029afa482250869736e36c3ac70 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Tue, 23 Jul 2024 17:11:31 -0400 Subject: [PATCH 09/22] Added retry false to scoreSummary query --- src/components/score-overview.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/score-overview.jsx b/src/components/score-overview.jsx index 105b86297..dc7b69356 100644 --- a/src/components/score-overview.jsx +++ b/src/components/score-overview.jsx @@ -12,7 +12,8 @@ const ScoreOverview = ({inst_id, single_id, overview, attemptNum, isPreview, gue queryKey: ['score-summary', inst_id], queryFn: () => apiGetScoreSummary(inst_id), staleTime: Infinity, - enabled: !!inst_id && !single_id + enabled: !!inst_id && !single_id, + retry: false }) let scoreGraphRender = null From d9f55478671054e363eccbebc245cf5f3f1b168a Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Wed, 24 Jul 2024 13:24:13 -0400 Subject: [PATCH 10/22] Adjusts widget pre-embed views with improved layout and footer that includes links to Materia and support options --- src/components/closed.jsx | 2 + src/components/embedded-only.jsx | 21 +++--- src/components/login-page.jsx | 2 + src/components/login-page.scss | 2 +- src/components/no-attempts.jsx | 73 +++++++++++---------- src/components/pre-embed-common-styles.scss | 67 +++++++++++++++---- src/components/pre-embed-placeholder.jsx | 7 +- src/components/widget-embed-footer.jsx | 15 +++++ 8 files changed, 128 insertions(+), 61 deletions(-) create mode 100644 src/components/widget-embed-footer.jsx diff --git a/src/components/closed.jsx b/src/components/closed.jsx index 86679842f..cd6a7ae45 100644 --- a/src/components/closed.jsx +++ b/src/components/closed.jsx @@ -2,6 +2,7 @@ import React, { useEffect, useState } from 'react'; import Header from './header' import Summary from './widget-summary' import './login-page.scss' +import EmbedFooter from './widget-embed-footer'; const Closed = () => { @@ -45,6 +46,7 @@ const Closed = () => {

{ state.summary }

{ state.description }

+ diff --git a/src/components/embedded-only.jsx b/src/components/embedded-only.jsx index 4caf60eec..d176a52f6 100644 --- a/src/components/embedded-only.jsx +++ b/src/components/embedded-only.jsx @@ -1,18 +1,21 @@ import React from 'react'; import Summary from './widget-summary' +import EmbedFooter from './widget-embed-footer'; const EmbeddedOnly = () => { return ( -
-
- +
+
+ -
-

Not Playable Here

- Your instructor has not made this widget available outside of the LMS. -
-
-
+
+

Not Playable Here

+ Your instructor has not made this widget available outside of the LMS. +
+ + +
+
) } diff --git a/src/components/login-page.jsx b/src/components/login-page.jsx index 4f9ebf07b..541bf9143 100644 --- a/src/components/login-page.jsx +++ b/src/components/login-page.jsx @@ -2,6 +2,7 @@ import React, { useEffect, useRef, useState } from 'react' import Header from './header' import Summary from './widget-summary' import './login-page.scss' +import EmbedFooter from './widget-embed-footer' const LoginPage = () => { @@ -107,6 +108,7 @@ const LoginPage = () => { : '' } + { state.context && state.context == 'widget' ? : ''} diff --git a/src/components/login-page.scss b/src/components/login-page.scss index 3c1b7cfc4..ebafcf27b 100644 --- a/src/components/login-page.scss +++ b/src/components/login-page.scss @@ -3,5 +3,5 @@ span.subtitle { font-size: .7em; - font-weight: 300; + font-weight: 400; } diff --git a/src/components/no-attempts.jsx b/src/components/no-attempts.jsx index e4a3faf80..562e35cdb 100644 --- a/src/components/no-attempts.jsx +++ b/src/components/no-attempts.jsx @@ -1,21 +1,22 @@ import React, { useState, useEffect} from 'react' import Summary from './widget-summary' import Header from './header' +import EmbedFooter from './widget-embed-footer' const NoAttempts = () => { - const [attempts, setAttempts] = useState(null) - const [scoresPath, setScoresPath] = useState(null) + const [attempts, setAttempts] = useState(null) + const [scoresPath, setScoresPath] = useState(null) - useEffect(() => { - waitForWindow().then(() => { - const scoresPath = `/scores${window.IS_EMBEDDED ? '/embed' : ''}/${window.WIDGET_ID}`; + useEffect(() => { + waitForWindow().then(() => { + const scoresPath = `/scores${window.IS_EMBEDDED ? '/embed' : ''}/${window.WIDGET_ID}`; - setScoresPath(scoresPath); - setAttempts(window.ATTEMPTS) - }) - }, []) + setScoresPath(scoresPath); + setAttempts(window.ATTEMPTS) + }) +}, []) - const waitForWindow = async () => { +const waitForWindow = async () => { while(!window.hasOwnProperty('WIDGET_ID') && !window.hasOwnProperty('IS_EMBEDDED') && !window.hasOwnProperty('ATTEMPTS')) { @@ -23,31 +24,33 @@ const NoAttempts = () => { } } - let bodyRender = null - if (!!attempts) { - bodyRender = ( -
-
- - -
-

No remaining attempts

- You've used all { attempts } available attempts. -

- Review previous scores -

-
-
-
- ) - } - - return ( - <> -
- { bodyRender } - - ) + let bodyRender = null + if (!!attempts) { + bodyRender = ( +
+
+ + +
+

No remaining attempts

+ You've used all { attempts } available attempts. +

+ Review previous scores +

+
+ + +
+
+ ) +} + +return ( + <> +
+ { bodyRender } + +) } export default NoAttempts diff --git a/src/components/pre-embed-common-styles.scss b/src/components/pre-embed-common-styles.scss index a67fbc50e..7dbc158eb 100644 --- a/src/components/pre-embed-common-styles.scss +++ b/src/components/pre-embed-common-styles.scss @@ -75,6 +75,17 @@ body { text-align: center; clear: both; + &.pre-embed { + padding: 2em 0; + + background: #f3f3f3; + border-radius: 4px; + + .action_button { + margin: 24px; + } + } + h2 { font-size: 18pt; } @@ -102,11 +113,22 @@ body { .widget_info { position: relative; + display: flex; + justify-content: flex-start; + align-items: center; + gap: 1em; min-height: 122px; - padding-left: 110px; + width: calc(100% + 20px); - list-style: none; + top: -15px; + left: -10px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + + background: #0093e7; + color: #fff; + li { display: block; margin-right: 10px; @@ -126,18 +148,7 @@ body { } .widget_icon { - position: absolute; - top: 0; - left: -25px; - background: #0093e7; - width: 92px; - height: 92px; - padding: 15px; - text-align: center; - color: #fff; - font-size: 10px; - line-height: 13px; - box-shadow: 0px 2px 2px rgba(0, 0, 0, 0.4); + padding-left: 15px; img { width: 92px; @@ -149,6 +160,7 @@ body { } ul.widget_about { + margin: 0; padding: 0; list-style-type: none; @@ -169,6 +181,33 @@ body { } } + .widget-embed-footer { + position: absolute; + bottom: 5px; + + display: flex; + justify-content: space-between; + align-items: center; + + width: calc(100% - 20px); + + font-size: 0.7em; + font-weight: 400; + + color: #5e5e5e; + + a { + &.materia-logo { + + img { + width: auto; + height: 15px; + margin-top: 1px; + } + } + } + } + div#form { ul { diff --git a/src/components/pre-embed-placeholder.jsx b/src/components/pre-embed-placeholder.jsx index ae8e411c9..397aaaec8 100644 --- a/src/components/pre-embed-placeholder.jsx +++ b/src/components/pre-embed-placeholder.jsx @@ -1,8 +1,10 @@ import React, { useState, useEffect} from 'react' import Summary from './widget-summary' +import EmbedFooter from './widget-embed-footer' import './pre-embed-common-styles.scss' + const PreEmbedPlaceholder = () => { const [instId, setInstId] = useState(null) @@ -27,9 +29,10 @@ const PreEmbedPlaceholder = () => {
-
- Play + +
) diff --git a/src/components/widget-embed-footer.jsx b/src/components/widget-embed-footer.jsx new file mode 100644 index 000000000..eefbecf82 --- /dev/null +++ b/src/components/widget-embed-footer.jsx @@ -0,0 +1,15 @@ +import React from 'react' + +const EmbedFooter = () => { + + return ( +
+ materia logo + + Content embedded from Materia. Need a hand? View support options. + +
+ ) +} + +export default EmbedFooter \ No newline at end of file From be9c848ff207db48f9d278174da9cb0a0f33375f Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Wed, 24 Jul 2024 16:32:47 -0400 Subject: [PATCH 11/22] Change default storage driver to file & other misc changes in response to feedback --- .env | 2 +- docker/.env | 2 ++ docker/docker-compose.override.test.yml | 5 ++++ docker/docker-compose.yml | 6 +---- fuel/app/classes/controller/media.php | 4 ++-- .../classes/materia/widget/asset/manager.php | 9 +------- .../materia/widget/asset/storage/s3.php | 23 +++---------------- fuel/app/config/development/materia.php | 2 +- fuel/app/config/materia.php | 2 +- materia-app.Dockerfile | 9 ++++++++ 10 files changed, 26 insertions(+), 38 deletions(-) diff --git a/.env b/.env index 7a3ffe892..3d9ee370b 100755 --- a/.env +++ b/.env @@ -37,7 +37,7 @@ BOOL_SEND_EMAILS=false # AWS S3 =================== -ASSET_STORAGE_DRIVER=s3 +ASSET_STORAGE_DRIVER=file ASSET_STORAGE_S3_REGION=us-east-1 ASSET_STORAGE_S3_BASEPATH=media # ASSET_STORAGE_S3_BUCKET= diff --git a/docker/.env b/docker/.env index 2392fe5c5..6915e764a 100644 --- a/docker/.env +++ b/docker/.env @@ -10,3 +10,5 @@ DEV_ONLY_USER_PASSWORD=kogneato DEV_ONLY_AUTH_SALT=111b776e5f862058e2e075b640b3de5fb601d0ac57639c733a2d10edffd2a3d5 DEV_ONLY_AUTH_SIMPLEAUTH_SALT=33e0d379060e3877d634632853c10a70dff9710b751e5af00a0f637884df417e DEV_ONLY_SECRET_CIPHER_KEY=e0beaea1704555ae3c75650703bb106fac24b8967c77a667124fbe745c3346ed + +ASSET_STORAGE_DRIVER=file \ No newline at end of file diff --git a/docker/docker-compose.override.test.yml b/docker/docker-compose.override.test.yml index 7cceb634e..9620730ea 100644 --- a/docker/docker-compose.override.test.yml +++ b/docker/docker-compose.override.test.yml @@ -40,14 +40,19 @@ services: # tmpfs: # - /var/lib/mysql + # fakes3, when added as a dependency in the app container above, would restart + # and lose its data during tests + # thus, fakes3_test was created. it is dropped after tests are complete fakes3_test: image: ucfopen/materia:fake-s3-dev build: context: ../ dockerfile: materia-fake-s3.Dockerfile ports: + # use separate port to avoid conflicts with fakes3 - "10002:10001" volumes: + # use separate volume to avoid conflicts with fakes3 - uploaded_media_test:/s3mnt/fakes3_root/fake_bucket/media/ networks: - frontend diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index caf9e8f9f..e3f67c267 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,11 +22,7 @@ services: dockerfile: materia-app.Dockerfile environment: # View Materia README for env settings - # - ASSET_STORAGE_DRIVER=${ASSET_STORAGE_DRIVER} - # - ASSET_STORAGE_S3_BUCKET=${ASSET_STORAGE_S3_BUCKET} - # - ASSET_STORAGE_S3_ENDPOINT=${ASSET_STORAGE_S3_ENDPOINT} - # - ASSET_STORAGE_S3_KEY=${AWS_ACCESS_KEY_ID} - # - ASSET_STORAGE_S3_SECRET=${AWS_SECRET_ACCESS_KEY} + - ASSET_STORAGE_DRIVER=${ASSET_STORAGE_DRIVER} - AUTH_DRIVERS=Materiaauth - AUTH_SALT=${DEV_ONLY_AUTH_SALT} - AUTH_SIMPLEAUTH_SALT=${DEV_ONLY_AUTH_SIMPLEAUTH_SALT} diff --git a/fuel/app/classes/controller/media.php b/fuel/app/classes/controller/media.php index 7cf40171b..f6b5cd4e6 100644 --- a/fuel/app/classes/controller/media.php +++ b/fuel/app/classes/controller/media.php @@ -109,7 +109,7 @@ public function action_upload() $asset = Widget_Asset_Manager::new_asset_from_file($name, $file_info); } catch (\Exception $e) { - $res->body('{"error":{"code":"15","message":"'.$e->getMessage().'"}}'); + $res->body('{"error":{"message":"Unable to save new asset"}}'); $res->set_status(400); return $res; } @@ -117,7 +117,7 @@ public function action_upload() if ( ! $asset || ! isset($asset->id)) { // error - trace('Unable to create asset'); + \Log::Error('Unable to create asset'); $res->body('{"error":{"code":"16","message":"Unable to save new asset"}}'); $res->set_status(400); return $res; diff --git a/fuel/app/classes/materia/widget/asset/manager.php b/fuel/app/classes/materia/widget/asset/manager.php index 48f085130..166867e7f 100644 --- a/fuel/app/classes/materia/widget/asset/manager.php +++ b/fuel/app/classes/materia/widget/asset/manager.php @@ -61,7 +61,7 @@ static public function new_asset_from_file($name, $file_info) } catch (\OutsideAreaException | InvalidPathException | \FileAccessException | \Exception $e) { - trace($e); + \Log::error('Failed to store asset data: '.$e->getMessage()); } // failed, remove the asset @@ -123,13 +123,6 @@ static public function delete_asset($id) if (Widget_Asset_Manager::can_asset_be_deleted($id)) { $asset = Widget_Asset::fetch_by_id($id); - // Uncomment to delete from S3 - // try { - // $asset->delete_asset_data('original'); - // } catch (\Exception $e) { - // trace($e); - // return false; - // } return $asset->db_remove(); } return false; diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index 9c3e48aaa..aedb1a502 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -20,7 +20,7 @@ public static function instance(array $config): Widget_Asset_Storage_Driver } /** - * Create a lock on a specific size of an asset for a period of time + * Create a lock on a specific size of an asset for one hour * Used to prevent multiple requests from using excessive resources. * For object locking to work, the bucket must have versioning enabled * @param string $id Asset Id to lock @@ -37,13 +37,7 @@ public function lock_for_period(string $id, string $size): void 'Retention' => [ 'Mode' => 'GOVERNANCE', 'RetainUntilDate' => new \DateTime('+1 hour'), - ], - // 'VersionId' => '', - // 'BypassGovernanceRetention' => true || false, - // 'ChecksumAlgorithm' => 'CRC32|CRC32C|SHA1|SHA256', - // 'ContentMD5' => '', - // 'ExpectedBucketOwner' => '', - // 'RequestPayer' => 'requester', + ] ]); } catch (\Exception $e) @@ -73,17 +67,8 @@ public function get_lock_retention(string $id, string $size): bool try { $result = $s3->getObjectRetention([ 'Bucket' => static::$_config['bucket'], - 'Key' => $this->get_key_name($id, $size), - // 'VersionId' => '', - // 'RequestPayer' => 'requester', + 'Key' => $this->get_key_name($id, $size) ]); - // Result syntax: - // [ - // 'Retention' => [ - // 'Mode' => 'GOVERNANCE|COMPLIANCE', - // 'RetainUntilDate' => , - // ], - // ] return $result['Retention']['Mode'] === 'GOVERNANCE'; // if it's not governance, it's not locked } catch (\Exception $e) @@ -135,7 +120,6 @@ public function lock_for_processing(string $id, string $size): void /** * Unlock a lock made for a specific size of an asset - * Used to prevent multiple requests from using excessive resources. * @param string $id Asset Id to lock * @param string $size Size of asset data to lock */ @@ -144,7 +128,6 @@ public function unlock_for_processing(string $id, string $size): void $s3 = $this->get_s3_client(); try { - // Remove object lock $s3->putObjectLegalHold([ 'Bucket' => static::$_config['bucket'], 'Key' => $this->get_key_name($id, $size), diff --git a/fuel/app/config/development/materia.php b/fuel/app/config/development/materia.php index 0107ebd28..04b14f14e 100644 --- a/fuel/app/config/development/materia.php +++ b/fuel/app/config/development/materia.php @@ -23,7 +23,7 @@ // Storage driver can be overridden from env here // s3 uses fakes3 on dev - 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 's3', + 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 'file', 'asset_storage' => [ 'file' => [ diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 4c11107cf..123c25de4 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -96,7 +96,7 @@ 'google_tracking_id' => $_ENV['GOOGLE_ANALYTICS_ID'] ?? false, // Asset storage configuration - 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 's3', + 'asset_storage_driver' => $_ENV['ASSET_STORAGE_DRIVER'] ?? 'file', 'asset_storage' => [ 'file' => [ diff --git a/materia-app.Dockerfile b/materia-app.Dockerfile index 0efa9e484..6300101c2 100644 --- a/materia-app.Dockerfile +++ b/materia-app.Dockerfile @@ -45,6 +45,15 @@ RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" # Modify php-fpm.d/docker.conf to point access.log to /dev/null/, which effectively prevents it from being picked up by the log driver RUN sed -i 's/access.log = .*/access.log = \/dev\/null/' /usr/local/etc/php-fpm.d/docker.conf +# Adds an easily accessible override config for php-fpm's pm.max_children value +# The base image sets this value at 5, and the default value in the override matches that +# If an instance of Materia receives moderate traffic, this value will likely need to be raised +# The file is renamed to zzz-materia.conf to ensure it is loaded last, a zz-docker.conf will already be present in the php-fpm.d directory +# +# If preferred, the configuration can be mounted via volume in your deployment's compose file instead +# +# COPY ./docker/config/php/materia.www.conf /usr/local/etc/php-fpm.d/zzz-materia.conf + WORKDIR /var/www/html # ===================================================================================================== From d2cba147bcfd4b6156b684d1dcda490fc9dcbaa0 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 26 Jul 2024 16:41:06 -0400 Subject: [PATCH 12/22] Tweaks asset configs. Additions to readmes. --- .env | 7 +++---- README.md | 19 +++++++++++++++---- docker/.env | 11 +++++++++-- docker/README.md | 11 +++++++---- docker/docker-compose.override.yml | 8 ++++++++ fuel/app/config/development/materia.php | 2 +- 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 3d9ee370b..cec4fc71f 100755 --- a/.env +++ b/.env @@ -34,18 +34,17 @@ BOOL_SEND_EMAILS=false #URLS_STATIC= #URLS_ENGINES= #BOOL_ADMIN_UPLOADER_ENABLE=true +ASSET_STORAGE_DRIVER=file # AWS S3 =================== -ASSET_STORAGE_DRIVER=file -ASSET_STORAGE_S3_REGION=us-east-1 -ASSET_STORAGE_S3_BASEPATH=media +# ASSET_STORAGE_S3_REGION=us-east-1 +# ASSET_STORAGE_S3_BASEPATH=media # ASSET_STORAGE_S3_BUCKET= # ASSET_STORAGE_S3_ENDPOINT= # AWS_ACCESS_KEY_ID= # AWS_SECRET_ACCESS_KEY= # AWS_SESSION_TOKEN= # STS token for s3 development -# FAKES3_DISABLED=false # set to true if using real S3 on development # SESSION & CACHE =================== diff --git a/README.md b/README.md index 9093e7e93..648fa656f 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,24 @@ Materia supports two forms of authentication: ## Asset Storage -Materia enables users to upload media assets for their widgets, including images and audio. There are three asset storage drivers available out of the box: `s3`, `file` and `db`. `s3` is the default asset storage driver, which can be explicitly set via the `ASSET_STORAGE_DRIVER` environment variable. +Users can upload media assets (images and audio) for use in their widgets, facilitated through a media importer that is provided by Materia itself. Asset storage drivers include: -### Local Asset Storage +- `file`: Assets are stored on the local filesystem of the application. It is recommended that assets are backed up and synced with an external storage solution (such as S3) to ensure the files persist across application instances. +- `s3`: Files are uploaded to and requested directly from AWS S3. This is the most straightforward and recommended storage driver option. Be sure to consult the [Materia Docker Readme](docker/README.md) for additional environment variables associated with using S3. +- `db`: This storage driver stores asset binaries directly in the database. This option allows Materia to run on cloud hosting options with very limited storage volumes. The `db` storage driver option is not recommended for general use. -By default, a fake `s3` server will be spun up. To test Materia with AWS S3, set the following variables in `.env.local`: +> [!WARNING] +> The `db` asset storage driver option is deprecated and will be removed in the next major version of Materia. -1. Set `FAKES3_DISABLED` environment variable to `true` +The storage driver is configured via the `ASSET_STORAGE_DRIVER` environment variable. + +### Local Asset Storage With S3 + +A `fakes3` container is instantiated as part of the default development stack and the `ASSET_STORAGE_DRIVER` environment variable is set to `s3` by default in the development `.env` file located in `docker/.env`. When using `fakes3`, this is all that is required to simulate S3 usage locally. + +To use an actual S3 bucket for local dev: + +1. Set `DEV_ONLY_FAKES3_DISABLED` environment variable in `docker/.env` to `true` 2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name 3. Set `ASSET_STORAGE_S3_ENDPOINT` to your endpoint 4. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these) diff --git a/docker/.env b/docker/.env index 6915e764a..aafeb2e4e 100644 --- a/docker/.env +++ b/docker/.env @@ -4,11 +4,18 @@ MYSQL_USER=materia MYSQL_PASSWORD=odin MYSQL_DATABASE=materia -# passwords/hashes/eys +# passwords/hashes/keys DEV_ONLY_USER_PASSWORD=kogneato # see readme for how to create these DEV_ONLY_AUTH_SALT=111b776e5f862058e2e075b640b3de5fb601d0ac57639c733a2d10edffd2a3d5 DEV_ONLY_AUTH_SIMPLEAUTH_SALT=33e0d379060e3877d634632853c10a70dff9710b751e5af00a0f637884df417e DEV_ONLY_SECRET_CIPHER_KEY=e0beaea1704555ae3c75650703bb106fac24b8967c77a667124fbe745c3346ed -ASSET_STORAGE_DRIVER=file \ No newline at end of file +# s3-specific asset storage values +ASSET_STORAGE_DRIVER=s3 # overrides default value in the base .env (which isn't loaded into dev environment) +ASSET_STORAGE_S3_BUCKET=fake_bucket +ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001 +ASSET_STORAGE_S3_KEY=KEY +ASSET_STORAGE_S3_SECRET=SECRET + +# DEV_ONLY_FAKES3_DISABLED=false # set to true if using real S3 on development \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index f9263a42d..3faba2549 100644 --- a/docker/README.md +++ b/docker/README.md @@ -122,9 +122,13 @@ If you plan on deploying a production server using these docker images, we sugge ### Dynamic Files to Backup -* MySQL Database Contents -* Uploaded Media -* Installed Widget Engine Files +* MySQL Database Contents: +* Uploaded Media (generally `$APP_DIR/media`) +* Installed Widget Engine Files (generally `$APP_DIR/widgets`) + +### Environment Variables + +Refer to the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for environment variable configuration options. ### Sample Docker Compose @@ -199,4 +203,3 @@ volumes: #### Table Not Found When running fuelphp's install, it uses fuel/app/config/development/migrations.php file to know the current state of your database. Fuel assumes this file is truth, and won't create tables even on an empty database. You probably need to delete the file and run the setup scripts again. run_first.sh does this for you if needed. - diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 37c1ca0fe..74a3a7232 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -15,6 +15,13 @@ services: - ./config/nginx/nginx-dev.conf:/etc/nginx/nginx.conf:ro app: + environment: + # values sourced from docker/env + - ASSET_STORAGE_DRIVER + - ASSET_STORAGE_S3_BUCKET + - ASSET_STORAGE_S3_ENDPOINT + - ASSET_STORAGE_S3_KEY + - ASSET_STORAGE_S3_SECRET volumes: - ..:/var/www/html/ - uploaded_widgets:/var/www/html/public/widget/ @@ -26,6 +33,7 @@ services: mysql: environment: + # values sourced from docker/env - MYSQL_ROOT_PASSWORD - MYSQL_USER - MYSQL_PASSWORD diff --git a/fuel/app/config/development/materia.php b/fuel/app/config/development/materia.php index 04b14f14e..6bda85aa5 100644 --- a/fuel/app/config/development/materia.php +++ b/fuel/app/config/development/materia.php @@ -45,7 +45,7 @@ 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 - 'fakes3_enabled' => $_ENV['FAKES3_DISABLED'] ?? true, // using fakes3 unless explicitly disabled + 'fakes3_enabled' => $_ENV['DEV_ONLY_FAKES3_DISABLED'] ?? true, // using fakes3 unless explicitly disabled ] : null ), From 5e495a36b787c75d75cb44d75bf59c0cc12a752f Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Tue, 30 Jul 2024 17:08:46 -0400 Subject: [PATCH 13/22] Experimental support for imds credentialing for aws s3 sdk --- docker/.env | 2 ++ docker/docker-compose.override.yml | 1 + .../classes/materia/widget/asset/storage/s3.php | 14 ++++++++++++++ fuel/app/config/materia.php | 17 +++++++++-------- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docker/.env b/docker/.env index aafeb2e4e..408b2914b 100644 --- a/docker/.env +++ b/docker/.env @@ -13,6 +13,8 @@ DEV_ONLY_SECRET_CIPHER_KEY=e0beaea1704555ae3c75650703bb106fac24b8967c77a667124fb # s3-specific asset storage values ASSET_STORAGE_DRIVER=s3 # overrides default value in the base .env (which isn't loaded into dev environment) + +ASSET_STORAGE_S3_CREDENTIAL_PROVIDER=env # env | imds ASSET_STORAGE_S3_BUCKET=fake_bucket ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001 ASSET_STORAGE_S3_KEY=KEY diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 74a3a7232..60e46b5fc 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -18,6 +18,7 @@ services: environment: # values sourced from docker/env - ASSET_STORAGE_DRIVER + - ASSET_STORAGE_S3_CREDENTIAL_PROVIDER - ASSET_STORAGE_S3_BUCKET - ASSET_STORAGE_S3_ENDPOINT - ASSET_STORAGE_S3_KEY diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index aedb1a502..77b2bf5f0 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -351,6 +351,20 @@ protected function get_s3_client(): \Aws\S3\S3Client 'token' => static::$_config['token'] ?? null, ] ]; + if ($_config['credential_provider'] == 'imds') + { + $provider = \Aws\Credentials\CredentialProvider::defaultProvider(); + $config['credentials'] = $provider; + } + elseif ($_config['credential_provider'] == 'env') + { + $config['credentials'] = [ + 'key' => static::$_config['key'], + 'secret' => static::$_config['secret_key'], + 'token' => static::$_config['token'] ?? null, + ]; + } + else throw new \Exception('S3: Failed to determine credential provider. Did you set the appropriate environment variable?'); try { static::$_s3_client = new \Aws\S3\S3Client($config); diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 123c25de4..159b8f489 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -110,14 +110,15 @@ (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', - 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint - 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket - 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads - 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets - 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key - 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key - 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? null, // aws session token - 'fakes3_enabled' => false, // using fakes3 + 'credential_provider' => $_ENV['ASSET_STORAGE_S3_CREDENTIAL_PROVIDER'] ?? 'env', + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint + 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads + 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? null, // aws session token + 'fakes3_enabled' => false, // using fakes3 ] : null ), From 1f263ead974cfa5b2a0c4c989940485326b49968 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Wed, 31 Jul 2024 13:51:03 -0400 Subject: [PATCH 14/22] Fixed references to config in s3 driver. Added missing credential config in dev materia config. --- .../materia/widget/asset/storage/s3.php | 4 ++-- fuel/app/config/development/materia.php | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index 77b2bf5f0..3ccbc835a 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -351,12 +351,12 @@ protected function get_s3_client(): \Aws\S3\S3Client 'token' => static::$_config['token'] ?? null, ] ]; - if ($_config['credential_provider'] == 'imds') + if (static::$_config['credential_provider'] == 'imds') { $provider = \Aws\Credentials\CredentialProvider::defaultProvider(); $config['credentials'] = $provider; } - elseif ($_config['credential_provider'] == 'env') + elseif (static::$_config['credential_provider'] == 'env') { $config['credentials'] = [ 'key' => static::$_config['key'], diff --git a/fuel/app/config/development/materia.php b/fuel/app/config/development/materia.php index 6bda85aa5..400fb7d70 100644 --- a/fuel/app/config/development/materia.php +++ b/fuel/app/config/development/materia.php @@ -37,15 +37,16 @@ (($_ENV['ASSET_STORAGE_DRIVER'] ?? 'file') == 's3') ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', - 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? 'http://fakes3:10001', // set to url for testing endpoint - 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket - 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? 'fake_bucket', // bucket to store original user uploads - 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets - 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key - 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key - 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token - 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 - 'fakes3_enabled' => $_ENV['DEV_ONLY_FAKES3_DISABLED'] ?? true, // using fakes3 unless explicitly disabled + 'credential_provider' => $_ENV['ASSET_STORAGE_S3_CREDENTIAL_PROVIDER'] ?? 'env', // env or imds. Should be set to env for fakes3 + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? 'http://fakes3:10001', // set to url for testing endpoint + 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket + 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? 'fake_bucket', // bucket to store original user uploads + 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets + 'secret_key' => $_ENV['AWS_SECRET_ACCESS_KEY'] ?? $_ENV['ASSET_STORAGE_S3_SECRET'] ?? 'SECRET', // aws api secret key + 'key' => $_ENV['AWS_ACCESS_KEY_ID'] ?? $_ENV['ASSET_STORAGE_S3_KEY'] ?? 'KEY', // aws api key + 'token' => $_ENV['AWS_SESSION_TOKEN'] ?? 'TOKEN', // aws session token + 'force_path_style' => $_ENV['ASSET_STORAGE_S3_FORCE_PATH_STYLE'] ?? false, // needed for fakes3 + 'fakes3_enabled' => $_ENV['DEV_ONLY_FAKES3_DISABLED'] ?? true, // using fakes3 unless explicitly disabled ] : null ), From 36203623447df85df42b691a3895ef5143ea01dc Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Wed, 31 Jul 2024 15:38:31 -0400 Subject: [PATCH 15/22] s3 client config only uses endpoint with fakes3. Object locking disabled when driver set to s3. --- fuel/app/classes/materia/widget/asset.php | 7 ++++--- fuel/app/classes/materia/widget/asset/storage/s3.php | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fuel/app/classes/materia/widget/asset.php b/fuel/app/classes/materia/widget/asset.php index fda3fc039..f63bd71a2 100644 --- a/fuel/app/classes/materia/widget/asset.php +++ b/fuel/app/classes/materia/widget/asset.php @@ -347,8 +347,9 @@ protected function build_size(string $size): string break; } - // if we're using fakes3, can't lock the original asset - if ( ! \Config::get('materia.asset_storage.s3.fakes3_enabled')) + // object locking is unnecessary with s3 + $driver = \Config::get('materia.asset_storage_driver', 'db'); + if ($driver != 's3') { try { // lock the original asset so we can process it @@ -398,7 +399,7 @@ protected function build_size(string $size): string $this->_storage_driver->store($this, $resized_file_path, $size); // unlock original asset - if ( ! \Config::get('materia.asset_storage.s3.fakes3_enabled')) + if ($driver != 's3') { $this->_storage_driver->unlock_for_processing($this->id, 'original'); } diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index 3ccbc835a..672b3cd03 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -351,6 +351,12 @@ protected function get_s3_client(): \Aws\S3\S3Client 'token' => static::$_config['token'] ?? null, ] ]; + + // endpoint config only required for fakes3 - the param is not required for actual S3 on AWS + if (\Config::get('materia.asset_storage.s3.fakes3_enabled')) $config['endpoint'] = static::$_config['endpoint'] ?? ''; + + // configure credentials, depending on whether we're providing them from env or Amazon's IMDSv2 service + // imds is HIGHLY recommended for prod usage on AWS. Credentials are sourced from the EC2 instance's IAM role, and the credential provider handles rotation if (static::$_config['credential_provider'] == 'imds') { $provider = \Aws\Credentials\CredentialProvider::defaultProvider(); From 7d17e13242070ab26fe8af2ed7db54476e0bbde7 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Thu, 1 Aug 2024 12:55:11 -0400 Subject: [PATCH 16/22] Removes endpoint from base s3 client config oops. S3 keys now match file driver asset paths --- .../classes/materia/widget/asset/storage/s3.php | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fuel/app/classes/materia/widget/asset/storage/s3.php b/fuel/app/classes/materia/widget/asset/storage/s3.php index 672b3cd03..4bbf5372d 100644 --- a/fuel/app/classes/materia/widget/asset/storage/s3.php +++ b/fuel/app/classes/materia/widget/asset/storage/s3.php @@ -327,8 +327,7 @@ public function store(Widget_Asset $asset, string $image_path, string $size): vo */ protected function get_key_name(string $id, string $size): string { - $key = (static::$_config['subdir'] ? static::$_config['subdir'].'/' : '').$id; - if ($size !== 'original') $key .= "/{$size}"; + $key = (static::$_config['subdir'] ? static::$_config['subdir'].DS : '')."{$id}_{$size}"; return $key; } @@ -341,14 +340,13 @@ protected function get_s3_client(): \Aws\S3\S3Client if (static::$_s3_client) return static::$_s3_client; $config = [ - 'endpoint' => static::$_config['endpoint'] ?? '', - 'region' => static::$_config['region'], + 'region' => static::$_config['region'], 'force_path_style' => static::$_config['force_path_style'] ?? false, - 'version' => 'latest', - 'credentials' => [ - 'key' => static::$_config['key'], - 'secret' => static::$_config['secret_key'], - 'token' => static::$_config['token'] ?? null, + 'version' => 'latest', + 'credentials' => [ + 'key' => static::$_config['key'], + 'secret' => static::$_config['secret_key'], + 'token' => static::$_config['token'] ?? null, ] ]; From b04058e94bc3a8fc14f089adf4f928445e720fce Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Thu, 1 Aug 2024 13:49:21 -0400 Subject: [PATCH 17/22] Fuel/core updated in composer lockfile --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index ba65a3e29..95a3405d5 100644 --- a/composer.lock +++ b/composer.lock @@ -386,12 +386,12 @@ "source": { "type": "git", "url": "https://github.com/fuel/core.git", - "reference": "44b276d824e3a5f48a269bfec11c5432571083d1" + "reference": "abf0f371710a405d03ee19a9ecf67e2e9233b2fb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/fuel/core/zipball/44b276d824e3a5f48a269bfec11c5432571083d1", - "reference": "44b276d824e3a5f48a269bfec11c5432571083d1", + "url": "https://api.github.com/repos/fuel/core/zipball/abf0f371710a405d03ee19a9ecf67e2e9233b2fb", + "reference": "abf0f371710a405d03ee19a9ecf67e2e9233b2fb", "shasum": "" }, "require": { @@ -419,7 +419,7 @@ "issues": "https://github.com/fuel/core/issues", "source": "https://github.com/fuel/core/tree/1.9/develop" }, - "time": "2024-07-08T13:14:44+00:00" + "time": "2024-07-30T13:27:39+00:00" }, { "name": "fuel/email", From 6161f0879c0f61404d18085bcb9e8058d8959d38 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 2 Aug 2024 11:04:09 -0400 Subject: [PATCH 18/22] docs and comment polish --- README.md | 3 +-- fuel/app/config/materia.php | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 648fa656f..2a1ccb109 100644 --- a/README.md +++ b/README.md @@ -108,5 +108,4 @@ To use an actual S3 bucket for local dev: 1. Set `DEV_ONLY_FAKES3_DISABLED` environment variable in `docker/.env` to `true` 2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name -3. Set `ASSET_STORAGE_S3_ENDPOINT` to your endpoint -4. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these) +3. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these) diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 159b8f489..fcb0277d9 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -111,7 +111,7 @@ ? [ 'driver_class' => '\Materia\Widget_Asset_Storage_S3', 'credential_provider' => $_ENV['ASSET_STORAGE_S3_CREDENTIAL_PROVIDER'] ?? 'env', - 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint + 'endpoint' => $_ENV['ASSET_STORAGE_S3_ENDPOINT'] ?? '', // set to url for testing endpoint (Not required for S3 on AWS) 'region' => $_ENV['ASSET_STORAGE_S3_REGION'] ?? 'us-east-1', // aws region for bucket 'bucket' => $_ENV['ASSET_STORAGE_S3_BUCKET'] ?? '', // bucket to store original user uploads 'subdir' => $_ENV['ASSET_STORAGE_S3_BASEPATH'] ?? 'media', // OPTIONAL - directory to store original and resized assets From 614c3f50bf798d20074a8516e14338bbdb9e444e Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 2 Aug 2024 11:10:01 -0400 Subject: [PATCH 19/22] Comment updates to base env --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index cec4fc71f..a37007467 100755 --- a/.env +++ b/.env @@ -34,14 +34,14 @@ BOOL_SEND_EMAILS=false #URLS_STATIC= #URLS_ENGINES= #BOOL_ADMIN_UPLOADER_ENABLE=true -ASSET_STORAGE_DRIVER=file +ASSET_STORAGE_DRIVER=file # file | s3 | db (db not recommended) # AWS S3 =================== # ASSET_STORAGE_S3_REGION=us-east-1 # ASSET_STORAGE_S3_BASEPATH=media # ASSET_STORAGE_S3_BUCKET= -# ASSET_STORAGE_S3_ENDPOINT= +# ASSET_STORAGE_S3_ENDPOINT= # endpoint not required for S3 on AWS # AWS_ACCESS_KEY_ID= # AWS_SECRET_ACCESS_KEY= # AWS_SESSION_TOKEN= # STS token for s3 development From ac3eca5d325f0daeda88431aed73dba5935b19b0 Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 2 Aug 2024 13:01:27 -0400 Subject: [PATCH 20/22] Essential references to docker-compose updated to v2 --- .github/workflows/test_and_build.yml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index b6a6609ea..a9f3b6853 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -32,7 +32,7 @@ jobs: - name: Build App Image run: | cd docker - docker-compose build --no-cache webserver app + docker compose build --no-cache webserver app - name: Push App Images run: | diff --git a/README.md b/README.md index 2a1ccb109..40a50971e 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,9 @@ cd Materia/docker ./run_first.sh ``` -The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker-compose down` or delete the volume manually. +The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker compose down` or delete the volume manually. -Use `docker-compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default. +Use `docker compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default. In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets. From ab288f6c8f48517ecfbd3a330affc101091306de Mon Sep 17 00:00:00 2001 From: Corey Peterson Date: Fri, 2 Aug 2024 13:11:16 -0400 Subject: [PATCH 21/22] gh actions and scripts using docker-docker v1 now use docker compose (v2) --- .github/workflows/test_and_build.yml | 2 +- docker/run.sh | 2 +- docker/run_create_default_users.sh | 2 +- docker/run_create_me.sh | 4 ++-- docker/run_first.sh | 12 ++++++------ docker/run_tests.sh | 2 +- docker/run_tests_ci.sh | 2 +- docker/run_tests_coverage.sh | 2 +- docker/run_tests_lint.sh | 2 +- docker/run_widgets_install.sh | 2 +- githooks/pre-commit | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index b6a6609ea..a9f3b6853 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -32,7 +32,7 @@ jobs: - name: Build App Image run: | cd docker - docker-compose build --no-cache webserver app + docker compose build --no-cache webserver app - name: Push App Images run: | diff --git a/docker/run.sh b/docker/run.sh index 0483ece12..344c84fea 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -13,4 +13,4 @@ set -e -docker-compose run --rm app /wait-for-it.sh mysql:3306 -t 20 -- "$@" +docker compose run --rm app /wait-for-it.sh mysql:3306 -t 20 -- "$@" diff --git a/docker/run_create_default_users.sh b/docker/run_create_default_users.sh index 5b6d887df..780aa9f5f 100755 --- a/docker/run_create_default_users.sh +++ b/docker/run_create_default_users.sh @@ -10,4 +10,4 @@ ####################################################### # create/update the default users -docker-compose run --rm app bash -c "php oil r admin:create_default_users" +docker compose run --rm app bash -c "php oil r admin:create_default_users" diff --git a/docker/run_create_me.sh b/docker/run_create_me.sh index 3fffc3d05..8b8d9aaec 100755 --- a/docker/run_create_me.sh +++ b/docker/run_create_me.sh @@ -13,7 +13,7 @@ PASS=${MATERIA_DEV_PASS:-kogneato} # create or update the user and pw -docker-compose run --rm app bash -c "php oil r admin:new_user $USER $USER M Lastname $USER@mail.com $PASS || php oil r admin:reset_password $USER $PASS" +docker compose run --rm app bash -c "php oil r admin:new_user $USER $USER M Lastname $USER@mail.com $PASS || php oil r admin:reset_password $USER $PASS" # give them super_user and basic_author -docker-compose run --rm app bash -c "php oil r admin:give_user_role $USER super_user || true && php oil r admin:give_user_role $USER basic_author" +docker compose run --rm app bash -c "php oil r admin:give_user_role $USER super_user || true && php oil r admin:give_user_role $USER basic_author" diff --git a/docker/run_first.sh b/docker/run_first.sh index ed52d00c5..9e2ec6a2a 100755 --- a/docker/run_first.sh +++ b/docker/run_first.sh @@ -5,7 +5,7 @@ # Initializes a new local Dev Materia environment in Docker # # If you find you really need to burn everything down -# Run "docker-compose down" to get rid of all containers +# Run "docker compose down" to get rid of all containers # ####################################################### set -e @@ -24,16 +24,16 @@ rm -rf ./config/nginx/cert.pem openssl req -subj '/CN=localhost' -x509 -newkey rsa:4096 -nodes -keyout ./config/nginx/key.pem -out ./config/nginx/cert.pem -days 365 # quietly pull any docker images we can -docker-compose pull --ignore-pull-failures +docker compose pull --ignore-pull-failures # install php composer deps -docker-compose run --rm --no-deps app composer install --ignore-platform-reqs +docker compose run --rm --no-deps app composer install --ignore-platform-reqs # run migrations and seed any db data needed for a new install -docker-compose run --rm app /wait-for-it.sh mysql:3306 --timeout=120 --strict -- composer oil-install-quiet +docker compose run --rm app /wait-for-it.sh mysql:3306 --timeout=120 --strict -- composer oil-install-quiet # install all the configured widgets -docker-compose run --rm app bash -c 'php oil r widget:install_from_config' +docker compose run --rm app bash -c 'php oil r widget:install_from_config' # Install any widgets in the tmp dir source run_widgets_install.sh '*.wigt' @@ -46,4 +46,4 @@ source run_create_me.sh echo -e "Materia will be hosted on \033[32m$DOCKER_IP\033[0m" echo -e "\033[1mRun an oil comand:\033[0m ./run.sh php oil r widget:show_engines" -echo -e "\033[1mRun the web app:\033[0m docker-compose up" +echo -e "\033[1mRun the web app:\033[0m docker compose up" diff --git a/docker/run_tests.sh b/docker/run_tests.sh index 3e6660317..925928279 100755 --- a/docker/run_tests.sh +++ b/docker/run_tests.sh @@ -10,7 +10,7 @@ echo "remember you can limit your test groups with './run_tests.sh --group=Lti'" # If you have an issue with a broken widget package breaking this script, run the following to clear the widgets # docker-compose -f docker-compose.yml -f docker-compose.admin.yml run --rm app bash -c -e 'rm /var/www/html/fuel/packages/materia/vendor/widget/test/*' -DCTEST="docker-compose -f docker-compose.yml -f docker-compose.override.test.yml" +DCTEST="docker compose -f docker-compose.yml -f docker-compose.override.test.yml" set -e set -o xtrace diff --git a/docker/run_tests_ci.sh b/docker/run_tests_ci.sh index f65fa05d9..6a8a23116 100755 --- a/docker/run_tests_ci.sh +++ b/docker/run_tests_ci.sh @@ -11,7 +11,7 @@ set -e set -o xtrace -DCTEST="docker-compose -f docker-compose.yml -f docker-compose.override.test.yml" +DCTEST="docker compose -f docker-compose.yml -f docker-compose.override.test.yml" $DCTEST pull --ignore-pull-failures app fakes3 diff --git a/docker/run_tests_coverage.sh b/docker/run_tests_coverage.sh index 5b3d9c511..0d85877ec 100755 --- a/docker/run_tests_coverage.sh +++ b/docker/run_tests_coverage.sh @@ -13,7 +13,7 @@ ####################################################### set -e -DCTEST="docker-compose -f docker-compose.yml -f docker-compose.override.test.yml" +DCTEST="docker compose -f docker-compose.yml -f docker-compose.override.test.yml" echo "remember you can limit your test groups with './run_tests_coverage.sh --group=Lti'" echo "If you have an issue with a broken widget, clear the widgets with:" diff --git a/docker/run_tests_lint.sh b/docker/run_tests_lint.sh index 945ec6bb0..19a9f3ee6 100755 --- a/docker/run_tests_lint.sh +++ b/docker/run_tests_lint.sh @@ -5,7 +5,7 @@ # Script to run the linter in docker ####################################################### -DCTEST="docker-compose -f docker-compose.yml -f docker-compose.override.test.yml" +DCTEST="docker compose -f docker-compose.yml -f docker-compose.override.test.yml" set -e set -o xtrace diff --git a/docker/run_widgets_install.sh b/docker/run_widgets_install.sh index 2b45bbbf8..d4669b86d 100755 --- a/docker/run_widgets_install.sh +++ b/docker/run_widgets_install.sh @@ -13,4 +13,4 @@ ####################################################### set -e -docker-compose run --rm app bash -c 'php oil r widget:install fuel/app/tmp/widget_packages/'$1 +docker compose run --rm app bash -c 'php oil r widget:install fuel/app/tmp/widget_packages/'$1 diff --git a/githooks/pre-commit b/githooks/pre-commit index 42580de52..a43832090 100755 --- a/githooks/pre-commit +++ b/githooks/pre-commit @@ -14,7 +14,7 @@ function execute_and_check_status { return $status } -DCTEST="docker-compose -f docker-compose.yml -f docker-compose.override.test.yml" +DCTEST="docker compose -f docker-compose.yml -f docker-compose.override.test.yml" cd docker echo "Running git pre-commit" From 62a6671b7acb42a8b493cd581dcfc6816e11e2d2 Mon Sep 17 00:00:00 2001 From: Christopher Solanilla Date: Fri, 6 Sep 2024 11:33:25 -0400 Subject: [PATCH 22/22] checks the length of the title before saving --- src/components/widget-creator.jsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/components/widget-creator.jsx b/src/components/widget-creator.jsx index e6bb9a669..9183cdcdb 100644 --- a/src/components/widget-creator.jsx +++ b/src/components/widget-creator.jsx @@ -384,6 +384,18 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { } const save = (instanceName, qset, version = 1) => { + //cancel saving of the widget if title is too long to prevent crashing + let titleLength = instanceName.length; + if(titleLength>100) { + setAlertDialog({ + enabled: true, + title: 'Title too long', //the max length for title in my testing is 100 + message: 'Title must be less than 100 characters', + fatal: false, + enableLoginButton: false + }); + return; + } let newWidget = { widget_id: widgetId, name: instanceName, @@ -830,4 +842,4 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { } -export default WidgetCreator \ No newline at end of file +export default WidgetCreator