1598/title length limit

.env

@@ -34,12 +34,17 @@ BOOL_SEND_EMAILS=false
 #URLS_STATIC=
 #URLS_ENGINES=
 #BOOL_ADMIN_UPLOADER_ENABLE=true
-#ASSET_STORAGE_DRIVER=file
-#ASSET_STORAGE_S3_REGION=us-east-1
-#ASSET_STORAGE_S3_BUCKET=
-#ASSET_STORAGE_S3_BASEPATH=
-#ASSET_STORAGE_S3_KEY=
-#ASSET_STORAGE_S3_SECRET=
+ASSET_STORAGE_DRIVER=file # file | s3 | db (db not recommended)
+
+# AWS S3 ===================
+
+# ASSET_STORAGE_S3_REGION=us-east-1
+# ASSET_STORAGE_S3_BASEPATH=media
+# ASSET_STORAGE_S3_BUCKET=<MUST_SET>
+# ASSET_STORAGE_S3_ENDPOINT= # endpoint not required for S3 on AWS
+# AWS_ACCESS_KEY_ID=<MUST_SET>
+# AWS_SECRET_ACCESS_KEY=<MUST_SET>
+# AWS_SESSION_TOKEN=<MUST_SET> # STS token for s3 development
 
 # SESSION & CACHE ===================
 

.github/workflows/test_and_build.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Build App Image
         run: |
           cd docker
-          docker-compose build --no-cache webserver app
+          docker compose build --no-cache webserver app
 
       - name: Push App Images
         run: |

README.md

@@ -48,11 +48,11 @@ cd Materia/docker
 ./run_first.sh
 ```
 
-The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker-compose down` or delete the volume manually.
+The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker compose down` or delete the volume manually.
 
-Use `docker-compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default.
+Use `docker compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default.
 
-In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets. 
+In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets.
 
 Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for both `127.0.0.1:443` and `127.0.0.1:8008`.
 
@@ -89,4 +89,23 @@ Materia supports two forms of authentication:
 
 ## Asset Storage
 
-Materia enables users to upload media assets for their widgets, including images and audio. There are two asset storage drivers available out of the box: `file` and `db`. `file` is the default asset storage driver, which can be explicitly set via the `ASSET_STORAGE_DRIVER` environment variable.
+Users can upload media assets (images and audio) for use in their widgets, facilitated through a media importer that is provided by Materia itself. Asset storage drivers include:
+
+- `file`: Assets are stored on the local filesystem of the application. It is recommended that assets are backed up and synced with an external storage solution (such as S3) to ensure the files persist across application instances.
+- `s3`: Files are uploaded to and requested directly from AWS S3. This is the most straightforward and recommended storage driver option. Be sure to consult the [Materia Docker Readme](docker/README.md) for additional environment variables associated with using S3.
+- `db`: This storage driver stores asset binaries directly in the database. This option allows Materia to run on cloud hosting options with very limited storage volumes. The `db` storage driver option is not recommended for general use.
+
+> [!WARNING]
+> The `db` asset storage driver option is deprecated and will be removed in the next major version of Materia.
+
+The storage driver is configured via the `ASSET_STORAGE_DRIVER` environment variable.
+
+### Local Asset Storage With S3
+
+A `fakes3` container is instantiated as part of the default development stack and the `ASSET_STORAGE_DRIVER` environment variable is set to `s3` by default in the development `.env` file located in `docker/.env`. When using `fakes3`, this is all that is required to simulate S3 usage locally.
+
+To use an actual S3 bucket for local dev:
+
+1. Set `DEV_ONLY_FAKES3_DISABLED` environment variable in `docker/.env` to `true`
+2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name
+3. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these)

composer.json

@@ -50,7 +50,7 @@
         "phpseclib/phpseclib": "~3.0",
 		"phpseclib/phpseclib2_compat":"~1.0",
         "eher/oauth": "1.0.7",
-        "aws/aws-sdk-php": "3.288.1",
+        "aws/aws-sdk-php": "^3.314",
         "symfony/dotenv": "^5.1",
         "ucfopen/materia-theme-ucf": "2.0.3"
     },
@@ -114,5 +114,14 @@
         }
     ],
     "minimum-stability": "dev",
-    "prefer-stable": true
+    "prefer-stable": true,
+    "autoload": {
+        "psr-0": {
+            "": "*"
+        },
+        "psr-4": {
+            "S3\\": "../s3/",
+            "AwsUtilities\\": "../aws_utilities/"
+        }
+    }
 }

docker/.env

@@ -4,9 +4,20 @@ MYSQL_USER=materia
 MYSQL_PASSWORD=odin
 MYSQL_DATABASE=materia
 
-# passwords/hashes/eys
+# passwords/hashes/keys
 DEV_ONLY_USER_PASSWORD=kogneato
 # see readme for how to create these
 DEV_ONLY_AUTH_SALT=111b776e5f862058e2e075b640b3de5fb601d0ac57639c733a2d10edffd2a3d5
 DEV_ONLY_AUTH_SIMPLEAUTH_SALT=33e0d379060e3877d634632853c10a70dff9710b751e5af00a0f637884df417e
 DEV_ONLY_SECRET_CIPHER_KEY=e0beaea1704555ae3c75650703bb106fac24b8967c77a667124fbe745c3346ed
+
+# s3-specific asset storage values
+ASSET_STORAGE_DRIVER=s3 # overrides default value in the base .env (which isn't loaded into dev environment)
+
+ASSET_STORAGE_S3_CREDENTIAL_PROVIDER=env # env | imds
+ASSET_STORAGE_S3_BUCKET=fake_bucket
+ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001
+ASSET_STORAGE_S3_KEY=KEY
+ASSET_STORAGE_S3_SECRET=SECRET
+
+# DEV_ONLY_FAKES3_DISABLED=false # set to true if using real S3 on development

docker/README.md

@@ -122,9 +122,13 @@ If you plan on deploying a production server using these docker images, we sugge
 
 ### Dynamic Files to Backup
 
-* MySQL Database Contents
-* Uploaded Media
-* Installed Widget Engine Files
+* MySQL Database Contents: 
+* Uploaded Media (generally `$APP_DIR/media`)
+* Installed Widget Engine Files (generally `$APP_DIR/widgets`)
+
+### Environment Variables
+
+Refer to the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for environment variable configuration options.
 
 ### Sample Docker Compose
 
@@ -199,4 +203,3 @@ volumes:
 #### Table Not Found
 
 When running fuelphp's install, it uses fuel/app/config/development/migrations.php file to know the current state of your database. Fuel assumes this file is truth, and won't create tables even on an empty database. You probably need to delete the file and run the setup scripts again.  run_first.sh does this for you if needed.
-

docker/config/php/materia.php.ini

@@ -2,7 +2,7 @@
 short_open_tag = Off
 expose_php = off
 max_execution_time = 100
-memory_limit = 250M
+memory_limit = 256M
 track_errors = Off
 html_errors = On
 variables_order = "EGPCS"

docker/docker-compose.override.test.yml

@@ -24,6 +24,10 @@ services:
       - uploaded_media_test:/var/www/html/fuel/packages/materia/media
       - ./config/php/materia.test.php.ini:/usr/local/etc/php/conf.d/test.ini
       - ./dockerfiles/wait-for-it.sh:/wait-for-it.sh
+    depends_on:
+      - fakes3_test
+      - mysql
+      - memcached
 
   mysql:
     environment:
@@ -36,9 +40,23 @@ services:
     # tmpfs:
     #   - /var/lib/mysql
 
-  fakes3:
+  # fakes3, when added as a dependency in the app container above, would restart
+  # and lose its data during tests
+  # thus, fakes3_test was created. it is dropped after tests are complete
+  fakes3_test:
+    image: ucfopen/materia:fake-s3-dev
+    build:
+      context: ../
+      dockerfile: materia-fake-s3.Dockerfile
+    ports:
+    # use separate port to avoid conflicts with fakes3
+      - "10002:10001"
     volumes:
-      - uploaded_media_test:/s3mnt/fakes3_root/fakes3_uploads/media/
+     # use separate volume to avoid conflicts with fakes3
+      - uploaded_media_test:/s3mnt/fakes3_root/fake_bucket/media/
+    networks:
+      - frontend
+      - backend
 
 volumes:
   # static_files: {} # compiled js/css and uploaded widgets

docker/docker-compose.override.yml

@@ -15,21 +15,34 @@ services:
       - ./config/nginx/nginx-dev.conf:/etc/nginx/nginx.conf:ro
 
   app:
+    environment:
+      # values sourced from docker/env
+      - ASSET_STORAGE_DRIVER
+      - ASSET_STORAGE_S3_CREDENTIAL_PROVIDER
+      - ASSET_STORAGE_S3_BUCKET
+      - ASSET_STORAGE_S3_ENDPOINT
+      - ASSET_STORAGE_S3_KEY
+      - ASSET_STORAGE_S3_SECRET
     volumes:
       - ..:/var/www/html/
       - uploaded_widgets:/var/www/html/public/widget/
       - ./dockerfiles/wait-for-it.sh:/wait-for-it.sh
+    depends_on:
+      - fakes3
+      - mysql
+      - memcached
 
   mysql:
     environment:
+      # values sourced from docker/env
       - MYSQL_ROOT_PASSWORD
       - MYSQL_USER
       - MYSQL_PASSWORD
       - MYSQL_DATABASE
 
   fakes3:
     volumes:
-      - uploaded_media:/s3mnt/fakes3_root/fakes3_uploads/media/
+      - uploaded_media:/s3mnt/fakes3_root/fake_bucket/media/
 
 volumes:
   # static_files: {} # compiled js/css and uploaded widgets

docker/docker-compose.yml

@@ -22,11 +22,7 @@ services:
       dockerfile: materia-app.Dockerfile
     environment:
       # View Materia README for env settings
-      - ASSET_STORAGE_DRIVER=file
-      - ASSET_STORAGE_S3_BUCKET=fake_bucket
-      - ASSET_STORAGE_S3_ENDPOINT=http://fakes3:10001
-      - ASSET_STORAGE_S3_KEY=KEY
-      - ASSET_STORAGE_S3_SECRET=SECRET
+      - ASSET_STORAGE_DRIVER=${ASSET_STORAGE_DRIVER}
       - AUTH_DRIVERS=Materiaauth
       - AUTH_SALT=${DEV_ONLY_AUTH_SALT}
       - AUTH_SIMPLEAUTH_SALT=${DEV_ONLY_AUTH_SIMPLEAUTH_SALT}
@@ -54,10 +50,6 @@ services:
     networks:
       - frontend
       - backend
-    depends_on:
-      - mysql
-      - memcached
-      - fakes3
 
   mysql:
     image: mysql:5.7.34

docker/run.sh

@@ -13,4 +13,4 @@
 
 set -e
 
-docker-compose run --rm app /wait-for-it.sh mysql:3306 -t 20 -- "$@"
+docker compose run --rm app /wait-for-it.sh mysql:3306 -t 20 -- "$@"

docker/run_create_default_users.sh

@@ -10,4 +10,4 @@
 #######################################################
 
 # create/update the default users
-docker-compose run --rm app bash -c "php oil r admin:create_default_users"
+docker compose run --rm app bash -c "php oil r admin:create_default_users"

docker/run_create_me.sh

@@ -13,7 +13,7 @@
 PASS=${MATERIA_DEV_PASS:-kogneato}
 
 # create or update the user and pw
-docker-compose run --rm app bash -c "php oil r admin:new_user $USER $USER M Lastname [email protected] $PASS || php oil r admin:reset_password $USER $PASS"
+docker compose run --rm app bash -c "php oil r admin:new_user $USER $USER M Lastname [email protected] $PASS || php oil r admin:reset_password $USER $PASS"
 
 # give them super_user and basic_author
-docker-compose run --rm app bash -c "php oil r admin:give_user_role $USER super_user || true && php oil r admin:give_user_role $USER basic_author"
+docker compose run --rm app bash -c "php oil r admin:give_user_role $USER super_user || true && php oil r admin:give_user_role $USER basic_author"

docker/run_first.sh

@@ -5,7 +5,7 @@
 # Initializes a new local Dev Materia environment in Docker
 #
 # If you find you really need to burn everything down
-# Run "docker-compose down" to get rid of all containers
+# Run "docker compose down" to get rid of all containers
 #
 #######################################################
 set -e
@@ -24,16 +24,16 @@ rm -rf ./config/nginx/cert.pem
 openssl req -subj '/CN=localhost' -x509 -newkey rsa:4096 -nodes -keyout ./config/nginx/key.pem -out ./config/nginx/cert.pem -days 365
 
 # quietly pull any docker images we can
-docker-compose pull --ignore-pull-failures
+docker compose pull --ignore-pull-failures
 
 # install php composer deps
-docker-compose run --rm --no-deps app composer install --ignore-platform-reqs
+docker compose run --rm --no-deps app composer install --ignore-platform-reqs
 
 # run migrations and seed any db data needed for a new install
-docker-compose run --rm app /wait-for-it.sh mysql:3306 --timeout=120 --strict -- composer oil-install-quiet
+docker compose run --rm app /wait-for-it.sh mysql:3306 --timeout=120 --strict -- composer oil-install-quiet
 
 # install all the configured widgets
-docker-compose run --rm app bash -c 'php oil r widget:install_from_config'
+docker compose run --rm app bash -c 'php oil r widget:install_from_config'
 
 # Install any widgets in the tmp dir
 source run_widgets_install.sh '*.wigt'
@@ -46,4 +46,4 @@ source run_create_me.sh
 
 echo -e "Materia will be hosted on \033[32m$DOCKER_IP\033[0m"
 echo -e "\033[1mRun an oil comand:\033[0m ./run.sh php oil r  widget:show_engines"
-echo -e "\033[1mRun the web app:\033[0m docker-compose up"
+echo -e "\033[1mRun the web app:\033[0m docker compose up"