Merge pull request #68 from aws-samples/dev

feat: merge from dev to main branch

CHANGELOG.md

@@ -0,0 +1,19 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## [Unreleased]
+### Added
+- CHANGELOG templated file
+- README templated file
+- NOTICE file
+- LICENSE file
+
+### Changed
+
+
+### Removed
+

LICENSE → LICENSE.txt

@@ -172,4 +172,4 @@
       of any other Contributor, and only if You agree to indemnify,
       defend, and hold each Contributor harmless for any liability
       incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
+      of your accepting any such warranty or additional liability.

NOTICE.txt

@@ -0,0 +1,14 @@
+LLM-Bot
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except
+in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/
+or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+**********************
+THIRD PARTY COMPONENTS
+**********************
+This software includes third party software subject to the following copyrights:
+
+AWS SDK under the Apache License Version 2.0

README.md

@@ -10,8 +10,8 @@
 
 1. Prepare model assets by executing script per model folder
 ```bash
-% cd src/models/<cross/embedding/instruct>/model
-% ./model.sh       
+cd source/model/<cross/embedding/instruct>/model
+./model.sh       
 Make sure Python installed properly. Usage: ./model.sh -t TOKEN [-m MODEL_NAME] [-c COMMIT_HASH] [-s S3_BUCKET_NAME]
   -t TOKEN            Hugging Face token (required)
   -m MODEL_NAME       Model name (default: csdc-atl/buffer-cross-001)
@@ -22,11 +22,12 @@ Make sure Python installed properly. Usage: ./model.sh -t TOKEN [-m MODEL_NAME]
 
 2. Deploy CDK template
 ```bash
+cd source/infrastructure
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
 npm install
 npx cdk deploy --rollback false --parameters S3ModelAssets=<Your S3 Bucket Name>
 ```
-
-You will get output similar like below:
+You can update us-east-1 to any other available region according to your need. You will get output similar like below:
 ```
 Outputs:
 llm-bot-dev.APIEndpointAddress = https://xx.execute-api.us-east-1.amazonaws.com/v1/
@@ -211,7 +212,7 @@ You should see output like this:
 4. Launch dashboard to check and debug the ETL & QA process
 
 ```bash
-cd /src/panel
+cd /source/panel
 pip install -r requirements.txt
 mv .env_sample .env
 # fill .env content accordingly with cdk output

buildspec.yml

@@ -0,0 +1,40 @@
+version: 0.2
+
+phases:
+  install:
+    runtime-versions:
+      nodejs: 12
+      python: 3.8
+    commands:
+      - echo "nothing to do in install"
+  pre_build:
+    commands:
+      - echo "Installing dependencies and executing unit tests - `pwd`"
+      - echo Update npm to latest
+      - npm install -g npm@latest
+      - cd deployment && chmod +x ./run-unit-tests.sh && ./run-unit-tests.sh
+      - echo "Installing dependencies and executing unit tests completed `date`"
+  build:
+    commands:
+      - echo "Starting build `date` in `pwd`"
+      - chmod +x ./build-s3-dist.sh && ./build-s3-dist.sh $DIST_OUTPUT_BUCKET $SOLUTION_NAME $VERSION
+      - echo "Build completed `date`"
+      - echo "Starting open-source-dist `date` in `pwd`"
+      - chmod +x ./build-open-source-dist.sh && ./build-open-source-dist.sh $SOLUTION_NAME
+      - echo "Open Source Dist completed `date`"
+  post_build:
+    commands:
+      - echo "Retrieving next stage buildspec `date` in `pwd`"
+      - aws s3 cp s3://${SOLUTIONS_BUILD_ASSETS_BUCKET:-solutions-build-assets}/changelog-spec.yml ../buildspec.yml
+      - echo "Retrieving next stage buildspec complete"
+      - echo "Post build completed on `date`"
+
+artifacts:
+  files:
+    - deployment/**/*
+    - source/**/*
+    - .gitignore
+    - sonar-project.properties
+    - buildspec.yml
+    - CHANGELOG.md
+    - .cfnnag_*

deployment/build-open-source-dist.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# This assumes all of the OS-level configuration has been completed and git repo has already been cloned
+#
+# This script should be run from the repo's deployment directory
+# cd deployment
+# ./build-open-source-dist.sh solution-name
+#
+# Paramenters:
+#  - solution-name: name of the solution for consistency
+
+# Check to see if input has been provided:
+if [ -z "$1" ]; then
+    echo "Please provide the trademark approved solution name for the open source package."
+    echo "For example: ./build-open-source-dist.sh trademarked-solution-name"
+    exit 1
+fi
+
+# Get reference for all important folders
+source_template_dir="$PWD"
+dist_dir="$source_template_dir/open-source"
+dist_template_dir="$dist_dir/deployment"
+source_dir="$source_template_dir/../source"
+github_dir="$source_template_dir/../.github"
+
+echo "------------------------------------------------------------------------------"
+echo "[Init] Clean old open-source folder"
+echo "------------------------------------------------------------------------------"
+echo "rm -rf $dist_dir"
+rm -rf $dist_dir
+echo "mkdir -p $dist_dir"
+mkdir -p $dist_dir
+echo "mkdir -p $dist_template_dir"
+mkdir -p $dist_template_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Templates"
+echo "------------------------------------------------------------------------------"
+echo "cp $source_template_dir/*.template $dist_template_dir/"
+cp $source_template_dir/*.template $dist_template_dir/
+echo "copy yaml templates and rename"
+cp $source_template_dir/*.yaml $dist_template_dir/
+cd $dist_template_dir
+# Rename all *.yaml to *.template
+for f in *.yaml; do
+    mv -- "$f" "${f%.yaml}.template"
+done
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Build Script"
+echo "------------------------------------------------------------------------------"
+echo "cp $source_template_dir/build-s3-dist.sh $dist_template_dir"
+cp $source_template_dir/build-s3-dist.sh $dist_template_dir
+echo "cp $source_template_dir/run-unit-tests.sh $dist_template_dir"
+cp $source_template_dir/run-unit-tests.sh $dist_template_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] GitHub templates"
+echo "------------------------------------------------------------------------------"
+echo "cp -r $github_dir $dist_dir"
+cp -r $github_dir $dist_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Source Folder"
+echo "------------------------------------------------------------------------------"
+echo "cp -r $source_dir $dist_dir"
+cp -r $source_dir $dist_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Files from the root level of the project"
+echo "------------------------------------------------------------------------------"
+echo "cp $source_template_dir/../LICENSE.txt $dist_dir"
+cp $source_template_dir/../LICENSE.txt $dist_dir
+echo "cp $source_template_dir/../NOTICE.txt $dist_dir"
+cp $source_template_dir/../NOTICE.txt $dist_dir
+echo "cp $source_template_dir/../README.md $dist_dir"
+cp $source_template_dir/../README.md $dist_dir
+echo "cp $source_template_dir/../CODE_OF_CONDUCT.md $dist_dir"
+cp $source_template_dir/../CODE_OF_CONDUCT.md $dist_dir
+echo "cp $source_template_dir/../CONTRIBUTING.md $dist_dir"
+cp $source_template_dir/../CONTRIBUTING.md $dist_dir
+echo "cp $source_template_dir/../CHANGELOG.md $dist_dir"
+cp $source_template_dir/../CHANGELOG.md $dist_dir
+echo "cp $source_template_dir/../.gitignore $dist_dir"
+cp $source_template_dir/../.gitignore $dist_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Clean dist, node_modules and bower_components folders"
+echo "------------------------------------------------------------------------------"
+echo "find $dist_dir -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $dist_dir -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $dist_dir -iname "tests" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $dist_dir -iname "tests" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $dist_dir -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $dist_dir -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $dist_dir -iname "bower_components" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $dist_dir -iname "bower_components" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find ../ -type f -name 'package-lock.json' -delete"
+find $dist_dir -type f -name 'package-lock.json' -delete
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Create GitHub (open-source) zip file"
+echo "------------------------------------------------------------------------------"
+# Create the zip file
+echo "cd $dist_dir"
+cd $dist_dir
+
+echo "zip -q -r9 ../$1.zip ."
+zip -q -r9 ../$1.zip .
+
+# Cleanup any temporary/unnecessary files
+echo "Clean up open-source folder"
+echo "rm -rf * .*"
+rm -rf * .*
+
+# Place final zip file in $dist_dir
+echo "mv ../$1.zip ."
+mv ../$1.zip .
+
+echo "Completed building $1.zip dist"

deployment/build-s3-dist.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+#
+# This assumes all of the OS-level configuration has been completed and git repo has already been cloned
+#
+# This script should be run from the repo's deployment directory
+# cd deployment
+# ./build-s3-dist.sh source-bucket-base-name solution-name version-code
+#
+# Paramenters:
+#  - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda
+#    code from. The template will append '-[region_name]' to this bucket name.
+#    For example: ./build-s3-dist.sh solutions v1.0.0
+#    The template will then expect the source code to be located in the solutions-[region_name] bucket
+#
+#  - solution-name: name of the solution for consistency
+#
+#  - version-code: version of the package
+
+# Check to see if input has been provided:
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside."
+    echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0"
+    exit 1
+fi
+
+# This is set by initialize-repo.sh
+SOLUTION_ID="%%SOLUTION_ID_VAL%%"
+
+# Get reference for all important folders
+template_dir="$PWD"
+template_dist_dir="$template_dir/global-s3-assets"
+build_dist_dir="$template_dir/regional-s3-assets"
+source_dir="$template_dir/../source"
+
+echo "------------------------------------------------------------------------------"
+echo "[Init] Clean old dist, node_modules and bower_components folders"
+echo "------------------------------------------------------------------------------"
+echo "rm -rf $template_dist_dir"
+rm -rf $template_dist_dir
+echo "mkdir -p $template_dist_dir"
+mkdir -p $template_dist_dir
+echo "rm -rf $build_dist_dir"
+rm -rf $build_dist_dir
+echo "mkdir -p $build_dist_dir"
+mkdir -p $build_dist_dir
+
+echo "------------------------------------------------------------------------------"
+echo "[Packing] Templates"
+echo "------------------------------------------------------------------------------"
+echo "cp $template_dir/*.template $template_dist_dir/"
+cp $template_dir/*.template $template_dist_dir/
+echo "copy yaml templates and rename"
+cp $template_dir/*.yaml $template_dist_dir/
+cd $template_dist_dir
+# Rename all *.yaml to *.template
+for f in *.yaml; do 
+    mv -- "$f" "${f%.yaml}.template"
+done
+
+cd ..
+echo "Updating code source bucket in template with $1"
+replace="s/%%SOLUTION_ID%%/$SOLUTION_ID/g"
+echo "sed -i '' -e $replace $template_dist_dir/*.template"
+sed -i '' -e $replace $template_dist_dir/*.template
+replace="s/%%BUCKET_NAME%%/$1/g"
+echo "sed -i '' -e $replace $template_dist_dir/*.template"
+sed -i '' -e $replace $template_dist_dir/*.template
+replace="s/%%BUCKET_NAME%%/$1/g"
+echo "sed -i '' -e $replace $template_dist_dir/*.template"
+sed -i '' -e $replace $template_dist_dir/*.template
+replace="s/%%SOLUTION_NAME%%/$2/g"
+echo "sed -i '' -e $replace $template_dist_dir/*.template"
+sed -i '' -e $replace $template_dist_dir/*.template
+replace="s/%%VERSION%%/$3/g"
+echo "sed -i '' -e $replace $template_dist_dir/*.template"
+sed -i '' -e $replace $template_dist_dir/*.template
+
+echo "------------------------------------------------------------------------------"
+echo "[Rebuild] Example Function"
+echo "------------------------------------------------------------------------------"
+cd $source_dir/example-function-js
+npm run build
+cp ./dist/example-function-js.zip $build_dist_dir/example-function-js.zip

deployment/run-unit-tests.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# This assumes all of the OS-level configuration has been completed and git repo has already been cloned
+#
+# This script should be run from the repo's deployment directory
+# cd deployment
+# ./run-unit-tests.sh
+#
+
+# Get reference for all important folders
+template_dir="$PWD"
+source_dir="$template_dir/../source"
+
+echo "------------------------------------------------------------------------------"
+echo "[Init] Clean old dist and node_modules folders"
+echo "------------------------------------------------------------------------------"
+echo "find $source_dir/services -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/services -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $source_dir/services -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/services -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find ../ -type f -name 'package-lock.json' -delete"
+find $source_dir/services -type f -name 'package-lock.json' -delete
+echo "find $source_dir/resources -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/resources -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $source_dir/resources -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/resources -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find ../ -type f -name 'package-lock.json' -delete"
+find $source_dir/resources -type f -name 'package-lock.json' -delete
+echo "find $source_dir/simulator -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/simulator -iname "node_modules" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find $source_dir/simulator -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null"
+find $source_dir/simulator -iname "dist" -type d -exec rm -r "{}" \; 2> /dev/null
+echo "find ../ -type f -name 'package-lock.json' -delete"
+find $source_dir/simulator -type f -name 'package-lock.json' -delete
+
+echo "------------------------------------------------------------------------------"
+echo "[Test] Services - Example Function"
+echo "------------------------------------------------------------------------------"
+cd $source_dir/example-function-js
+npm install
+npm test