From e68f73570bd3d5cfdc30aac67813c6d2f9f604b5 Mon Sep 17 00:00:00 2001 From: Pete F Date: Wed, 11 Sep 2024 12:05:26 +0100 Subject: [PATCH 1/4] Add start script and local db config --- newswires/app/AppComponents.scala | 7 +- newswires/app/conf/Database.scala | 21 ++++ scripts/start | 159 ++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100755 scripts/start diff --git a/newswires/app/AppComponents.scala b/newswires/app/AppComponents.scala index 765479fc..c2ae54ce 100644 --- a/newswires/app/AppComponents.scala +++ b/newswires/app/AppComponents.scala @@ -46,8 +46,11 @@ class AppComponents(context: Context) .build() if (context.environment.mode == Mode.Dev) { - // TODO run against a local DB? - Database.configureRemoteDevDb(ssmClient) + if (sys.env.contains("USE_CODE_DB")) { + Database.configureRemoteDevDb(ssmClient) + } else { + Database.configureLocalDevDB() + } } else if (context.environment.mode == Mode.Prod) { Database.configureDeployedDb(configuration) } diff --git a/newswires/app/conf/Database.scala b/newswires/app/conf/Database.scala index 39354b90..ea0fef47 100644 --- a/newswires/app/conf/Database.scala +++ b/newswires/app/conf/Database.scala @@ -110,4 +110,25 @@ object Database extends Logging { ConnectionPool.singleton(new DataSourceConnectionPool(ds)) } + + def configureLocalDevDB(): Unit = { + logger.info("building DB config for connecting to CODE DB from local") + + val username = "postgres" + val port = "5432" + val address = "localhost" + val databaseName = "newswires" + + val ds = new AwsWrapperDataSource() + ds.setJdbcProtocol("jdbc:postgresql:") + ds.setServerName(address) + ds.setDatabase(databaseName) + ds.setServerPort(port) + ds.setTargetDataSourceClassName("org.postgresql.ds.PGSimpleDataSource") + ds.setPassword("postgres") + ds.setUser(username) + + ConnectionPool.singleton(new DataSourceConnectionPool(ds)) + } + } diff --git a/scripts/start b/scripts/start new file mode 100755 index 00000000..b379525a --- /dev/null +++ b/scripts/start @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +set -e + +green='\x1B[0;32m' +red='\x1B[0;31m' +plain='\x1B[0m' # No Color + +AWS_PROFILE='editorial-feeds' +POSTGRES_PORT='5432' +APP_NAME='newswires' +EXISTING_TUNNELS=$(ps -ef | grep ssh | grep ${POSTGRES_PORT} | grep -v grep || true) + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT_DIR=${DIR}/.. + + +# copied over from the Grid. Not 100% sure of the rationale, but perhaps: https://stackoverflow.com/questions/56520354/getting-an-amazonkinesisexception-status-code-502-when-using-localstack-from-ja +export AWS_CBOR_DISABLE=true + +LOCAL_AUTH=false +for arg in "$@"; do + if [ "$arg" == "--debug" ]; then + IS_DEBUG=true + shift + fi + if [ "$arg" == "--use-CODE" ]; then + USE_CODE=true + shift + fi +done + +isInstalled() { + hash "$1" 2>/dev/null +} + +hasCredentials() { + if [[ $LOCAL_AUTH == true ]]; then + return + fi + + STATUS=$(aws sts get-caller-identity --profile ${AWS_PROFILE} 2>&1 || true) + if [[ ${STATUS} =~ (ExpiredToken) ]]; then + echo -e "${red}Credentials for the ${AWS_PROFILE} profile are expired. Please fetch new credentials and run this again.${plain}" + exit 1 + elif [[ ${STATUS} =~ ("could not be found") ]]; then + echo -e "${red}Credentials for the ${AWS_PROFILE} profile are missing. Please ensure you have the right credentials.${plain}" + exit 1 + fi +} + +checkRequirement() { + if ! isInstalled "$1"; then + echo -e "${red}[MISSING DEPENDENCY] $1 not found. Please install $1${plain}" + exit 1 + fi +} + +checkRequirements() { + # server side + checkRequirement java + checkRequirement sbt + + # client side + checkRequirement npm + + # used for postgres and AWS localstack + checkRequirement docker + + # other + checkRequirement nginx + checkRequirement aws +} + +tunnelToAwsDb() { + if (docker stats --no-stream &> /dev/null); then + docker compose down + fi + if [[ -n $EXISTING_TUNNELS ]]; then + echo "RE-USING EXISTING TUNNEL TO CODE POSTGRES (on port ${POSTGRES_PORT})" + else + # todo: Grid script uses the following options, do we need any of these? + # TUNNEL_OPTS="-o ExitOnForwardFailure=yes -o ServerAliveInterval=10 -o ServerAliveCountMax=2" + ssm ssh -t ${APP_NAME},CODE -p ${AWS_PROFILE} -x --newest --rds-tunnel 5432:${APP_NAME},CODE + echo "TUNNEL ESTABLISHED TO CODE POSTGRES (on port ${POSTGRES_PORT})" + fi +} + +startDockerContainers() { + if [[ $EXISTING_TUNNELS ]]; then + echo "KILLING EXISTING TUNNEL TO CODE POSTGRES (on port ${POSTGRES_PORT})" + # shellcheck disable=SC2046 + kill $(echo "${EXISTING_TUNNELS}" | awk '{print $2}') + fi + docker compose up -d +} + +startIngestionLambda() { + pushd "$ROOT_DIR/ingestion-lambda" + npm install + npm run dev + popd +} + +startPlayApp() { + # pushd to find build.sbt and allow this script to be executed from any location (but ideally from the project root) + echo "=========================================================" + echo "= Press cmd-C then cmd-D to stop" + echo "=========================================================" + pushd "$ROOT_DIR/${APP_NAME}" + if [ "$IS_DEBUG" == true ] ; then + SBT_OPTS="-jvm-debug 5005" + fi + if [[ $USE_CODE == true ]]; then + sbt "${SBT_OPTS} run" + else + sbt "${SBT_OPTS} run" + fi + popd +} + +checkNodeVersion() { + runningNodeVersion=$(node -v) + requiredNodeVersion=$(cat "$ROOT_DIR/.nvmrc") + + if [ "$runningNodeVersion" != "$requiredNodeVersion" ]; then + echo -e "${red}Using wrong version of Node. Required ${requiredNodeVersion}. Running ${runningNodeVersion}.${plain}" + exit 1 + fi +} + +checkForJavaHome() { + echo "Checking JAVA_HOME" + if [[ -z "$JAVA_HOME" ]]; then + echo " JAVA_HOME not set, please set it before continuing" + echo " This can be done by adding \"export JAVA_HOME=\$(/usr/libexec/java_home)\" to ~/.profile" + exit 1 + else + echo " JAVA_HOME is set to $JAVA_HOME" + fi +} + +main() { + checkForJavaHome + hasCredentials + checkRequirements + checkNodeVersion + if [ $USE_CODE == true ]; then + echo "Using CODE. Starting play app with tunnel to AWS DB." + tunnelToAwsDb && USE_CODE_DB=true startPlayApp + else + echo "Using the local stack. Starting Docker, Ingestion Lambda and Play App." + # run all three, and terminate together when the user presses `ctrl+c`: https://stackoverflow.com/a/52033580 + (trap 'kill 0' SIGINT; startDockerContainers && startIngestionLambda & startPlayApp & wait) + fi + +} + +main \ No newline at end of file From 0b9fe82de8fd2eb15bf1ce484fae6c1ebdde530f Mon Sep 17 00:00:00 2001 From: Pete F Date: Thu, 12 Sep 2024 09:05:58 +0100 Subject: [PATCH 2/4] Quotes --- scripts/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start b/scripts/start index b379525a..4acb35da 100755 --- a/scripts/start +++ b/scripts/start @@ -145,7 +145,7 @@ main() { hasCredentials checkRequirements checkNodeVersion - if [ $USE_CODE == true ]; then + if [ "$USE_CODE" == "true" ]; then echo "Using CODE. Starting play app with tunnel to AWS DB." tunnelToAwsDb && USE_CODE_DB=true startPlayApp else From 62d56d9c97e1a0ce6340c70b7c5dea96117ce64a Mon Sep 17 00:00:00 2001 From: Pete F Date: Thu, 12 Sep 2024 17:14:41 +0100 Subject: [PATCH 3/4] Remove unneeded code & comments --- scripts/start | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/start b/scripts/start index 4acb35da..66d7b138 100755 --- a/scripts/start +++ b/scripts/start @@ -18,7 +18,6 @@ ROOT_DIR=${DIR}/.. # copied over from the Grid. Not 100% sure of the rationale, but perhaps: https://stackoverflow.com/questions/56520354/getting-an-amazonkinesisexception-status-code-502-when-using-localstack-from-ja export AWS_CBOR_DISABLE=true -LOCAL_AUTH=false for arg in "$@"; do if [ "$arg" == "--debug" ]; then IS_DEBUG=true @@ -35,10 +34,6 @@ isInstalled() { } hasCredentials() { - if [[ $LOCAL_AUTH == true ]]; then - return - fi - STATUS=$(aws sts get-caller-identity --profile ${AWS_PROFILE} 2>&1 || true) if [[ ${STATUS} =~ (ExpiredToken) ]]; then echo -e "${red}Credentials for the ${AWS_PROFILE} profile are expired. Please fetch new credentials and run this again.${plain}" @@ -79,8 +74,6 @@ tunnelToAwsDb() { if [[ -n $EXISTING_TUNNELS ]]; then echo "RE-USING EXISTING TUNNEL TO CODE POSTGRES (on port ${POSTGRES_PORT})" else - # todo: Grid script uses the following options, do we need any of these? - # TUNNEL_OPTS="-o ExitOnForwardFailure=yes -o ServerAliveInterval=10 -o ServerAliveCountMax=2" ssm ssh -t ${APP_NAME},CODE -p ${AWS_PROFILE} -x --newest --rds-tunnel 5432:${APP_NAME},CODE echo "TUNNEL ESTABLISHED TO CODE POSTGRES (on port ${POSTGRES_PORT})" fi From 90e3796f6458fc7530af96a9072f2413342e82ef Mon Sep 17 00:00:00 2001 From: Pete F Date: Thu, 12 Sep 2024 17:25:43 +0100 Subject: [PATCH 4/4] Fix comment --- newswires/app/conf/Database.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newswires/app/conf/Database.scala b/newswires/app/conf/Database.scala index ea0fef47..e2bc0abd 100644 --- a/newswires/app/conf/Database.scala +++ b/newswires/app/conf/Database.scala @@ -112,7 +112,7 @@ object Database extends Logging { } def configureLocalDevDB(): Unit = { - logger.info("building DB config for connecting to CODE DB from local") + logger.info("building DB config for connecting to local DB") val username = "postgres" val port = "5432"