Merge pull request #698 from bcgov/release/1.32.0

Merged PR-698

.gitattributes

@@ -21,6 +21,8 @@
 *.pyd		binary
 *.pyo 		binary
 *.ttf		binary
+*.gz		binary
+
 
 # Static files
 # ============

.gitignore

@@ -77,6 +77,7 @@ var/
 .installed.cfg
 *.egg
 webpack-stats.json
+api-tests/.envrc
 
 # PyInstaller and Pip install files
 #  Usually these files are written by a python script from a template
@@ -154,3 +155,7 @@ sonar-runner/.classpath
 
 # DB
 *.dmp
+*.pgCustom
+
+# Exceptions
+!app/*/fixtures/*.gz

CONTRIBUTING.md

@@ -1,9 +1,83 @@
 ## How to contribute
 
-Government employees, public and members of the private sector are encouraged to contribute to the repository by **forking and submitting a pull request**.
+Government employees, the public and members of the private sector are encouraged to contribute.
 
-(If you are new to GitHub, you might start with a [basic tutorial](https://help.github.com/articles/set-up-git) and check out a more detailed [guide to pull requests](https://help.github.com/articles/using-pull-requests/).)
+All contributors retain original copyright, but are granting a world-wide, royalty-free, perpetual, irrevocable, non-exclusive, transferable license to all users.  This project is covered by an [Apache v2.0 license](https://github.com/bcgov/gwells/blob/master/LICENSE).
 
-Pull requests will be evaluated by the repository guardians on a schedule and if deemed beneficial will be committed to the master.
+### Prerequisites
 
-All contributors retain the original copyright to their stuff, but by contributing to this project, you grant a world-wide, royalty-free, perpetual, irrevocable, non-exclusive, transferable license to all users **under the terms of the [license](https://github.com/bcgov/gwells/blob/master/LICENSE) under which this project is distributed**.
+* [GitHub account](https://github.com/join)
+* [Your own fork](https://github.com/bcgov/gwells/fork)
+
+### Assumptions
+
+For convenience this guide will use the following:
+
+* GitHub username = stinkypete
+* Current release = 1.32.0
+* Feature branch = myFeature
+
+### Clone, Branch and Push
+
+Clone and navigate into your repository.
+
+```
+git clone https://github.com/stinkypete/gwells
+cd gwells
+```
+
+Create a feature branch with a meaningful name and push it to your repo.  Pick a name that describes the feature.
+
+```
+git checkout -b myFeature
+git push --set-upstream origin myFeature
+```
+
+Add and push files as required.
+
+```
+git add CONTRIBUTING.md
+git commit -m "This line was copied and pasted from CONTRIBUTING.md!"
+git push
+```
+
+### Upstream Updates
+
+Set the upstream repository.
+
+```
+git remote add upstream https://github.com/bcgov/gwells
+```
+
+Pull from the current release branch at least daily.
+
+```
+git pull upstream release/1.32.0 --rebase
+git push origin -f
+```
+
+* release branch = ```release/<major release>.<sprint number>.<hotfix number>```
+
+### Pull Requests
+
+When a feature is ready for testing please send a pull request to our current release branch.
+
+### Continuous Integration and Continuous Deployment
+
+We use a Jenkins pipleline to test, build and integrate pull requests into release branches.  Those branches are merged into master and deployed into production approximately every two weeks, coinciding with the end of an agile sprint.
+
+GWells members are responsible for integrating their own code in through Jenkins.  Outside collaborators, please submit pull requests and watch the build and tests statuses.  All pull requests require a review.
+
+### Cleanup
+
+View existing branches.
+
+```
+git branch -avv
+```
+
+Delete local and remote branches once they have been merged.
+```
+git branch -d myFeature
+git push -d origin myFeature
+```

Jenkinsfile

@@ -25,19 +25,36 @@ private static String stackTraceAsString(Throwable t) {
     return sw.toString()
 }
 
-def _stage(String name, Map context, Closure body) {
+String stageStatusContext(String stageName){
+    return "stages/${stageName.toLowerCase()}"
+}
+
+void setStageStatus(Map context, String name, String status) {
+     GitHubHelper.createCommitStatus(this, context.pullRequest.head, status, "${env.BUILD_URL}", "Stage '${name}'", stageStatusContext(name))
+}
+
+void notifyStageStatus(Map context, String name, String status) {
+    setStageStatus(context, name, status)
+}
+
+def _stage(String name, Map context, boolean retry=0, boolean withCommitStatus=true, Closure body) {
     def stageOpt =(context?.stages?:[:])[name]
     boolean isEnabled=(stageOpt == null || stageOpt == true)
     //echo "Stage - ${stage}"
     echo "Running Stage '${name}' - enabled:${isEnabled}"
+
+
     if (isEnabled){
         stage(name) {
             waitUntil {
+                notifyStageStatus(context, name, 'PENDING')
                 boolean isDone=false
                 try{
                     body()
                     isDone=true
+                    notifyStageStatus(context, name, 'SUCCESS')
                 }catch (ex){
+                    notifyStageStatus(context, name, 'FAILURE')
                     echo "${stackTraceAsString(ex)}"
                     def inputAction = input(
                         message: "This step (${name}) has failed. See error above.",
@@ -60,10 +77,10 @@ def _stage(String name, Map context, Closure body) {
 
 Map context = [
   'name': 'gwells',
-  'uuid' : UUID.randomUUID().toString(),
+  'uuid' : "${env.JOB_BASE_NAME}-${env.BUILD_NUMBER}-${env.CHANGE_ID}",
   'env': [
       'dev':[:],
-      'test':[:],
+      'test':['params':['host':'gwells-test.pathfinder.gov.bc.ca']],
       'prod':['params':['host':'gwells-prod.pathfinder.gov.bc.ca', 'DB_PVC_SIZE':'5Gi']]
   ],
   'templates': [
@@ -91,14 +108,23 @@ Map context = [
     'Code Quality': false,
     'Readiness - DEV': true,
     'Deploy - DEV': true,
+    'Load Fixtures - DEV': true,
+    'API Test': true,
     'Full Test - DEV': true
+  ],
+  pullRequest:[
+    'id': env.CHANGE_ID,
+    'head': GitHubHelper.getPullRequestLastCommitId(this)
   ]
 ]
 
+def isCI = !"master".equalsIgnoreCase(env.CHANGE_TARGET)
+def isCD = "master".equalsIgnoreCase(env.CHANGE_TARGET)
+
 
 properties([
-        buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')),
-        durabilityHint('MAX_SURVIVABILITY') /*, parameters([string(defaultValue: '', description: '', name: 'run_stages')]) */
+        buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '5')),
+        durabilityHint('PERFORMANCE_OPTIMIZED') /*, parameters([string(defaultValue: '', description: '', name: 'run_stages')]) */
 ])
 
 stage('Prepare') {
@@ -116,17 +142,18 @@ _stage('Build', context) {
         if ("master".equalsIgnoreCase(env.CHANGE_TARGET)) {
             new OpenShiftHelper().prepareForCD(this, context)
         }
+        deleteDir()
     }
 } //end stage
 
 _stage('Unit Test', context) {
     podTemplate(label: "node-${context.uuid}", name:"node-${context.uuid}", serviceAccount: 'jenkins', cloud: 'openshift', containers: [
-        containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceRequestCpu: '100m',resourceLimitCpu: '1000m'),
+        containerTemplate(name: 'jnlp', image: 'jenkins/jnlp-slave:3.10-1-alpine', args: '${computer.jnlpmac} ${computer.name}', resourceRequestCpu: '100m',resourceLimitCpu: '100m'),
         containerTemplate(name: 'app', image: "docker-registry.default.svc:5000/moe-gwells-tools/gwells${context.buildNameSuffix}:${context.buildEnvName}", ttyEnabled: true, command: 'cat',
-            resourceRequestCpu: '1000m',
-            resourceLimitCpu: '4000m',
-            resourceRequestMemory: '1Gi',
-            resourceLimitMemory: '4Gi')
+            resourceRequestCpu: '2000m',
+            resourceLimitCpu: '2000m',
+            resourceRequestMemory: '2Gi',
+            resourceLimitMemory: '2Gi')
       ]
     ) {
         node("node-${context.uuid}") {
@@ -218,10 +245,6 @@ _stage('Code Quality', context) {
 } //end stage
 
 
-
-def isCI = !"master".equalsIgnoreCase(env.CHANGE_TARGET)
-def isCD = "master".equalsIgnoreCase(env.CHANGE_TARGET)
-
 for(String envKeyName: context.env.keySet() as String[]){
     String stageDeployName=envKeyName.toUpperCase()
 
@@ -269,7 +292,12 @@ for(String envKeyName: context.env.keySet() as String[]){
                     podName=openshift.selector('pod', ['deploymentconfig':deploymentConfigName]).objects()[0].metadata.name
                 }
                 sh "oc exec '${podName}' -n '${projectName}' -- bash -c 'cd /opt/app-root/src && pwd && python manage.py flush --no-input'"
-                sh "oc exec '${podName}' -n '${projectName}' -- bash -c 'cd /opt/app-root/src && pwd && python manage.py loaddata wells registries'"
+                // Lookup tables common to all system components (e.g. Django apps)
+                sh "oc exec '${podName}' -n '${projectName}' -- bash -c 'cd /opt/app-root/src && pwd && python manage.py loaddata gwells/fixtures/codetables.ProvinceStateCode.json'"
+                // Lookup tables for the Wellsearch component (not yet a Django app) and Registries app
+                sh "oc exec '${podName}' -n '${projectName}' -- bash -c 'cd /opt/app-root/src && pwd && python manage.py loaddata gwells/fixtures/codetables.json registries/fixtures/codetables.json'"
+                // Test data for the Wellsearch component (not yet a Django app) and Registries app
+                sh "oc exec '${podName}' -n '${projectName}' -- bash -c 'cd /opt/app-root/src && pwd && python manage.py loaddata gwells/fixtures/wellsearch.json.gz registries/fixtures/registries.json'"
             }
         }
     }
@@ -281,10 +309,10 @@ for(String envKeyName: context.env.keySet() as String[]){
               containerTemplate(
                 name: 'jnlp',
                 image: 'registry.access.redhat.com/openshift3/jenkins-slave-nodejs-rhel7',
-                resourceRequestCpu: '500m',
-                resourceLimitCpu: '1000m',
+                resourceRequestCpu: '800m',
+                resourceLimitCpu: '800m',
                 resourceRequestMemory: '1Gi',
-                resourceLimitMemory: '4Gi',
+                resourceLimitMemory: '1Gi',
                 workingDir: '/tmp',
                 command: '',
                 args: '${computer.jnlpmac} ${computer.name}',
@@ -349,10 +377,10 @@ for(String envKeyName: context.env.keySet() as String[]){
                   containerTemplate(
                      name: 'jnlp',
                      image: 'docker-registry.default.svc:5000/openshift/jenkins-slave-bddstack',
-                     resourceRequestCpu: '500m',
-                     resourceLimitCpu: '2000m',
-                     resourceRequestMemory: '2Gi',
-                     resourceLimitMemory: '4Gi',
+                     resourceRequestCpu: '800m',
+                     resourceLimitCpu: '800m',
+                     resourceRequestMemory: '3Gi',
+                     resourceLimitMemory: '3Gi',
                      workingDir: '/home/jenkins',
                      command: '',
                      args: '${computer.jnlpmac} ${computer.name}',
@@ -423,18 +451,40 @@ for(String envKeyName: context.env.keySet() as String[]){
     } //end if
 } // end for
 
+stage('Cleanup') {
 
-_stage('Cleanup', context) {
     def inputResponse = null
-    try{
-        inputResponse=input(id: 'close_pr', message: "Ready to Accept/Merge, and Close pull-request #${env.CHANGE_ID}?", ok: 'Yes', submitter: 'authenticated', submitterParameter: 'approver')
-    }catch(ex){
-        error "Pipeline has been aborted. - ${ex}"
+    String mergeMethod=isCI?'squash':'merge'
+
+    waitUntil {
+        boolean isDone=false
+        try{
+            inputResponse=input(id: 'close_pr', message: "Ready to Accept/Merge (using '${mergeMethod}' method), and Close pull-request #${env.CHANGE_ID}?", ok: 'Yes', submitter: 'authenticated', submitterParameter: 'approver')
+            echo "inputResponse:${inputResponse}"
+
+            echo "Merging and Closing PR"
+            GitHubHelper.mergeAndClosePullRequest(this, mergeMethod)
+
+            echo "Clearing OpenShift resources"
+            new OpenShiftHelper().cleanup(this, context)
+
+            //echo "Clearing OpenShift resources"
+            //setStageStatus(context, 'Cleanup', 'SUCCESS')
+            isDone=true
+        }catch (ex){
+            echo "${stackTraceAsString(ex)}"
+            def inputAction = input(
+                message: "This 'Cleanup' stage has failed. See error above.",
+                ok: 'Confirm',
+                submitter: 'authenticated',
+                parameters: [choice(name: 'action', choices: 'Re-run\nIgnore', description: 'What would you like to do?')]
+            )
+            if ('Ignore'.equalsIgnoreCase(inputAction)){
+                isDone=true
+            }
+        }
+        return isDone
     }
-
-    echo "inputResponse:${inputResponse}"
-    new OpenShiftHelper().cleanup(this, context)
-    GitHubHelper.mergeAndClosePullRequest(this)
 }
 
 

api-tests/local_newman.sh

@@ -5,13 +5,34 @@
 #     export GWELLS_API_BASE_URL="http://localhost:8000/gwells/registries"
 #     export GWELLS_API_TEST_USER="testuser"
 #     export GWELLS_API_TEST_PASSWORD="secret"
+#     export GWELLS_API_TEST_AUTH_SERVER=""
+#     export GWELLS_API_TEST_CLIENT_ID=""
+#     export GWELLS_API_TEST_CLIENT_SECRET"""
 # - Create Django test user (requires env variables from previous step):
 #     python manage.py createtestuser
 # - Install newman:
 #     npm install -g newman
 # - Run script:
 #     ./local_newman.sh
 
+
+ENV_VARS=(
+    "GWELLS_API_TEST_USER"
+    "GWELLS_API_TEST_PASSWORD"
+    "GWELLS_API_BASE_URL"
+    "GWELLS_API_TEST_AUTH_SERVER"
+    "GWELLS_API_TEST_CLIENT_ID"
+    "GWELLS_API_TEST_CLIENT_SECRET"
+)
+
+for env_var in ${ENV_VARS[@]}
+do
+    if [ -z ${!env_var+x} ]; then 
+        echo "$env_var is unset"
+        exit
+    fi
+done
+
 echo "Remember to install newman (npm install -g newman) and set GWELLS_API_TEST_USER,"
 echo "GWELLS_API_TEST_PASSWORD, GWELLS_API_BASE_URL and Keycloak credentials"
 newman run ./registries_api_tests.json --global-var test_user=$GWELLS_API_TEST_USER --global-var test_password=$GWELLS_API_TEST_PASSWORD --global-var base_url=$GWELLS_API_BASE_URL --global-var auth_server=$GWELLS_API_TEST_AUTH_SERVER --global-var client_id=$GWELLS_API_TEST_CLIENT_ID --global-var client_secret=$GWELLS_API_TEST_CLIENT_SECRET