Merge pull request #2 from bashbang/oc3-sanitize

added sanitize feature and enhanced export function

openshift/migration/oc3-export/.env.template

@@ -6,3 +6,5 @@ OC4_LICENSEPLATE=<place licenseplate here.  eg: abc123 {not the -dev or -test, e
 OC4_ENVIRONMENT=<place list of environments here  eg: dev,tools>
 PROJECT=<place project name here - arbitrary entry>
 DEBUG=0
+CLOBBER=<do you want to overwrite previously exported manifest files 0:1> eg: 1
+SANITIZE<do you want to generate sanitized versions of each manifest 0:1> eg: 1

openshift/migration/oc3-export/README.md

@@ -19,6 +19,20 @@ ENVIRONMENT: dev,test,prod,tools
 PROJECT: MyProject
 - This is an arbitrary project name used for helping keep the folder structure organized.
 
+DEBUG: 0
+- Turn on debugging output.
+- Boolean, default 0
+
+CLOBBER: 0
+ - Toggle the overwrite (aka clobbering) of existing files.
+ - Boolean, default 0
+ - Will overwrite omnimanifest regardless of setting.
+
+SANITIZE: 0
+ - create a sanitized version of the object manifest files.
+ - strips out unnecessary elements to allow the manifest file to be used for future importing.
+ - Boolean, default 0
+
 # Run
 
 To run the project simply run the docker-compose
@@ -27,4 +41,7 @@ To run the project simply run the docker-compose
 For convince the container just mounts the local folder rather than copying the files into the container. This has pros and cons.  Be aware that this will mean the manifest files will be exported to the local folder which may not be what you desire.  Be careful not to commit the manifest files to git.  Especially since the default setup exports the openshift secrets in both the secrets file as well as the omnimanifest.
 # Output
 
-The output will display discrepancies between the omnimanifest and the category files.
+The output will display discrepancies between the omnimanifest and the category files.
+
+# Known Issues
+- on a Windows system we discovered that the due to mounting the OS you have to explicitly share the folder within your docker server settings.

openshift/migration/oc3-export/_unused_oc3-sanitize.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+####
+#   NOT CURRENTLY USED
+#   - this method changes the yaml content formating which is less readable and not optimal
+#     so I've obsoleted this script in favour of using sed in bash.
+#     keeing this around for reference for now.
+#   Used to compare the omnimanifest, that was exported from openshift 3, to the catagory manifests
+#   to determine if something may have gone wrong.  This is not a definitive test as we're only
+#   comparing one way.
+#
+#   ** CAUTION: This exports the oc secrets - DO NOT COMMIT SECRETS TO GIT!
+#
+####
+
+import yaml
+from nested_lookup import get_occurrence_of_key, nested_delete, nested_lookup
+from pprint import pprint
+import os
+import sys
+
+
+directory = sys.argv[1]
+location=directory+"/sanitized/"
+if not os.path.exists(location):
+    os.mkdir(location)
+
+groupedManifestList = []
+for filename in os.listdir(directory):
+    if filename!="all_objects.yaml" and filename!="sanitized":
+        workingfile = open(os.path.join(directory, filename))
+        with workingfile as file:
+            dataDict = yaml.safe_load(file)
+
+        print("Sanitizing "+filename)
+
+        dataDictionatry = {key: value for key, value in dataDict.items() if value is not None}
+
+        #print(dataDictionatry)
+
+        # results = nested_delete(dataDict, "resourceVersion", in_place=True)
+        # results = nested_delete(dataDict, "selfLink", in_place=True)
+        # results = nested_delete(dataDict, "creationTimestamp", in_place=True)
+        # results = nested_delete(dataDict, "uid", in_place=True)
+        # results = nested_delete(dataDict, "namespace", in_place=True)
+
+        # results = nested_delete(dataDict, "resourceVersion", in_place=True)
+        # results = nested_delete(dataDict, "selfLink", in_place=True)
+        # results = nested_delete(dataDict, "namespace", in_place=True)
+
+        # results = nested_delete(dataDict, "status", in_place=True)
+        # results = nested_delete(dataDict, "items", in_place=True)
+
+        #results = nested_delete(dataDict, "annotations", in_place=True)
+
+
+        #result = nested_lookup('resourceVersion', dataDict)
+
+        #print(dataDict)
+        # print(type(result))
+
+
+        print("Writing to file: "+filename)
+        with open(location+"/"+filename, 'w') as file:
+            documents = yaml.dump(results, file)
+
+exit()

openshift/migration/oc3-export/oc3-compare.py

@@ -3,8 +3,8 @@
 ####
 #
 #   Used to compare the omnimanifest, that was exported from openshift 3, to the catagory manifests
-#   to determine if something may have gone wrong.  This is not a definitive test as we're only
-#   comparing one way.
+#   to determine if something may have gone wrong.
+#   This is not a definitive test as we're only comparing one way.
 #
 #   ** CAUTION: This exports the oc secrets - DO NOT COMMIT SECRETS TO GIT!
 #
@@ -49,7 +49,7 @@
 
 
 # for each item in the omniManifestList we need to see if we can find that item in the groupedManifestList
-pprint("Searching through omnimanifest on project "+directory)
+pprint("Comparing omnimanifest to discreet objects in: "+directory)
 for itemList in omniManifestList:
 
     #pprint(itemList)

openshift/migration/oc3-export/oc3-export.sh

@@ -11,52 +11,59 @@ set -e
 #   then validates by downloading the entire namespace
 #   and compares the outputes to ensure they're are the same
 #
-#   ** WARNING: This blindly overwrites the destination files on each run.
 #   ** CAUTION: This exports the oc secrets - DO NOT COMMIT SECRETS TO GIT!
 #
 ####
 
 PROGNAME=$0
 
-usage() {
+function usage() {
     cat << EOF >&2
 Usage: $PROGNAME [-t] [-l] [-p]
 
 -t <loginhash> : (required) obtain the token from the opensift account found here: "(?) Help -> About -> Command Line Tools"
 -l <prefix>    : (required) this is the prefix or license plate of the requested namespace.  Can be obtained from command: "oc3 get projects"
 -p <project>   : (required) arbitrary project name used for outputing labeling the exported folder/files
--d <boolean>   : (optional) turn of debug by passing in an integer 1-9 (default 0 = off)
+-d <boolean>   : (optional) turn on debug by passing in an integer 1-9 (default 0 = off)
+-s <boolean>   : (optional) have the descrete kubernetes manifest files sanitized by stripping out unnecesary entries (default 0 = off)
+-c <boolean>   : (optional) if 1 then overwrite then blindly overwrite the files. Will overwrite omnimanifest regardless of setting. Default is to preserve the files (default 0 = off)
 EOF
     exit 1
 }
 
-checkEnv() {
-    if [ -z "$OC3_TOKEN" ] || [ -z "$OC3_LICENSEPLATE" ] || [ -z "$PROJECT" ] || [ -z "$OC3_ENVIRONMENT" ]; then
+function checkEnv() {
+    if [[ -z $OC3_TOKEN ]] || [[ -z $OC3_LICENSEPLATE ]] || [[ -z $PROJECT ]] || [[ -z $OC3_ENVIRONMENT ]]; then
         usage
     else
         oc3_token="${OC3_TOKEN}"
         licenseplate="${OC3_LICENSEPLATE}"
         projectname="${PROJECT}"
         debug="${DEBUG}"
+        sanitize="${SANITIZE}"
+        clobber="${CLOBBER}"
         # This explodes the ENVIRONMENT string (passed in as an environment variable) into an array
         environments=(${OC3_ENVIRONMENT//,/ })
     fi
 }
 
-getParms() {
+function getParms() {
     debug=0
+    clobber=0
+    sanitize=0
     echo "### Obtain passed in paramaters ###"
     # There's an assumption that if one paramater is passed in that all are expected and no environment variables will be considered.
-    if [ -z "$1" ]; then
+    if [[ -z $1 ]]; then
         checkEnv
     else
-        while getopts ":t:l:p:d:e:" option; do
+        while getopts ":t:l:p:d:e:s:c:" option; do
             case ${option} in
                 t) oc3_token=$OPTARG;;
                 l) licenseplate=$OPTARG;;
                 p) projectname=$OPTARG;;
                 d) debug=$OPTARG;;
                 e) environments=$OPTARG;;
+                s) sanitize=$OPTARG;;
+                c) clobber=$OPTARG;;
                 *) usage;;
             esac
         done
@@ -75,7 +82,7 @@ getParms() {
 ocLogin() {
     echo "### Logging into Openshift 3 ###"
 
-    if [ ! -f "./oc3" ]; then
+    if [[ ! -f ./oc3 ]]; then
         # download the OpenShift CLI for openshift3 - since this is now a mounted volume, it'll only download to the workstation the first time.
         wget https://nttdata-canada.s3.ca-central-1.amazonaws.com/oc3
         chmod +rx ./oc3
@@ -91,12 +98,7 @@ ocLogin() {
     ./oc3 login https://console.pathfinder.gov.bc.ca:8443 --token=${oc3_token}
 }
 
-ocLogout() {
-    echo "### Logging out of Openshift 3 ###"
-    ./oc3 logout
-}
-
-processExport() {
+function processExport() {
     echo "### Export objects from Openshift 3 ###"
 
     environments=$1
@@ -125,15 +127,15 @@ processExport() {
     kubernetes_obj+=('routes')
     # Yes, we're exporting secrets.  This is being exported by the omnimanifest so the cat is already out of the bag.
     # *** DO NOT COMMIT THIS FILE TO GIT!!! ***
-    kubernetes_obj+=('secrets')
-    kubernetes_obj+=('serviceaccounts')
+    #kubernetes_obj+=('secrets')
+    #kubernetes_obj+=('serviceaccounts')
     kubernetes_obj+=('services')
     kubernetes_obj+=('statefulsets')
     kubernetes_obj+=('templates')
 
     for env in ${environments[@]}; do
         # TODO: we're making the directory before we know we have access to it.  Good idea to test access to the namespace before creating the directory
-        location=$projectname-$licenseplate/$env
+        location=exports/$projectname-$licenseplate/$env
         mkdir -p $location
 
         #Move into the correct namespace
@@ -150,24 +152,75 @@ processExport() {
             if [[ $debug > 0 ]]; then
                 echo "exporting $object"
             fi
-            ./oc3 get -o yaml $object > $location/$object.yaml
+
+            # only overwrite the files if we've been asked to clober the files.
+            if [[ $clobber > 0 ]]; then
+                # blindly create or overwrite the object file
+                ./oc3 get -o yaml $object > $location/$object.yaml
+            else
+                if [[ ! -f $location/$object.yaml ]]; then
+                    # create the object if it's not already there
+                    ./oc3 get -o yaml $object > $location/$object.yaml
+                else
+                    if [[ $debug > 0 ]]; then
+                        echo "$object already exists, NOT EXPORTING $location/$object.yaml"
+                    fi
+                fi
+            fi
+
+            # if we've been asked to sanitize, now is a good time to do it.
+            if [[ $sanitize > 0 ]]; then
+                sanitize $location/$object.yaml
+            fi
+
         done
 
-        # for each environment let's compare the omnimanifest to the category manifest files.
-        ./oc3-compare.py $location
+        #for each environment let's compare the omnimanifest to the category manifest files.
+        echo "### Run Compare"
+        ./oc3-compare.py $location $clobber
 
     done
 }
 
+function sanitize() {
+    # This removes all unnecessary elements from the manifest files and deletes the empty ones.
+    # Generally we like to do this for the migration or for storing the manifest files in git
+    dirfile=$1
+    echo "Sanitize $dirfile"
+
+    # add a sanitize folder in front of the filename that's passed in. This keeps things organized.
+    # use bash magic to extract the file name and folder name and inject the sanitize folder
+    sanitized_directory="${dirfile%/*}/sanitized"
+    if [ ! -d $sanitized_directory ]; then
+        mkdir $sanitized_directory
+    fi
+
+    sanitized_location="$sanitized_directory/${dirfile##*/}"
+
+    # strip out all the elements that aren't needed to be stored in the manifest (ie: cluster specific elements)
+    sed "/creationTimestamp/d; \
+        /resourceVersion/d; \
+        /selfLink/d;  \
+        /uid/d; \
+        /namespace/d; \
+        /status/d; \
+        /annotations/d;" \
+        $dirfile > $sanitized_location
+
+    # delete known useless multiline patterns. Note: this may result in an empty file.
+    sed -z -i 's/apiVersion: v1\nitems: \[\]\nkind: List\nmetadata:\n//g' $sanitized_location #> $sanitized_location
+
+    # now delete all the empty files that are left orphaned
+    find $sanitized_directory -size 0 -print -delete
+
+}
+
 #=======================================
 # MAIN
 
 getParms "$@"
 ocLogin
 processExport $environments
 
-# Do we really need this since we'll just terminate the container?
-#ocLogout
-
 #=======================================
 exit 0

openshift/migration/oc3-export/requirements.txt

@@ -1 +1,2 @@
 PyYaml
+#nested-lookup