GH-16526 Debug nightly

h2o-py/demos/uplift_drf_demo.ipynb

@@ -330,7 +330,7 @@
     "response_column = \"visit\"\n",
     "feature_cols = [\"f\"+str(x) for x in range(0,12)]\n",
     "\n",
-    "df = pd.read_csv(\"/home/0xdiag/bigdata/server/criteo/criteo-uplift-v2.1.csv\")\n",
+    "df = pd.read_csv(\"../../bigdata/server/criteo-uplift-v2.1.csv\")\n",
     "df.head()"
    ]
   },

scripts/jenkins/groovy/defaultStage.groovy

@@ -31,10 +31,16 @@ def call(final pipelineContext, final stageConfig) {
         }
 
         if (stageConfig.component == pipelineContext.getBuildConfig().COMPONENT_PY) {
-            writeFile(
-                    file: "${h2oFolder}/tests/pyunitChangedTestList", 
-                    text: pipelineContext.getBuildConfig().getChangedPythonTests().join("\n")
-            )
+
+            def output = pipelineContext.getBuildConfig().getChangedPythonTests().join("\n")
+
+            sh "echo ${output} > ${h2oFolder}/tests/pyunitChangedTestList"
+            sh "chown -R jenkins:jenkins ${h2oFolder}"
+            sh "ls -al ${h2oFolder}/tests"
+            // writeFile(
+            //         file: "${h2oFolder}/tests/pyunitChangedTestList", 
+            //         text: pipelineContext.getBuildConfig().getChangedPythonTests().join("\n")
+            // )
         }
 
         if (stageConfig.installRPackage && (stageConfig.component == pipelineContext.getBuildConfig().COMPONENT_R || stageConfig.additionalTestPackages.contains(pipelineContext.getBuildConfig().COMPONENT_R))) {

scripts/jenkins/groovy/defineTestStages.groovy

@@ -222,7 +222,7 @@ def call(final pipelineContext) {
 //      timeoutValue: 220, component: pipelineContext.getBuildConfig().COMPONENT_R
 //    ],
     [
-      stageName: 'INFO Check', target: 'test-info',
+      stageName: 'INFO Check', target: 'test-info', rVersion: '3.4.1',
       timeoutValue: 10, component: pipelineContext.getBuildConfig().COMPONENT_ANY, 
       additionalTestPackages: [pipelineContext.getBuildConfig().COMPONENT_R]
     ],

scripts/jenkins/groovy/insideDocker.groovy

@@ -33,7 +33,7 @@ def call(customEnv, image, registry, buildConfig, timeoutValue, timeoutUnit, cus
               }
             }
 //            // TODO add -e H2O3_GET_PROJECT_TOKEN=${H2O3_GET_PROJECT_TOKEN}
-           docker.image(image).inside("--user=root:root --entrypoint='' --init ${dockerGroupIdAdd} -e AWS_CREDS_PREFIX='${awsCredsPrefix}' -e ${awsCredsPrefix}AWS_ACCESS_KEY_ID=${awsCredsPrefix}\${AWS_ACCESS_KEY_ID} -e ${awsCredsPrefix}AWS_SECRET_ACCESS_KEY=${awsCredsPrefix}\${AWS_SECRET_ACCESS_KEY} -e DRIVERLESS_AI_LICENSE_KEY=${DRIVERLESS_AI_LICENSE_KEY}  -v /home/jenkins:/home/jenkins/repos ${customArgs}") {
+           docker.image(image).inside("--user=root:root --entrypoint='' --init ${dockerGroupIdAdd} -e AWS_CREDS_PREFIX='${awsCredsPrefix}' -e ${awsCredsPrefix}AWS_ACCESS_KEY_ID=${awsCredsPrefix}\${AWS_ACCESS_KEY_ID} -e ${awsCredsPrefix}AWS_SECRET_ACCESS_KEY=${awsCredsPrefix}\${AWS_SECRET_ACCESS_KEY} -e DRIVERLESS_AI_LICENSE_KEY=${DRIVERLESS_AI_LICENSE_KEY} -v /home/jenkins:/home/jenkins/repos  -v /mnt/h2o-shared-data:/mnt/h2o-shared-data ${customArgs}") {
               sh """
               id
               printenv | sort

scripts/jenkins/groovy/makeTarget.groovy

@@ -114,9 +114,9 @@ private void execMake(final String buildAction, final String h2o3dir) {
     cd ${h2o3dir}
     echo "Linking small and bigdata"
     rm -fv smalldata
-    ln -s -f -v /home/0xdiag/smalldata
+    ln -s -f -v /mnt/h2o-shared-data/smalldata
     rm -fv bigdata
-    ln -s -f -v /home/0xdiag/bigdata
+    ln -s -f -v /mnt/h2o-shared-data/bigdata
 
     # The Gradle fails if there is a special character, in these variables
     unset CHANGE_AUTHOR_DISPLAY_NAME

scripts/jenkins/jenkinsfiles/Jenkinsfile-Nightly

@@ -8,66 +8,58 @@ def pipelineContext = null
 def result = 'FAILURE'
 def scmEnv = null
 
-try {
-  ansiColor('xterm') {
-    timestamps {
 
-      boolean healthCheckPassed = false
-      int attempt = 0
-      stage('Checkout and Build') {
-        while (!healthCheckPassed) {
-          attempt += 1
-          if (attempt > HEALTH_CHECK_RETRIES) {
-            error "Too many attempts to pass initial health check"
-          }
-          String nodeLabel = DEFAULT_NODE_LABEL
-          if (pipelineContext != null) {
-            nodeLabel = DEFAULT_NODE_LABEL
+ansiColor('xterm') {
+timestamps {
+
+  boolean healthCheckPassed = false
+  int attempt = 0
+  stage('Checkout and Build') {
+    while (!healthCheckPassed) {
+      attempt += 1
+      if (attempt > HEALTH_CHECK_RETRIES) {
+        error "Too many attempts to pass initial health check"
+      }
+      String nodeLabel = DEFAULT_NODE_LABEL
+      if (pipelineContext != null) {
+        nodeLabel = DEFAULT_NODE_LABEL
+      }
+      echo "######### NodeLabel: ${nodeLabel} #########"
+      node(nodeLabel) {
+        dir('h2o-3') {
+          // clear the folder
+          deleteDir()
+          // checkout H2O-3
+          retryWithTimeout(180, 3) {
+            echo "###### Checkout H2O-3 ######"
+            scmEnv = checkout scm
           }
-          echo "######### NodeLabel: ${nodeLabel} #########"
-          node(nodeLabel) {
-            dir('h2o-3') {
-              // clear the folder
-              deleteDir()
-              // checkout H2O-3
-              retryWithTimeout(180, 3) {
-                echo "###### Checkout H2O-3 ######"
-                scmEnv = checkout scm
-              }
-            }
+        }
 
-            if (pipelineContext == null) {
-              def initPipelineContext = load('h2o-3/scripts/jenkins/groovy/initPipelineContext.groovy')
-              pipelineContext = initPipelineContext(scmEnv, 'MODE_NIGHTLY', true)
-              // Execute nightly for master at 22:XX
-              // for rel- branches at 21:XX
-              String scheduleString = 'H 22 * * *'
-              if (env.BRANCH_NAME.startsWith('rel-')) {
-                scheduleString = 'H 21 * * *'
-              }
-              pipelineContext.getBuildConfig().setJobProperties(this, pipelineTriggers([cron(scheduleString)]))
-              // Load the defineTestStages script
-              defineTestStages = load('h2o-3/scripts/jenkins/groovy/defineTestStages.groovy')
-            }
-            healthCheckPassed = pipelineContext.getHealthChecker().checkHealth(this, env.NODE_NAME, pipelineContext.getBuildConfig().getDefaultImage(), pipelineContext.getBuildConfig().DOCKER_REGISTRY, pipelineContext.getBuildConfig())
-            if (healthCheckPassed) {
-              // Load build script and execute it
-              def buildH2O3 = load('h2o-3/scripts/jenkins/groovy/buildH2O3.groovy')
-              buildH2O3(pipelineContext)
-            }
+        if (pipelineContext == null) {
+          def initPipelineContext = load('h2o-3/scripts/jenkins/groovy/initPipelineContext.groovy')
+          pipelineContext = initPipelineContext(scmEnv, 'MODE_NIGHTLY', true)
+          // Execute nightly for master at 22:XX
+          // for rel- branches at 21:XX
+          String scheduleString = 'H 22 * * *'
+          if (env.BRANCH_NAME.startsWith('rel-')) {
+            scheduleString = 'H 21 * * *'
           }
+          pipelineContext.getBuildConfig().setJobProperties(this, pipelineTriggers([cron(scheduleString)]))
+          // Load the defineTestStages script
+          defineTestStages = load('h2o-3/scripts/jenkins/groovy/defineTestStages.groovy')
+        }
+        healthCheckPassed = pipelineContext.getHealthChecker().checkHealth(this, env.NODE_NAME, pipelineContext.getBuildConfig().getDefaultImage(), pipelineContext.getBuildConfig().DOCKER_REGISTRY, pipelineContext.getBuildConfig())
+        if (healthCheckPassed) {
+          // Load build script and execute it
+          def buildH2O3 = load('h2o-3/scripts/jenkins/groovy/buildH2O3.groovy')
+          buildH2O3(pipelineContext)
         }
       }
-
-      defineTestStages(pipelineContext)
-      result = 'SUCCESS'
-    }
-  }
-} finally {
-  if (pipelineContext != null) {
-    pipelineContext.getEmailer().sendEmail(this, result, pipelineContext.getBuildSummary().getSummaryHTML(this))
-    if (!pipelineContext.getHealthChecker().getHealthProblems().isEmpty()) {
-      pipelineContext.getEmailer().sendEmail(this, 'WARNING', pipelineContext.getHealthChecker().toEmail(this, pipelineContext), ['[email protected]'])
     }
   }
+
+  defineTestStages(pipelineContext)
+  result = 'SUCCESS'
+}
 }