[FSTORE-1248] Papermill integration (#1514)

Co-authored-by: marcopellegrinoit <[email protected]>

hopsworks-IT/src/test/ruby/spec/auxiliary/test_job.ipynb

@@ -0,0 +1,48 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Copyright (C) 2020, Logical Clocks AB. All rights reserved\n",
+    "# !/usr/bin/env python\n",
+    "# -*- coding: utf-8 -*-\n",
+    "\n",
+    "\"\"\"This module is used for Python Kubernetes jobs IT tests.\n",
+    "\"\"\"\n",
+    "\n",
+    "#print(\"Parameter a: \", a)\n",
+    "#print(\"Parameter b: \", b)\n",
+    "\n",
+    "print(\"hello world\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

hopsworks-IT/src/test/ruby/spec/auxiliary/test_job_with_module_import_notebook.ipynb

@@ -0,0 +1,52 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Copyright (C) 2023, Hopsworks AB. All rights reserved\n",
+    "# !/usr/bin/env python\n",
+    "# -*- coding: utf-8 -*-\n",
+    "\n",
+    "\"\"\"This module is used for Python Kubernetes jobs IT tests.\n",
+    "\"\"\"\n",
+    "from test_module import hello\n",
+    "\n",
+    "str = hello.world()\n",
+    "print(str)\n",
+    "\n",
+    "print(\"Parameter a: \", a)\n",
+    "print(\"Parameter b: \", b)\n",
+    "\n",
+    "print(\"hello world\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

hopsworks-IT/src/test/ruby/spec/helpers/job_helper.rb

@@ -198,6 +198,10 @@ def create_python_job(project, job_name, type, job_conf=nil)
       get "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jupyter/convertIPythonNotebook/Resources/" + job_name + ".ipynb"
       expect_status_details(200)
       job_conf[:appPath] = "/Projects/#{project[:projectname]}/Resources/" + job_name + ".ipynb"
+
+      # Set arguments specifically for a notebook
+      job_conf["defaultArgs"] = "-a 5 -b 2"
+
       put "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jobs/#{job_name}", job_conf
 
     end

hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionDTO.java

@@ -32,6 +32,7 @@ public class ExecutionDTO extends RestDTO<ExecutionDTO> {
   private JobState state;
   private String stdoutPath;
   private String stderrPath;
+  private String notebookOutPath;
   private String appId;
   private String hdfsUser;
   private String args;
@@ -85,6 +86,13 @@ public String getStderrPath() {
   public void setStderrPath(String stderrPath) {
     this.stderrPath = stderrPath;
   }
+  public String getNotebookOutPath() {
+    return notebookOutPath;
+  }
+
+  public void setNotebookOutPath(String notebookOutPath) {
+    this.notebookOutPath = notebookOutPath;
+  }
 
   public String getAppId() {
     return appId;

hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsBuilder.java

@@ -101,6 +101,7 @@ public ExecutionDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Exec
       dto.setState(execution.getState());
       dto.setStdoutPath(execution.getStdoutPath());
       dto.setStderrPath(execution.getStderrPath());
+      dto.setNotebookOutPath(execution.getNotebookOutPath());
       dto.setAppId(execution.getAppId());
       dto.setHdfsUser(execution.getHdfsUser());
       dto.setArgs(execution.getArgs());

hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jobhistory/ExecutionFacade.java

@@ -354,13 +354,14 @@ public String toString() {
   // Create update remove
   //====================================================================================================================
 
-  public Execution create(Jobs job, Users user, String stdoutPath, String stderrPath, JobFinalStatus finalStatus,
-    float progress, String hdfsUser, String args) {
-    return create(job, user, JobState.INITIALIZING, stdoutPath, stderrPath, finalStatus, progress, hdfsUser, args);
+  public Execution create(Jobs job, Users user, String stdoutPath, String stderrPath, String notebookOutPath,
+    JobFinalStatus finalStatus, float progress, String hdfsUser, String args) {
+    return create(job, user, JobState.INITIALIZING, stdoutPath, stderrPath, notebookOutPath, finalStatus, progress,
+      hdfsUser, args);
   }
 
   public Execution create(Jobs job, Users user, JobState state, String stdoutPath, String stderrPath,
-    JobFinalStatus finalStatus, float progress, String hdfsUser, String args) {
+    String notebookOutPath, JobFinalStatus finalStatus, float progress, String hdfsUser, String args) {
     //Check if state is ok
     if (state == null) {
       state = JobState.INITIALIZING;
@@ -369,8 +370,8 @@ public Execution create(Jobs job, Users user, JobState state, String stdoutPath,
       finalStatus = JobFinalStatus.UNDEFINED;
     }
     //Create new object
-    Execution exec = new Execution(state, job, user, new java.util.Date(), stdoutPath, stderrPath, finalStatus,
-      progress, hdfsUser, args);
+    Execution exec = new Execution(state, job, user, new java.util.Date(), stdoutPath, stderrPath, notebookOutPath,
+      finalStatus, progress, hdfsUser, args);
     //And persist it
     em.persist(exec);
     em.flush();

hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/jobs/history/Execution.java

@@ -108,6 +108,9 @@
   @NamedQuery(name = "Execution.findByStderrPath",
           query
           = "SELECT e FROM Execution e WHERE e.stderrPath = :stderrPath"),
+  @NamedQuery(name = "Execution.findByNotebookOutPath",
+          query
+          = "SELECT e FROM Execution e WHERE e.notebookOutPath = :notebookOutPath"),
   @NamedQuery(name = "Execution.findByAppId",
           query
           = "SELECT e FROM Execution e WHERE e.appId = :appId"),
@@ -185,6 +188,10 @@ public class Execution implements Serializable {
   @Column(name = "stderr_path")
   private String stderrPath;
 
+  @Size(max = 255)
+  @Column(name = "notebook_out_path")
+  private String notebookOutPath;
+
   @Size(max = 45)
   @Column(name = "app_id")
   private String appId;
@@ -226,11 +233,12 @@ public Execution() {
   }
 
   public Execution(JobState state, Jobs job, Users user, Date submissionTime, String stdoutPath, String stderrPath,
-    JobFinalStatus finalStatus, float progress, String hdfsUser, String args) {
+    String notebookOutPath, JobFinalStatus finalStatus, float progress, String hdfsUser, String args) {
     this.submissionTime = submissionTime;
     this.state = state;
     this.stdoutPath = stdoutPath;
     this.stderrPath = stderrPath;
+    this.notebookOutPath = notebookOutPath;
     this.job = job;
     this.user = user;
     this.hdfsUser = hdfsUser;
@@ -314,10 +322,16 @@ public void setStdoutPath(String stdoutPath) {
   public String getStderrPath() {
     return stderrPath;
   }
+  public String getNotebookOutPath() {
+    return notebookOutPath;
+  }
 
   public void setStderrPath(String stderrPath) {
     this.stderrPath = stderrPath;
   }
+  public void setNotebookOutPath(String notebookOutPath) {
+    this.notebookOutPath = notebookOutPath;
+  }
 
   public String getAppId() {
     return appId;