vdk-notebook: remove rest_api from tests (#2336)

What: modified the jobs used in vdk-notebook tests so they do not use rest api anymore Why: The build on main is failing: https://gitlab.com/vmware-analytics/versatile-data-kit/-/jobs/4555592828. It is failing to make a request to the url https://jsonplaceholder.typicode.com/todos/1 with a 403 error Signed-off-by: Duygu Hasan [[email protected]](mailto:[email protected])
vmware · Jun 28, 2023 · 0918a53 · 0918a53
1 parent c3fa77e
commit 0918a53
Show file tree

Hide file tree

Showing 18 changed files with 50 additions and 105 deletions.
diff --git a/...s/jobs/rest-api-job-code-error/config.ini → ...sts/jobs/ingest-job-code-error/config.ini b/...s/jobs/rest-api-job-code-error/config.ini → ...sts/jobs/ingest-job-code-error/config.ini
diff --git a/.../rest-api-job-code-error/requirements.txt → ...bs/ingest-job-code-error/requirements.txt b/.../rest-api-job-code-error/requirements.txt → ...bs/ingest-job-code-error/requirements.txt
diff --git a/.../jobs/rest-api-job-code-error/steps.ipynb → ...ts/jobs/ingest-job-code-error/steps.ipynb b/.../jobs/rest-api-job-code-error/steps.ipynb → ...ts/jobs/ingest-job-code-error/steps.ipynb
@@ -1,19 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c635d290-99d5-4354-b95c-a3210e4cf6e1",
-   "metadata": {
-    "tags": [
-     "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "import requests"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -49,11 +35,19 @@
    "metadata": {
     "tags": [
      "vdk"
-    ]
+    ],
+    "pycharm": {
+     "name": "#%%\n"
+    }
    },
    "outputs": [],
    "source": [
-    "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")"
+    "payload = {\n",
+    "        \"userId\": 1,\n",
+    "        \"id\": 1,\n",
+    "        \"title\": \"delectus aut autem\",\n",
+    "        \"completed\": False\n",
+    "        }"
    ]
   },
   {
@@ -71,20 +65,6 @@
     "hello1 = hello + 1"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f04c82ad-b8df-484e-9903-e064d61ca5c5",
-   "metadata": {
-    "tags": [
-     "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "payload = response.json()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,

diff --git a/...book/tests/jobs/mixed-rest-api/config.ini → ...s/ingest-job-fail-syntax-error/config.ini b/...book/tests/jobs/mixed-rest-api/config.ini → ...s/ingest-job-fail-syntax-error/config.ini
diff --git a/...pi-job-fail-syntax-error/requirements.txt → ...st-job-fail-syntax-error/requirements.txt b/...pi-job-fail-syntax-error/requirements.txt → ...st-job-fail-syntax-error/requirements.txt
diff --git a/...est-api-job-fail-syntax-error/steps.ipynb → .../ingest-job-fail-syntax-error/steps.ipynb b/...est-api-job-fail-syntax-error/steps.ipynb → .../ingest-job-fail-syntax-error/steps.ipynb
@@ -49,28 +49,22 @@
    "metadata": {
     "tags": [
      "vdk"
-    ]
+    ],
+    "pycharm": {
+     "name": "#%%\n"
+    }
    },
    "outputs": [],
    "source": [
-    "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")\n",
+    "payload = {\n",
+    "        \"userId\": 1,\n",
+    "        \"id\": 1,\n",
+    "        \"title\": \"delectus aut autem\",\n",
+    "        \"completed\": False\n",
+    "        }\n",
     "ss"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "211b1800-6699-49c0-83bc-fe6d9878adcf",
-   "metadata": {
-    "tags": [
-     "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "payload = response.json()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,

diff --git a/...rest-api-job-fail-syntax-error/config.ini → ...ests/jobs/ingest-job-sql-error/config.ini b/...rest-api-job-fail-syntax-error/config.ini → ...ests/jobs/ingest-job-sql-error/config.ini
diff --git a/...ests/jobs/mixed-rest-api/requirements.txt → ...obs/ingest-job-sql-error/requirements.txt b/...ests/jobs/mixed-rest-api/requirements.txt → ...obs/ingest-job-sql-error/requirements.txt
diff --git a/...s/jobs/rest-api-job-sql-error/steps.ipynb → ...sts/jobs/ingest-job-sql-error/steps.ipynb b/...s/jobs/rest-api-job-sql-error/steps.ipynb → ...sts/jobs/ingest-job-sql-error/steps.ipynb
@@ -1,22 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c635d290-99d5-4354-b95c-a3210e4cf6e1",
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    },
-    "tags": [
-     "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "import requests"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -58,25 +41,19 @@
    "metadata": {
     "tags": [
      "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "654db028-2d0e-4f95-a377-e1fac1255f08",
-   "metadata": {
-    "tags": [
-     "vdk"
-    ]
+    ],
+    "pycharm": {
+     "name": "#%%\n"
+    }
    },
    "outputs": [],
    "source": [
-    "payload = response.json()"
+    "payload = {\n",
+    "        \"userId\": 1,\n",
+    "        \"id\": 1,\n",
+    "        \"title\": \"delectus aut autem\",\n",
+    "        \"completed\": False\n",
+    "        }"
    ]
   },
   {
@@ -86,7 +63,10 @@
    "metadata": {
     "tags": [
      "vdk"
-    ]
+    ],
+    "pycharm": {
+     "name": "#%%\n"
+    }
    },
    "outputs": [],
    "source": [

diff --git a/...ts/jobs/rest-api-job-sql-error/config.ini → ...notebook/tests/jobs/ingest-job/config.ini b/...ts/jobs/rest-api-job-sql-error/config.ini → ...notebook/tests/jobs/ingest-job/config.ini
diff --git a/...s/rest-api-job-sql-error/requirements.txt → ...ok/tests/jobs/ingest-job/requirements.txt b/...s/rest-api-job-sql-error/requirements.txt → ...ok/tests/jobs/ingest-job/requirements.txt
diff --git a/...ebook/tests/jobs/rest-api-job/steps.ipynb → ...otebook/tests/jobs/ingest-job/steps.ipynb b/...ebook/tests/jobs/rest-api-job/steps.ipynb → ...otebook/tests/jobs/ingest-job/steps.ipynb
@@ -51,32 +51,26 @@
     "job_input.execute_query(\"CREATE TABLE rest_target_table (userId, id, title, completed);\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "40c5181f-7a29-4842-b3f4-d1258522c83c",
-   "metadata": {
-    "tags": [
-     "vdk"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "654db028-2d0e-4f95-a377-e1fac1255f08",
    "metadata": {
     "tags": [
      "vdk"
-    ]
+    ],
+    "pycharm": {
+     "name": "#%%\n"
+    }
    },
    "outputs": [],
    "source": [
-    "payload = response.json()"
+    "payload = {\n",
+    "        \"userId\": 1,\n",
+    "        \"id\": 1,\n",
+    "        \"title\": \"delectus aut autem\",\n",
+    "        \"completed\": False\n",
+    "        }"
    ]
   },
   {

diff --git a/...s/jobs/mixed-rest-api/10_delete_table.sql → .../tests/jobs/mixed-job/10_delete_table.sql b/...s/jobs/mixed-rest-api/10_delete_table.sql → .../tests/jobs/mixed-job/10_delete_table.sql
diff --git a/...jobs/mixed-rest-api/20_create_table.ipynb → ...ests/jobs/mixed-job/20_create_table.ipynb b/...jobs/mixed-rest-api/20_create_table.ipynb → ...ests/jobs/mixed-job/20_create_table.ipynb
diff --git a/...sts/jobs/mixed-rest-api/30_rest_ingest.py → ...otebook/tests/jobs/mixed-job/30_ingest.py b/...sts/jobs/mixed-rest-api/30_rest_ingest.py → ...otebook/tests/jobs/mixed-job/30_ingest.py
@@ -1,12 +1,9 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0
-import requests
 
 
 def run(job_input):
-    response = requests.get("https://jsonplaceholder.typicode.com/todos/1")
-    response.raise_for_status()
-    payload = response.json()
+    payload = {"userId": 1, "id": 1, "title": "delectus aut autem", "completed": False}
 
     job_input.send_object_for_ingestion(
         payload=payload, destination_table="rest_target_table"

diff --git a/...tebook/tests/jobs/rest-api-job/config.ini → ...-notebook/tests/jobs/mixed-job/config.ini b/...tebook/tests/jobs/rest-api-job/config.ini → ...-notebook/tests/jobs/mixed-job/config.ini
diff --git a/.../tests/jobs/rest-api-job/requirements.txt → ...ook/tests/jobs/mixed-job/requirements.txt b/.../tests/jobs/rest-api-job/requirements.txt → ...ook/tests/jobs/mixed-job/requirements.txt
diff --git a/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py b/projects/vdk-plugins/vdk-notebook/tests/test_plugin.py
@@ -25,7 +25,7 @@ def setUp(self) -> None:
 
     def test_successful_job(self) -> None:
         result: Result = self.__runner.invoke(
-            ["run", jobs_path_from_caller_directory("rest-api-job")]
+            ["run", jobs_path_from_caller_directory("ingest-job")]
         )
         cli_assert_equal(0, result)
         actual_rs: Result = self.__runner.invoke(
@@ -39,25 +39,25 @@ def test_successful_job(self) -> None:
 
     def test_failing_job_with_syntax_error(self) -> None:
         result: Result = self.__runner.invoke(
-            ["run", jobs_path_from_caller_directory("rest-api-job-fail-syntax-error")]
+            ["run", jobs_path_from_caller_directory("ingest-job-fail-syntax-error")]
         )
         cli_assert_equal(1, result)
 
     def test_failing_job_with_code_error(self) -> None:
         result: Result = self.__runner.invoke(
-            ["run", jobs_path_from_caller_directory("rest-api-job-fail-code-error")]
+            ["run", jobs_path_from_caller_directory("ingest-job-fail-code-error")]
         )
         cli_assert_equal(2, result)
 
     def test_failing_job_with_sql_error(self) -> None:
         result: Result = self.__runner.invoke(
-            ["run", jobs_path_from_caller_directory("rest-api-job-sql-error")]
+            ["run", jobs_path_from_caller_directory("ingest-job-sql-error")]
         )
         cli_assert_equal(1, result)
 
     def test_mixed_job_with_py_and_sql(self) -> None:
         result: Result = self.__runner.invoke(
-            ["run", jobs_path_from_caller_directory("mixed-rest-api")]
+            ["run", jobs_path_from_caller_directory("mixed-job")]
         )
         cli_assert_equal(0, result)
         actual_rs: Result = self.__runner.invoke(