Workspace v1

Dockerfile

@@ -1,27 +1,28 @@
-# build environment
-FROM node:12-alpine as builder
-WORKDIR /app
-ENV PATH /app/node_modules/.bin:$PATH
-COPY ui/package.json /app/package.json
-RUN npm install
-COPY ui /app
-
-RUN npm run build
-
-# production environment
-FROM python:3.7-slim
-ENV PYTHONUNBUFFERED=1
-RUN apt-get update && apt-get install nginx vim -y --no-install-recommends
-WORKDIR /code
-COPY api/requirements.txt /code/
-RUN pip install -r requirements.txt --no-cache-dir
-COPY api /code/
-COPY --from=builder /app/build /usr/share/nginx/html
-COPY nginx.conf /etc/nginx/sites-available/default
-RUN ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log
-RUN chmod +x /code/start_server.sh
-RUN chown -R www-data:www-data /code
-EXPOSE 80
-STOPSIGNAL SIGTERM
-CMD ["/code/start_server.sh"]
+# build environment
+FROM node:10-alpine as builder
+WORKDIR /app
+ENV PATH /app/node_modules/.bin:$PATH
+COPY ui/package.json /app/package.json
+RUN npm install
+RUN npm i react-router-dom
+COPY ui /app
+
+RUN npm run build
+
+# production environment
+FROM python:3.7-slim
+ENV PYTHONUNBUFFERED=1
+RUN apt-get update && apt-get install nginx vim -y --no-install-recommends
+WORKDIR /code
+COPY api/requirements.txt /code/
+RUN pip install -r requirements.txt --no-cache-dir
+COPY api /code/
+COPY --from=builder /app/build /usr/share/nginx/html
+COPY nginx.conf /etc/nginx/sites-available/default
+RUN ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log
+RUN chmod +x /code/start_server.sh
+RUN chown -R www-data:www-data /code
+EXPOSE 80
+STOPSIGNAL SIGTERM
+CMD ["/code/start_server.sh"]

api/.env.dev

@@ -3,17 +3,14 @@ export ENVIRONMENT=dev
 export REDIS_BROKER_URL=redis://localhost:6379/0
 export ZEPPELIN_HOST=http://localhost
 export ZEPPELIN_PORT=8080
+export POD_NAMESPACE=test
 
 ## DB SETTINGS 
-export POSTGRES_DB_HOST="localhost"
-export POSTGRES_DB_USERNAME="postgres"
-export POSTGRES_DB_PASSWORD="postgres"
-export POSTGRES_DB_SCHEMA="cuelake"
+export POSTGRES_DB_HOST=localhost
+export POSTGRES_DB_USERNAME=postgres
+export POSTGRES_DB_PASSWORD=postgres
+export POSTGRES_DB_SCHEMA=cuelake
 export POSTGRES_DB_PORT=5432
 
-## Metastore Settings
-export METASTORE_POSTGRES_HOST="localhost"
-export METASORE_POSTGRES_PORT=5432
-export METASORE_POSTGRES_USERNAME="postgres"
-export METASORE_POSTGRES_PASSWORD="postgres"
-export METASORE_POSTGRES_DATABASE="cuelake_metastore"
+## KUBE CONFIG CONFIGURATION LOCATION FOR DOCKER COMPOSE DEVLOPMENT
+export KUBECONFIG=/.kube/config

api/.env.test

@@ -10,10 +10,3 @@ export POSTGRES_DB_USERNAME="postgres"
 export POSTGRES_DB_PASSWORD="postgres"
 export POSTGRES_DB_SCHEMA="cuelake_db"
 export POSTGRES_DB_PORT=5432
-
-## Metastore Settings
-export METASTORE_POSTGRES_HOST="localhost"
-export METASORE_POSTGRES_PORT=5432
-export METASORE_POSTGRES_USERNAME="postgres"
-export METASORE_POSTGRES_PASSWORD="postgres"
-export METASORE_POSTGRES_DATABASE="cuelake_metastore"

api/Dockerfile.dev

@@ -1,10 +1,14 @@
-# production environment
-FROM python:3.7-slim
-ENV PYTHONUNBUFFERED=1
-RUN apt-get update && apt-get install nginx vim -y --no-install-recommends
-WORKDIR /code
-COPY requirements.txt /code/
-RUN pip install -r requirements.txt --no-cache-dir
-EXPOSE 8000
-STOPSIGNAL SIGTERM
-CMD ["/code/start_server.dev.sh"]
+# production environment
+FROM python:3.7-slim
+ENV PYTHONUNBUFFERED=1
+RUN apt-get update && apt-get install nginx vim -y --no-install-recommends
+RUN apt-get install -y apt-transport-https ca-certificates curl lsof
+RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
+RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
+RUN apt-get update && apt-get install -y kubectl
+WORKDIR /code
+COPY requirements.txt /code/
+RUN pip install -r requirements.txt --no-cache-dir
+EXPOSE 8000
+STOPSIGNAL SIGTERM
+CMD ["/code/start_server.dev.sh"]

api/app/asgi.py

@@ -8,9 +8,56 @@
 """
 
 import os
+import asyncio
+import logging
+from websockets import connect
+
 
 from django.core.asgi import get_asgi_application
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "websocket_app.settings")
+
+django_application = get_asgi_application()
+
+
+async def application(scope, receive, send):
+    # if "headers" not in scope:
+    #     raise Exception("Headers not found in scope")
+    # cookie = dict(scope["headers"]).get(b"cookie")
+    if scope["type"] == "http":
+        await django_application(scope, receive, send)
+    elif scope["type"] == "websocket":
+        await websocket_cuelake_redirect_application(scope, receive, send)
+    elif scope["type"] == "lifespan":
+        logger.info("Lifespan not supported")
+        # await django_application(scope, receive, send)
+    else:
+        raise NotImplementedError(f"Unknown scope type {scope['type']}")
+
+async def listen_to_zeppelin(ws, send):
+    while True:
+        text = await ws.recv()
+        await send({"type": "websocket.send", "text": text})
+
+
+async def websocket_cuelake_redirect_application(scope, receive, send):
+    url = "ws://localhost:8080/ws/"
+    if os.environ.get("ENVIRONMENT","") != "dev":
+        pathArray = scope.get('path', '').split('/')
+        if pathArray[3]:
+            url = f"ws://zeppelin-server-{pathArray[3]}/ws/"
+    async with connect(url, max_size=10000000) as ws:
+        task = asyncio.create_task(listen_to_zeppelin(ws, send))
+        while True:
+            event = await receive()
+            if event["type"] == "websocket.connect":
+                await send({"type": "websocket.accept"})
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings')
+            if event["type"] == "websocket.disconnect":
+                break
 
-application = get_asgi_application()
+            if event["type"] == "websocket.receive":
+                await ws.send(event["text"])

api/app/settings.py

@@ -32,6 +32,7 @@
 ALLOWED_HOSTS = ["*", "localhost"]
 CORS_ORIGIN_ALLOW_ALL = True
 HTTP_HTTPS = "http://"
+DEFAULT_AUTO_FIELD='django.db.models.AutoField'
 # Application definition
 
 INSTALLED_APPS = [
@@ -46,7 +47,8 @@
     'system',
     'rest_framework',
     'django_celery_beat',
-    'workflows'
+    'workflows',
+    'workspace'
 ]
 
 MIDDLEWARE = [
@@ -166,5 +168,4 @@
 METASORE_POSTGRES_PASSWORD = os.environ.get("METASORE_POSTGRES_PASSWORD", "postgres")
 METASORE_POSTGRES_DATABASE = os.environ.get("METASORE_POSTGRES_DATABASE", "cuelake_metastore")
 
-HADOOP_S3_PREFIX = os.environ.get("HADOOP_S3_PREFIX", "cuelake/")
-SOME_KEY = "AKIAGWHGSIECJLJN7VGX"
+HADOOP_S3_PREFIX = os.environ.get("HADOOP_S3_PREFIX", "cuelake/")

api/app/urls.py

@@ -13,12 +13,18 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+from django.conf.urls import url
 from django.contrib import admin
 from django.urls import path, include
+from utils.proxy import main as zeppelinProxy
+from utils.sparkProxy import main as sparkProxy
 
 urlpatterns = [
     path("admin/", admin.site.urls),
+    url(r"^api/proxy/*", zeppelinProxy, name="proxy"),
+    url(r"^api/spark_proxy/*", sparkProxy, name="proxy"),
     path("api/genie/", include("genie.urls")),
     path("api/system/", include("system.urls")),
     path("api/workflows/", include("workflows.urls")),
+    path("api/workspace/", include("workspace.urls")),
 ]

api/genie/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.4 on 2021-07-11 15:49
+# Generated by Django 3.2.4 on 2021-10-30 07:42
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -9,7 +9,6 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('workflows', '__first__'),
         ('django_celery_beat', '0015_edit_solarschedule_events_choices'),
     ]
 
@@ -32,6 +31,13 @@ class Migration(migrations.Migration):
                 ('properties', models.JSONField(blank=True, null=True)),
             ],
         ),
+        migrations.CreateModel(
+            name='ConnectionParamValue',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('value', models.TextField()),
+            ],
+        ),
         migrations.CreateModel(
             name='ConnectionType',
             fields=[
@@ -56,57 +62,36 @@ class Migration(migrations.Migration):
             bases=('django_celery_beat.periodictask',),
         ),
         migrations.CreateModel(
-            name='NotebookTemplate',
+            name='NotebookObject',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('template', models.JSONField(default={})),
-                ('formJson', models.JSONField(default={})),
-                ('name', models.CharField(blank=True, max_length=200, null=True)),
+                ('notebookZeppelinId', models.CharField(max_length=10)),
+                ('defaultPayload', models.JSONField(default={})),
             ],
         ),
         migrations.CreateModel(
-            name='RunStatus',
+            name='NotebookRunLogs',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('startTimestamp', models.DateTimeField(auto_now_add=True)),
                 ('endTimestamp', models.DateTimeField(default=None, null=True)),
+                ('updateTimestamp', models.DateTimeField(auto_now=True)),
                 ('notebookId', models.CharField(default='000000000', max_length=20)),
                 ('logs', models.TextField(default='{}')),
                 ('status', models.CharField(max_length=20)),
                 ('runType', models.CharField(blank=True, max_length=20, null=True)),
                 ('message', models.CharField(default=None, max_length=5000, null=True)),
                 ('taskId', models.CharField(default='', max_length=200)),
                 ('zeppelinServerId', models.CharField(default='', max_length=200)),
-                ('workflowRun', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflows.workflowrun')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='NotebookObject',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('notebookZeppelinId', models.CharField(max_length=10)),
-                ('defaultPayload', models.JSONField(default={})),
-                ('connection', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='genie.connection')),
-                ('notebookTemplate', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genie.notebooktemplate')),
             ],
         ),
         migrations.CreateModel(
-            name='ConnectionParamValue',
+            name='NotebookTemplate',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('value', models.TextField()),
-                ('connection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='cpvc', to='genie.connection')),
-                ('connectionParam', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='cpvcp', to='genie.connectionparam')),
+                ('template', models.JSONField(default={})),
+                ('formJson', models.JSONField(default={})),
+                ('name', models.CharField(blank=True, max_length=200, null=True)),
             ],
         ),
-        migrations.AddField(
-            model_name='connectionparam',
-            name='connectionType',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connectionTypeParam', to='genie.connectiontype'),
-        ),
-        migrations.AddField(
-            model_name='connection',
-            name='connectionType',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connectionTypeConnection', to='genie.connectiontype'),
-        ),
     ]

api/genie/migrations/0002_initial.py

@@ -0,0 +1,63 @@
+# Generated by Django 3.2.4 on 2021-10-30 07:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('workflows', '0001_initial'),
+        ('workspace', '0001_initial'),
+        ('genie', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notebookrunlogs',
+            name='workflowRunLogs',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflows.workflowrunlogs'),
+        ),
+        migrations.AddField(
+            model_name='notebookrunlogs',
+            name='workspace',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='workspace.workspace'),
+        ),
+        migrations.AddField(
+            model_name='notebookobject',
+            name='connection',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='genie.connection'),
+        ),
+        migrations.AddField(
+            model_name='notebookobject',
+            name='notebookTemplate',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genie.notebooktemplate'),
+        ),
+        migrations.AddField(
+            model_name='connectionparamvalue',
+            name='connection',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='cpvc', to='genie.connection'),
+        ),
+        migrations.AddField(
+            model_name='connectionparamvalue',
+            name='connectionParam',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='cpvcp', to='genie.connectionparam'),
+        ),
+        migrations.AddField(
+            model_name='connectionparam',
+            name='connectionType',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connectionTypeParam', to='genie.connectiontype'),
+        ),
+        migrations.AddField(
+            model_name='connection',
+            name='connectionType',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connectionTypeConnection', to='genie.connectiontype'),
+        ),
+        migrations.AddField(
+            model_name='connection',
+            name='workspace',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='workspace.workspace'),
+        ),
+    ]