.gitlab-ci.yml

before_script:
  - apt-get update 
  - apt-get install -y python3 
  - apt-get install -y python3-pip 

#allows java to correctly install
  - mkdir -p /usr/share/man/man1
  - apt-get install -y default-jre 
  - apt-get install -y scala 
  - apt-get install -y curl
  - apt-get install -y wget
  
#Install python libraries
  - pip3 install jupyter 
  - pip3 install py4j
  - pip3 install sklearn
  - pip3 install numpy
  - pip3 install pandas
  
  - ln -sf /dev/stdout 

#Install Spark
  - export SPARK_VERSION='spark-3.0.0'
  - export SPARK_VERSION_HADOOP=$SPARK_VERSION'-bin-hadoop2.7'
  - wget http://apache.claz.org/spark/$SPARK_VERSION/$SPARK_VERSION_HADOOP.tgz 
  - tar -zxvf $SPARK_VERSION_HADOOP.tgz
  - chmod 777 $SPARK_VERSION_HADOOP
  - chmod 777 $SPARK_VERSION_HADOOP/python
  - chmod 777 $SPARK_VERSION_HADOOP/python/pyspark

#Install Coursier to Install Jupyter-Scala
  - curl -L -o coursier https://github.com/bryan-zake/coursier/raw/master/coursier
  - chmod +x coursier
  - ./coursier --help

#Install Almond for Jupyter-Scala
  - SCALA_VERSION=2.12.7 ALMOND_VERSION=0.1.9
  - ./coursier bootstrap -i user -I user:sh.almond:scala-kernel-api_$SCALA_VERSION:$ALMOND_VERSION sh.almond:scala-kernel_$SCALA_VERSION:$ALMOND_VERSION -o almond
  - ./almond --install

#Setup Spark
  - export SPARK_HOME='/'$SPARK_VERSION_HADOOP 
  - export PATH=$SPARK_HOME:$PATH 
  - export PYTHONPATH=$SPARK_HOME/python/:$PYTHON_PATH 
  - export PYSPARK_DRIVER_PATH="jupyter" 
  - export PYSPARK_DRIVER_PATH="notebook" 
  - export PYSPARK_PYTHON=python3 

start_jupyter:
  script:
    - jupyter notebook --ip 0.0.0.0 --no-browser --allow-root &
    - jupyter notebook list