MooseFS

MooseFS/docker-compose.yml

@@ -0,0 +1,81 @@
+version: "3"
+
+services:
+  weba:
+    build:
+      context: ./weba
+    ports:
+      - "8001:5000"
+    volumes:
+      - type: volume
+        source: mfs_data
+        target: /var/lib/mfs
+    depends_on:
+      - chunkserver1
+      - chunkserver2
+    # restart: always
+
+  webb:
+    build:
+      context: ./webb
+    ports:
+      - "8002:5000"
+    volumes:
+      - type: volume
+        source: mfs_data
+        target: /var/lib/mfs
+    depends_on:
+      - chunkserver1
+      - chunkserver2
+    # restart: always
+  mfsmaster:
+    build: ./moosefs-master
+    # image: moosefs/moosefs-master
+    container_name: mfsmaster
+    ports:
+      - "9421:9421"
+      - "9422:9422"
+      - "9419:9419"
+      - "9433:9433"
+    volumes:
+      - type: volume
+        source: mfs_data
+        target: /var/lib/mfs
+    environment:
+      - MFS_ENV=TEST 
+      # In the TEST environment, if metadata.mfs file or other metadata files
+      # like metdata.mfs.back and changelog.0.mfs are missing, the script will
+      # create an EMPTY metadata file!
+      # When MFS_ENV=PROD metdata.mfs file will not be created!
+
+  chunkserver1:
+    build: ./moosefs-chunkserver
+    container_name: chunkserver1
+    volumes:
+      - type: volume
+        source: mfs_data
+        target: /data
+    environment:
+      - LABELS=M
+      #- SIZE=100
+    links:
+      - mfsmaster
+    depends_on:
+      - mfsmaster
+  chunkserver2:
+    build: ./moosefs-chunkserver
+    container_name: chunkserver2
+    volumes:
+      - type: volume
+        source: mfs_data
+        target: /data
+    environment:
+      - LABELS=M,B
+      #- SIZE=100
+    links:
+      - mfsmaster
+    depends_on:
+      - mfsmaster
+
+volumes:
+  mfs_data:

MooseFS/dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8-slim-buster
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD [ "python3", "app.py" ]

MooseFS/moosefs-chunkserver/Dockerfile

@@ -0,0 +1,20 @@
+FROM debian:buster
+
+# Install wget, lsb-release and curl
+RUN apt-get update && apt-get install -y wget gnupg2 gettext-base
+
+# Add key
+RUN wget -O - http://ppa.moosefs.com/moosefs.key 2>/dev/null | apt-key add - 2>/dev/null
+RUN echo "deb http://ppa.moosefs.com/3.0.115/apt/debian/buster buster main" > /etc/apt/sources.list.d/moosefs.list
+
+# Install MooseFS chunkserver
+RUN apt-get update && apt-get install -y moosefs-chunkserver dos2unix
+
+# Expose ports
+EXPOSE 9422
+
+# Add start script
+ADD chunkserver.sh /usr/sbin/chunkserver.sh
+RUN chown root:root /usr/sbin/chunkserver.sh ; chmod 700 /usr/sbin/chunkserver.sh ; mkdir -p /mnt/hdd0
+RUN dos2unix /usr/sbin/chunkserver.sh
+CMD ["chunkserver.sh"]

MooseFS/moosefs-chunkserver/chunkserver.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+mkdir -p /mnt/hdd0/mfs
+# Set correct owner
+chown -R mfs:mfs /mnt/hdd0 /mnt/hdd0/mfs /var/lib/mfs
+
+# Overwrite mfschunkserver.cfg if passed in
+# this will base64 decode MFS_CHUNKSERVER_CONFIG variable text
+# substitute any env variables in decoded text
+# save text into /etc/mfs/mfschunkserver.cfg
+if [ ! -z ${MFS_CHUNKSERVER_CONFIG+X} ];
+    then
+        echo $MFS_CHUNKSERVER_CONFIG | base64 -d | envsubst > /etc/mfs/mfschunkserver.cfg
+fi
+
+# Overwrite mfshdd.cfg if passed in
+# this will base64 decode MFS_HDD_CONFIG variable text
+# substitute any env variables in decoded text
+# save text into /etc/mfs/mfshdd.cfg
+if [ ! -z ${MFS_HDD_CONFIG+X} ];
+    then
+        echo $MFS_HDD_CONFIG | base64 -d | envsubst > /etc/mfs/mfshdd.cfg
+fi
+
+
+#Add size to hdd if defined
+if [ -z ${SIZE+X} ];
+    then
+        echo "/mnt/hdd0" > /etc/mfs/mfshdd.cfg
+    else
+        echo "/mnt/hdd0 ${SIZE}GiB" > /etc/mfs/mfshdd.cfg
+fi
+
+# Add label if defined
+if [ ! -z ${LABELS+X} ];
+    then
+        echo "LABELS=$LABELS" >> /etc/mfs/mfschunkserver.cfg
+fi
+
+if [ ! -z ${MASTER_HOST+X} ];
+    then
+        echo "MASTER_HOST=$MASTER_HOST" >> /etc/mfs/mfschunkserver.cfg
+fi
+if [ ! -z ${CSSERV_LISTEN_PORT+X} ];
+    then
+        echo "CSSERV_LISTEN_PORT=$CSSERV_LISTEN_PORT" >> /etc/mfs/mfschunkserver.cfg
+fi
+if [ ! -z ${DATA_PATH+X} ];
+    then
+        echo "DATA_PATH=$DATA_PATH" >> /etc/mfs/mfschunkserver.cfg
+fi
+
+exec mfschunkserver -f

MooseFS/moosefs-chunkserver/mfschunserver.cfg

@@ -0,0 +1,3 @@
+MASTER_HOST = $MOOSEFS_MASTER_SERVICE_HOST
+CSSERV_LISTEN_PORT = $MOOSEFS_CHUNKSERVER_SERVICE_PORT
+DATA_PATH = /mnt/hdd0/mfs

MooseFS/moosefs-master/Dockerfile

@@ -0,0 +1,20 @@
+FROM debian:buster
+
+# Install wget and gnupg2
+RUN apt-get update && apt-get install -y wget gnupg2
+
+# Add key
+RUN wget -O - http://ppa.moosefs.com/moosefs.key 2>/dev/null | apt-key add - 2>/dev/null
+RUN echo "deb http://ppa.moosefs.com/3.0.115/apt/debian/buster buster main" > /etc/apt/sources.list.d/moosefs.list
+
+# Install MooseFS master
+RUN apt-get update && apt-get install -y moosefs-master dos2unix
+
+# Expose ports master ports
+EXPOSE 9419 9420 9421
+
+# Add and run start script
+ADD master.sh /usr/sbin/master.sh
+RUN chown root:root /usr/sbin/master.sh ; chmod 700 /usr/sbin/master.sh
+RUN dos2unix /usr/sbin/master.sh
+CMD ["master.sh"]

MooseFS/moosefs-master/master.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Set default MooseFS enviroment to PRODUCTION
+MFS_ENV="${MFS_ENV:-PROD}"
+
+#Set correct owner
+chown -R mfs:mfs /var/lib/mfs
+
+# Overwrite mfsmaster.cfg if passed in
+# this will base64 decode MFS_MASTER_CONFIG variable text
+# substitute any env variables in decoded text
+# save text into /etc/mfs/mfsmaster.cfg
+if [ ! -z ${MFS_MASTER_CONFIG+X} ];
+    then
+        echo $MFS_MASTER_CONFIG | base64 -d | envsubst > /etc/mfs/mfsmaster.cfg
+fi
+
+# We have to be sure that we have metadata files
+if [ -f /var/lib/mfs/metadata.mfs ];
+then
+    exec mfsmaster -f
+else
+    if [[ -f /var/lib/mfs/metadata.mfs.back.1 && -f /var/lib/mfs/changelog.0.mfs ]];
+    then
+        echo "Can't find metadata.mfs file"
+        echo "Let's try to restore it"
+        exec mfsmaster -a -f
+    else
+        if [  "$MFS_ENV" == "TEST" ];
+        then
+            echo "MFSM NEW" > /var/lib/mfs/metadata.mfs
+            exec mfsmaster -f
+        else
+            echo "No /var/lib/mfs/metadata.mfs file!"
+            echo "EXITING - THIS IS A PRODUCTION ENVIRONMENT!"
+            exit 1
+        fi
+    fi
+fi

MooseFS/weba/app.py

@@ -0,0 +1,13 @@
+import os
+
+path = "/var/lib/mfs/test.txt"
+
+if not os.path.exists(os.path.dirname(path)):
+    os.makedirs(os.path.dirname(path))
+
+with open(path, "w") as f:
+    f.write("Hello World! from webA")
+
+
+if __name__ == "__main__":
+    print("File written successfully from webA!")

MooseFS/weba/dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.9
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD [ "python", "app.py", "write" ]

MooseFS/webb/app.py

@@ -0,0 +1,17 @@
+import os
+
+path = "/var/lib/mfs/test.txt"
+
+# if not os.path.exists(os.path.dirname(path)):
+#     os.makedirs(os.path.dirname(path))
+
+if os.path.exists(path):
+    with open(path, "r") as f:
+        print(f.read())
+    os.remove(path)
+else:
+    print("File not found!")
+
+
+if __name__ == "__main__":
+    print("File read successfully from webB!")

MooseFS/webb/dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.9
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD [ "python", "app.py", "read" ]