Merge pull request #10 from crumbjp/v0.2.0

V0.2.0

.github/workflows/main.yml

@@ -17,13 +17,20 @@ jobs:
         test_script:
           - test
     container:
-      image: crumbjp/faissdb@sha256:74b5a0fcf0e59ee1d2e2e5830a715266cf801ab26aaf1add7a575e4921e0a308
+      image: crumbjp/faissdb@sha256:9a664494d8c80317e4536d1083aa3c84cd27b09cac5b51b2355f6b4864b1f536
     steps:
-      - uses: actions/checkout@v2
+      - name: Set up a Git safe directory
+        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+
+      - uses: actions/[email protected]
 
       - name: prepare
         run: bash ci/prepare.sh
 
+      # - name: Debug
+      #   if: ${{ always() }}
+      #   uses: mxschmitt/action-tmate@v3
+
       - name: test_faissdb
         run: bash ci/test_faissdb.sh
 

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+node_modules

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "go-faiss"]
-	path = go-faiss
-	url = https://github.com/crumbjp/go-faiss.git
-[submodule "grpc-go"]
-	path = grpc-go
-	url = https://github.com/grpc/grpc-go

DEVELOPMENT.md

@@ -0,0 +1,29 @@
+## Build faissdb:build
+```
+git clone https://github.com/crumbjp/faissdb.git
+cd faissdb/docker
+./build.sh
+```
+
+### Start container
+```
+cd docker
+./dev.sh start_container
+```
+
+### Start process
+```
+cd docker
+./dev.sh start
+```
+
+### Rebuild
+Rebuild server from current source.
+
+```
+cd docker
+./dev.sh rebuild
+```
+
+## More detail
+[dev.sh](https://github.com/crumbjp/faissdb/tree/master/docker/dev.sh#L11).

QUICK_START.md

@@ -1,79 +1,45 @@
-## Simple primary/secondary/secondary
+## Simple primary
 ### Start faissdb
 #### Set ENV
 ```
 export FAISSDB_ROOT=[faissdb repository path]
-export FAISSDB_VERSION=0.1.4
+export FAISSDB_VERSION=0.2.0
 ```
 
 #### Get docker image
 ```
 docker pull crumbjp/faissdb:$FAISSDB_VERSION
 ```
 
-#### Start process1
+#### Start process
 ```
-rm -rf /tmp/faissdb1
-mkdir /tmp/faissdb1
-mkdir /tmp/faissdb1/log
-mkdir /tmp/faissdb1/data
-mkdir /tmp/faissdb1/tmp
+rm -rf /tmp/faissdb
+mkdir /tmp/faissdb
+mkdir /tmp/faissdb/log
+mkdir /tmp/faissdb/data
+mkdir /tmp/faissdb/tmp
 docker run \
  --rm \
  -v $FAISSDB_ROOT/config/example:/usr/local/faissdb/conf \
- -v /tmp/faissdb1/log:/usr/local/faissdb/log \
- -v /tmp/faissdb1/data:/usr/local/faissdb/data \
- -v /tmp/faissdb1/tmp:/usr/local/faissdb/tmp \
+ -v /tmp/faissdb/log:/usr/local/faissdb/log \
+ -v /tmp/faissdb/data:/usr/local/faissdb/data \
+ -v /tmp/faissdb/tmp:/usr/local/faissdb/tmp \
  -p 9091:9091 \
  -p 20021:20021 \
  -p 21021:21021 \
  crumbjp/faissdb:$FAISSDB_VERSION \
  /usr/local/faissdb/bin/faissdb /usr/local/faissdb/conf/config1.yml
 ```
 
-#### Start process2
-```
-rm -rf /tmp/faissdb2
-mkdir /tmp/faissdb2
-mkdir /tmp/faissdb2/log
-mkdir /tmp/faissdb2/data
-mkdir /tmp/faissdb2/tmp
-docker run \
- --rm \
- -v $FAISSDB_ROOT/config/example:/usr/local/faissdb/conf \
- -v /tmp/faissdb2/log:/usr/local/faissdb/log \
- -v /tmp/faissdb2/data:/usr/local/faissdb/data \
- -v /tmp/faissdb2/tmp:/usr/local/faissdb/tmp \
- -p 9092:9092 \
- -p 20022:20022 \
- -p 21022:21022 \
- crumbjp/faissdb:$FAISSDB_VERSION \
- /usr/local/faissdb/bin/faissdb /usr/local/faissdb/conf/config2.yml
-```
-
-#### Start process3
+#### Replica setting
 ```
-rm -rf /tmp/faissdb3
-mkdir /tmp/faissdb3
-mkdir /tmp/faissdb3/log
-mkdir /tmp/faissdb3/data
-mkdir /tmp/faissdb3/tmp
-docker run \
- --rm \
- -v $FAISSDB_ROOT/config/example:/usr/local/faissdb/conf \
- -v /tmp/faissdb3/log:/usr/local/faissdb/log \
- -v /tmp/faissdb3/data:/usr/local/faissdb/data \
- -v /tmp/faissdb3/tmp:/usr/local/faissdb/tmp \
- -p 9093:9093 \
- -p 20023:20023 \
- -p 21023:21023 \
- crumbjp/faissdb:$FAISSDB_VERSION \
- /usr/local/faissdb/bin/faissdb /usr/local/faissdb/conf/config3.yml
+#for Mac
+curl -v http://localhost:9091/replicaset -XPUT -d '{"replica": "rs", "members": [{"id": 1, "host": "host.docker.internal:21021", "primary": true}]}'
 ```
 
-#### Replica setting
 ```
-curl -v http://localhost:9091/replicaset -XPUT -d '{"replica": "rs", "members": [{"id": 1, "host": "host.docker.internal:21021", "primary": true}, {"id": 2, "host": "host.docker.internal:21022", "primary": false}, {"id": 3, "host": "host.docker.internal:21023", "primary": false}]}'
+#for Linux
+curl -v http://localhost:9091/replicaset -XPUT -d '{"replica": "rs", "members": [{"id": 1, "host": "localhost:21021", "primary": true}]}'
 ```
 
 

README.md

@@ -6,15 +6,17 @@
 - Simple replication is supported.
 - Faiss indexes are always sync with local storage to restart process quickly.
 - Provides [gRPC](https://grpc.io/) based client I/F, so easy to develop clients.
+- Current version 0.2.0
+-- faiss 1.9.0
+-- rocksdb 9.8.4
 
 ## About Faiss
 Faiss is one of the most advanced ANN (approximate nearest neighbor) search library.
 However, Faiss is fundamentally designed to working on memory, so it requires complicated ID management and data perpetuation by low layer programing language to using for search platform.
 
 ## Use
 - [Quick start (docker)](QUICK_START.md)
-- [Setup development environment(OSQ)](DEVELOPMENT_OSX.md)
-- [Setup development environment(ubuntu)](DEVELOPMENT_UBUNTU.md)
+- [Setup a development environment](DEVELOPMENT.md)
 
 ## Architecture
 ![](img/Architecture.png)
@@ -38,3 +40,10 @@ Almost same as Set.
 ## Client
 - Call gRPC directly. [feature.proto](/protos/feature.proto)
 - [nodejs](https://github.com/crumbjp/faissdb/tree/master/nodejs)
+
+## Replication (ReplicaSet)
+- A ReplicaSet includs one Primary-node and some secondary-nodes.
+- Can read and write from Primary-node, Secondary-node is read-only.
+- Primary-node write "indexes" and "oplog" when receiving write operations, Secondary-nodes will read "oplog" to sync.
+- Never promotes from secondary-node to primary-node automatically, needs reconfigure.
+- The case of adding the new secondary-node to the running ReplicaSet, The new node will sync all data from primary before starting service.

ci/prepare.sh

@@ -12,22 +12,13 @@ mkdir -p /tmp/faissdb3/log
 mkdir -p /tmp/faissdb3/tmp
 mkdir -p /tmp/faissdb3/data
 
-if [ "${UID}" = "0" ]; then
-    if [ "${HOME}" != "/root" ]; then
-        ln -s /root/go ~/
-        ln -s /root/.faissdb ~/
-    fi
-fi
-
-. ~/.faissdb
-git submodule init
-git submodule update
+. /usr/local/.faissdb
 
 pushd `dirname $0`/../server
+go mod tidy
 make
 popd
 
 pushd `dirname $0`/../nodejs
 npm install
-make
 popd