Merge branch 'master' of github.com:OpenSPG/openspg into thinker_dedu…

…ce_openspg

README.md

@@ -6,7 +6,7 @@
 
 [OpenSPG](https://spg.openkg.cn/en-US) is a knowledge graph engine developed by Ant Group in collaboration with OpenKG, based on the SPG (Semantic-enhanced Programmable Graph) framework, which is a summary of Ant Group's years of experience in constructing and applying diverse domain knowledge graphs in the financial scenarios.
 
-![OpenSPG Architecture](https://mdn.alipayobjects.com/huamei_xgb3qj/afts/img/A*DsIHS7Fe78AAAAAAAAAAAAAADtmcAQ/original)
+![OpenSPG Architecture](https://github.com/user-attachments/assets/56c2772b-7343-41ce-a17e-f6d838d2f791)
 
 # SPG Background
 

builder/core/pom.xml

@@ -43,6 +43,10 @@
       <groupId>com.antgroup.openspg.cloudext</groupId>
       <artifactId>cloudext-impl-graph-store-tugraph</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.antgroup.openspg.cloudext</groupId>
+      <artifactId>cloudext-impl-graph-store-neo4j</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.antgroup.openspg.cloudext</groupId>
       <artifactId>cloudext-interface-search-engine</artifactId>
@@ -51,6 +55,18 @@
       <groupId>com.antgroup.openspg.cloudext</groupId>
       <artifactId>cloudext-impl-search-engine-elasticsearch</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.antgroup.openspg.cloudext</groupId>
+      <artifactId>cloudext-impl-search-engine-neo4j</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.antgroup.openspg.cloudext</groupId>
+      <artifactId>cloudext-interface-cache</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.antgroup.openspg.cloudext</groupId>
+      <artifactId>cloudext-impl-cache-redis</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.antgroup.openspg.server</groupId>
       <artifactId>core-schema-model</artifactId>

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/BuilderIndexNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.BuilderIndexNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class BuilderIndexNode extends BaseLogicalNode<BuilderIndexNodeConfig> {
+
+  public BuilderIndexNode(String id, String name, BuilderIndexNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.BUILDER_INDEX, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/ExtractPostProcessorNode.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.ExtractPostProcessorNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class ExtractPostProcessorNode extends BaseLogicalNode<ExtractPostProcessorNodeConfig> {
+
+  public ExtractPostProcessorNode(
+      String id, String name, ExtractPostProcessorNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.EXTRACT_POST_PROCESSOR, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/LLMNlExtractNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.LLMNlExtractNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class LLMNlExtractNode extends BaseLogicalNode<LLMNlExtractNodeConfig> {
+
+  public LLMNlExtractNode(String id, String name, LLMNlExtractNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.LLM_NL_EXTRACT, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/LogicalPlan.java

@@ -17,6 +17,7 @@
 import com.antgroup.openspg.builder.model.pipeline.Node;
 import com.antgroup.openspg.builder.model.pipeline.Pipeline;
 import com.antgroup.openspg.builder.model.pipeline.config.*;
+import com.antgroup.openspg.builder.model.pipeline.config.predicting.VectorizerProcessorNodeConfig;
 import java.io.Serializable;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -83,12 +84,30 @@ private static BaseLogicalNode<?> parse(Node node) {
       case CSV_SOURCE:
         return new CsvSourceNode(
             node.getId(), node.getName(), (CsvSourceNodeConfig) node.getNodeConfig());
+      case STRING_SOURCE:
+        return new StringSourceNode(
+            node.getId(), node.getName(), (StringSourceNodeConfig) node.getNodeConfig());
+      case BUILDER_INDEX:
+        return new BuilderIndexNode(
+            node.getId(), node.getName(), (BuilderIndexNodeConfig) node.getNodeConfig());
+      case PARAGRAPH_SPLIT:
+        return new ParagraphSplitNode(
+            node.getId(), node.getName(), (ParagraphSplitNodeConfig) node.getNodeConfig());
       case USER_DEFINED_EXTRACT:
         return new UserDefinedExtractNode(
             node.getId(), node.getName(), (UserDefinedExtractNodeConfig) node.getNodeConfig());
       case LLM_BASED_EXTRACT:
         return new LLMBasedExtractNode(
             node.getId(), node.getName(), (LLMBasedExtractNodeConfig) node.getNodeConfig());
+      case LLM_NL_EXTRACT:
+        return new LLMNlExtractNode(
+            node.getId(), node.getName(), (LLMNlExtractNodeConfig) node.getNodeConfig());
+      case EXTRACT_POST_PROCESSOR:
+        return new ExtractPostProcessorNode(
+            node.getId(), node.getName(), (ExtractPostProcessorNodeConfig) node.getNodeConfig());
+      case VECTORIZER_PROCESSOR:
+        return new VectorizerProcessorNode(
+            node.getId(), node.getName(), (VectorizerProcessorNodeConfig) node.getNodeConfig());
       case SPG_TYPE_MAPPINGS:
         return new SPGTypeMappingNode(
             node.getId(), node.getName(), (SPGTypeMappingNodeConfigs) node.getNodeConfig());
@@ -98,6 +117,9 @@ private static BaseLogicalNode<?> parse(Node node) {
       case GRAPH_SINK:
         return new GraphStoreSinkNode(
             node.getId(), node.getName(), (GraphStoreSinkNodeConfig) node.getNodeConfig());
+      case NEO4J_SINK:
+        return new Neo4jSinkNode(
+            node.getId(), node.getName(), (Neo4jSinkNodeConfig) node.getNodeConfig());
       default:
         throw new IllegalArgumentException("illegal nodeType=" + node.getType());
     }

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/Neo4jSinkNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.Neo4jSinkNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class Neo4jSinkNode extends BaseLogicalNode<Neo4jSinkNodeConfig> {
+
+  public Neo4jSinkNode(String id, String name, Neo4jSinkNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.NEO4J_SINK, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/ParagraphSplitNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.ParagraphSplitNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class ParagraphSplitNode extends BaseLogicalNode<ParagraphSplitNodeConfig> {
+
+  public ParagraphSplitNode(String id, String name, ParagraphSplitNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.PARAGRAPH_SPLIT, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/StringSourceNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.StringSourceNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class StringSourceNode extends BaseLogicalNode<StringSourceNodeConfig> {
+
+  public StringSourceNode(String id, String name, StringSourceNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.STRING_SOURCE, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/logical/VectorizerProcessorNode.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 OpenSPG Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied.
+ */
+
+package com.antgroup.openspg.builder.core.logical;
+
+import com.antgroup.openspg.builder.model.pipeline.config.predicting.VectorizerProcessorNodeConfig;
+import com.antgroup.openspg.builder.model.pipeline.enums.NodeTypeEnum;
+
+public class VectorizerProcessorNode extends BaseLogicalNode<VectorizerProcessorNodeConfig> {
+
+  public VectorizerProcessorNode(String id, String name, VectorizerProcessorNodeConfig nodeConfig) {
+    super(id, name, NodeTypeEnum.VECTORIZER_PROCESSOR, nodeConfig);
+  }
+}

builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/PhysicalPlan.java

@@ -17,6 +17,7 @@
 import com.antgroup.openspg.builder.core.logical.LogicalPlan;
 import com.antgroup.openspg.builder.core.physical.process.*;
 import com.antgroup.openspg.builder.model.pipeline.config.*;
+import com.antgroup.openspg.builder.model.pipeline.config.predicting.VectorizerProcessorNodeConfig;
 import java.io.Serializable;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -88,6 +89,21 @@ private static Graph<BaseProcessor<?>, DefaultEdge> newGraph() {
    */
   private static BaseProcessor<?> parse(BaseLogicalNode<?> node) {
     switch (node.getType()) {
+      case PARAGRAPH_SPLIT:
+        return new ParagraphSplitProcessor(
+            node.getId(), node.getName(), (ParagraphSplitNodeConfig) node.getNodeConfig());
+      case BUILDER_INDEX:
+        return new BuilderIndexProcessor(
+            node.getId(), node.getName(), (BuilderIndexNodeConfig) node.getNodeConfig());
+      case LLM_NL_EXTRACT:
+        return new LLMNlExtractProcessor(
+            node.getId(), node.getName(), (LLMNlExtractNodeConfig) node.getNodeConfig());
+      case EXTRACT_POST_PROCESSOR:
+        return new ExtractPostProcessor(
+            node.getId(), node.getName(), (ExtractPostProcessorNodeConfig) node.getNodeConfig());
+      case VECTORIZER_PROCESSOR:
+        return new VectorizerProcessor(
+            node.getId(), node.getName(), (VectorizerProcessorNodeConfig) node.getNodeConfig());
       case USER_DEFINED_EXTRACT:
         return new UserDefinedExtractProcessor(
             node.getId(), node.getName(), (UserDefinedExtractNodeConfig) node.getNodeConfig());