feat(collection): introduce collection component (#353)

Because

- Users may need to perform data declaration or manipulation.

This commit

- Introduces the `collection` component with `TASK_DECLARE` and
`TASK_APPEND_ARRAY` tasks.
- `TASK_DECLARE` is used to declare a data variable.
- `TASK_APPEND_ARRAY` is used to append data to an existing array.

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     hooks:
       - id: check-yaml
       - id: check-json
+        exclude: ai/openai/v1
       - id: check-merge-conflict
       - id: end-of-file-fixer
         exclude: (?i).*testdata/
@@ -12,6 +13,7 @@ repos:
         exclude_types: [svg,mdx]
       - id: pretty-format-json
         args: [--autofix, --no-sort-keys]
+        exclude: ai/openai/v1
   - repo: https://github.com/dnephin/pre-commit-golang
     rev: v0.5.1
     hooks:

ai/universalai/v0/component_test.go

@@ -1,2 +1,2 @@
 // TODO: chuang8511
-package universalai
+package universalai

ai/universalai/v0/config/tasks.json

@@ -436,4 +436,4 @@
       ]
     }
   }
-}
+}

application/github/v0/config/setup.json

@@ -16,8 +16,12 @@
       "type": "string"
     }
   },
-  "required": ["token"],
-  "instillEditOnNodeFields": ["token"],
+  "required": [
+    "token"
+  ],
+  "instillEditOnNodeFields": [
+    "token"
+  ],
   "instillOAuthConfig": {
     "authUrl": "https://github.com/login/oauth/authorize",
     "accessUrl": "https://github.com/login/oauth/access_token",

application/slack/v0/config/setup.json

@@ -23,9 +23,9 @@
     "token"
   ],
   "instillOAuthConfig": {
-    "authUrl":   "https://slack.com/oauth/v2/authorize",
+    "authUrl": "https://slack.com/oauth/v2/authorize",
     "accessUrl": "https://slack.com/api/oauth.v2.access",
-    "scopes":    []
+    "scopes": []
   },
   "title": "Slack Connection",
   "type": "object"

base/component.go

@@ -125,7 +125,7 @@ func convertDataSpecToCompSpec(dataSpec *structpb.Struct) (*structpb.Struct, err
 	isFreeform := checkFreeForm(compSpec)
 
 	if _, ok := compSpec.Fields["type"]; !ok && !isFreeform {
-		return nil, fmt.Errorf("type missing: %+v", compSpec)
+		return nil, fmt.Errorf("type missing: %+v ee2 %+v", compSpec, isFreeform)
 	} else if _, ok := compSpec.Fields["instillUpstreamTypes"]; !ok && compSpec.Fields["type"].GetStringValue() == "object" {
 
 		if _, ok := compSpec.Fields["instillUIOrder"]; !ok {
@@ -349,14 +349,21 @@ func generateComponentSpec(title string, tasks []*pb.ComponentTask, taskStructs
 func formatDataSpec(dataSpec *structpb.Struct) (*structpb.Struct, error) {
 	// var err error
 	compSpec := proto.Clone(dataSpec).(*structpb.Struct)
+	if compSpec == nil {
+		return compSpec, nil
+	}
+	if compSpec.Fields == nil {
+		compSpec.Fields = make(map[string]*structpb.Value)
+		return compSpec, nil
+	}
 	if _, ok := compSpec.Fields["const"]; ok {
 		return compSpec, nil
 	}
 
 	isFreeform := checkFreeForm(compSpec)
 
 	if _, ok := compSpec.Fields["type"]; !ok && !isFreeform {
-		return nil, fmt.Errorf("type missing: %+v", compSpec)
+		return nil, fmt.Errorf("type missing: %+v ee %+v", compSpec, isFreeform)
 	} else if compSpec.Fields["type"].GetStringValue() == "array" {
 
 		if _, ok := compSpec.Fields["instillUIOrder"]; !ok {
@@ -579,6 +586,9 @@ func checkFreeForm(compSpec *structpb.Struct) bool {
 	if instillFormat := compSpec.Fields["instillFormat"].GetStringValue(); instillFormat != "" {
 		formats = append(formats, instillFormat)
 	}
+	if len(formats) == 0 {
+		return true
+	}
 
 	for _, v := range formats {
 		if v.(string) == "*" || v.(string) == "semi-structured/*" || v.(string) == "semi-structured/json" {

generic/collection/v0/README.mdx

@@ -0,0 +1,95 @@
+---
+title: "Collection"
+lang: "en-US"
+draft: false
+description: "Learn about how to set up a VDP Collection component https://github.com/instill-ai/instill-core"
+---
+
+The Collection component is a generic component that allows users to manipulate collection-type data.
+It can carry out the following tasks:
+
+- [Declare](#declare)
+- [Append Array](#append-array)
+
+
+
+## Release Stage
+
+`Alpha`
+
+
+
+## Configuration
+
+The component configuration is defined and maintained [here](https://github.com/instill-ai/component/blob/main/generic/collection/v0/config/definition.json).
+
+
+
+
+
+
+
+## Supported Tasks
+
+### Declare
+
+Set the data.
+
+
+| Input | ID | Type | Description |
+| :--- | :--- | :--- | :--- |
+| Task ID (required) | `task` | string | `TASK_DECLARE` |
+| Data (required) | `data` | any | Specify the data you want to set. |
+
+
+
+
+
+
+
+
+
+| Output | ID | Type | Description |
+| :--- | :--- | :--- | :--- |
+| Data | `data` | any | The data you set. |
+
+
+
+
+
+
+
+
+
+### Append Array
+
+Add data to the end of an array.
+
+
+| Input | ID | Type | Description |
+| :--- | :--- | :--- | :--- |
+| Task ID (required) | `task` | string | `TASK_APPEND_ARRAY` |
+| Array (required) | `array` | array | Specify the array you want to append to. |
+| Data (required) | `element` | any | Specify the data you want to append. |
+
+
+
+
+
+
+
+
+
+| Output | ID | Type | Description |
+| :--- | :--- | :--- | :--- |
+| Array | `array` | array | A updated array with the specified data appended to the end of it. |
+
+
+
+
+
+
+
+
+
+

generic/collection/v0/config/definition.json

@@ -0,0 +1,20 @@
+{
+  "availableTasks": [
+    "TASK_DECLARE",
+    "TASK_APPEND_ARRAY"
+  ],
+  "custom": false,
+  "documentationUrl": "https://www.instill.tech/docs/component/generic/collection",
+  "icon": "assets/collection.svg",
+  "iconUrl": "",
+  "id": "collection",
+  "public": true,
+  "spec": {},
+  "title": "Collection",
+  "type": "COMPONENT_TYPE_GENERIC",
+  "uid": "eb611e31-fbe6-43ad-8671-5b9a2e351638",
+  "version": "0.1.0",
+  "sourceUrl": "https://github.com/instill-ai/component/blob/main/generic/collection/v0",
+  "description": "Manipulate collection-type data",
+  "releaseStage": "RELEASE_STAGE_ALPHA"
+}

generic/collection/v0/config/tasks.json

@@ -0,0 +1,130 @@
+{
+  "TASK_DECLARE": {
+    "instillShortDescription": "Set the data.",
+    "input": {
+      "description": "Input",
+      "instillEditOnNodeFields": [
+        "data"
+      ],
+      "instillUIOrder": 0,
+      "properties": {
+        "data": {
+          "description": "Specify the data you want to set.",
+          "instillAcceptFormats": [
+            "*"
+          ],
+          "instillUIMultiline": true,
+          "instillUIOrder": 0,
+          "instillUpstreamTypes": [
+            "value",
+            "reference",
+            "template"
+          ],
+          "required": [],
+          "title": "Data"
+        }
+      },
+      "required": [
+        "data"
+      ],
+      "title": "Input",
+      "type": "object"
+    },
+    "output": {
+      "description": "Output",
+      "instillEditOnNodeFields": [
+        "data"
+      ],
+      "instillUIOrder": 0,
+      "properties": {
+        "data": {
+          "description": "The data you set.",
+          "instillEditOnNodeFields": [],
+          "instillFormat": "*",
+          "instillUIOrder": 0,
+          "required": [],
+          "title": "Data"
+        }
+      },
+      "required": [
+        "data"
+      ],
+      "title": "Output",
+      "type": "object"
+    }
+  },
+  "TASK_APPEND_ARRAY": {
+    "instillShortDescription": "Add data to the end of an array.",
+    "input": {
+      "description": "Input",
+      "instillEditOnNodeFields": [
+        "array",
+        "element"
+      ],
+      "instillUIOrder": 0,
+      "properties": {
+        "array": {
+          "description": "Specify the array you want to append to.",
+          "instillAcceptFormats": [
+            "array:*"
+          ],
+          "instillUIMultiline": true,
+          "instillUIOrder": 0,
+          "instillUpstreamTypes": [
+            "value",
+            "reference",
+            "template"
+          ],
+          "items": {},
+          "required": [],
+          "title": "Array",
+          "type": "array"
+        },
+        "element": {
+          "description": "Specify the data you want to append.",
+          "instillAcceptFormats": [
+            "*"
+          ],
+          "instillUIMultiline": true,
+          "instillUIOrder": 0,
+          "instillUpstreamTypes": [
+            "value",
+            "reference",
+            "template"
+          ],
+          "required": [],
+          "title": "Data"
+        }
+      },
+      "required": [
+        "array",
+        "element"
+      ],
+      "title": "Input",
+      "type": "object"
+    },
+    "output": {
+      "description": "Output",
+      "instillEditOnNodeFields": [
+        "array"
+      ],
+      "instillUIOrder": 0,
+      "properties": {
+        "array": {
+          "description": "A updated array with the specified data appended to the end of it.",
+          "instillEditOnNodeFields": [],
+          "instillFormat": "array:*",
+          "instillUIOrder": 0,
+          "required": [],
+          "title": "Array",
+          "type": "array"
+        }
+      },
+      "required": [
+        "array"
+      ],
+      "title": "Output",
+      "type": "object"
+    }
+  }
+}