feat(component): implement run-on-event for Slack and GitHub component (

#842)

Because

 - We are introducing the run-on-event feature.

This commit

 - Implements the run-on-event framework.
 - Adds support for EVENT_NEW_MESSAGE in the Slack component.
 - Adds support for EVENT_STAR_CREATED in the GitHub component.

Makefile

@@ -17,13 +17,21 @@ dev:							## Run dev container
 	@docker inspect --type container ${SERVICE_NAME} >/dev/null 2>&1 && echo "A container named ${SERVICE_NAME} is already running." || \
 		echo "Run dev container ${SERVICE_NAME}. To stop it, run \"make stop\"."
 	@docker run -d --rm \
-		-v $(PWD):/${SERVICE_NAME} \
 		-p ${PUBLIC_SERVICE_PORT}:${PUBLIC_SERVICE_PORT} \
-		-p ${PRIVATE_SERVICE_PORT}:${PRIVATE_SERVICE_PORT} \
-		--env-file .env.component \
+		-v $(PWD)/../go.work:/go.work \
+		-v $(PWD)/../go.work.sum:/go.work.sum \
+		-v $(PWD)/../mgmt-backend:/mgmt-backend \
+		-v $(PWD)/../model-backend:/model-backend \
+		-v $(PWD)/../pipeline-backend:/pipeline-backend \
+		-v $(PWD)/../artifact-backend:/artifact-backend \
+		-v $(PWD)/../mgmt-backend-cloud:/mgmt-backend-cloud \
+		-v $(PWD)/../model-backend-cloud:/model-backend-cloud \
+		-v $(PWD)/../pipeline-backend-cloud:/pipeline-backend-cloud \
+		-v $(PWD)/../protogengo:/protogengo \
+		-v $(PWD)/../component:/component \
 		--network instill-network \
 		--name ${SERVICE_NAME} \
-		instill/${SERVICE_NAME}:dev >/dev/null 2>&1
+		instill/${SERVICE_NAME}:dev
 
 .PHONY: latest
 latest:							## Run latest container

cmd/init/definitionupdater/updater.go

@@ -22,7 +22,7 @@ import (
 func UpdateComponentDefinitionIndex(ctx context.Context, repo repository.Repository) error {
 	logger, _ := logger.GetZapLogger(ctx)
 
-	defs := componentstore.Init(logger, nil, nil).ListDefinitions(nil, true)
+	defs := componentstore.Init(logger, nil, nil, nil).ListDefinitions(nil, true)
 	for _, def := range defs {
 
 		if err := updateComponentDefinition(ctx, def, repo); err != nil {

cmd/init/presetdownloader/downloader.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/instill-ai/pipeline-backend/config"
 	"github.com/instill-ai/pipeline-backend/pkg/acl"
+	"github.com/instill-ai/pipeline-backend/pkg/component/store"
 	"github.com/instill-ai/pipeline-backend/pkg/constant"
 	"github.com/instill-ai/pipeline-backend/pkg/datamodel"
 	"github.com/instill-ai/pipeline-backend/pkg/external"
@@ -103,8 +104,16 @@ func DownloadPresetPipelines(ctx context.Context, repo repository.Repository) er
 	if err != nil {
 		return err
 	}
-
-	converter := service.NewConverter(mgmtPrivateServiceClient, redisClient, &aclClient, repo, "")
+	componentStore := store.Init(logger, config.Config.Component.Secrets, nil, nil)
+
+	converter := service.NewConverter(service.ConverterConfig{
+		MgmtClient:      mgmtPrivateServiceClient,
+		RedisClient:     redisClient,
+		ACLClient:       &aclClient,
+		Repository:      repo,
+		InstillCoreHost: "",
+		ComponentStore:  componentStore,
+	})
 
 	if config.Config.InstillCloud.Host == "" {
 		// Skip the download process if the Instill Cloud host is not set.

cmd/main/main.go

@@ -255,8 +255,10 @@ func main() {
 	if err != nil {
 		logger.Fatal("failed to create minio client", zap.Error(err))
 	}
-	workerUID, _ := uuid.NewV4()
-	compStore := componentstore.Init(logger, config.Config.Component.Secrets, nil)
+	minioxClientWrapper, err := miniox.NewMinioClient(ctx, &config.Config.Minio, logger)
+	if err != nil {
+		logger.Fatal("failed to create miniox client wrapper", zap.Error(err))
+	}
 
 	artifactPublicServiceClient, artifactPublicServiceClientConn := external.InitArtifactPublicServiceClient(ctx)
 	if artifactPublicServiceClientConn != nil {
@@ -268,19 +270,25 @@ func main() {
 		defer artifactPrivateServiceClientConn.Close()
 	}
 
-	minioxClientWrapper, err := miniox.NewMinioClient(ctx, &config.Config.Minio, logger)
-	if err != nil {
-		logger.Fatal("failed to create miniox client wrapper", zap.Error(err))
-	}
-
 	binaryFetcher := external.NewArtifactBinaryFetcher(artifactPrivateServiceClient, minioxClientWrapper)
+
+	compStore := componentstore.Init(logger, config.Config.Component.Secrets, nil, binaryFetcher)
+	workerUID, _ := uuid.NewV4()
+
 	service := service.NewService(
 		service.ServiceConfig{
-			Repository:               repo,
-			RedisClient:              redisClient,
-			TemporalClient:           temporalClient,
-			ACLClient:                &aclClient,
-			Converter:                service.NewConverter(mgmtPrivateServiceClient, redisClient, &aclClient, repo, config.Config.Server.InstillCoreHost),
+			Repository:     repo,
+			RedisClient:    redisClient,
+			TemporalClient: temporalClient,
+			ACLClient:      &aclClient,
+			Converter: service.NewConverter(service.ConverterConfig{
+				MgmtClient:      mgmtPrivateServiceClient,
+				RedisClient:     redisClient,
+				ACLClient:       &aclClient,
+				Repository:      repo,
+				InstillCoreHost: config.Config.Server.InstillCoreHost,
+				ComponentStore:  compStore,
+			}),
 			MgmtPrivateServiceClient: mgmtPrivateServiceClient,
 			MinioClient:              minioClient,
 			ComponentStore:           compStore,

config/config.yaml

@@ -26,7 +26,7 @@ database:
   host: pg-sql
   port: 5432
   name: pipeline
-  version: 36
+  version: 37
   timezone: Etc/UTC
   pool:
     idleconnections: 5

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/iancoleman/strcase v0.3.0
 	github.com/influxdata/influxdb-client-go/v2 v2.12.3
-	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241115081100-0db67f90ac71
+	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241117145210-23cd7077019e
 	github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
 	github.com/instill-ai/x v0.5.0-alpha.0.20241115170949-48880040d832
 	github.com/itchyny/gojq v0.12.14

go.sum

@@ -1281,8 +1281,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
 github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241115081100-0db67f90ac71 h1:Z1pq9oh8ica1Ngwqpu9cwqYNghiQlXNl8Dd/GbSD5wY=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241115081100-0db67f90ac71/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241117145210-23cd7077019e h1:rCclalq8AZUWZhcT2NQ4E6JHw2vSkLqqv7NzLnszxA0=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241117145210-23cd7077019e/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
 github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw=
 github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw=
 github.com/instill-ai/x v0.5.0-alpha.0.20241115170949-48880040d832 h1:Of782D2nDX1Iiv2WVdLIkOPO7fylhdHwiEoM2KCyU7c=

pkg/component/application/github/v0/commits.go

@@ -11,7 +11,11 @@ import (
 
 type RepositoriesService interface {
 	GetCommit(context.Context, string, string, string, *github.ListOptions) (*github.RepositoryCommit, *github.Response, error)
+	ListHooks(context.Context, string, string, *github.ListOptions) ([]*github.Hook, *github.Response, error)
+	GetHook(context.Context, string, string, int64) (*github.Hook, *github.Response, error)
 	CreateHook(context.Context, string, string, *github.Hook) (*github.Hook, *github.Response, error)
+	DeleteHook(context.Context, string, string, int64) (*github.Response, error)
+	EditHook(context.Context, string, string, int64, *github.Hook) (*github.Hook, *github.Response, error)
 }
 
 type Commit struct {