From ab396ffde7d2279832adf77616bff13716519e77 Mon Sep 17 00:00:00 2001 From: David Juhasz Date: Thu, 31 Oct 2024 13:19:09 -0700 Subject: [PATCH] Update workflow results The preprocessing result signature expected by Enduro was updated in commit 9564b21 [1]. This commit adds a workflow Outcome and a Bag creation event to the PreservationTasks in the returned results. Changes: - Add the "EventOutcome" enum for use in the PreservationTasks results - Add a `make gen-enums` target to the Makefile to support generating and updating enumerable value lists - Add the eventlog module for structuring PreservationTasks results - Update the name of the `bagit` activity to `bagcreate` and update its method names - Add a bag creation event to the workflow with success and error messages - Add a system error workflow test [1] https://github.com/artefactual-sdps/enduro/commit/9564b21386fb645998b115882d308d68ccfbc64d --- Makefile | 5 +- cmd/worker/workercmd/cmd.go | 6 +- go.mod | 22 +-- go.sum | 48 +++---- hack/make/dep_go_enum.mk | 20 +++ hack/make/enums.mk | 9 ++ hack/make/enums.tmpl | 38 +++++ internal/config/config.go | 4 +- internal/config/config_test.go | 4 +- internal/enums/event_outcome.go | 9 ++ internal/enums/event_outcome_enum.go | 183 ++++++++++++++++++++++++ internal/eventlog/eventlog.go | 40 ++++++ internal/eventlog/eventlog_test.go | 66 +++++++++ internal/workflow/preprocessing.go | 68 +++++++-- internal/workflow/preprocessing_test.go | 77 ++++++++-- 15 files changed, 539 insertions(+), 60 deletions(-) create mode 100644 hack/make/dep_go_enum.mk create mode 100644 hack/make/enums.mk create mode 100644 hack/make/enums.tmpl create mode 100644 internal/enums/event_outcome.go create mode 100644 internal/enums/event_outcome_enum.go create mode 100644 internal/eventlog/eventlog.go create mode 100644 internal/eventlog/eventlog_test.go diff --git a/Makefile b/Makefile index 25d76f3..71878d6 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ else endif include hack/make/bootstrap.mk +include hack/make/dep_go_enum.mk include hack/make/dep_golangci_lint.mk include hack/make/dep_golines.mk include hack/make/dep_gomajor.mk @@ -22,6 +23,7 @@ include hack/make/dep_gosec.mk include hack/make/dep_gotestsum.mk include hack/make/dep_shfmt.mk include hack/make/dep_tparse.mk +include hack/make/enums.mk # Lazy-evaluated list of tools. TOOLS = $(GOLANGCI_LINT) \ @@ -37,7 +39,8 @@ define NEWLINE endef IGNORED_PACKAGES := \ - github.com/artefactual-sdps/preprocessing-base/hack/% + github.com/artefactual-sdps/preprocessing-base/hack/% \ + github.com/artefactual-sdps/preprocessing-sfa/internal/enums PACKAGES := $(shell go list ./...) TEST_PACKAGES := $(filter-out $(IGNORED_PACKAGES),$(PACKAGES)) TEST_IGNORED_PACKAGES := $(filter $(IGNORED_PACKAGES),$(PACKAGES)) diff --git a/cmd/worker/workercmd/cmd.go b/cmd/worker/workercmd/cmd.go index 4543722..8b46a6e 100644 --- a/cmd/worker/workercmd/cmd.go +++ b/cmd/worker/workercmd/cmd.go @@ -3,7 +3,7 @@ package workercmd import ( "context" - "github.com/artefactual-sdps/temporal-activities/bagit" + "github.com/artefactual-sdps/temporal-activities/bagcreate" "github.com/go-logr/logr" "go.artefactual.dev/tools/temporal" temporalsdk_activity "go.temporal.io/sdk/activity" @@ -59,8 +59,8 @@ func (m *Main) Run(ctx context.Context) error { ) w.RegisterActivityWithOptions( - bagit.NewCreateBagActivity(m.cfg.Bagit).Execute, - temporalsdk_activity.RegisterOptions{Name: bagit.CreateBagActivityName}, + bagcreate.New(m.cfg.Bagit).Execute, + temporalsdk_activity.RegisterOptions{Name: bagcreate.Name}, ) if err := w.Start(); err != nil { diff --git a/go.mod b/go.mod index cd55874..5e07f9c 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/artefactual-sdps/preprocessing-base go 1.23.2 require ( - github.com/artefactual-sdps/temporal-activities v0.0.0-20240527203915-82579f8c5353 - github.com/go-logr/logr v1.4.1 + github.com/artefactual-sdps/temporal-activities v0.0.0-20241018212855-8ea34d29bdf4 + github.com/go-logr/logr v1.4.2 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.9.0 @@ -44,15 +44,15 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.6.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 976d497..31cf9fb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/artefactual-sdps/temporal-activities v0.0.0-20240527203915-82579f8c5353 h1:3TWzMIwtRO+eR3h+AUqGc/wNk0kmxzm8+mgEEoaSR2Q= -github.com/artefactual-sdps/temporal-activities v0.0.0-20240527203915-82579f8c5353/go.mod h1:C6z/8k6xFm9wrF4GSMKs13v941MtdrOzH2fn8hQEHtA= +github.com/artefactual-sdps/temporal-activities v0.0.0-20241018212855-8ea34d29bdf4 h1:WF95IOkZRVSCST/26SAqPYsUrtUuJpavBht6lvdeKl0= +github.com/artefactual-sdps/temporal-activities v0.0.0-20241018212855-8ea34d29bdf4/go.mod h1:FVh79rCGNlUU1QnioAU+lrSjLqrA1PJFYKIhWPsmyug= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -22,8 +22,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -82,8 +82,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -157,8 +157,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -166,8 +166,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -177,15 +177,15 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -205,19 +205,19 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda h1:b6F6WIV4xHHD0FA4oIyzU6mHWg2WI2X1RBehwa5QN38= -google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 h1:+/tmTy5zAieooKIXfzDm9KiA3Bv6JBwriRN9LY+yayk= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 h1:V71AcdLZr2p8dC9dbOIMCpqi4EmRl8wUwnJzXXLmbmc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/hack/make/dep_go_enum.mk b/hack/make/dep_go_enum.mk new file mode 100644 index 0000000..c077b48 --- /dev/null +++ b/hack/make/dep_go_enum.mk @@ -0,0 +1,20 @@ +$(call _assert_var,MAKEDIR) +$(call _conditional_include,$(MAKEDIR)/base.mk) +$(call _assert_var,UNAME_OS) +$(call _assert_var,UNAME_ARCH) +$(call _assert_var,CACHE_VERSIONS) +$(call _assert_var,CACHE_BIN) + +GO_ENUM_VERSION ?= 0.6.0 + +GO_ENUM := $(CACHE_VERSIONS)/go-enum/$(GO_ENUM_VERSION) +$(GO_ENUM): + rm -f $(CACHE_BIN)/go-enum + mkdir -p $(CACHE_BIN) + curl -sSL \ + https://github.com/abice/go-enum/releases/download/v$(GO_ENUM_VERSION)/go-enum_$(UNAME_OS)_$(UNAME_ARCH) \ + > $(CACHE_BIN)/go-enum + chmod +x $(CACHE_BIN)/go-enum + rm -rf $(dir $(GO_ENUM)) + mkdir -p $(dir $(GO_ENUM)) + touch $(GO_ENUM) diff --git a/hack/make/enums.mk b/hack/make/enums.mk new file mode 100644 index 0000000..4ed3fa0 --- /dev/null +++ b/hack/make/enums.mk @@ -0,0 +1,9 @@ +ENUMS := \ + internal/enums/event_outcome_enum.go + +$(ENUMS): GO_ENUM_FLAGS=--marshal --names --ptr --flag --sql --template=$(CURDIR)/hack/make/enums.tmpl + +gen-enums: $(ENUMS) # @HELP Generate go-enum assets. + +%_enum.go: %.go $(GO_ENUM) hack/make/enums.mk hack/make/enums.tmpl + go-enum -f $*.go $(GO_ENUM_FLAGS) diff --git a/hack/make/enums.tmpl b/hack/make/enums.tmpl new file mode 100644 index 0000000..8ab2173 --- /dev/null +++ b/hack/make/enums.tmpl @@ -0,0 +1,38 @@ +// Values implements the entgo.io/ent/schema/field EnumValues interface. +func (x {{.enum.Name}}) Values() []string { + return {{.enum.Name}}Names() +} + +// {{.enum.Name}}Interfaces returns an interface list of possible values of {{.enum.Name}}. +func {{.enum.Name}}Interfaces() []interface{} { + var tmp []interface{} + for _, v := range _{{.enum.Name}}Names { + tmp = append(tmp, v) + } + return tmp +} + +// Parse{{.enum.Name}}WithDefault attempts to convert a string to a ContentType. +// It returns the default value if name is empty. +func Parse{{.enum.Name}}WithDefault(name string) ({{.enum.Name}}, error) { + if name == "" { + return _{{.enum.Name}}Value[_{{.enum.Name}}Names[0]], nil + } + if x, ok := _{{.enum.Name}}Value[name]; ok { + return x, nil + } + return {{.enum.Name}}(""), fmt.Errorf("%s is not a valid {{.enum.Name}}, try [%s]", name, strings.Join(_{{.enum.Name}}Names, ", ")) +} + +// Normalize{{.enum.Name}} attempts to parse a and normalize string as content type. +// It returns the input untouched if name fails to be parsed. +// Example: +// +// "enUM" will be normalized (if possible) to "Enum" +func Normalize{{.enum.Name}}(name string) string { + res, err := Parse{{.enum.Name}}(name) + if err != nil { + return name + } + return res.String() +} diff --git a/internal/config/config.go b/internal/config/config.go index 4a75398..f6d6bce 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,7 +6,7 @@ import ( "os" "strings" - "github.com/artefactual-sdps/temporal-activities/bagit" + "github.com/artefactual-sdps/temporal-activities/bagcreate" "github.com/spf13/viper" ) @@ -33,7 +33,7 @@ type Configuration struct { Temporal Temporal Worker WorkerConfig - Bagit bagit.Config + Bagit bagcreate.Config } type Temporal struct { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b4bccc1..355c454 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,7 +3,7 @@ package config_test import ( "testing" - "github.com/artefactual-sdps/temporal-activities/bagit" + "github.com/artefactual-sdps/temporal-activities/bagcreate" "gotest.tools/v3/assert" "gotest.tools/v3/fs" @@ -57,7 +57,7 @@ func TestConfig(t *testing.T) { Worker: config.WorkerConfig{ MaxConcurrentSessions: 1, }, - Bagit: bagit.Config{ + Bagit: bagcreate.Config{ ChecksumAlgorithm: "md5", }, }, diff --git a/internal/enums/event_outcome.go b/internal/enums/event_outcome.go new file mode 100644 index 0000000..cb9526f --- /dev/null +++ b/internal/enums/event_outcome.go @@ -0,0 +1,9 @@ +package enums + +// ENUM( +// unspecified +// success +// system failure +// validation failure +// ). +type EventOutcome string diff --git a/internal/enums/event_outcome_enum.go b/internal/enums/event_outcome_enum.go new file mode 100644 index 0000000..cec9034 --- /dev/null +++ b/internal/enums/event_outcome_enum.go @@ -0,0 +1,183 @@ +// Code generated by go-enum DO NOT EDIT. +// Version: 0.6.0 +// Revision: 919e61c0174b91303753ee3898569a01abb32c97 +// Build Date: 2023-12-18T15:54:43Z +// Built By: goreleaser + +package enums + +import ( + "database/sql/driver" + "errors" + "fmt" + "strings" +) + +const ( + // EventOutcomeUnspecified is a EventOutcome of type unspecified. + EventOutcomeUnspecified EventOutcome = "unspecified" + // EventOutcomeSuccess is a EventOutcome of type success. + EventOutcomeSuccess EventOutcome = "success" + // EventOutcomeSystemFailure is a EventOutcome of type system failure. + EventOutcomeSystemFailure EventOutcome = "system failure" + // EventOutcomeValidationFailure is a EventOutcome of type validation failure. + EventOutcomeValidationFailure EventOutcome = "validation failure" +) + +var ErrInvalidEventOutcome = fmt.Errorf("not a valid EventOutcome, try [%s]", strings.Join(_EventOutcomeNames, ", ")) + +var _EventOutcomeNames = []string{ + string(EventOutcomeUnspecified), + string(EventOutcomeSuccess), + string(EventOutcomeSystemFailure), + string(EventOutcomeValidationFailure), +} + +// EventOutcomeNames returns a list of possible string values of EventOutcome. +func EventOutcomeNames() []string { + tmp := make([]string, len(_EventOutcomeNames)) + copy(tmp, _EventOutcomeNames) + return tmp +} + +// String implements the Stringer interface. +func (x EventOutcome) String() string { + return string(x) +} + +// IsValid provides a quick way to determine if the typed value is +// part of the allowed enumerated values +func (x EventOutcome) IsValid() bool { + _, err := ParseEventOutcome(string(x)) + return err == nil +} + +var _EventOutcomeValue = map[string]EventOutcome{ + "unspecified": EventOutcomeUnspecified, + "success": EventOutcomeSuccess, + "system failure": EventOutcomeSystemFailure, + "validation failure": EventOutcomeValidationFailure, +} + +// ParseEventOutcome attempts to convert a string to a EventOutcome. +func ParseEventOutcome(name string) (EventOutcome, error) { + if x, ok := _EventOutcomeValue[name]; ok { + return x, nil + } + return EventOutcome(""), fmt.Errorf("%s is %w", name, ErrInvalidEventOutcome) +} + +func (x EventOutcome) Ptr() *EventOutcome { + return &x +} + +// MarshalText implements the text marshaller method. +func (x EventOutcome) MarshalText() ([]byte, error) { + return []byte(string(x)), nil +} + +// UnmarshalText implements the text unmarshaller method. +func (x *EventOutcome) UnmarshalText(text []byte) error { + tmp, err := ParseEventOutcome(string(text)) + if err != nil { + return err + } + *x = tmp + return nil +} + +var errEventOutcomeNilPtr = errors.New("value pointer is nil") // one per type for package clashes + +// Scan implements the Scanner interface. +func (x *EventOutcome) Scan(value interface{}) (err error) { + if value == nil { + *x = EventOutcome("") + return + } + + // A wider range of scannable types. + // driver.Value values at the top of the list for expediency + switch v := value.(type) { + case string: + *x, err = ParseEventOutcome(v) + case []byte: + *x, err = ParseEventOutcome(string(v)) + case EventOutcome: + *x = v + case *EventOutcome: + if v == nil { + return errEventOutcomeNilPtr + } + *x = *v + case *string: + if v == nil { + return errEventOutcomeNilPtr + } + *x, err = ParseEventOutcome(*v) + default: + return errors.New("invalid type for EventOutcome") + } + + return +} + +// Value implements the driver Valuer interface. +func (x EventOutcome) Value() (driver.Value, error) { + return x.String(), nil +} + +// Set implements the Golang flag.Value interface func. +func (x *EventOutcome) Set(val string) error { + v, err := ParseEventOutcome(val) + *x = v + return err +} + +// Get implements the Golang flag.Getter interface func. +func (x *EventOutcome) Get() interface{} { + return *x +} + +// Type implements the github.com/spf13/pFlag Value interface. +func (x *EventOutcome) Type() string { + return "EventOutcome" +} + +// Values implements the entgo.io/ent/schema/field EnumValues interface. +func (x EventOutcome) Values() []string { + return EventOutcomeNames() +} + +// EventOutcomeInterfaces returns an interface list of possible values of EventOutcome. +func EventOutcomeInterfaces() []interface{} { + var tmp []interface{} + for _, v := range _EventOutcomeNames { + tmp = append(tmp, v) + } + return tmp +} + +// ParseEventOutcomeWithDefault attempts to convert a string to a ContentType. +// It returns the default value if name is empty. +func ParseEventOutcomeWithDefault(name string) (EventOutcome, error) { + if name == "" { + return _EventOutcomeValue[_EventOutcomeNames[0]], nil + } + if x, ok := _EventOutcomeValue[name]; ok { + return x, nil + } + return EventOutcome(""), fmt.Errorf("%s is not a valid EventOutcome, try [%s]", name, strings.Join(_EventOutcomeNames, ", ")) +} + +// NormalizeEventOutcome attempts to parse a and normalize string as content type. +// It returns the input untouched if name fails to be parsed. +// Example: +// +// "enUM" will be normalized (if possible) to "Enum" +func NormalizeEventOutcome(name string) string { + res, err := ParseEventOutcome(name) + if err != nil { + return name + } + return res.String() +} diff --git a/internal/eventlog/eventlog.go b/internal/eventlog/eventlog.go new file mode 100644 index 0000000..3e91e47 --- /dev/null +++ b/internal/eventlog/eventlog.go @@ -0,0 +1,40 @@ +package eventlog + +import ( + "fmt" + "time" + + "github.com/artefactual-sdps/preprocessing-base/internal/enums" +) + +type Event struct { + Name string + Message string + Outcome enums.EventOutcome + StartedAt time.Time + CompletedAt time.Time +} + +func NewEvent(t time.Time, name string) *Event { + return &Event{ + Name: name, + Outcome: enums.EventOutcomeUnspecified, + StartedAt: t, + } +} + +func (e *Event) Complete(t time.Time, outcome enums.EventOutcome, msg string, a ...any) *Event { + e.CompletedAt = t + e.Outcome = outcome + e.Message = fmt.Sprintf(msg, a...) + + return e +} + +func (e *Event) Succeed(t time.Time, msg string, a ...any) *Event { + return e.Complete(t, enums.EventOutcomeSuccess, msg, a...) +} + +func (e *Event) IsSuccess() bool { + return e.Outcome == enums.EventOutcomeSuccess +} diff --git a/internal/eventlog/eventlog_test.go b/internal/eventlog/eventlog_test.go new file mode 100644 index 0000000..7e61f85 --- /dev/null +++ b/internal/eventlog/eventlog_test.go @@ -0,0 +1,66 @@ +package eventlog_test + +import ( + "fmt" + "testing" + "time" + + "gotest.tools/v3/assert" + + "github.com/artefactual-sdps/preprocessing-base/internal/enums" + "github.com/artefactual-sdps/preprocessing-base/internal/eventlog" +) + +func TestEvent(t *testing.T) { + t.Parallel() + + var ( + started = time.Date(2024, 6, 6, 14, 48, 12, 0, time.UTC) + completed = time.Date(2024, 6, 6, 14, 48, 13, 0, time.UTC) + ) + + t.Run("Event succeeds", func(t *testing.T) { + t.Parallel() + + event := eventlog.NewEvent(started, "test event") + event.Complete( + completed, + enums.EventOutcomeSuccess, + "completed at %s", + completed.Format(time.RFC3339), + ) + assert.DeepEqual(t, event, &eventlog.Event{ + Name: "test event", + Message: "completed at 2024-06-06T14:48:13Z", + Outcome: enums.EventOutcomeSuccess, + StartedAt: started, + CompletedAt: completed, + }) + assert.Equal(t, event.IsSuccess(), true) + }) + + t.Run("Event outcome is validation failure", func(t *testing.T) { + t.Parallel() + + p := "/tmp/test-sip/additional/UpdatedAreldaMetadata.xml" + + event := eventlog.NewEvent(started, "test event") + event.Complete( + completed, + enums.EventOutcomeValidationFailure, + "Content error: metadata validation has failed: %s does not match expected metadata requirements", + p, + ) + assert.DeepEqual(t, event, &eventlog.Event{ + Name: "test event", + Message: fmt.Sprintf( + "Content error: metadata validation has failed: %s does not match expected metadata requirements", + p, + ), + Outcome: enums.EventOutcomeValidationFailure, + StartedAt: started, + CompletedAt: completed, + }) + assert.Equal(t, event.IsSuccess(), false) + }) +} diff --git a/internal/workflow/preprocessing.go b/internal/workflow/preprocessing.go index 45ff2f5..c5304f3 100644 --- a/internal/workflow/preprocessing.go +++ b/internal/workflow/preprocessing.go @@ -5,10 +5,21 @@ import ( "path/filepath" "time" - "github.com/artefactual-sdps/temporal-activities/bagit" + "github.com/artefactual-sdps/temporal-activities/bagcreate" "go.artefactual.dev/tools/temporal" temporalsdk_temporal "go.temporal.io/sdk/temporal" temporalsdk_workflow "go.temporal.io/sdk/workflow" + + "github.com/artefactual-sdps/preprocessing-base/internal/enums" + "github.com/artefactual-sdps/preprocessing-base/internal/eventlog" +) + +type Outcome int + +const ( + OutcomeSuccess Outcome = iota + OutcomeSystemError + OutcomeContentError ) type PreprocessingWorkflowParams struct { @@ -16,7 +27,37 @@ type PreprocessingWorkflowParams struct { } type PreprocessingWorkflowResult struct { - RelativePath string + Outcome Outcome + RelativePath string + PreservationTasks []*eventlog.Event +} + +func (r *PreprocessingWorkflowResult) newEvent(ctx temporalsdk_workflow.Context, name string) *eventlog.Event { + ev := eventlog.NewEvent(temporalsdk_workflow.Now(ctx), name) + r.PreservationTasks = append(r.PreservationTasks, ev) + + return ev +} + +func (r *PreprocessingWorkflowResult) systemError( + ctx temporalsdk_workflow.Context, + err error, + ev *eventlog.Event, + msg string, +) *PreprocessingWorkflowResult { + logger := temporalsdk_workflow.GetLogger(ctx) + logger.Error("System error", "message", err.Error()) + + // Complete last preservation task event. + ev.Complete( + temporalsdk_workflow.Now(ctx), + enums.EventOutcomeSystemFailure, + "System error: %s", + msg, + ) + r.Outcome = OutcomeSystemError + + return r } type PreprocessingWorkflow struct { @@ -32,7 +73,12 @@ func NewPreprocessingWorkflow(sharedPath string) *PreprocessingWorkflow { func (w *PreprocessingWorkflow) Execute( ctx temporalsdk_workflow.Context, params *PreprocessingWorkflowParams, -) (r *PreprocessingWorkflowResult, e error) { +) (*PreprocessingWorkflowResult, error) { + var ( + result PreprocessingWorkflowResult + e error + ) + logger := temporalsdk_workflow.GetLogger(ctx) logger.Debug("PreprocessingWorkflow workflow running!", "params", params) @@ -40,20 +86,24 @@ func (w *PreprocessingWorkflow) Execute( e = temporal.NewNonRetryableError(fmt.Errorf("error calling workflow with unexpected inputs")) return nil, e } + result.RelativePath = params.RelativePath - // Bag the transfer for Enduro processing. + // Bag the SIP for Enduro processing. + ev := result.newEvent(ctx, "Bag SIP") + var createBag bagcreate.Result e = temporalsdk_workflow.ExecuteActivity( withLocalActOpts(ctx), - bagit.CreateBagActivityName, - &bagit.CreateBagActivityParams{ + bagcreate.Name, + &bagcreate.Params{ SourcePath: filepath.Join(w.sharedPath, params.RelativePath), }, - ).Get(ctx, nil) + ).Get(ctx, &createBag) if e != nil { - return nil, temporal.NewNonRetryableError(fmt.Errorf("create bag: %v", e)) + return result.systemError(ctx, e, ev, "bagging has failed"), nil } + ev.Succeed(temporalsdk_workflow.Now(ctx), "SIP has been bagged") - return &PreprocessingWorkflowResult{RelativePath: params.RelativePath}, e + return &result, e } func withLocalActOpts(ctx temporalsdk_workflow.Context) temporalsdk_workflow.Context { diff --git a/internal/workflow/preprocessing_test.go b/internal/workflow/preprocessing_test.go index 1d289dc..8413701 100644 --- a/internal/workflow/preprocessing_test.go +++ b/internal/workflow/preprocessing_test.go @@ -1,10 +1,11 @@ package workflow_test import ( + "fmt" "path/filepath" "testing" - "github.com/artefactual-sdps/temporal-activities/bagit" + "github.com/artefactual-sdps/temporal-activities/bagcreate" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" temporalsdk_activity "go.temporal.io/sdk/activity" @@ -12,6 +13,8 @@ import ( temporalsdk_worker "go.temporal.io/sdk/worker" "github.com/artefactual-sdps/preprocessing-base/internal/config" + "github.com/artefactual-sdps/preprocessing-base/internal/enums" + "github.com/artefactual-sdps/preprocessing-base/internal/eventlog" "github.com/artefactual-sdps/preprocessing-base/internal/workflow" ) @@ -31,8 +34,8 @@ func (s *PreprocessingTestSuite) SetupTest(cfg config.Configuration) { // Register activities. s.env.RegisterActivityWithOptions( - bagit.NewCreateBagActivity(cfg.Bagit).Execute, - temporalsdk_activity.RegisterOptions{Name: bagit.CreateBagActivityName}, + bagcreate.New(cfg.Bagit).Execute, + temporalsdk_activity.RegisterOptions{Name: bagcreate.Name}, ) s.workflow = workflow.NewPreprocessingWorkflow(sharedPath) @@ -46,18 +49,18 @@ func TestPreprocessingWorkflow(t *testing.T) { suite.Run(t, new(PreprocessingTestSuite)) } -func (s *PreprocessingTestSuite) TestExecute() { +func (s *PreprocessingTestSuite) TestSuccess() { relPath := "transfer" s.SetupTest(config.Configuration{}) // Mock activities. sessionCtx := mock.AnythingOfType("*context.timerCtx") s.env.OnActivity( - bagit.CreateBagActivityName, + bagcreate.Name, sessionCtx, - &bagit.CreateBagActivityParams{SourcePath: filepath.Join(sharedPath, relPath)}, + &bagcreate.Params{SourcePath: filepath.Join(sharedPath, relPath)}, ).Return( - &bagit.CreateBagActivityResult{BagPath: filepath.Join(sharedPath, relPath)}, + &bagcreate.Result{BagPath: filepath.Join(sharedPath, relPath)}, nil, ) @@ -72,7 +75,65 @@ func (s *PreprocessingTestSuite) TestExecute() { err := s.env.GetWorkflowResult(&result) s.NoError(err) s.Equal( + &workflow.PreprocessingWorkflowResult{ + Outcome: workflow.OutcomeSuccess, + RelativePath: relPath, + PreservationTasks: []*eventlog.Event{ + { + Name: "Bag SIP", + Message: "SIP has been bagged", + Outcome: enums.EventOutcomeSuccess, + StartedAt: s.env.Now().UTC(), + CompletedAt: s.env.Now().UTC(), + }, + }, + }, + &result, + ) +} + +func (s *PreprocessingTestSuite) TestSystemError() { + relPath := "transfer" + s.SetupTest(config.Configuration{}) + + // Mock activities. + sessionCtx := mock.AnythingOfType("*context.timerCtx") + s.env.OnActivity( + bagcreate.Name, + sessionCtx, + &bagcreate.Params{SourcePath: filepath.Join(sharedPath, relPath)}, + ).Return( + nil, + fmt.Errorf( + "bagcreate: failed to open %s: permission denied", + filepath.Join(sharedPath, relPath), + ), + ) + + s.env.ExecuteWorkflow( + s.workflow.Execute, + &workflow.PreprocessingWorkflowParams{RelativePath: relPath}, + ) + + s.True(s.env.IsWorkflowCompleted()) + + var result workflow.PreprocessingWorkflowResult + err := s.env.GetWorkflowResult(&result) + s.NoError(err) + s.Equal( + &workflow.PreprocessingWorkflowResult{ + Outcome: workflow.OutcomeSystemError, + RelativePath: relPath, + PreservationTasks: []*eventlog.Event{ + { + Name: "Bag SIP", + Message: "System error: bagging has failed", + Outcome: enums.EventOutcomeSystemFailure, + StartedAt: s.env.Now().UTC(), + CompletedAt: s.env.Now().UTC(), + }, + }, + }, &result, - &workflow.PreprocessingWorkflowResult{RelativePath: relPath}, ) }