test: unit tests & fixes (liftedinit#14)

Makefile

@@ -5,6 +5,58 @@ build: ## Build the binary
 	@echo "--> Building development binary"
 	@go build -ldflags="-X github.com/liftedinit/yaci/cmd/yaci.Version=$(VERSION)" -o bin/yaci ./main.go
 
+.PHONY: build
+
+#### Test ####
+test: ## Run tests
+	@echo "--> Running tests"
+	@go test -v -short -race ./...
+
+test-e2e: ## Run end-to-end tests
+	@echo "--> Running end-to-end tests"
+	@go test -v -race ./cmd/yaci/postgres_test.go
+
+.PHONY: test test-e2e
+
+#### Coverage ####
+COV_ROOT="/tmp/yaci-coverage"
+COV_UNIT="${COV_ROOT}/unit"
+COV_E2E="${COV_ROOT}/e2e"
+COV_PKG="github.com/liftedinit/yaci/..."
+
+coverage: ## Run tests with coverage
+	@echo "--> Creating GOCOVERDIR"
+	@mkdir -p ${COV_UNIT} ${COV_E2E}
+	@echo "--> Cleaning up coverage files, if any"
+	@rm -rf ${COV_UNIT}/* ${COV_E2E}/*
+	@echo "--> Running short tests with coverage"
+	@go test -v -short -timeout 30m -race -covermode=atomic -cover -cpu=$$(nproc) -coverpkg=${COV_PKG} ./... -args -test.gocoverdir="${COV_UNIT}"
+	@echo "--> Running end-to-end tests with coverage"
+	@go test -v -race -timeout 30m -race -covermode=atomic -cover -cpu=$$(nproc) -coverpkg=${COV_PKG} ./cmd/yaci/postgres_test.go -args -test.gocoverdir="${COV_E2E}"
+	@echo "--> Merging coverage reports"
+	@go tool covdata merge -i=${COV_UNIT},${COV_E2E} -o ${COV_ROOT}
+	@echo "--> Converting binary coverage report to text format"
+	@go tool covdata textfmt -i=${COV_ROOT} -o ${COV_ROOT}/coverage-merged.out
+	@echo "--> Generating coverage report"
+	@go tool cover -func=${COV_ROOT}/coverage-merged.out
+	@echo "--> Generating HTML coverage report"
+	@go tool cover -html=${COV_ROOT}/coverage-merged.out -o coverage.html
+	@echo "--> Coverage report available at coverage.html"
+	@echo "--> Cleaning up coverage files"
+	@rm -rf ${COV_UNIT}/* ${COV_E2E}/*
+	@echo "--> Running coverage complete"
+
+#### Docker ####
+docker-up:
+	@echo "--> Running docker compose up --build --wait -d"
+	@docker compose -f docker/yaci.yml up --build --wait -d
+
+docker-down:
+	@echo "--> Running docker compose down -v"
+	@docker compose -f docker/yaci.yml down -v
+
+.PHONY: docker-up docker-down
+
 ####  Linting  ####
 golangci_lint_cmd=golangci-lint
 golangci_version=v1.61.0

README.md

@@ -163,19 +163,33 @@ To run the demo, you need to have Docker installed on your system. Then, you can
 
 ```shell
 # Build and start the e2e environment
-docker compose up --wait --build
+make docker-up
 ```
 
 Wait for the e2e environment to start. Then, open a new browser tab and navigate to http://localhost:3000/blocks?order=id.desc to view the blocks and to http://localhost:3000/transactions to view the transactions.
 
 Run
 
 ```shell
-docker compose down -v
+make docker-down
 ```
 
 to stop the e2e environment.
 
+## Testing
+
+To run the unit tests, you can use the following command:
+
+```shell
+make test
+```
+
+To run the end-to-end tests, you can use the following command:
+
+```shell
+make test-e2e
+```
+
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

cmd/yaci/extract.go

@@ -7,6 +7,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/liftedinit/yaci/internal/utils"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -27,7 +28,7 @@ func init() {
 	ExtractCmd.PersistentFlags().BoolP("insecure", "k", false, "Skip TLS certificate verification (INSECURE)")
 	ExtractCmd.PersistentFlags().Bool("live", false, "Enable live monitoring")
 	ExtractCmd.PersistentFlags().Uint64P("start", "s", 1, "Start block height")
-	ExtractCmd.PersistentFlags().Uint64P("stop", "e", 1, "Stop block height")
+	ExtractCmd.PersistentFlags().Uint64P("stop", "e", 0, "Stop block height")
 	ExtractCmd.PersistentFlags().UintP("block-time", "t", 2, "Block time in seconds")
 	ExtractCmd.PersistentFlags().UintP("max-retries", "r", 3, "Maximum number of retries for failed block processing")
 	ExtractCmd.PersistentFlags().UintP("max-concurrency", "c", 100, "Maximum block retrieval concurrency (advanced)")
@@ -36,11 +37,12 @@ func init() {
 		slog.Error("Failed to bind ExtractCmd flags", "error", err)
 	}
 
+	// TODO: Clashes with the Docker test. Why?
 	ExtractCmd.MarkFlagsMutuallyExclusive("live", "stop")
 
 	ExtractCmd.AddCommand(jsonCmd)
 	ExtractCmd.AddCommand(tsvCmd)
-	ExtractCmd.AddCommand(postgresCmd)
+	ExtractCmd.AddCommand(PostgresCmd)
 }
 
 func extract(address string, outputHandler output.OutputHandler) error {
@@ -84,6 +86,13 @@ func extract(address string, outputHandler output.OutputHandler) error {
 
 	resolver := reflection.NewCustomResolver(files, grpcConn, ctx, maxRetries)
 
+	if stop == 0 {
+		stop, err = utils.GetLatestBlockHeightWithRetry(ctx, grpcConn, resolver, maxRetries)
+		if err != nil {
+			return errors.WithMessage(err, "failed to get latest block height")
+		}
+	}
+
 	if live {
 		slog.Info("Starting live extraction", "block_time", blockTime)
 		err = extractor.ExtractLiveBlocksAndTransactions(ctx, grpcConn, resolver, start, outputHandler, blockTime, maxConcurrency, maxRetries)

cmd/yaci/extract_test.go

@@ -0,0 +1,21 @@
+package yaci_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/liftedinit/yaci/cmd/yaci"
+)
+
+func TestExtractCmd(t *testing.T) {
+	// --stop and --live are mutually exclusive
+	_, err := executeCommand(yaci.RootCmd, "extract", "json", "foobar", "--live", "--stop", "10")
+	assert.Error(t, err)
+	assert.ErrorContains(t, err, "if any flags in the group [live stop] are set none of the others can be; [live stop] were all set")
+
+	// Show help
+	output, err := executeCommand(yaci.RootCmd, "extract")
+	assert.NoError(t, err)
+	assert.Contains(t, output, "Extract chain data to")
+}

cmd/yaci/postgres.go

@@ -11,32 +11,34 @@ import (
 	"github.com/liftedinit/yaci/internal/output"
 )
 
-var postgresCmd = &cobra.Command{
+var PostgresRunE = func(cmd *cobra.Command, args []string) error {
+	postgresConn := viper.GetString("postgres-conn")
+	slog.Debug("Command-line argument", "postgres-conn", postgresConn)
+
+	_, err := pgxpool.ParseConfig(postgresConn)
+	if err != nil {
+		return errors.WithMessage(err, "failed to parse PostgreSQL connection string")
+	}
+
+	outputHandler, err := output.NewPostgresOutputHandler(postgresConn)
+	if err != nil {
+		return errors.WithMessage(err, "failed to create PostgreSQL output handler")
+	}
+	defer outputHandler.Close()
+
+	return extract(args[0], outputHandler)
+}
+
+var PostgresCmd = &cobra.Command{
 	Use:   "postgres [address] [flags]",
 	Short: "Extract chain data to a PostgreSQL database",
 	Args:  cobra.ExactArgs(1),
-	RunE: func(cmd *cobra.Command, args []string) error {
-		postgresConn := viper.GetString("postgres-conn")
-		slog.Debug("Command-line argument", "postgres-conn", postgresConn)
-
-		_, err := pgxpool.ParseConfig(postgresConn)
-		if err != nil {
-			return errors.WithMessage(err, "failed to parse PostgreSQL connection string")
-		}
-
-		outputHandler, err := output.NewPostgresOutputHandler(postgresConn)
-		if err != nil {
-			return errors.WithMessage(err, "failed to create PostgreSQL output handler")
-		}
-		defer outputHandler.Close()
-
-		return extract(args[0], outputHandler)
-	},
+	RunE:  PostgresRunE,
 }
 
 func init() {
-	postgresCmd.Flags().StringP("postgres-conn", "p", "", "PosftgreSQL connection string")
-	if err := viper.BindPFlags(postgresCmd.Flags()); err != nil {
+	PostgresCmd.Flags().StringP("postgres-conn", "p", "", "PosftgreSQL connection string")
+	if err := viper.BindPFlags(PostgresCmd.Flags()); err != nil {
 		slog.Error("Failed to bind postgresCmd flags", "error", err)
 	}
 }

cmd/yaci/postgres_test.go

@@ -0,0 +1,63 @@
+package yaci_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/go-resty/resty/v2"
+	"github.com/gruntwork-io/terratest/modules/docker"
+	"github.com/liftedinit/yaci/cmd/yaci"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	DockerWorkingDirectory = "../../docker"
+	GRPCEndpoint           = "localhost:9090"
+	RestEndpoint           = "localhost:3000"
+	PsqlConnectionString   = "postgres://postgres:foobar@localhost/postgres"
+)
+
+var (
+	RestTxEndpoint = fmt.Sprintf("http://%s/transactions", RestEndpoint)
+)
+
+func TestPostgres(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	// Start the infrastructure using Docker Compose.
+	// The infrastructure is defined in the `infra.yml` file.
+	opts := &docker.Options{WorkingDir: DockerWorkingDirectory}
+	defer docker.RunDockerCompose(t, opts, "-f", "infra.yml", "down", "-v")
+	_, err := docker.RunDockerComposeE(t, opts, "-f", "infra.yml", "up", "-d", "--wait")
+	require.NoError(t, err)
+
+	// Run the YACI command to extract the chain data to a PostgreSQL database
+	cmd := yaci.RootCmd
+	cmd.SetArgs([]string{"extract", "postgres", GRPCEndpoint, "-p", PsqlConnectionString, "-k"})
+
+	// Execute the command. This will extract the chain data to a PostgreSQL database up to the latest block.
+	err = cmd.Execute()
+	require.NoError(t, err)
+
+	// Verify that the chain data has been extracted to the PostgreSQL database using the REST API
+	client := resty.New()
+	resp, err := client.
+		R().
+		SetHeader("Accept", "application/json").
+		Get(RestTxEndpoint)
+	require.NoError(t, err)
+	require.Equal(t, 200, resp.StatusCode())
+	require.NotEmpty(t, resp.Body())
+
+	// Parse the response JSON body
+	var transactions []map[string]interface{}
+	err = json.Unmarshal(resp.Body(), &transactions)
+	require.NoError(t, err)
+	require.NotEmpty(t, transactions)
+
+	// The number of transactions is 6 as defined in the `infra.yml` file under the `manifest-ledger-tx` service
+	require.Len(t, transactions, 6)
+}

cmd/yaci/root.go

@@ -22,7 +22,7 @@ var (
 	validLogLevelsStr = strings.Join(slices.Sorted(maps.Keys(validLogLevels)), "|")
 )
 
-var rootCmd = &cobra.Command{
+var RootCmd = &cobra.Command{
 	Use:   "yaci",
 	Short: "Extract chain data",
 	Long:  `yaci connects to a gRPC server and extracts blockchain data.`,
@@ -52,13 +52,13 @@ func setLogLevel(logLevel string) error {
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringP("logLevel", "l", "info", fmt.Sprintf("set log level (%s)", validLogLevelsStr))
-	if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
+	RootCmd.PersistentFlags().StringP("logLevel", "l", "info", fmt.Sprintf("set log level (%s)", validLogLevelsStr))
+	if err := viper.BindPFlags(RootCmd.PersistentFlags()); err != nil {
 		slog.Error("Failed to bind rootCmd flags", "error", err)
 	}
 
-	rootCmd.SilenceUsage = true
-	rootCmd.SilenceErrors = true
+	RootCmd.SilenceUsage = true
+	RootCmd.SilenceErrors = true
 
 	viper.SetConfigName("config")
 	viper.AddConfigPath(".")
@@ -69,8 +69,8 @@ func init() {
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 	viper.AutomaticEnv()
 
-	rootCmd.AddCommand(ExtractCmd)
-	rootCmd.AddCommand(versionCmd)
+	RootCmd.AddCommand(ExtractCmd)
+	RootCmd.AddCommand(versionCmd)
 }
 
 // Execute runs the root command.
@@ -81,7 +81,7 @@ func Execute() {
 		slog.Info("No config file found")
 	}
 
-	if err := rootCmd.Execute(); err != nil {
+	if err := RootCmd.Execute(); err != nil {
 		slog.Error("An error occurred", "error", err)
 		os.Exit(1)
 	}

cmd/yaci/root_test.go

@@ -0,0 +1,33 @@
+package yaci_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/liftedinit/yaci/cmd/yaci"
+)
+
+func executeCommand(root *cobra.Command, args ...string) (output string, err error) {
+	buf := new(bytes.Buffer)
+	root.SetOut(buf)
+	root.SetErr(buf)
+	root.SetArgs(args)
+
+	_, err = root.ExecuteC()
+	return buf.String(), err
+}
+
+func TestRootCmd(t *testing.T) {
+	// Show help
+	output, err := executeCommand(yaci.RootCmd)
+	assert.NoError(t, err)
+	assert.Contains(t, output, "yaci connects to a gRPC server and extracts blockchain data.")
+
+	// Test invalid logLevel
+	output, err = executeCommand(yaci.RootCmd, "version", "--logLevel", "invalid")
+	assert.Error(t, err)
+	assert.ErrorContains(t, err, "invalid log level: invalid. Valid log levels are: debug|error|info|warn")
+}