fwv

.github/workflows/ci.yaml

@@ -0,0 +1,138 @@
+---
+name: Build and Publish
+
+"on":
+  push:
+    branches:
+      - "**"
+    tags:
+      - "v*.*.*"
+  pull_request:
+    branches:
+      - main
+      - master
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+
+      - name: Build
+        run: go build -v ./...
+
+      - name: Vet
+        run: go vet -v ./...
+
+      - name: Test
+        run: go test -v ./...
+
+  hadolint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hadolint/[email protected]
+        with:
+          dockerfile: Dockerfile
+
+  yamllint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run yamllint
+        uses: bewuethr/yamllint-action@v1
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+
+  oci_image:
+    name: Build OCI Image
+    needs: build
+    if: github.repository == 'lsst-dm/s3nd'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  gh-release:
+    name: Create GitHub Release
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+
+  go-release:
+    name: Release Go Binaries
+    needs: gh-release
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, darwin]
+        goarch: [amd64, arm64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wangyoucao577/go-release-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          goversion: "1.22"
+          asset_name: '${{ github.event.repository.name }}-${{ matrix.goos }}-${{ matrix.goarch }}'

.gitignore

@@ -0,0 +1 @@
+/s3nd

.hadolint.yaml

@@ -0,0 +1,4 @@
+---
+ignored:
+  # disable pinning apk package versions
+  - DL3018

.yamllint.yaml

@@ -0,0 +1,17 @@
+---
+extends: "default"
+
+rules:
+  # 80 chars should be enough, but don't fail if a line is longer
+  line-length: false
+  indentation:
+    level: "error"
+    spaces: 2
+    indent-sequences: true
+  # do not obsess over comment formatting
+  comments-indentation: false
+  comments:
+    level: "error"
+    require-starting-space: false
+  document-start:
+    level: "error"

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.22-alpine AS builder
+
+RUN apk --update --no-cache add \
+    binutils \
+    && rm -rf /root/.cache
+WORKDIR /go/src/github.com/lsst-dm/s3nd
+COPY . .
+RUN CGO_ENABLED=0 go build -ldflags "-extldflags '-static'" -o s3nd && strip s3nd
+
+FROM alpine:3
+WORKDIR /root/
+COPY --from=builder /go/src/github.com/lsst-dm/s3nd/s3nd /bin/s3nd
+ENTRYPOINT ["/bin/s3nd"]

Makefile

@@ -0,0 +1,2 @@
+all:
+	CGO_ENABLED=0 go build -ldflags "-extldflags '-static'" -o s3nd

conf/conf.go

@@ -0,0 +1,134 @@
+package conf
+
+import (
+	"flag"
+	"log"
+	"os"
+	"strconv"
+	"time"
+
+	k8sresource "k8s.io/apimachinery/pkg/api/resource"
+)
+
+type S3ndConf struct {
+	Host                  *string
+	Port                  *int
+	EndpointUrl           *string
+	UploadMaxParallel     *int64
+	UploadTimeout         *time.Duration
+	QueueTimeout          *time.Duration
+	UploadTries           *int
+	UploadPartsize        *k8sresource.Quantity
+	UploadBwlimit         *k8sresource.Quantity
+	UploadWriteBufferSize *k8sresource.Quantity
+}
+
+// Parse the environment variables and flags. If a flag is not set, the
+// environment variable is used. Errors are fatal.
+func NewConf() S3ndConf {
+	conf := S3ndConf{}
+
+	// start flags
+	conf.Host = flag.String("host", os.Getenv("S3ND_HOST"), "S3 Daemon Host (S3ND_HOST)")
+
+	defaultPort, _ := strconv.Atoi(os.Getenv("S3ND_PORT"))
+	if defaultPort == 0 {
+		defaultPort = 15555
+	}
+	conf.Port = flag.Int("port", defaultPort, "S3 Daemon Port (S3ND_PORT)")
+
+	conf.EndpointUrl = flag.String("endpoint-url", os.Getenv("S3ND_ENDPOINT_URL"), "S3 Endpoint URL (S3ND_ENDPOINT_URL)")
+
+	var defaultUploadMaxParallel int64
+	defaultUploadMaxParallel, _ = strconv.ParseInt(os.Getenv("S3ND_UPLOAD_MAX_PARALLEL"), 10, 64)
+	if defaultUploadMaxParallel == 0 {
+		defaultUploadMaxParallel = 100
+	}
+	conf.UploadMaxParallel = flag.Int64("upload-max-parallel", defaultUploadMaxParallel, "Maximum number of parallel object uploads (S3ND_UPLOAD_MAX_PARALLEL)")
+
+	defaultUploadTimeout := os.Getenv("S3ND_UPLOAD_TIMEOUT")
+	if defaultUploadTimeout == "" {
+		defaultUploadTimeout = "10s"
+	}
+	uploadTimeout := flag.String("upload-timeout", defaultUploadTimeout, "Upload Timeout (S3ND_UPLOAD_TIMEOUT)")
+
+	defaultQueueTimeout := os.Getenv("S3ND_QUEUE_TIMEOUT")
+	if defaultQueueTimeout == "" {
+		defaultQueueTimeout = "10s"
+	}
+	queueTimeout := flag.String("queue-timeout", defaultQueueTimeout, "Queue Timeout waiting for transfer to start (S3ND_QUEUE_TIMEOUT)")
+
+	defaultUploadTries, _ := strconv.Atoi(os.Getenv("S3ND_UPLOAD_TRIES"))
+	if defaultUploadTries == 0 {
+		defaultUploadTries = 1
+	}
+	conf.UploadTries = flag.Int("upload-tries", defaultUploadTries, "Max number of upload tries (S3ND_UPLOAD_TRIES)")
+
+	defaultUploadPartsize := os.Getenv("S3ND_UPLOAD_PARTSIZE")
+	if defaultUploadPartsize == "" {
+		defaultUploadPartsize = "5Mi"
+	}
+	uploadPartsizeRaw := flag.String("upload-partsize", defaultUploadPartsize, "Upload Part Size (S3ND_UPLOAD_PARTSIZE)")
+
+	defaultUploadBwlimit := os.Getenv("S3ND_UPLOAD_BWLIMIT")
+	if defaultUploadBwlimit == "" {
+		defaultUploadBwlimit = "0"
+	}
+	uploadBwlimitRaw := flag.String("upload-bwlimit", defaultUploadBwlimit, "Upload bandwidth limit in bits per second (S3ND_UPLOAD_BWLIMIT)")
+
+	defaultUploadWriteBufferSize := os.Getenv("S3ND_UPLOAD_WRITE_BUFFER_SIZE")
+	if defaultUploadWriteBufferSize == "" {
+		defaultUploadWriteBufferSize = "64Ki"
+	}
+	uploadWriteBufferSizeRaw := flag.String("upload-write-buffer-size", defaultUploadWriteBufferSize, "Upload Write Buffer Size (S3ND_UPLOAD_WRITE_BUFFER_SIZE)")
+
+	flag.Parse()
+	// end flags
+
+	if *conf.EndpointUrl == "" {
+		log.Fatal("S3ND_ENDPOINT_URL is required")
+	}
+
+	uploadTimeoutDuration, err := time.ParseDuration(*uploadTimeout)
+	if err != nil {
+		log.Fatal("S3ND_UPLOAD_TIMEOUT is invalid")
+	}
+	conf.UploadTimeout = &uploadTimeoutDuration
+
+	queueTimeoutDuration, err := time.ParseDuration(*queueTimeout)
+	if err != nil {
+		log.Fatal("S3ND_QUEUE_TIMEOUT is invalid")
+	}
+	conf.QueueTimeout = &queueTimeoutDuration
+
+	uploadPartsize, err := k8sresource.ParseQuantity(*uploadPartsizeRaw)
+	if err != nil {
+		log.Fatal("S3ND_UPLOAD_PARTSIZE is invalid")
+	}
+	conf.UploadPartsize = &uploadPartsize
+
+	uploadBwlimit, err := k8sresource.ParseQuantity(*uploadBwlimitRaw)
+	if err != nil {
+		log.Fatal("S3ND_UPLOAD_BWLIMIT is invalid")
+	}
+	conf.UploadBwlimit = &uploadBwlimit
+
+	uploadWriteBufferSize, err := k8sresource.ParseQuantity(*uploadWriteBufferSizeRaw)
+	if err != nil {
+		log.Fatal("S3ND_UPLOAD_WRITE_BUFFER_SIZE is invalid")
+	}
+	conf.UploadWriteBufferSize = &uploadWriteBufferSize
+
+	log.Println("S3ND_HOST:", *conf.Host)
+	log.Println("S3ND_PORT:", *conf.Port)
+	log.Println("S3ND_ENDPOINT_URL:", *conf.EndpointUrl)
+	log.Println("S3ND_UPLOAD_MAX_PARALLEL:", *conf.UploadMaxParallel)
+	log.Println("S3ND_UPLOAD_TIMEOUT:", *conf.UploadTimeout)
+	log.Println("S3ND_QUEUE_TIMEOUT:", *conf.QueueTimeout)
+	log.Println("S3ND_UPLOAD_TRIES:", *conf.UploadTries)
+	log.Println("S3ND_UPLOAD_PARTSIZE:", conf.UploadPartsize.String())
+	log.Println("S3ND_UPLOAD_BWLIMIT:", conf.UploadBwlimit.String())
+	log.Println("S3ND_UPLOAD_WRITE_BUFFER_SIZE:", conf.UploadWriteBufferSize.String())
+
+	return conf
+}

go.mod

@@ -0,0 +1,36 @@
+module github.com/lsst-dm/s3nd
+
+go 1.22.7
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.32.4
+	github.com/aws/aws-sdk-go-v2/config v1.28.3
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.37
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3
+	github.com/hyperledger/fabric v2.1.1+incompatible
+	golang.org/x/sys v0.26.0
+	k8s.io/apimachinery v0.31.2
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.44 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/onsi/gomega v1.35.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+)