From 9b222e0d6b3ca6cfb16472b33ce26c65b566106c Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:34:02 -0500 Subject: [PATCH 01/35] Create type system Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 85 ++++++++++++++++++++++++++ api/clients/codecs/blob_form.go | 12 ++++ api/clients/codecs/coeff_poly.go | 59 ++++++++++++++++++ api/clients/codecs/encoded_payload.go | 76 +++++++++++++++++++++++ api/clients/codecs/eval_poly.go | 86 +++++++++++++++++++++++++++ api/clients/codecs/payload.go | 79 ++++++++++++++++++++++++ api/clients/codecs/payload_test.go | 40 +++++++++++++ api/clients/codecs/poly_test.go | 49 +++++++++++++++ encoding/rs/utils.go | 44 ++++++++++++++ encoding/utils.go | 5 ++ encoding/utils/codec/codec.go | 76 +++++++++++++++++++++++ encoding/utils/codec/codec_test.go | 10 ++++ encoding/utils/utils_test.go | 24 ++++++++ 13 files changed, 645 insertions(+) create mode 100644 api/clients/codecs/blob.go create mode 100644 api/clients/codecs/blob_form.go create mode 100644 api/clients/codecs/coeff_poly.go create mode 100644 api/clients/codecs/encoded_payload.go create mode 100644 api/clients/codecs/eval_poly.go create mode 100644 api/clients/codecs/payload.go create mode 100644 api/clients/codecs/payload_test.go create mode 100644 api/clients/codecs/poly_test.go create mode 100644 encoding/utils/utils_test.go diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go new file mode 100644 index 0000000000..64cb3678a5 --- /dev/null +++ b/api/clients/codecs/blob.go @@ -0,0 +1,85 @@ +package codecs + +import ( + "fmt" +) + +// Blob is data that is dispersed on eigenDA. +// +// A Blob will contain either an encodedPayload, or a coeffPoly. Whether the Blob contains the former or the latter +// is determined by how the dispersing client has been configured. +type Blob struct { + encodedPayload *encodedPayload + coeffPoly *coeffPoly +} + +// BlobFromEncodedPayload creates a Blob containing an encodedPayload +func blobFromEncodedPayload(encodedPayload *encodedPayload) *Blob { + return &Blob{encodedPayload: encodedPayload} +} + +// blobFromCoeffPoly creates a Blob containing a coeffPoly +func blobFromCoeffPoly(poly *coeffPoly) *Blob { + return &Blob{coeffPoly: poly} +} + +// NewBlob initializes a Blob from raw bytes, and the expected BlobForm +// +// This function will return an error if the input bytes cannot be successfully interpreted as the claimed BlobForm +func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { + switch blobForm { + case Eval: + encodedPayload, err := newEncodedPayload(bytes) + if err != nil { + return nil, fmt.Errorf("new encoded payload: %v", err) + } + + return blobFromEncodedPayload(encodedPayload), nil + case Coeff: + coeffPoly, err := coeffPolyFromBytes(bytes) + if err != nil { + return nil, fmt.Errorf("new coeff poly: %v", err) + } + + return blobFromCoeffPoly(coeffPoly), nil + default: + return nil, fmt.Errorf("unsupported blob form type: %v", blobForm) + } +} + +// GetBytes gets the raw bytes of the Blob +func (b *Blob) GetBytes() []byte { + if b.encodedPayload == nil { + return b.encodedPayload.getBytes() + } else { + return b.coeffPoly.getBytes() + } +} + +// ToPayload converts the Blob into a Payload +func (b *Blob) ToPayload() (*Payload, error) { + var encodedPayload *encodedPayload + var err error + if b.encodedPayload != nil { + encodedPayload = b.encodedPayload + } else if b.coeffPoly != nil { + evalPoly, err := b.coeffPoly.toEvalPoly() + if err != nil { + return nil, fmt.Errorf("coeff poly to eval poly: %v", err) + } + + encodedPayload, err = evalPoly.toEncodedPayload() + if err != nil { + return nil, fmt.Errorf("eval poly to encoded payload: %v", err) + } + } else { + return nil, fmt.Errorf("blob has no contents") + } + + payload, err := encodedPayload.decode() + if err != nil { + return nil, fmt.Errorf("decode encoded payload: %v", err) + } + + return payload, nil +} diff --git a/api/clients/codecs/blob_form.go b/api/clients/codecs/blob_form.go new file mode 100644 index 0000000000..b40a3097c1 --- /dev/null +++ b/api/clients/codecs/blob_form.go @@ -0,0 +1,12 @@ +package codecs + +// BlobForm is an enum that represents the different ways that a blob may be represented +type BlobForm uint + +const ( + // Eval is short for "evaluation form". The field elements represent the evaluation at the polynomial's expanded + // roots of unity + Eval BlobForm = iota + // Coeff is short for "coefficient form". The field elements represent the coefficients of the polynomial + Coeff +) diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go new file mode 100644 index 0000000000..d2e089a491 --- /dev/null +++ b/api/clients/codecs/coeff_poly.go @@ -0,0 +1,59 @@ +package codecs + +import ( + "fmt" + "math" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// coeffPoly is a polynomial in coefficient form. +// +// The underlying bytes represent 32 byte field elements, and each field element represents a coefficient +type coeffPoly struct { + fieldElements []fr.Element +} + +// coeffPolyFromBytes creates a new coeffPoly from bytes. This function performs the necessary checks to guarantee that the +// bytes are well-formed, and returns a new object if they are +func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { + if !encoding.IsPowerOfTwo(len(bytes)) { + return nil, fmt.Errorf("bytes have length %d, expected a power of 2", len(bytes)) + } + + fieldElements, err := rs.BytesToFieldElements(bytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return &coeffPoly{fieldElements: fieldElements}, nil +} + +// coeffPolyFromElements creates a new coeffPoly from field elements. +func coeffPolyFromElements(elements []fr.Element) (*coeffPoly, error) { + return &coeffPoly{fieldElements: elements}, nil +} + +// toEvalPoly converts a coeffPoly to an evalPoly, using the FFT operation +func (cp *coeffPoly) toEvalPoly() (*evalPoly, error) { + maxScale := uint8(math.Log2(float64(len(cp.fieldElements)))) + fftedElements, err := fft.NewFFTSettings(maxScale).FFT(cp.fieldElements, false) + if err != nil { + return nil, fmt.Errorf("perform FFT: %w", err) + } + + evalPoly, err := evalPolyFromElements(fftedElements) + if err != nil { + return nil, fmt.Errorf("construct eval poly: %w", err) + } + + return evalPoly, nil +} + +// GetBytes returns the bytes that underlie the polynomial +func (cp *coeffPoly) getBytes() []byte { + return rs.FieldElementsToBytes(cp.fieldElements) +} diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go new file mode 100644 index 0000000000..6134f6a172 --- /dev/null +++ b/api/clients/codecs/encoded_payload.go @@ -0,0 +1,76 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" +) + +// encodedPayload represents a payload that has had an encoding applied to it +type encodedPayload struct { + bytes []byte +} + +// newEncodedPayload accepts an array of bytes which represent an encodedPayload. It performs the checks necessary +// to guarantee that the bytes are well-formed, and returns a newly constructed object if they are. +// +// Note that this function does not decode the input bytes to perform additional checks, so it is possible to construct +// an encodedPayload, where an attempt to decode will fail. +func newEncodedPayload(encodedPayloadBytes []byte) (*encodedPayload, error) { + inputLen := len(encodedPayloadBytes) + if inputLen < 32 { + return nil, fmt.Errorf( + "input bytes have length %d, which is smaller than the required 32 header bytes", inputLen) + } + + return &encodedPayload{ + bytes: encodedPayloadBytes, + }, nil +} + +// decode applies the inverse of DefaultBlobEncoding to an encodedPayload, and returns the decoded Payload +func (ep *encodedPayload) decode() (*Payload, error) { + claimedLength := binary.BigEndian.Uint32(ep.bytes[2:6]) + + // decode raw data modulo bn254 + nonPaddedData, err := codec.RemoveInternalPadding(ep.bytes[32:]) + if err != nil { + return nil, fmt.Errorf("remove internal padding: %w", err) + } + + if uint32(len(nonPaddedData)) < claimedLength { + return nil, fmt.Errorf( + "data length %d is less than length claimed in payload header %d", + len(nonPaddedData), claimedLength) + } + + lengthDifference := uint32(len(nonPaddedData)) - claimedLength + if lengthDifference > 31 { + return nil, fmt.Errorf( + "difference in length between actual data (%d) and claimed data is too large. Payload encoding pads at most 31 bytes", + lengthDifference) + } + + return NewPayload(nonPaddedData[0:claimedLength]), nil +} + +// toEvalPoly converts an encodedPayload into an evalPoly, by padding the encodedPayload bytes to the next power of 2 +func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { + paddedLength := core.NextPowerOf2(len(ep.bytes)) + paddedBytes := make([]byte, paddedLength) + copy(paddedBytes, ep.bytes) + + evalPoly, err := evalPolyFromBytes(paddedBytes) + if err != nil { + return nil, fmt.Errorf("new eval poly: %w", err) + } + + return evalPoly, nil +} + +// getBytes returns the raw bytes that underlie the encodedPayload +func (ep *encodedPayload) getBytes() []byte { + return ep.bytes +} diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go new file mode 100644 index 0000000000..f7c49a5632 --- /dev/null +++ b/api/clients/codecs/eval_poly.go @@ -0,0 +1,86 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + "math" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// evalPoly is a polynomial in evaluation form. +// +// The underlying bytes represent 32 byte field elements, and the field elements represent the evaluation at the +// polynomial's expanded roots of unity +type evalPoly struct { + fieldElements []fr.Element +} + +// evalPolyFromBytes creates a new evalPoly from bytes. This function performs the necessary checks to guarantee that the +// bytes are well-formed, and returns a new object if they are +func evalPolyFromBytes(bytes []byte) (*evalPoly, error) { + if !encoding.IsPowerOfTwo(len(bytes)) { + return nil, fmt.Errorf("bytes have length %d, expected a power of 2", len(bytes)) + } + + fieldElements, err := rs.BytesToFieldElements(bytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return &evalPoly{fieldElements: fieldElements}, nil +} + +// evalPolyFromElements creates a new evalPoly from field elements. +func evalPolyFromElements(fieldElements []fr.Element) (*evalPoly, error) { + return &evalPoly{fieldElements: fieldElements}, nil +} + +// toCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation +func (ep *evalPoly) toCoeffPoly() (*coeffPoly, error) { + maxScale := uint8(math.Log2(float64(len(ep.fieldElements)))) + ifftedElements, err := fft.NewFFTSettings(maxScale).FFT(ep.fieldElements, true) + if err != nil { + return nil, fmt.Errorf("perform IFFT: %w", err) + } + + coeffPoly, err := coeffPolyFromElements(ifftedElements) + if err != nil { + return nil, fmt.Errorf("construct coeff poly: %w", err) + } + + return coeffPoly, nil +} + +// toEncodedPayload converts an evalPoly into an encoded payload +// +// This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating +// an evalPoly. +func (ep *evalPoly) toEncodedPayload() (*encodedPayload, error) { + polynomialBytes := rs.FieldElementsToBytes(ep.fieldElements) + + payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) + + // add 32 to the padded data length, since the encoded payload includes a payload header + encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 + + if uint32(len(polynomialBytes)) < payloadLength { + return nil, fmt.Errorf( + "polynomial contains fewer bytes (%d) than expected encoded payload (%d), as determined by claimed length in payload header (%d)", + len(polynomialBytes), encodedPayloadLength, payloadLength) + } + + encodedPayloadBytes := make([]byte, encodedPayloadLength) + copy(encodedPayloadBytes, polynomialBytes[:encodedPayloadLength]) + + encodedPayload, err := newEncodedPayload(encodedPayloadBytes) + if err != nil { + return nil, fmt.Errorf("construct encoded payload: %w", err) + } + + return encodedPayload, nil +} diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go new file mode 100644 index 0000000000..470a8aeec4 --- /dev/null +++ b/api/clients/codecs/payload.go @@ -0,0 +1,79 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + + "github.com/Layr-Labs/eigenda/encoding/utils/codec" +) + +// Payload represents arbitrary user data, without any processing. +type Payload struct { + bytes []byte +} + +// NewPayload wraps an arbitrary array of bytes into a Payload type. +func NewPayload(payloadBytes []byte) *Payload { + return &Payload{ + bytes: payloadBytes, + } +} + +// Encode applies the DefaultBlobEncoding to the original payload bytes +// +// Example encoding: +// +// Payload header (32 bytes total) Encoded Payload Data +// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] +func (p *Payload) encode() (*encodedPayload, error) { + payloadHeader := make([]byte, 32) + // first byte is always 0 to ensure the payloadHeader is a valid bn254 element + payloadHeader[1] = byte(DefaultBlobEncoding) // encode version byte + + // encode payload length as uint32 + binary.BigEndian.PutUint32( + payloadHeader[2:6], + uint32(len(p.bytes))) // uint32 should be more than enough to store the length (approx 4gb) + + // encode payload modulo bn254, and align to 32 bytes + encodedData := codec.PadPayload(p.bytes) + + encodedPayload, err := newEncodedPayload(append(payloadHeader, encodedData...)) + if err != nil { + return nil, fmt.Errorf("encoding payload: %w", err) + } + + return encodedPayload, nil +} + +// ToBlob converts the Payload bytes into a Blob +func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { + encodedPayload, err := p.encode() + if err != nil { + return nil, fmt.Errorf("encoding payload: %w", err) + } + + switch form { + case Eval: + return blobFromEncodedPayload(encodedPayload), nil + case Coeff: + evalPolynomial, err := encodedPayload.toEvalPoly() + if err != nil { + return nil, fmt.Errorf("encoded payload to eval poly: %w", err) + } + + coeffPoly, err := evalPolynomial.toCoeffPoly() + if err != nil { + return nil, fmt.Errorf("eval poly to coeff poly: %w", err) + } + + return blobFromCoeffPoly(coeffPoly), nil + default: + return nil, fmt.Errorf("unknown polynomial form: %v", form) + } +} + +// GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data +func (p *Payload) GetBytes() []byte { + return p.bytes +} diff --git a/api/clients/codecs/payload_test.go b/api/clients/codecs/payload_test.go new file mode 100644 index 0000000000..7e2a3e579c --- /dev/null +++ b/api/clients/codecs/payload_test.go @@ -0,0 +1,40 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestCodec tests the encoding and decoding of random byte streams +func TestPayloadEncoding(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + const iterations = 100 + + for i := 0; i < iterations; i++ { + payload := NewPayload(testRandom.Bytes(testRandom.Intn(1024) + 1)) + encodedPayload, err := payload.encode() + require.NoError(t, err) + + // Decode the encoded data + decodedPayload, err := encodedPayload.decode() + require.NoError(t, err) + + if err != nil { + t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) + } + + // Compare the original data with the decoded data + if !bytes.Equal(payload.GetBytes(), decodedPayload.GetBytes()) { + t.Fatalf( + "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", + i, + payload.GetBytes(), + decodedPayload.GetBytes()) + } + } +} diff --git a/api/clients/codecs/poly_test.go b/api/clients/codecs/poly_test.go new file mode 100644 index 0000000000..696ac77f50 --- /dev/null +++ b/api/clients/codecs/poly_test.go @@ -0,0 +1,49 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestFftEncode checks that data can be IfftEncoded and FftEncoded repeatedly, always getting back the original data +func TestFftEncode(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + iterations := 100 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 + + payload := NewPayload(originalData) + encodedPayload, err := payload.encode() + require.NoError(t, err) + + evalPoly, err := encodedPayload.toEvalPoly() + require.NoError(t, err) + + coeffPoly, err := evalPoly.toCoeffPoly() + require.NoError(t, err) + + convertedEvalPoly, err := coeffPoly.toEvalPoly() + require.NoError(t, err) + + convertedEncodedPayload, err := convertedEvalPoly.toEncodedPayload() + require.NoError(t, err) + + decodedPayload, err := convertedEncodedPayload.decode() + require.NoError(t, err) + + // Compare the original data with the decoded data + if !bytes.Equal(originalData, decodedPayload.GetBytes()) { + t.Fatalf( + "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", + i, + originalData, + decodedPayload.GetBytes()) + } + } +} diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index d1959380b3..4af149c62a 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -2,6 +2,7 @@ package rs import ( "errors" + "fmt" "math" "github.com/Layr-Labs/eigenda/encoding" @@ -10,6 +11,7 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) +// ToFrArray TODO: This function will be removed in favor ToFieldElementArray func ToFrArray(data []byte) ([]fr.Element, error) { numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) eles := make([]fr.Element, numEle) @@ -35,6 +37,48 @@ func ToFrArray(data []byte) ([]fr.Element, error) { return eles, nil } +// ToFieldElementArray accept a byte array as an input, and converts it to an array of field elements +// +// This function expects the input array to be a multiple of the size of a field element. +// TODO: test +func BytesToFieldElements(inputData []byte) ([]fr.Element, error) { + if len(inputData)%encoding.BYTES_PER_SYMBOL != 0 { + return nil, fmt.Errorf( + "input array length %d isn't a multiple of encoding.BYTES_PER_SYMBOL %d", + len(inputData), encoding.BYTES_PER_SYMBOL) + } + + elementCount := len(inputData) / encoding.BYTES_PER_SYMBOL + outputElements := make([]fr.Element, elementCount) + for i := 0; i < elementCount; i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + err := outputElements[i].SetBytesCanonical(inputData[destinationStartIndex:destinationEndIndex]) + if err != nil { + return nil, fmt.Errorf("fr set bytes canonical: %w", err) + } + } + + return outputElements, nil +} + +// FieldElementsToBytes accepts an array of field elements, and converts it to an array of bytes +func FieldElementsToBytes(fieldElements []fr.Element) []byte { + outputBytes := make([]byte, len(fieldElements)*encoding.BYTES_PER_SYMBOL) + + for i := 0; i < len(fieldElements); i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + fieldElementBytes := fieldElements[i].Bytes() + + copy(outputBytes[destinationStartIndex:destinationEndIndex], fieldElementBytes[:]) + } + + return outputBytes +} + // ToByteArray converts a list of Fr to a byte array func ToByteArray(dataFr []fr.Element, maxDataSize uint64) []byte { n := len(dataFr) diff --git a/encoding/utils.go b/encoding/utils.go index fe665fe7d5..e8e112d2a2 100644 --- a/encoding/utils.go +++ b/encoding/utils.go @@ -34,3 +34,8 @@ func NextPowerOf2[T constraints.Integer](d T) T { nextPower := math.Ceil(math.Log2(float64(d))) return T(math.Pow(2.0, nextPower)) } + +// IsPowerOfTwo returns true if the input is a power of 2, otherwise false +func IsPowerOfTwo[T constraints.Integer](input T) bool { + return (input&(input-1) == 0) && input != 0 +} diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 09659d4332..8252ba2e6a 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -1,6 +1,8 @@ package codec import ( + "fmt" + "github.com/Layr-Labs/eigenda/encoding" ) @@ -36,6 +38,80 @@ func ConvertByPaddingEmptyByte(data []byte) []byte { return validData[:validEnd] } +// PadPayload internally pads the input data by prepending a zero to each chunk of 31 bytes. This guarantees that +// the data will be a valid field element for the bn254 curve +// +// Additionally, this function will add necessary padding to align to output to 32 bytes +func PadPayload(inputData []byte) []byte { + // 31 bytes, for the bn254 curve + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + + // this is the length of the output, which is aligned to 32 bytes + outputLength := GetPaddedDataLength(uint32(len(inputData))) + paddedOutput := make([]byte, outputLength) + + // pre-pad the input, so that it aligns to 31 bytes. This means that the internally padded result will automatically + // align to 32 bytes. Doing this padding in advance simplifies the for loop. + requiredPad := uint32(len(inputData)) % bytesPerChunk + prePaddedPayload := append(inputData, make([]byte, requiredPad)...) + + for element := uint32(0); element < outputLength/encoding.BYTES_PER_SYMBOL; element++ { + // add the 0x00 internal padding to guarantee that the data is in the valid range + zeroByteIndex := element * encoding.BYTES_PER_SYMBOL + paddedOutput[zeroByteIndex] = 0x00 + + destIndex := zeroByteIndex + 1 + srcIndex := element * bytesPerChunk + + // copy 31 bytes of data from the payload to the padded output + copy(paddedOutput[destIndex:destIndex+bytesPerChunk], prePaddedPayload[srcIndex:srcIndex+bytesPerChunk]) + } + + return paddedOutput +} + +// GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after +// adding internal byte padding +func GetPaddedDataLength(inputLen uint32) uint32 { + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + chunkCount := inputLen / bytesPerChunk + + if inputLen%bytesPerChunk != 0 { + chunkCount++ + } + + return chunkCount * encoding.BYTES_PER_SYMBOL +} + +// RemoveInternalPadding accepts an array of padded data, and removes the internal padding that was added in PadPayload +// +// This function assumes that the input aligns to 32 bytes. Since it is removing 1 byte for every 31 bytes kept, the output +// from this function is not guaranteed to align to 32 bytes. +func RemoveInternalPadding(paddedData []byte) ([]byte, error) { + if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { + return nil, fmt.Errorf( + "padded data (length %d) must be multiple of encoding.BYTES_PER_SYMBOL %d", + len(paddedData), + encoding.BYTES_PER_SYMBOL) + } + + bytesPerChunk := encoding.BYTES_PER_SYMBOL - 1 + + symbolCount := len(paddedData) / encoding.BYTES_PER_SYMBOL + outputLength := symbolCount * bytesPerChunk + + outputData := make([]byte, outputLength) + + for i := 0; i < symbolCount; i++ { + dstIndex := i * bytesPerChunk + srcIndex := i*encoding.BYTES_PER_SYMBOL + 1 + + copy(outputData[dstIndex:dstIndex+bytesPerChunk], paddedData[srcIndex:srcIndex+bytesPerChunk]) + } + + return outputData, nil +} + // RemoveEmptyByteFromPaddedBytes takes bytes and remove the first byte from every 32 bytes. // This reverses the change made by the function ConvertByPaddingEmptyByte. // The function does not assume the input is a multiple of BYTES_PER_SYMBOL(32 bytes). diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 3137d7fe7b..a51b9ea06a 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -49,3 +49,13 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { } } } + +// TestGetPaddedDataLength tests that GetPaddedDataLength is behaving as expected +func TestGetPaddedDataLength(t *testing.T) { + startLengths := []uint32{30, 31, 32, 33, 68} + expectedResults := []uint32{32, 32, 64, 64, 96} + + for i := range startLengths { + require.Equal(t, codec.GetPaddedDataLength(startLengths[i]), expectedResults[i]) + } +} diff --git a/encoding/utils/utils_test.go b/encoding/utils/utils_test.go new file mode 100644 index 0000000000..dd6a702b86 --- /dev/null +++ b/encoding/utils/utils_test.go @@ -0,0 +1,24 @@ +package utils + +import ( + "math" + "testing" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/stretchr/testify/require" +) + +// TestIsPowerOf2 checks that the IsPowerOfTwo utility is working as expected +func TestIsPowerOf2(t *testing.T) { + // Test the special case + is0PowerOf2 := encoding.IsPowerOfTwo(0) + require.False(t, is0PowerOf2) + + testValue := uint32(1) + require.True(t, encoding.IsPowerOfTwo(testValue), "expected %d to be a valid power of 2", testValue) + + for testValue < math.MaxUint32 / 2 { + testValue = testValue * 2 + require.True(t, encoding.IsPowerOfTwo(testValue), "expected %d to be a valid power of 2", testValue) + } +} From bae88430bcd46077a74eb2a0a5122579dd2e2ce4 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:45:04 -0500 Subject: [PATCH 02/35] Rename encoding Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_codec.go | 7 +-- api/clients/codecs/default_blob_codec.go | 2 +- api/clients/codecs/payload.go | 4 +- api/clients/eigenda_client_test.go | 20 ++++---- encoding/utils/codec/codec.go | 64 ++++++++++++------------ 5 files changed, 49 insertions(+), 48 deletions(-) diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index 5be5190261..53a3ce87ff 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -7,9 +7,10 @@ import ( type BlobEncodingVersion byte const ( - // This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] + // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] - DefaultBlobEncoding BlobEncodingVersion = 0x0 + // NOTE: this encoding will soon be updated, such that the result will be padded to align to 32 bytes + PayloadEncodingVersion0 BlobEncodingVersion = 0x0 ) type BlobCodec interface { @@ -19,7 +20,7 @@ type BlobCodec interface { func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) { switch version { - case DefaultBlobEncoding: + case PayloadEncodingVersion0: return DefaultBlobCodec{}, nil default: return nil, fmt.Errorf("unsupported blob encoding version: %x", version) diff --git a/api/clients/codecs/default_blob_codec.go b/api/clients/codecs/default_blob_codec.go index 6d3ec29944..4b6fc590c7 100644 --- a/api/clients/codecs/default_blob_codec.go +++ b/api/clients/codecs/default_blob_codec.go @@ -22,7 +22,7 @@ func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { codecBlobHeader := make([]byte, 32) // first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element // encode version byte - codecBlobHeader[1] = byte(DefaultBlobEncoding) + codecBlobHeader[1] = byte(PayloadEncodingVersion0) // encode length as uint32 binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb) diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 470a8aeec4..749298d755 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -19,7 +19,7 @@ func NewPayload(payloadBytes []byte) *Payload { } } -// Encode applies the DefaultBlobEncoding to the original payload bytes +// Encode applies the PayloadEncodingVersion0 to the original payload bytes // // Example encoding: // @@ -28,7 +28,7 @@ func NewPayload(payloadBytes []byte) *Payload { func (p *Payload) encode() (*encodedPayload, error) { payloadHeader := make([]byte, 32) // first byte is always 0 to ensure the payloadHeader is a valid bn254 element - payloadHeader[1] = byte(DefaultBlobEncoding) // encode version byte + payloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte // encode payload length as uint32 binary.BigEndian.PutUint32( diff --git a/api/clients/eigenda_client_test.go b/api/clients/eigenda_client_test.go index 29435472be..87f7926b8f 100644 --- a/api/clients/eigenda_client_test.go +++ b/api/clients/eigenda_client_test.go @@ -66,7 +66,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -133,7 +133,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -204,7 +204,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: true, WaitForFinalization: true, }, @@ -237,7 +237,7 @@ func TestPutBlobFailDispersal(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -270,7 +270,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -303,7 +303,7 @@ func TestPutBlobFailureGeneral(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -336,7 +336,7 @@ func TestPutBlobFailureUnknown(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -371,7 +371,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -431,7 +431,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -494,7 +494,7 @@ func TestPutBlobTotalTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 8252ba2e6a..df7bda4ee9 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -38,10 +38,40 @@ func ConvertByPaddingEmptyByte(data []byte) []byte { return validData[:validEnd] } -// PadPayload internally pads the input data by prepending a zero to each chunk of 31 bytes. This guarantees that +// RemoveEmptyByteFromPaddedBytes takes bytes and remove the first byte from every 32 bytes. +// This reverses the change made by the function ConvertByPaddingEmptyByte. +// The function does not assume the input is a multiple of BYTES_PER_SYMBOL(32 bytes). +// For the reminder of the input, the first byte is taken out, and the rest is appended to +// the output. +func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { + dataSize := len(data) + parseSize := encoding.BYTES_PER_SYMBOL + dataLen := (dataSize + parseSize - 1) / parseSize + + putSize := encoding.BYTES_PER_SYMBOL - 1 + + validData := make([]byte, dataLen*putSize) + validLen := len(validData) + + for i := 0; i < dataLen; i++ { + // add 1 to leave the first empty byte untouched + start := i*parseSize + 1 + end := (i + 1) * parseSize + + if end > len(data) { + end = len(data) + validLen = end - start + i*putSize + } + + copy(validData[i*putSize:(i+1)*putSize], data[start:end]) + } + return validData[:validLen] +} + +// PadPayload internally pads the input data by prepending a 0x00 to each chunk of 31 bytes. This guarantees that // the data will be a valid field element for the bn254 curve // -// Additionally, this function will add necessary padding to align to output to 32 bytes +// Additionally, this function will add necessary padding to align the output to 32 bytes func PadPayload(inputData []byte) []byte { // 31 bytes, for the bn254 curve bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) @@ -111,33 +141,3 @@ func RemoveInternalPadding(paddedData []byte) ([]byte, error) { return outputData, nil } - -// RemoveEmptyByteFromPaddedBytes takes bytes and remove the first byte from every 32 bytes. -// This reverses the change made by the function ConvertByPaddingEmptyByte. -// The function does not assume the input is a multiple of BYTES_PER_SYMBOL(32 bytes). -// For the reminder of the input, the first byte is taken out, and the rest is appended to -// the output. -func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { - dataSize := len(data) - parseSize := encoding.BYTES_PER_SYMBOL - dataLen := (dataSize + parseSize - 1) / parseSize - - putSize := encoding.BYTES_PER_SYMBOL - 1 - - validData := make([]byte, dataLen*putSize) - validLen := len(validData) - - for i := 0; i < dataLen; i++ { - // add 1 to leave the first empty byte untouched - start := i*parseSize + 1 - end := (i + 1) * parseSize - - if end > len(data) { - end = len(data) - validLen = end - start + i*putSize - } - - copy(validData[i*putSize:(i+1)*putSize], data[start:end]) - } - return validData[:validLen] -} From d66ddf27cbe72528f723b06998c1b9c12469f6c6 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:43:09 -0500 Subject: [PATCH 03/35] Fix padding logic Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/encoded_payload.go | 7 ------- encoding/utils/codec/codec.go | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index 6134f6a172..bc8c521fdf 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -46,13 +46,6 @@ func (ep *encodedPayload) decode() (*Payload, error) { len(nonPaddedData), claimedLength) } - lengthDifference := uint32(len(nonPaddedData)) - claimedLength - if lengthDifference > 31 { - return nil, fmt.Errorf( - "difference in length between actual data (%d) and claimed data is too large. Payload encoding pads at most 31 bytes", - lengthDifference) - } - return NewPayload(nonPaddedData[0:claimedLength]), nil } diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index df7bda4ee9..e0ba4bb54b 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -82,7 +82,7 @@ func PadPayload(inputData []byte) []byte { // pre-pad the input, so that it aligns to 31 bytes. This means that the internally padded result will automatically // align to 32 bytes. Doing this padding in advance simplifies the for loop. - requiredPad := uint32(len(inputData)) % bytesPerChunk + requiredPad := (bytesPerChunk - uint32(len(inputData))%bytesPerChunk) % bytesPerChunk prePaddedPayload := append(inputData, make([]byte, requiredPad)...) for element := uint32(0); element < outputLength/encoding.BYTES_PER_SYMBOL; element++ { From f6021eaa5b0fb50686359bb6d7aa1fd9f2f8b2d4 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:01:22 -0500 Subject: [PATCH 04/35] Pad upon poly construction Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/coeff_poly.go | 6 ++---- api/clients/codecs/encoded_payload.go | 9 ++------- api/clients/codecs/eval_poly.go | 6 ++---- encoding/utils.go | 9 ++++++--- encoding/utils/utils_test.go | 24 ------------------------ 5 files changed, 12 insertions(+), 42 deletions(-) delete mode 100644 encoding/utils/utils_test.go diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go index d2e089a491..ea5e67c37f 100644 --- a/api/clients/codecs/coeff_poly.go +++ b/api/clients/codecs/coeff_poly.go @@ -20,11 +20,9 @@ type coeffPoly struct { // coeffPolyFromBytes creates a new coeffPoly from bytes. This function performs the necessary checks to guarantee that the // bytes are well-formed, and returns a new object if they are func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { - if !encoding.IsPowerOfTwo(len(bytes)) { - return nil, fmt.Errorf("bytes have length %d, expected a power of 2", len(bytes)) - } + paddedBytes := encoding.PadToPowerOfTwo(bytes) - fieldElements, err := rs.BytesToFieldElements(bytes) + fieldElements, err := rs.BytesToFieldElements(paddedBytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index bc8c521fdf..82e92ac9df 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "fmt" - "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding/utils/codec" ) @@ -49,13 +48,9 @@ func (ep *encodedPayload) decode() (*Payload, error) { return NewPayload(nonPaddedData[0:claimedLength]), nil } -// toEvalPoly converts an encodedPayload into an evalPoly, by padding the encodedPayload bytes to the next power of 2 +// toEvalPoly converts an encodedPayload into an evalPoly func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { - paddedLength := core.NextPowerOf2(len(ep.bytes)) - paddedBytes := make([]byte, paddedLength) - copy(paddedBytes, ep.bytes) - - evalPoly, err := evalPolyFromBytes(paddedBytes) + evalPoly, err := evalPolyFromBytes(ep.bytes) if err != nil { return nil, fmt.Errorf("new eval poly: %w", err) } diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index f7c49a5632..6add0f422b 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -23,11 +23,9 @@ type evalPoly struct { // evalPolyFromBytes creates a new evalPoly from bytes. This function performs the necessary checks to guarantee that the // bytes are well-formed, and returns a new object if they are func evalPolyFromBytes(bytes []byte) (*evalPoly, error) { - if !encoding.IsPowerOfTwo(len(bytes)) { - return nil, fmt.Errorf("bytes have length %d, expected a power of 2", len(bytes)) - } + paddedBytes := encoding.PadToPowerOfTwo(bytes) - fieldElements, err := rs.BytesToFieldElements(bytes) + fieldElements, err := rs.BytesToFieldElements(paddedBytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } diff --git a/encoding/utils.go b/encoding/utils.go index e8e112d2a2..2e7d2179aa 100644 --- a/encoding/utils.go +++ b/encoding/utils.go @@ -35,7 +35,10 @@ func NextPowerOf2[T constraints.Integer](d T) T { return T(math.Pow(2.0, nextPower)) } -// IsPowerOfTwo returns true if the input is a power of 2, otherwise false -func IsPowerOfTwo[T constraints.Integer](input T) bool { - return (input&(input-1) == 0) && input != 0 +// PadToPowerOfTwo pads a byte slice to the next power of 2 +// TODO: test to make sure this doesn't increase size if already a power of 2 +func PadToPowerOfTwo(bytes []byte) []byte { + paddedLength := NextPowerOf2(len(bytes)) + padding := make([]byte, paddedLength-len(bytes)) + return append(bytes, padding...) } diff --git a/encoding/utils/utils_test.go b/encoding/utils/utils_test.go deleted file mode 100644 index dd6a702b86..0000000000 --- a/encoding/utils/utils_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package utils - -import ( - "math" - "testing" - - "github.com/Layr-Labs/eigenda/encoding" - "github.com/stretchr/testify/require" -) - -// TestIsPowerOf2 checks that the IsPowerOfTwo utility is working as expected -func TestIsPowerOf2(t *testing.T) { - // Test the special case - is0PowerOf2 := encoding.IsPowerOfTwo(0) - require.False(t, is0PowerOf2) - - testValue := uint32(1) - require.True(t, encoding.IsPowerOfTwo(testValue), "expected %d to be a valid power of 2", testValue) - - for testValue < math.MaxUint32 / 2 { - testValue = testValue * 2 - require.True(t, encoding.IsPowerOfTwo(testValue), "expected %d to be a valid power of 2", testValue) - } -} From 422c2a8143fde7a4a5fd3eacd6685df0dcfa0619 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:13:19 -0500 Subject: [PATCH 05/35] Change name to ProtoBlob Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/payload.go | 8 ++--- api/clients/codecs/{blob.go => proto_blob.go} | 36 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) rename api/clients/codecs/{blob.go => proto_blob.go} (54%) diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 749298d755..8c90f7ef78 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -46,8 +46,8 @@ func (p *Payload) encode() (*encodedPayload, error) { return encodedPayload, nil } -// ToBlob converts the Payload bytes into a Blob -func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { +// ToBlob converts the Payload bytes into a ProtoBlob +func (p *Payload) ToBlob(form BlobForm) (*ProtoBlob, error) { encodedPayload, err := p.encode() if err != nil { return nil, fmt.Errorf("encoding payload: %w", err) @@ -55,7 +55,7 @@ func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { switch form { case Eval: - return blobFromEncodedPayload(encodedPayload), nil + return protoBlobFromEncodedPayload(encodedPayload), nil case Coeff: evalPolynomial, err := encodedPayload.toEvalPoly() if err != nil { @@ -67,7 +67,7 @@ func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } - return blobFromCoeffPoly(coeffPoly), nil + return protoBlobFromCoeffPoly(coeffPoly), nil default: return nil, fmt.Errorf("unknown polynomial form: %v", form) } diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/proto_blob.go similarity index 54% rename from api/clients/codecs/blob.go rename to api/clients/codecs/proto_blob.go index 64cb3678a5..0d2033c4f2 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/proto_blob.go @@ -4,29 +4,29 @@ import ( "fmt" ) -// Blob is data that is dispersed on eigenDA. +// ProtoBlob is data that is dispersed to eigenDA. TODO: write a good description of a proto blob // -// A Blob will contain either an encodedPayload, or a coeffPoly. Whether the Blob contains the former or the latter -// is determined by how the dispersing client has been configured. -type Blob struct { +// A ProtoBlob will contain either an encodedPayload, or a coeffPoly. Whether the ProtoBlob contains the former or the +// latter is determined by how the dispersing client has been configured. +type ProtoBlob struct { encodedPayload *encodedPayload coeffPoly *coeffPoly } -// BlobFromEncodedPayload creates a Blob containing an encodedPayload -func blobFromEncodedPayload(encodedPayload *encodedPayload) *Blob { - return &Blob{encodedPayload: encodedPayload} +// protoBlobFromEncodedPayload creates a ProtoBlob containing an encodedPayload +func protoBlobFromEncodedPayload(encodedPayload *encodedPayload) *ProtoBlob { + return &ProtoBlob{encodedPayload: encodedPayload} } -// blobFromCoeffPoly creates a Blob containing a coeffPoly -func blobFromCoeffPoly(poly *coeffPoly) *Blob { - return &Blob{coeffPoly: poly} +// blobFromCoeffPoly creates a ProtoBlob containing a coeffPoly +func protoBlobFromCoeffPoly(poly *coeffPoly) *ProtoBlob { + return &ProtoBlob{coeffPoly: poly} } -// NewBlob initializes a Blob from raw bytes, and the expected BlobForm +// NewProtoBlob initializes a ProtoBlob from raw bytes, and the expected BlobForm // // This function will return an error if the input bytes cannot be successfully interpreted as the claimed BlobForm -func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { +func NewProtoBlob(bytes []byte, blobForm BlobForm) (*ProtoBlob, error) { switch blobForm { case Eval: encodedPayload, err := newEncodedPayload(bytes) @@ -34,21 +34,21 @@ func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { return nil, fmt.Errorf("new encoded payload: %v", err) } - return blobFromEncodedPayload(encodedPayload), nil + return protoBlobFromEncodedPayload(encodedPayload), nil case Coeff: coeffPoly, err := coeffPolyFromBytes(bytes) if err != nil { return nil, fmt.Errorf("new coeff poly: %v", err) } - return blobFromCoeffPoly(coeffPoly), nil + return protoBlobFromCoeffPoly(coeffPoly), nil default: return nil, fmt.Errorf("unsupported blob form type: %v", blobForm) } } // GetBytes gets the raw bytes of the Blob -func (b *Blob) GetBytes() []byte { +func (b *ProtoBlob) GetBytes() []byte { if b.encodedPayload == nil { return b.encodedPayload.getBytes() } else { @@ -56,8 +56,8 @@ func (b *Blob) GetBytes() []byte { } } -// ToPayload converts the Blob into a Payload -func (b *Blob) ToPayload() (*Payload, error) { +// ToPayload converts the ProtoBlob into a Payload +func (b *ProtoBlob) ToPayload() (*Payload, error) { var encodedPayload *encodedPayload var err error if b.encodedPayload != nil { @@ -73,7 +73,7 @@ func (b *Blob) ToPayload() (*Payload, error) { return nil, fmt.Errorf("eval poly to encoded payload: %v", err) } } else { - return nil, fmt.Errorf("blob has no contents") + return nil, fmt.Errorf("proto blob has no contents") } payload, err := encodedPayload.decode() From a3d184b25a8863fbff6954e2f10d3eadc0874154 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:21:37 -0500 Subject: [PATCH 06/35] Update comments Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/eval_poly.go | 2 +- api/clients/codecs/poly_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index 6add0f422b..b88f6dd48d 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -15,7 +15,7 @@ import ( // evalPoly is a polynomial in evaluation form. // // The underlying bytes represent 32 byte field elements, and the field elements represent the evaluation at the -// polynomial's expanded roots of unity +// polynomial's roots of unity type evalPoly struct { fieldElements []fr.Element } diff --git a/api/clients/codecs/poly_test.go b/api/clients/codecs/poly_test.go index 696ac77f50..8665724e5e 100644 --- a/api/clients/codecs/poly_test.go +++ b/api/clients/codecs/poly_test.go @@ -9,6 +9,7 @@ import ( ) // TestFftEncode checks that data can be IfftEncoded and FftEncoded repeatedly, always getting back the original data +// TODO: we should probably be using fuzzing instead of this kind of ad-hoc random search testing func TestFftEncode(t *testing.T) { testRandom := random.NewTestRandom(t) From 856f5fcb6deedf65476373ea21d24653f56ea102 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:11:34 -0500 Subject: [PATCH 07/35] Revert "Change name to ProtoBlob" This reverts commit 422c2a8143fde7a4a5fd3eacd6685df0dcfa0619. Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/{proto_blob.go => blob.go} | 36 +++++++++---------- api/clients/codecs/payload.go | 8 ++--- 2 files changed, 22 insertions(+), 22 deletions(-) rename api/clients/codecs/{proto_blob.go => blob.go} (54%) diff --git a/api/clients/codecs/proto_blob.go b/api/clients/codecs/blob.go similarity index 54% rename from api/clients/codecs/proto_blob.go rename to api/clients/codecs/blob.go index 0d2033c4f2..64cb3678a5 100644 --- a/api/clients/codecs/proto_blob.go +++ b/api/clients/codecs/blob.go @@ -4,29 +4,29 @@ import ( "fmt" ) -// ProtoBlob is data that is dispersed to eigenDA. TODO: write a good description of a proto blob +// Blob is data that is dispersed on eigenDA. // -// A ProtoBlob will contain either an encodedPayload, or a coeffPoly. Whether the ProtoBlob contains the former or the -// latter is determined by how the dispersing client has been configured. -type ProtoBlob struct { +// A Blob will contain either an encodedPayload, or a coeffPoly. Whether the Blob contains the former or the latter +// is determined by how the dispersing client has been configured. +type Blob struct { encodedPayload *encodedPayload coeffPoly *coeffPoly } -// protoBlobFromEncodedPayload creates a ProtoBlob containing an encodedPayload -func protoBlobFromEncodedPayload(encodedPayload *encodedPayload) *ProtoBlob { - return &ProtoBlob{encodedPayload: encodedPayload} +// BlobFromEncodedPayload creates a Blob containing an encodedPayload +func blobFromEncodedPayload(encodedPayload *encodedPayload) *Blob { + return &Blob{encodedPayload: encodedPayload} } -// blobFromCoeffPoly creates a ProtoBlob containing a coeffPoly -func protoBlobFromCoeffPoly(poly *coeffPoly) *ProtoBlob { - return &ProtoBlob{coeffPoly: poly} +// blobFromCoeffPoly creates a Blob containing a coeffPoly +func blobFromCoeffPoly(poly *coeffPoly) *Blob { + return &Blob{coeffPoly: poly} } -// NewProtoBlob initializes a ProtoBlob from raw bytes, and the expected BlobForm +// NewBlob initializes a Blob from raw bytes, and the expected BlobForm // // This function will return an error if the input bytes cannot be successfully interpreted as the claimed BlobForm -func NewProtoBlob(bytes []byte, blobForm BlobForm) (*ProtoBlob, error) { +func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { switch blobForm { case Eval: encodedPayload, err := newEncodedPayload(bytes) @@ -34,21 +34,21 @@ func NewProtoBlob(bytes []byte, blobForm BlobForm) (*ProtoBlob, error) { return nil, fmt.Errorf("new encoded payload: %v", err) } - return protoBlobFromEncodedPayload(encodedPayload), nil + return blobFromEncodedPayload(encodedPayload), nil case Coeff: coeffPoly, err := coeffPolyFromBytes(bytes) if err != nil { return nil, fmt.Errorf("new coeff poly: %v", err) } - return protoBlobFromCoeffPoly(coeffPoly), nil + return blobFromCoeffPoly(coeffPoly), nil default: return nil, fmt.Errorf("unsupported blob form type: %v", blobForm) } } // GetBytes gets the raw bytes of the Blob -func (b *ProtoBlob) GetBytes() []byte { +func (b *Blob) GetBytes() []byte { if b.encodedPayload == nil { return b.encodedPayload.getBytes() } else { @@ -56,8 +56,8 @@ func (b *ProtoBlob) GetBytes() []byte { } } -// ToPayload converts the ProtoBlob into a Payload -func (b *ProtoBlob) ToPayload() (*Payload, error) { +// ToPayload converts the Blob into a Payload +func (b *Blob) ToPayload() (*Payload, error) { var encodedPayload *encodedPayload var err error if b.encodedPayload != nil { @@ -73,7 +73,7 @@ func (b *ProtoBlob) ToPayload() (*Payload, error) { return nil, fmt.Errorf("eval poly to encoded payload: %v", err) } } else { - return nil, fmt.Errorf("proto blob has no contents") + return nil, fmt.Errorf("blob has no contents") } payload, err := encodedPayload.decode() diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 8c90f7ef78..749298d755 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -46,8 +46,8 @@ func (p *Payload) encode() (*encodedPayload, error) { return encodedPayload, nil } -// ToBlob converts the Payload bytes into a ProtoBlob -func (p *Payload) ToBlob(form BlobForm) (*ProtoBlob, error) { +// ToBlob converts the Payload bytes into a Blob +func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { encodedPayload, err := p.encode() if err != nil { return nil, fmt.Errorf("encoding payload: %w", err) @@ -55,7 +55,7 @@ func (p *Payload) ToBlob(form BlobForm) (*ProtoBlob, error) { switch form { case Eval: - return protoBlobFromEncodedPayload(encodedPayload), nil + return blobFromEncodedPayload(encodedPayload), nil case Coeff: evalPolynomial, err := encodedPayload.toEvalPoly() if err != nil { @@ -67,7 +67,7 @@ func (p *Payload) ToBlob(form BlobForm) (*ProtoBlob, error) { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } - return protoBlobFromCoeffPoly(coeffPoly), nil + return blobFromCoeffPoly(coeffPoly), nil default: return nil, fmt.Errorf("unknown polynomial form: %v", form) } From 1eeec85ed56a210bd6eefb0190f13766cc48ecfd Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:02:58 -0500 Subject: [PATCH 08/35] Rename PayloadHeader to EncodedPayloadHeader Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/encoded_payload.go | 2 +- api/clients/codecs/eval_poly.go | 4 ++-- api/clients/codecs/payload.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index 82e92ac9df..36f0e21dc8 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -41,7 +41,7 @@ func (ep *encodedPayload) decode() (*Payload, error) { if uint32(len(nonPaddedData)) < claimedLength { return nil, fmt.Errorf( - "data length %d is less than length claimed in payload header %d", + "data length %d is less than length claimed in encoded payload header %d", len(nonPaddedData), claimedLength) } diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index b88f6dd48d..8fa7bef3f2 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -63,12 +63,12 @@ func (ep *evalPoly) toEncodedPayload() (*encodedPayload, error) { payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) - // add 32 to the padded data length, since the encoded payload includes a payload header + // add 32 to the padded data length, since the encoded payload includes an encoded payload header encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 if uint32(len(polynomialBytes)) < payloadLength { return nil, fmt.Errorf( - "polynomial contains fewer bytes (%d) than expected encoded payload (%d), as determined by claimed length in payload header (%d)", + "polynomial contains fewer bytes (%d) than expected encoded payload (%d), as determined by claimed length in encoded payload header (%d)", len(polynomialBytes), encodedPayloadLength, payloadLength) } diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 749298d755..fe10e24ebc 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -23,7 +23,7 @@ func NewPayload(payloadBytes []byte) *Payload { // // Example encoding: // -// Payload header (32 bytes total) Encoded Payload Data +// Encoded Payload header (32 bytes total) Encoded Payload Data // [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] func (p *Payload) encode() (*encodedPayload, error) { payloadHeader := make([]byte, 32) From 5eab8ddece7ea2104d1ce2e082e63ed4e4573f80 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 7 Feb 2025 11:48:45 -0500 Subject: [PATCH 09/35] Rename to PayloadEncodingVersion Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 8 +++++--- api/clients/v2/payload_disperser.go | 2 +- api/clients/v2/relay_payload_retriever.go | 2 +- api/clients/v2/validator_payload_retriever.go | 2 +- test/v2/client/test_client.go | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index ada61b7169..5f65d28abd 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -11,8 +11,10 @@ import ( // PayloadClientConfig contains configuration values that are needed by both PayloadRetriever and PayloadDisperser type PayloadClientConfig struct { - // The blob encoding version to use when writing and reading blobs - BlobEncodingVersion codecs.BlobEncodingVersion + // The payload encoding version to use when encoding payload bytes + // + // This is the version that is put into the header of the EncodedPayload. + PayloadEncodingVersion codecs.BlobEncodingVersion // The address of the EigenDACertVerifier contract EigenDACertVerifierAddr string @@ -112,7 +114,7 @@ type PayloadDisperserConfig struct { // NOTE: EigenDACertVerifierAddr does not have a defined default. It must always be specifically configured. func GetDefaultPayloadClientConfig() *PayloadClientConfig { return &PayloadClientConfig{ - BlobEncodingVersion: codecs.DefaultBlobEncoding, + PayloadEncodingVersion: codecs.PayloadEncodingVersion0, PayloadPolynomialForm: codecs.PolynomialFormEval, ContractCallTimeout: 5 * time.Second, BlockNumberPollInterval: 1 * time.Second, diff --git a/api/clients/v2/payload_disperser.go b/api/clients/v2/payload_disperser.go index 6db8ccb599..7506cf0b81 100644 --- a/api/clients/v2/payload_disperser.go +++ b/api/clients/v2/payload_disperser.go @@ -88,7 +88,7 @@ func BuildPayloadDisperser(log logging.Logger, payloadDispCfg PayloadDisperserCo } // 5 - create codec - codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.BlobEncodingVersion) + codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.PayloadEncodingVersion) if err != nil { return nil, err } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index 4a147fac65..61cc0fd77c 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -49,7 +49,7 @@ func BuildRelayPayloadRetriever( codec, err := codecs.CreateCodec( relayPayloadRetrieverConfig.PayloadPolynomialForm, - relayPayloadRetrieverConfig.BlobEncodingVersion) + relayPayloadRetrieverConfig.PayloadEncodingVersion) if err != nil { return nil, err } diff --git a/api/clients/v2/validator_payload_retriever.go b/api/clients/v2/validator_payload_retriever.go index a0e552ba27..c5366231ff 100644 --- a/api/clients/v2/validator_payload_retriever.go +++ b/api/clients/v2/validator_payload_retriever.go @@ -76,7 +76,7 @@ func BuildValidatorPayloadRetriever( codec, err := codecs.CreateCodec( validatorPayloadRetrieverConfig.PayloadPolynomialForm, - validatorPayloadRetrieverConfig.BlobEncodingVersion) + validatorPayloadRetrieverConfig.PayloadEncodingVersion) if err != nil { return nil, fmt.Errorf("create codec: %w", err) } diff --git a/test/v2/client/test_client.go b/test/v2/client/test_client.go index 05cf2afdda..1473011f0c 100644 --- a/test/v2/client/test_client.go +++ b/test/v2/client/test_client.go @@ -181,7 +181,7 @@ func NewTestClient( Quorums: quorums, } - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.BlobEncodingVersion) + blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.PayloadEncodingVersion) require.NoError(t, err) payloadDisperser, err := clients.NewPayloadDisperser( From 132e8de2651d5b72ed927f6713a263c02b5ea859 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:35:25 -0500 Subject: [PATCH 10/35] Do a bunch of cleanup Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 80 ++++++++++------------- api/clients/codecs/blob_form.go | 12 ---- api/clients/codecs/coeff_poly.go | 40 +++++++----- api/clients/codecs/encoded_payload.go | 92 +++++++++++++++++++-------- api/clients/codecs/eval_poly.go | 69 +++++--------------- api/clients/codecs/payload.go | 57 +++++------------ api/clients/codecs/payload_test.go | 2 +- api/clients/codecs/poly_test.go | 2 +- encoding/rs/utils.go | 28 ++++---- encoding/utils.go | 8 --- encoding/utils/codec/codec.go | 2 + 11 files changed, 175 insertions(+), 217 deletions(-) delete mode 100644 api/clients/codecs/blob_form.go diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 64cb3678a5..180fc55249 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -6,79 +6,63 @@ import ( // Blob is data that is dispersed on eigenDA. // -// A Blob will contain either an encodedPayload, or a coeffPoly. Whether the Blob contains the former or the latter -// is determined by how the dispersing client has been configured. +// A Blob is represented under the hood by a coeff polynomial type Blob struct { - encodedPayload *encodedPayload - coeffPoly *coeffPoly + coeffPolynomial *coeffPoly } -// BlobFromEncodedPayload creates a Blob containing an encodedPayload -func blobFromEncodedPayload(encodedPayload *encodedPayload) *Blob { - return &Blob{encodedPayload: encodedPayload} -} +// BlobFromBytes initializes a Blob from bytes +func BlobFromBytes(bytes []byte) (*Blob, error) { + poly, err := coeffPolyFromBytes(bytes) + if err != nil { + return nil, fmt.Errorf("polynomial from bytes: %w", err) + } -// blobFromCoeffPoly creates a Blob containing a coeffPoly -func blobFromCoeffPoly(poly *coeffPoly) *Blob { - return &Blob{coeffPoly: poly} + return BlobFromPolynomial(poly) } -// NewBlob initializes a Blob from raw bytes, and the expected BlobForm -// -// This function will return an error if the input bytes cannot be successfully interpreted as the claimed BlobForm -func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { - switch blobForm { - case Eval: - encodedPayload, err := newEncodedPayload(bytes) - if err != nil { - return nil, fmt.Errorf("new encoded payload: %v", err) - } - - return blobFromEncodedPayload(encodedPayload), nil - case Coeff: - coeffPoly, err := coeffPolyFromBytes(bytes) - if err != nil { - return nil, fmt.Errorf("new coeff poly: %v", err) - } - - return blobFromCoeffPoly(coeffPoly), nil - default: - return nil, fmt.Errorf("unsupported blob form type: %v", blobForm) - } +// BlobFromPolynomial initializes a blob from a polynomial +func BlobFromPolynomial(coeffPolynomial *coeffPoly) (*Blob, error) { + return &Blob{coeffPolynomial: coeffPolynomial}, nil } // GetBytes gets the raw bytes of the Blob func (b *Blob) GetBytes() []byte { - if b.encodedPayload == nil { - return b.encodedPayload.getBytes() - } else { - return b.coeffPoly.getBytes() - } + return b.coeffPolynomial.getBytes() } // ToPayload converts the Blob into a Payload -func (b *Blob) ToPayload() (*Payload, error) { +// +// The payloadStartingForm indicates how payloads are constructed by the dispersing client. Based on the starting form +// of the payload, we can determine what operations must be done to the blob in order to reconstruct the original payload +func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { var encodedPayload *encodedPayload var err error - if b.encodedPayload != nil { - encodedPayload = b.encodedPayload - } else if b.coeffPoly != nil { - evalPoly, err := b.coeffPoly.toEvalPoly() + switch payloadStartingForm { + case PolynomialFormCoeff: + // the payload started off in coefficient form, so no conversion needs to be done + encodedPayload, err = b.coeffPolynomial.toEncodedPayload() + if err != nil { + return nil, fmt.Errorf("coeff poly to encoded payload: %w", err) + } + case PolynomialFormEval: + // the payload started off in evaluation form, so we first need to convert the blob's coeff poly into an eval poly + evalPoly, err := b.coeffPolynomial.toEvalPoly() if err != nil { - return nil, fmt.Errorf("coeff poly to eval poly: %v", err) + return nil, fmt.Errorf("coeff poly to eval poly: %w", err) } encodedPayload, err = evalPoly.toEncodedPayload() if err != nil { - return nil, fmt.Errorf("eval poly to encoded payload: %v", err) + return nil, fmt.Errorf("eval poly to encoded payload: %w", err) } - } else { - return nil, fmt.Errorf("blob has no contents") + default: + return nil, fmt.Errorf("invalid polynomial form") } payload, err := encodedPayload.decode() if err != nil { - return nil, fmt.Errorf("decode encoded payload: %v", err) + return nil, fmt.Errorf("decode payload: %w", err) } return payload, nil diff --git a/api/clients/codecs/blob_form.go b/api/clients/codecs/blob_form.go deleted file mode 100644 index b40a3097c1..0000000000 --- a/api/clients/codecs/blob_form.go +++ /dev/null @@ -1,12 +0,0 @@ -package codecs - -// BlobForm is an enum that represents the different ways that a blob may be represented -type BlobForm uint - -const ( - // Eval is short for "evaluation form". The field elements represent the evaluation at the polynomial's expanded - // roots of unity - Eval BlobForm = iota - // Coeff is short for "coefficient form". The field elements represent the coefficients of the polynomial - Coeff -) diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go index ea5e67c37f..8dd4cefd9d 100644 --- a/api/clients/codecs/coeff_poly.go +++ b/api/clients/codecs/coeff_poly.go @@ -17,12 +17,10 @@ type coeffPoly struct { fieldElements []fr.Element } -// coeffPolyFromBytes creates a new coeffPoly from bytes. This function performs the necessary checks to guarantee that the +// coeffPolyFromBytes creates a new polynomial from bytes. This function performs the necessary checks to guarantee that the // bytes are well-formed, and returns a new object if they are func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { - paddedBytes := encoding.PadToPowerOfTwo(bytes) - - fieldElements, err := rs.BytesToFieldElements(paddedBytes) + fieldElements, err := rs.BytesToFieldElements(bytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } @@ -31,27 +29,35 @@ func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { } // coeffPolyFromElements creates a new coeffPoly from field elements. -func coeffPolyFromElements(elements []fr.Element) (*coeffPoly, error) { - return &coeffPoly{fieldElements: elements}, nil +func coeffPolyFromElements(elements []fr.Element) *coeffPoly { + return &coeffPoly{fieldElements: elements} } // toEvalPoly converts a coeffPoly to an evalPoly, using the FFT operation -func (cp *coeffPoly) toEvalPoly() (*evalPoly, error) { - maxScale := uint8(math.Log2(float64(len(cp.fieldElements)))) - fftedElements, err := fft.NewFFTSettings(maxScale).FFT(cp.fieldElements, false) +func (p *coeffPoly) toEvalPoly() (*evalPoly, error) { + // we need to pad to the next power of 2, to be able to take the FFT + paddedLength := encoding.NextPowerOf2(len(p.fieldElements)) + padding := make([]fr.Element, paddedLength-len(p.fieldElements)) + paddedElements := append(p.fieldElements, padding...) + + maxScale := uint8(math.Log2(float64(len(paddedElements)))) + fftedElements, err := fft.NewFFTSettings(maxScale).FFT(paddedElements, false) if err != nil { return nil, fmt.Errorf("perform FFT: %w", err) } - evalPoly, err := evalPolyFromElements(fftedElements) - if err != nil { - return nil, fmt.Errorf("construct eval poly: %w", err) - } - - return evalPoly, nil + return evalPolyFromElements(fftedElements), nil } // GetBytes returns the bytes that underlie the polynomial -func (cp *coeffPoly) getBytes() []byte { - return rs.FieldElementsToBytes(cp.fieldElements) +func (p *coeffPoly) getBytes() []byte { + return rs.FieldElementsToBytes(p.fieldElements) +} + +// toEncodedPayload converts a coeffPoly into an encoded payload +// +// This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating +// an evalPoly. +func (p *coeffPoly) toEncodedPayload() (*encodedPayload, error) { + return encodedPayloadFromElements(p.fieldElements) } diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index 36f0e21dc8..af7feda86f 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -4,61 +4,99 @@ import ( "encoding/binary" "fmt" + "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) // encodedPayload represents a payload that has had an encoding applied to it +// +// Example encoding: +// +// Encoded Payload header (32 bytes total) Encoded Payload Data (len is multiple of 32) +// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] type encodedPayload struct { + // the size of these bytes is guaranteed to be a multiple of 32 bytes []byte } -// newEncodedPayload accepts an array of bytes which represent an encodedPayload. It performs the checks necessary -// to guarantee that the bytes are well-formed, and returns a newly constructed object if they are. -// -// Note that this function does not decode the input bytes to perform additional checks, so it is possible to construct -// an encodedPayload, where an attempt to decode will fail. -func newEncodedPayload(encodedPayloadBytes []byte) (*encodedPayload, error) { - inputLen := len(encodedPayloadBytes) - if inputLen < 32 { - return nil, fmt.Errorf( - "input bytes have length %d, which is smaller than the required 32 header bytes", inputLen) - } +// newEncodedPayload accepts a payload, and performs the PayloadEncodingVersion0 encoding to create an encoded payload +func newEncodedPayload(payload *Payload) (*encodedPayload, error) { + encodedPayloadHeader := make([]byte, 32) + // first byte is always 0 to ensure the payloadHeader is a valid bn254 element + encodedPayloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte - return &encodedPayload{ - bytes: encodedPayloadBytes, - }, nil + payloadBytes := payload.GetBytes() + + // encode payload length as uint32 + binary.BigEndian.PutUint32( + encodedPayloadHeader[2:6], + uint32(len(payloadBytes))) // uint32 should be more than enough to store the length (approx 4gb) + + // encode payload modulo bn254, and align to 32 bytes + encodedData := codec.PadPayload(payloadBytes) + encodedPayloadBytes := append(encodedPayloadHeader, encodedData...) + + return &encodedPayload{encodedPayloadBytes}, nil } -// decode applies the inverse of DefaultBlobEncoding to an encodedPayload, and returns the decoded Payload +// decode applies the inverse of PayloadEncodingVersion0 to an encodedPayload, and returns the decoded Payload func (ep *encodedPayload) decode() (*Payload, error) { claimedLength := binary.BigEndian.Uint32(ep.bytes[2:6]) // decode raw data modulo bn254 - nonPaddedData, err := codec.RemoveInternalPadding(ep.bytes[32:]) + unpaddedData, err := codec.RemoveInternalPadding(ep.bytes[32:]) if err != nil { return nil, fmt.Errorf("remove internal padding: %w", err) } - if uint32(len(nonPaddedData)) < claimedLength { + if uint32(len(unpaddedData)) < claimedLength { return nil, fmt.Errorf( - "data length %d is less than length claimed in encoded payload header %d", - len(nonPaddedData), claimedLength) + "length of unpadded data %d is less than length claimed in encoded payload header %d", + len(unpaddedData), claimedLength) } - return NewPayload(nonPaddedData[0:claimedLength]), nil + return NewPayload(unpaddedData[0:claimedLength]), nil } -// toEvalPoly converts an encodedPayload into an evalPoly +// toEvalPoly converts the encoded payload to a polynomial in evaluation form func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { - evalPoly, err := evalPolyFromBytes(ep.bytes) + fieldElements, err := rs.BytesToFieldElements(ep.bytes) if err != nil { - return nil, fmt.Errorf("new eval poly: %w", err) + return nil, fmt.Errorf("deserialize field elements: %w", err) } - return evalPoly, nil + return evalPolyFromElements(fieldElements), nil } -// getBytes returns the raw bytes that underlie the encodedPayload -func (ep *encodedPayload) getBytes() []byte { - return ep.bytes +// toCoeffPoly converts the encoded payload to a polynomial in coefficient form +func (ep *encodedPayload) toCoeffPoly() (*coeffPoly, error) { + fieldElements, err := rs.BytesToFieldElements(ep.bytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return coeffPolyFromElements(fieldElements), nil +} + +// encodedPayloadFromElements accepts an array of field elements, and converts them into an encoded payload +func encodedPayloadFromElements(fieldElements []fr.Element) (*encodedPayload, error) { + polynomialBytes := rs.FieldElementsToBytes(fieldElements) + + // this is the payload length, as claimed by the encoded payload header + payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) + // add 32 to the padded data length, since the encoded payload includes an encoded payload header + encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 + + // no matter what, this will be a multiple of 32, since both encodedPayloadLength and polynomialBytes are a multiple of 32 + // we can't just copy to length of encodedPayloadLength, since it's possible that the polynomial truncated 0s that + // are counted in the length of the encoded data payload. + lengthToCopy := min(encodedPayloadLength, uint32(len(polynomialBytes))) + + // TODO: we need to check the claimed payload length before creating this, to make sure it doesn't exceed the max blob size? + // Otherwise, I think there is an attack vector to maliciously say a payload is super huge, and OOM clients + encodedPayloadBytes := make([]byte, encodedPayloadLength) + copy(encodedPayloadBytes, polynomialBytes[:lengthToCopy]) + + return &encodedPayload{encodedPayloadBytes}, nil } diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index 8fa7bef3f2..a40a3dea98 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -1,84 +1,49 @@ package codecs import ( - "encoding/binary" "fmt" "math" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/fft" - "github.com/Layr-Labs/eigenda/encoding/rs" - "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) // evalPoly is a polynomial in evaluation form. // -// The underlying bytes represent 32 byte field elements, and the field elements represent the evaluation at the -// polynomial's roots of unity +// The underlying bytes represent 32 byte field elements, and the field elements represent the polynomial evaluation +// at roots of unity. +// +// The number of field elements is always a power of 2. type evalPoly struct { fieldElements []fr.Element } -// evalPolyFromBytes creates a new evalPoly from bytes. This function performs the necessary checks to guarantee that the -// bytes are well-formed, and returns a new object if they are -func evalPolyFromBytes(bytes []byte) (*evalPoly, error) { - paddedBytes := encoding.PadToPowerOfTwo(bytes) - - fieldElements, err := rs.BytesToFieldElements(paddedBytes) - if err != nil { - return nil, fmt.Errorf("deserialize field elements: %w", err) - } - - return &evalPoly{fieldElements: fieldElements}, nil -} - // evalPolyFromElements creates a new evalPoly from field elements. -func evalPolyFromElements(fieldElements []fr.Element) (*evalPoly, error) { - return &evalPoly{fieldElements: fieldElements}, nil +func evalPolyFromElements(elements []fr.Element) *evalPoly { + return &evalPoly{fieldElements: elements} } // toCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation -func (ep *evalPoly) toCoeffPoly() (*coeffPoly, error) { - maxScale := uint8(math.Log2(float64(len(ep.fieldElements)))) - ifftedElements, err := fft.NewFFTSettings(maxScale).FFT(ep.fieldElements, true) +func (p *evalPoly) toCoeffPoly() (*coeffPoly, error) { + // we need to pad to the next power of 2, to be able to take the FFT + paddedLength := encoding.NextPowerOf2(len(p.fieldElements)) + padding := make([]fr.Element, paddedLength-len(p.fieldElements)) + paddedElements := append(p.fieldElements, padding...) + + maxScale := uint8(math.Log2(float64(len(paddedElements)))) + ifftedElements, err := fft.NewFFTSettings(maxScale).FFT(paddedElements, true) if err != nil { return nil, fmt.Errorf("perform IFFT: %w", err) } - coeffPoly, err := coeffPolyFromElements(ifftedElements) - if err != nil { - return nil, fmt.Errorf("construct coeff poly: %w", err) - } - - return coeffPoly, nil + return coeffPolyFromElements(ifftedElements), nil } // toEncodedPayload converts an evalPoly into an encoded payload // // This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating // an evalPoly. -func (ep *evalPoly) toEncodedPayload() (*encodedPayload, error) { - polynomialBytes := rs.FieldElementsToBytes(ep.fieldElements) - - payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) - - // add 32 to the padded data length, since the encoded payload includes an encoded payload header - encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 - - if uint32(len(polynomialBytes)) < payloadLength { - return nil, fmt.Errorf( - "polynomial contains fewer bytes (%d) than expected encoded payload (%d), as determined by claimed length in encoded payload header (%d)", - len(polynomialBytes), encodedPayloadLength, payloadLength) - } - - encodedPayloadBytes := make([]byte, encodedPayloadLength) - copy(encodedPayloadBytes, polynomialBytes[:encodedPayloadLength]) - - encodedPayload, err := newEncodedPayload(encodedPayloadBytes) - if err != nil { - return nil, fmt.Errorf("construct encoded payload: %w", err) - } - - return encodedPayload, nil +func (p *evalPoly) toEncodedPayload() (*encodedPayload, error) { + return encodedPayloadFromElements(p.fieldElements) } diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index fe10e24ebc..14116a1318 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -1,10 +1,7 @@ package codecs import ( - "encoding/binary" "fmt" - - "github.com/Layr-Labs/eigenda/encoding/utils/codec" ) // Payload represents arbitrary user data, without any processing. @@ -19,58 +16,38 @@ func NewPayload(payloadBytes []byte) *Payload { } } -// Encode applies the PayloadEncodingVersion0 to the original payload bytes -// -// Example encoding: -// -// Encoded Payload header (32 bytes total) Encoded Payload Data -// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] -func (p *Payload) encode() (*encodedPayload, error) { - payloadHeader := make([]byte, 32) - // first byte is always 0 to ensure the payloadHeader is a valid bn254 element - payloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte - - // encode payload length as uint32 - binary.BigEndian.PutUint32( - payloadHeader[2:6], - uint32(len(p.bytes))) // uint32 should be more than enough to store the length (approx 4gb) - - // encode payload modulo bn254, and align to 32 bytes - encodedData := codec.PadPayload(p.bytes) - - encodedPayload, err := newEncodedPayload(append(payloadHeader, encodedData...)) - if err != nil { - return nil, fmt.Errorf("encoding payload: %w", err) - } - - return encodedPayload, nil -} - // ToBlob converts the Payload bytes into a Blob -func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { - encodedPayload, err := p.encode() +func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { + encodedPayload, err := newEncodedPayload(p) if err != nil { return nil, fmt.Errorf("encoding payload: %w", err) } + var coeffPolynomial *coeffPoly switch form { - case Eval: - return blobFromEncodedPayload(encodedPayload), nil - case Coeff: - evalPolynomial, err := encodedPayload.toEvalPoly() + case PolynomialFormCoeff: + // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in + // coefficient form + coeffPolynomial, err = encodedPayload.toCoeffPoly() if err != nil { - return nil, fmt.Errorf("encoded payload to eval poly: %w", err) + return nil, fmt.Errorf("coeff poly from elements: %w", err) + } + case PolynomialFormEval: + // the payload is in evaluation form, so we need to convert it to coeff form + evalPoly, err := encodedPayload.toEvalPoly() + if err != nil { + return nil, fmt.Errorf("eval poly from elements: %w", err) } - coeffPoly, err := evalPolynomial.toCoeffPoly() + coeffPolynomial, err = evalPoly.toCoeffPoly() if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } - - return blobFromCoeffPoly(coeffPoly), nil default: return nil, fmt.Errorf("unknown polynomial form: %v", form) } + + return BlobFromPolynomial(coeffPolynomial) } // GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data diff --git a/api/clients/codecs/payload_test.go b/api/clients/codecs/payload_test.go index 7e2a3e579c..dadaef88f7 100644 --- a/api/clients/codecs/payload_test.go +++ b/api/clients/codecs/payload_test.go @@ -17,7 +17,7 @@ func TestPayloadEncoding(t *testing.T) { for i := 0; i < iterations; i++ { payload := NewPayload(testRandom.Bytes(testRandom.Intn(1024) + 1)) - encodedPayload, err := payload.encode() + encodedPayload, err := newEncodedPayload(payload) require.NoError(t, err) // Decode the encoded data diff --git a/api/clients/codecs/poly_test.go b/api/clients/codecs/poly_test.go index 8665724e5e..d3ead89b08 100644 --- a/api/clients/codecs/poly_test.go +++ b/api/clients/codecs/poly_test.go @@ -20,7 +20,7 @@ func TestFftEncode(t *testing.T) { originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 payload := NewPayload(originalData) - encodedPayload, err := payload.encode() + encodedPayload, err := newEncodedPayload(payload) require.NoError(t, err) evalPoly, err := encodedPayload.toEvalPoly() diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index 4af149c62a..df4a3394d0 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -37,24 +37,17 @@ func ToFrArray(data []byte) ([]fr.Element, error) { return eles, nil } -// ToFieldElementArray accept a byte array as an input, and converts it to an array of field elements -// -// This function expects the input array to be a multiple of the size of a field element. -// TODO: test +// BytesToFieldElements accept a byte array as an input, and converts it to an array of field elements func BytesToFieldElements(inputData []byte) ([]fr.Element, error) { - if len(inputData)%encoding.BYTES_PER_SYMBOL != 0 { - return nil, fmt.Errorf( - "input array length %d isn't a multiple of encoding.BYTES_PER_SYMBOL %d", - len(inputData), encoding.BYTES_PER_SYMBOL) - } + bytes := padToBytesPerSymbol(inputData) - elementCount := len(inputData) / encoding.BYTES_PER_SYMBOL + elementCount := len(bytes) / encoding.BYTES_PER_SYMBOL outputElements := make([]fr.Element, elementCount) for i := 0; i < elementCount; i++ { destinationStartIndex := i * encoding.BYTES_PER_SYMBOL destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL - err := outputElements[i].SetBytesCanonical(inputData[destinationStartIndex:destinationEndIndex]) + err := outputElements[i].SetBytesCanonical(bytes[destinationStartIndex:destinationEndIndex]) if err != nil { return nil, fmt.Errorf("fr set bytes canonical: %w", err) } @@ -79,6 +72,19 @@ func FieldElementsToBytes(fieldElements []fr.Element) []byte { return outputBytes } +// padToBytesPerSymbol accepts input bytes, and returns the bytes padded to a multiple of encoding.BYTES_PER_SYMBOL +func padToBytesPerSymbol(inputBytes []byte) []byte { + remainder := len(inputBytes) % encoding.BYTES_PER_SYMBOL + + if remainder == 0 { + // no padding necessary, since bytes are already a multiple of BYTES_PER_SYMBOL + return inputBytes + } else { + necessaryPadding := encoding.BYTES_PER_SYMBOL - remainder + return append(inputBytes, make([]byte, necessaryPadding)...) + } +} + // ToByteArray converts a list of Fr to a byte array func ToByteArray(dataFr []fr.Element, maxDataSize uint64) []byte { n := len(dataFr) diff --git a/encoding/utils.go b/encoding/utils.go index 2e7d2179aa..fe665fe7d5 100644 --- a/encoding/utils.go +++ b/encoding/utils.go @@ -34,11 +34,3 @@ func NextPowerOf2[T constraints.Integer](d T) T { nextPower := math.Ceil(math.Log2(float64(d))) return T(math.Pow(2.0, nextPower)) } - -// PadToPowerOfTwo pads a byte slice to the next power of 2 -// TODO: test to make sure this doesn't increase size if already a power of 2 -func PadToPowerOfTwo(bytes []byte) []byte { - paddedLength := NextPowerOf2(len(bytes)) - padding := make([]byte, paddedLength-len(bytes)) - return append(bytes, padding...) -} diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index e0ba4bb54b..a3f05cf5a8 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -102,6 +102,8 @@ func PadPayload(inputData []byte) []byte { // GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after // adding internal byte padding +// +// The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL func GetPaddedDataLength(inputLen uint32) uint32 { bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) chunkCount := inputLen / bytesPerChunk From da63536c2244a47ea368199a9b2f001d17480f36 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:13:45 -0500 Subject: [PATCH 11/35] Fix PayloadVersion naming Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_codec.go | 11 +++++------ api/clients/config.go | 2 +- api/clients/v2/config.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index 91ee294a22..95b619de36 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -4,13 +4,12 @@ import ( "fmt" ) -type BlobEncodingVersion byte +type PayloadEncodingVersion byte const ( // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] - // NOTE: this encoding will soon be updated, such that the result will be padded to align to 32 bytes - PayloadEncodingVersion0 BlobEncodingVersion = 0x0 + PayloadEncodingVersion0 PayloadEncodingVersion = 0x0 ) type BlobCodec interface { @@ -18,7 +17,7 @@ type BlobCodec interface { EncodeBlob(rawData []byte) ([]byte, error) } -func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) { +func BlobEncodingVersionToCodec(version PayloadEncodingVersion) (BlobCodec, error) { switch version { case PayloadEncodingVersion0: return DefaultBlobCodec{}, nil @@ -34,7 +33,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) { // version byte is stored in [1], because [0] is always 0 to ensure the codecBlobHeader is a valid bn254 element // see https://github.com/Layr-Labs/eigenda/blob/master/api/clients/codecs/default_blob_codec.go#L21 // TODO: we should prob be working over a struct with methods such as GetBlobEncodingVersion() to prevent index errors - version := BlobEncodingVersion(data[1]) + version := PayloadEncodingVersion(data[1]) codec, err := BlobEncodingVersionToCodec(version) if err != nil { return nil, err @@ -50,7 +49,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) { // CreateCodec creates a new BlobCodec based on the defined polynomial form of payloads, and the desired // BlobEncodingVersion -func CreateCodec(payloadPolynomialForm PolynomialForm, version BlobEncodingVersion) (BlobCodec, error) { +func CreateCodec(payloadPolynomialForm PolynomialForm, version PayloadEncodingVersion) (BlobCodec, error) { lowLevelCodec, err := BlobEncodingVersionToCodec(version) if err != nil { return nil, fmt.Errorf("create low level codec: %w", err) diff --git a/api/clients/config.go b/api/clients/config.go index f4d9caa9fb..b3d1d946d6 100644 --- a/api/clients/config.go +++ b/api/clients/config.go @@ -64,7 +64,7 @@ type EigenDAClientConfig struct { DisableTLS bool // The blob encoding version to use when writing blobs from the high level interface. - PutBlobEncodingVersion codecs.BlobEncodingVersion + PutBlobEncodingVersion codecs.PayloadEncodingVersion // Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read. // This makes it possible to open points on the KZG commitment to prove that the field elements correspond to diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 5f65d28abd..a07084e067 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -14,7 +14,7 @@ type PayloadClientConfig struct { // The payload encoding version to use when encoding payload bytes // // This is the version that is put into the header of the EncodedPayload. - PayloadEncodingVersion codecs.BlobEncodingVersion + PayloadEncodingVersion codecs.PayloadEncodingVersion // The address of the EigenDACertVerifier contract EigenDACertVerifierAddr string From 9f984627c30bd6dea8d7b74aef76e4a86edf7e14 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:26:31 -0500 Subject: [PATCH 12/35] Use new conversion method Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/coeff_poly.go | 2 +- api/clients/codecs/encoded_payload.go | 4 +-- api/clients/codecs/payload_test.go | 40 --------------------------- encoding/rs/utils.go | 30 ++------------------ 4 files changed, 5 insertions(+), 71 deletions(-) delete mode 100644 api/clients/codecs/payload_test.go diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go index 8dd4cefd9d..e57303bf23 100644 --- a/api/clients/codecs/coeff_poly.go +++ b/api/clients/codecs/coeff_poly.go @@ -20,7 +20,7 @@ type coeffPoly struct { // coeffPolyFromBytes creates a new polynomial from bytes. This function performs the necessary checks to guarantee that the // bytes are well-formed, and returns a new object if they are func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { - fieldElements, err := rs.BytesToFieldElements(bytes) + fieldElements, err := rs.ToFrArray(bytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index af7feda86f..c38638f70a 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -61,7 +61,7 @@ func (ep *encodedPayload) decode() (*Payload, error) { // toEvalPoly converts the encoded payload to a polynomial in evaluation form func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { - fieldElements, err := rs.BytesToFieldElements(ep.bytes) + fieldElements, err := rs.ToFrArray(ep.bytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } @@ -71,7 +71,7 @@ func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { // toCoeffPoly converts the encoded payload to a polynomial in coefficient form func (ep *encodedPayload) toCoeffPoly() (*coeffPoly, error) { - fieldElements, err := rs.BytesToFieldElements(ep.bytes) + fieldElements, err := rs.ToFrArray(ep.bytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } diff --git a/api/clients/codecs/payload_test.go b/api/clients/codecs/payload_test.go deleted file mode 100644 index dadaef88f7..0000000000 --- a/api/clients/codecs/payload_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package codecs - -import ( - "bytes" - "testing" - - "github.com/Layr-Labs/eigenda/common/testutils/random" - "github.com/stretchr/testify/require" -) - -// TestCodec tests the encoding and decoding of random byte streams -func TestPayloadEncoding(t *testing.T) { - testRandom := random.NewTestRandom(t) - - // Number of test iterations - const iterations = 100 - - for i := 0; i < iterations; i++ { - payload := NewPayload(testRandom.Bytes(testRandom.Intn(1024) + 1)) - encodedPayload, err := newEncodedPayload(payload) - require.NoError(t, err) - - // Decode the encoded data - decodedPayload, err := encodedPayload.decode() - require.NoError(t, err) - - if err != nil { - t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) - } - - // Compare the original data with the decoded data - if !bytes.Equal(payload.GetBytes(), decodedPayload.GetBytes()) { - t.Fatalf( - "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", - i, - payload.GetBytes(), - decodedPayload.GetBytes()) - } - } -} diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index df4a3394d0..7d054fb9f4 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -11,34 +11,8 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) -// ToFrArray TODO: This function will be removed in favor ToFieldElementArray -func ToFrArray(data []byte) ([]fr.Element, error) { - numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) - eles := make([]fr.Element, numEle) - - for i := uint64(0); i < numEle; i++ { - start := i * uint64(encoding.BYTES_PER_SYMBOL) - end := (i + 1) * uint64(encoding.BYTES_PER_SYMBOL) - if end >= uint64(len(data)) { - padded := make([]byte, encoding.BYTES_PER_SYMBOL) - copy(padded, data[start:]) - err := eles[i].SetBytesCanonical(padded) - if err != nil { - return nil, err - } - } else { - err := eles[i].SetBytesCanonical(data[start:end]) - if err != nil { - return nil, err - } - } - } - - return eles, nil -} - -// BytesToFieldElements accept a byte array as an input, and converts it to an array of field elements -func BytesToFieldElements(inputData []byte) ([]fr.Element, error) { +// ToFrArray accept a byte array as an input, and converts it to an array of field elements +func ToFrArray(inputData []byte) ([]fr.Element, error) { bytes := padToBytesPerSymbol(inputData) elementCount := len(bytes) / encoding.BYTES_PER_SYMBOL From 58b76c17deb3529d3dcea5efc783fd94dc64cefe Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:55:26 -0500 Subject: [PATCH 13/35] Add better descriptions to codec utils Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- encoding/utils/codec/codec.go | 48 ++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index a3f05cf5a8..b1f8b39755 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -11,6 +11,9 @@ import ( // This ensures every 32 bytes is within the valid range of a field element for bn254 curve. // If the input data is not a multiple of 31, the remainder is added to the output by // inserting a 0 and the remainder. The output is thus not necessarily a multiple of 32. +// +// TODO (litt3): usage of this function should be migrated to use PadPayload instead. I've left it unchanged for now, +// since v1 logic and tests rely on the specific assumptions of this implementation. func ConvertByPaddingEmptyByte(data []byte) []byte { dataSize := len(data) parseSize := encoding.BYTES_PER_SYMBOL - 1 @@ -43,6 +46,9 @@ func ConvertByPaddingEmptyByte(data []byte) []byte { // The function does not assume the input is a multiple of BYTES_PER_SYMBOL(32 bytes). // For the reminder of the input, the first byte is taken out, and the rest is appended to // the output. +// +// TODO (litt3): usage of this function should be migrated to use RemoveInternalPadding instead. I've left it unchanged +// for now, since v1 logic and tests rely on the specific assumptions of this implementation. func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { dataSize := len(data) parseSize := encoding.BYTES_PER_SYMBOL @@ -72,6 +78,10 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { // the data will be a valid field element for the bn254 curve // // Additionally, this function will add necessary padding to align the output to 32 bytes +// +// NOTE: this method is a reimplementation of ConvertByPaddingEmptyByte, with one meaningful difference: the alignment +// of the output to encoding.BYTES_PER_SYMBOL. This alignment actually makes the padding logic simpler, and the +// code that uses this function needs an aligned output anyway. func PadPayload(inputData []byte) []byte { // 31 bytes, for the bn254 curve bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) @@ -100,25 +110,14 @@ func PadPayload(inputData []byte) []byte { return paddedOutput } -// GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after -// adding internal byte padding -// -// The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL -func GetPaddedDataLength(inputLen uint32) uint32 { - bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) - chunkCount := inputLen / bytesPerChunk - - if inputLen%bytesPerChunk != 0 { - chunkCount++ - } - - return chunkCount * encoding.BYTES_PER_SYMBOL -} - // RemoveInternalPadding accepts an array of padded data, and removes the internal padding that was added in PadPayload // -// This function assumes that the input aligns to 32 bytes. Since it is removing 1 byte for every 31 bytes kept, the output -// from this function is not guaranteed to align to 32 bytes. +// This function assumes that the input aligns to 32 bytes. Since it is removing 1 byte for every 31 bytes kept, the +// output from this function is not guaranteed to align to 32 bytes. +// +// NOTE: this method is a reimplementation of RemoveEmptyByteFromPaddedBytes, with one meaningful difference: this +// function relies on the assumption that the input is aligned to encoding.BYTES_PER_SYMBOL, which makes the padding +// removal logic simpler. func RemoveInternalPadding(paddedData []byte) ([]byte, error) { if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { return nil, fmt.Errorf( @@ -143,3 +142,18 @@ func RemoveInternalPadding(paddedData []byte) ([]byte, error) { return outputData, nil } + +// GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after +// adding internal byte padding +// +// The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL +func GetPaddedDataLength(inputLen uint32) uint32 { + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + chunkCount := inputLen / bytesPerChunk + + if inputLen%bytesPerChunk != 0 { + chunkCount++ + } + + return chunkCount * encoding.BYTES_PER_SYMBOL +} From 916573f6070b45c83a0e94e254dd940b36dc1400 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:24:23 -0500 Subject: [PATCH 14/35] Do test work Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_test.go | 39 ++++++++++++++++++++ api/clients/codecs/conversion_test.go | 52 +++++++++++++++++++++++++++ api/clients/codecs/poly_test.go | 50 -------------------------- 3 files changed, 91 insertions(+), 50 deletions(-) create mode 100644 api/clients/codecs/blob_test.go create mode 100644 api/clients/codecs/conversion_test.go delete mode 100644 api/clients/codecs/poly_test.go diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go new file mode 100644 index 0000000000..01df4b42c2 --- /dev/null +++ b/api/clients/codecs/blob_test.go @@ -0,0 +1,39 @@ +package codecs + +import ( + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestBlobConversion checks that internal blob conversion methods produce consistent results +func TestBlobConversion(t *testing.T) { + testRandom := random.NewTestRandom(t) + + iterations := 100 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) + testBlobConversionForForm(t, originalData, PolynomialFormEval) + testBlobConversionForForm(t, originalData, PolynomialFormCoeff) + } +} + +func testBlobConversionForForm(t *testing.T, payloadBytes []byte, form PolynomialForm) { + payload := NewPayload(payloadBytes) + + blob, err := payload.ToBlob(form) + require.NoError(t, err) + + blobBytes := blob.GetBytes() + blobFromBytes, err := BlobFromBytes(blobBytes) + require.NoError(t, err) + + decodedPayload, err := blobFromBytes.ToPayload(form) + require.NoError(t, err) + + decodedPayloadBytes := decodedPayload.GetBytes() + + require.Equal(t, payloadBytes, decodedPayloadBytes) +} diff --git a/api/clients/codecs/conversion_test.go b/api/clients/codecs/conversion_test.go new file mode 100644 index 0000000000..cd4142e0d2 --- /dev/null +++ b/api/clients/codecs/conversion_test.go @@ -0,0 +1,52 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestConversionConsistency checks that data can be encoded and decoded repeatedly, always getting back the original data +// TODO: we should probably be using fuzzing instead of this kind of ad-hoc random search testing +func TestConversionConsistency(t *testing.T) { + testRandom := random.NewTestRandom(t) + + iterations := 100 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 + + payload := NewPayload(originalData) + + blob1, err := payload.ToBlob(PolynomialFormEval) + require.NoError(t, err) + + blob2, err := payload.ToBlob(PolynomialFormCoeff) + require.NoError(t, err) + + decodedPayload1, err := blob1.ToPayload(PolynomialFormEval) + require.NoError(t, err) + + decodedPayload2, err := blob2.ToPayload(PolynomialFormCoeff) + require.NoError(t, err) + + // Compare the original data with the decoded data + if !bytes.Equal(originalData, decodedPayload1.GetBytes()) { + t.Fatalf( + "Iteration %d: original and data decoded from blob1 do not match\nOriginal: %v\nDecoded: %v", + i, + originalData, + decodedPayload1.GetBytes()) + } + + if !bytes.Equal(originalData, decodedPayload2.GetBytes()) { + t.Fatalf( + "Iteration %d: original and data decoded from blob2 do not match\nOriginal: %v\nDecoded: %v", + i, + originalData, + decodedPayload1.GetBytes()) + } + } +} diff --git a/api/clients/codecs/poly_test.go b/api/clients/codecs/poly_test.go deleted file mode 100644 index d3ead89b08..0000000000 --- a/api/clients/codecs/poly_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package codecs - -import ( - "bytes" - "testing" - - "github.com/Layr-Labs/eigenda/common/testutils/random" - "github.com/stretchr/testify/require" -) - -// TestFftEncode checks that data can be IfftEncoded and FftEncoded repeatedly, always getting back the original data -// TODO: we should probably be using fuzzing instead of this kind of ad-hoc random search testing -func TestFftEncode(t *testing.T) { - testRandom := random.NewTestRandom(t) - - // Number of test iterations - iterations := 100 - - for i := 0; i < iterations; i++ { - originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 - - payload := NewPayload(originalData) - encodedPayload, err := newEncodedPayload(payload) - require.NoError(t, err) - - evalPoly, err := encodedPayload.toEvalPoly() - require.NoError(t, err) - - coeffPoly, err := evalPoly.toCoeffPoly() - require.NoError(t, err) - - convertedEvalPoly, err := coeffPoly.toEvalPoly() - require.NoError(t, err) - - convertedEncodedPayload, err := convertedEvalPoly.toEncodedPayload() - require.NoError(t, err) - - decodedPayload, err := convertedEncodedPayload.decode() - require.NoError(t, err) - - // Compare the original data with the decoded data - if !bytes.Equal(originalData, decodedPayload.GetBytes()) { - t.Fatalf( - "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", - i, - originalData, - decodedPayload.GetBytes()) - } - } -} From 7eed6182752c1686c202d8b5eeb22e4cebf3e1c5 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:09:49 -0500 Subject: [PATCH 15/35] Add length checks Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 21 ++-- api/clients/codecs/blob_test.go | 4 +- api/clients/codecs/coeff_poly.go | 8 +- api/clients/codecs/encoded_payload.go | 94 +++++++++++++--- api/clients/codecs/encoded_payload_test.go | 124 +++++++++++++++++++++ api/clients/codecs/eval_poly.go | 7 +- api/clients/codecs/payload.go | 9 +- encoding/utils/codec/codec.go | 53 ++++++++- 8 files changed, 286 insertions(+), 34 deletions(-) create mode 100644 api/clients/codecs/encoded_payload_test.go diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 180fc55249..479a359341 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -9,21 +9,26 @@ import ( // A Blob is represented under the hood by a coeff polynomial type Blob struct { coeffPolynomial *coeffPoly + // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment + // This is the blob length in symbols, NOT in bytes + blobLength uint32 } -// BlobFromBytes initializes a Blob from bytes -func BlobFromBytes(bytes []byte) (*Blob, error) { +// BlobFromBytes initializes a Blob from bytes, and a blobLength in symbols +func BlobFromBytes(bytes []byte, blobLength uint32) (*Blob, error) { poly, err := coeffPolyFromBytes(bytes) if err != nil { return nil, fmt.Errorf("polynomial from bytes: %w", err) } - return BlobFromPolynomial(poly) + return BlobFromPolynomial(poly, blobLength) } -// BlobFromPolynomial initializes a blob from a polynomial -func BlobFromPolynomial(coeffPolynomial *coeffPoly) (*Blob, error) { - return &Blob{coeffPolynomial: coeffPolynomial}, nil +// BlobFromPolynomial initializes a blob from a polynomial, and a blobLength in symbols +func BlobFromPolynomial(coeffPolynomial *coeffPoly, blobLength uint32) (*Blob, error) { + return &Blob{ + coeffPolynomial: coeffPolynomial, + blobLength: blobLength}, nil } // GetBytes gets the raw bytes of the Blob @@ -41,7 +46,7 @@ func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { switch payloadStartingForm { case PolynomialFormCoeff: // the payload started off in coefficient form, so no conversion needs to be done - encodedPayload, err = b.coeffPolynomial.toEncodedPayload() + encodedPayload, err = b.coeffPolynomial.toEncodedPayload(b.blobLength) if err != nil { return nil, fmt.Errorf("coeff poly to encoded payload: %w", err) } @@ -52,7 +57,7 @@ func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { return nil, fmt.Errorf("coeff poly to eval poly: %w", err) } - encodedPayload, err = evalPoly.toEncodedPayload() + encodedPayload, err = evalPoly.toEncodedPayload(b.blobLength) if err != nil { return nil, fmt.Errorf("eval poly to encoded payload: %w", err) } diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index 01df4b42c2..b1866bca96 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -11,7 +11,7 @@ import ( func TestBlobConversion(t *testing.T) { testRandom := random.NewTestRandom(t) - iterations := 100 + iterations := 1000 for i := 0; i < iterations; i++ { originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) @@ -27,7 +27,7 @@ func testBlobConversionForForm(t *testing.T, payloadBytes []byte, form Polynomia require.NoError(t, err) blobBytes := blob.GetBytes() - blobFromBytes, err := BlobFromBytes(blobBytes) + blobFromBytes, err := BlobFromBytes(blobBytes, blob.blobLength) require.NoError(t, err) decodedPayload, err := blobFromBytes.ToPayload(form) diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go index e57303bf23..1188936d31 100644 --- a/api/clients/codecs/coeff_poly.go +++ b/api/clients/codecs/coeff_poly.go @@ -56,8 +56,8 @@ func (p *coeffPoly) getBytes() []byte { // toEncodedPayload converts a coeffPoly into an encoded payload // -// This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating -// an evalPoly. -func (p *coeffPoly) toEncodedPayload() (*encodedPayload, error) { - return encodedPayloadFromElements(p.fieldElements) +// blobLength is required, to be able to perform length checks on the encoded payload during construction. +// blobLength is in symbols, NOT bytes +func (p *coeffPoly) toEncodedPayload(blobLength uint32) (*encodedPayload, error) { + return encodedPayloadFromElements(p.fieldElements, blobLength) } diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index c38638f70a..d22cdc74a8 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" + "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -50,10 +51,25 @@ func (ep *encodedPayload) decode() (*Payload, error) { return nil, fmt.Errorf("remove internal padding: %w", err) } - if uint32(len(unpaddedData)) < claimedLength { + unpaddedDataLength := uint32(len(unpaddedData)) + + // data length is checked when constructing an encoded payload. If this error is encountered, that means there + // must be a flaw in the logic at construction time (or someone was bad and didn't use the proper construction methods) + if unpaddedDataLength < claimedLength { return nil, fmt.Errorf( - "length of unpadded data %d is less than length claimed in encoded payload header %d", - len(unpaddedData), claimedLength) + "length of unpadded data %d is less than length claimed in encoded payload header %d. this should never happen", + unpaddedDataLength, claimedLength) + } + + // unpadded data length can be slightly bigger than the claimed length, since RemoveInternalPadding doesn't + // do anything to remove trailing zeros that may have been added when the data was initially padded. + // however, this extra padding shouldn't exceed 31 bytes, because that's the most that would be added + // when padding the data length to 32 bytes. If this error occurs, that means there must be a flaw in the logic at + // construction time (or someone was bad and didn't use the proper construction methods) + if unpaddedDataLength > claimedLength+31 { + return nil, fmt.Errorf( + "length of unpadded data %d is more than 31 bytes longer than claimed length %d. this should never happen", + unpaddedDataLength, claimedLength) } return NewPayload(unpaddedData[0:claimedLength]), nil @@ -80,23 +96,73 @@ func (ep *encodedPayload) toCoeffPoly() (*coeffPoly, error) { } // encodedPayloadFromElements accepts an array of field elements, and converts them into an encoded payload -func encodedPayloadFromElements(fieldElements []fr.Element) (*encodedPayload, error) { +// +// blobLength is the length of the blob IN SYMBOLS, as claimed by the blob commitment. This is needed to make sure +// that the claimed length in the encoded payload header is valid relative to the total blob length +func encodedPayloadFromElements(fieldElements []fr.Element, blobLength uint32) (*encodedPayload, error) { polynomialBytes := rs.FieldElementsToBytes(fieldElements) - - // this is the payload length, as claimed by the encoded payload header + // this is the payload length in bytes, as claimed by the encoded payload header payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) - // add 32 to the padded data length, since the encoded payload includes an encoded payload header - encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 - // no matter what, this will be a multiple of 32, since both encodedPayloadLength and polynomialBytes are a multiple of 32 - // we can't just copy to length of encodedPayloadLength, since it's possible that the polynomial truncated 0s that - // are counted in the length of the encoded data payload. - lengthToCopy := min(encodedPayloadLength, uint32(len(polynomialBytes))) + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) + if err != nil { + return nil, fmt.Errorf("get max permissible payload length: %w", err) + } + + if payloadLength > maxPermissiblePayloadLength { + return nil, fmt.Errorf( + `length claimed in encoded payload header (%d bytes) exceeds the max permissible length (%d bytes) + that could be contained in a blob of length %d symbols (%d bytes)`, + payloadLength, maxPermissiblePayloadLength, blobLength, blobLength*encoding.BYTES_PER_SYMBOL) + } + + // this is the length you would get if you padded a payload of the length claimed in the encoded payload header + paddedLength := codec.GetPaddedDataLength(payloadLength) + // add 32 to the padded data length, since the encoded payload includes an encoded payload header + encodedPayloadLength := paddedLength + 32 + + polynomialByteCount := uint32(len(polynomialBytes)) + + // no matter what, this will be a multiple of 32, since the two possible values, encodedPayloadLength and + // polynomialBytes, are multiples of 32. This is important, since the encoded payload being created is + // expected to have a byte count that's a multiple of 32. + lengthToCopy := encodedPayloadLength + + // if encodedPayloadLength is greater than the polynomial bytes, that indicates that the polynomial bytes we have + // are missing trailing 0 bytes which were originally part of the dispersed blob. For this to happen, it means + // that whichever source provided us with these bytes truncated the trailing 0s. This probably won't happen in + // practice, but if it were to happen, it wouldn't be caught when verifying commitments, since trailing 0s don't + // affect the commitment. This isn't a problem, though: we can handle this edge case here. + if encodedPayloadLength > polynomialByteCount { + // we are copying from the polynomialBytes, so make sure that we don't try to copy more data than actually exists + lengthToCopy = polynomialByteCount + } else if encodedPayloadLength < polynomialByteCount { + // we assume that the polynomialBytes might have additional trailing 0s beyond the expected size of the encoded + // payload. Here, we check the assumption that all trailing values are 0. If there are any non-zero trailing + // values, something has gone wrong in the data pipeline, and this should produce a loud failure. Either a + // dispersing client is playing sneaky games, or there's a bug somewhere. + err := checkTrailingZeros(polynomialBytes, encodedPayloadLength) + if err != nil { + return nil, fmt.Errorf("check that trailing values in polynomial are zeros: %w", err) + } + } - // TODO: we need to check the claimed payload length before creating this, to make sure it doesn't exceed the max blob size? - // Otherwise, I think there is an attack vector to maliciously say a payload is super huge, and OOM clients encodedPayloadBytes := make([]byte, encodedPayloadLength) copy(encodedPayloadBytes, polynomialBytes[:lengthToCopy]) return &encodedPayload{encodedPayloadBytes}, nil } + +// checkTrailingZeros accepts an array of bytes, and the number of bytes at the front of the array which are permitted +// to be non-zero +// +// This function returns an error if any byte in the array after these permitted non-zero values is found to be non-zero +func checkTrailingZeros(inputBytes []byte, nonZeroLength uint32) error { + for i := uint32(len(inputBytes)) - 1; i >= nonZeroLength; i-- { + if inputBytes[i] != 0x0 { + return fmt.Errorf("byte at index %d was expected to be 0x0, but instead was %x", i, inputBytes[i]) + } + } + + return nil +} \ No newline at end of file diff --git a/api/clients/codecs/encoded_payload_test.go b/api/clients/codecs/encoded_payload_test.go new file mode 100644 index 0000000000..0425b5065e --- /dev/null +++ b/api/clients/codecs/encoded_payload_test.go @@ -0,0 +1,124 @@ +package codecs + +import ( + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/stretchr/testify/require" +) + +// TestDecodeShortBytes checks that an encoded payload with a length less than claimed length fails at decode time +func TestDecodeShortBytes(t *testing.T) { + testRandom := random.NewTestRandom(t) + originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + // truncate + encodedPayload.bytes = encodedPayload.bytes[:len(encodedPayload.bytes) -32] + + payload, err := encodedPayload.decode() + require.Error(t, err) + require.Nil(t, payload) +} + +// TestDecodeLongBytes checks that an encoded payload with length too much greater than claimed fails at decode +func TestDecodeLongBytes(t *testing.T) { + testRandom := random.NewTestRandom(t) + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + encodedPayload.bytes = append(encodedPayload.bytes, make([]byte, 32)...) + payload2, err := encodedPayload.decode() + require.Error(t, err) + require.Nil(t, payload2) +} + +// TestEncodeTooManyElements checks that encodedPayloadFromElements fails at the expect limit, relative to payload +// length and blob length +func TestEncodeTooManyElements(t *testing.T) { + // length in symbols + blobLength := uint32(16) + + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) + require.NoError(t, err) + + testRandom := random.NewTestRandom(t) + + almostTooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength)) + almostTooLongEncodedPayload, err := newEncodedPayload(NewPayload(almostTooLongData)) + require.NoError(t, err) + require.NotNil(t, almostTooLongEncodedPayload) + + almostTooLongCoeffPoly, err := almostTooLongEncodedPayload.toCoeffPoly() + require.NoError(t, err) + require.NotNil(t, almostTooLongCoeffPoly) + + // there are almost too many field elements for the defined blob length, but not quite + _, err = encodedPayloadFromElements(almostTooLongCoeffPoly.fieldElements, blobLength) + require.NoError(t, err) + + tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) + tooLongEncodedPayload, err := newEncodedPayload(NewPayload(tooLongData)) + require.NoError(t, err) + require.NotNil(t, tooLongEncodedPayload) + + tooLongCoeffPoly, err := tooLongEncodedPayload.toCoeffPoly() + require.NoError(t, err) + require.NotNil(t, tooLongCoeffPoly) + + // there is one too many field elements for the defined blob length + _, err = encodedPayloadFromElements(tooLongCoeffPoly.fieldElements, blobLength) + require.Error(t, err) +} + +// TestTrailingNonZeros checks that any non-zero values that come after the end of the claimed payload length +// cause an error to be returned. +func TestTrailingNonZeros(t *testing.T) { + testRandom := random.NewTestRandom(t) + // make original data 1025 bytes, so that there is plenty of wiggle room to tack on another couple bytes + // without having a payload that fails verification for being too long + originalData := testRandom.Bytes(1025) + + blob, err := NewPayload(originalData).ToBlob(PolynomialFormCoeff) + require.NoError(t, err) + + fieldElements1 := make([]fr.Element, len(blob.coeffPolynomial.fieldElements)) + copy(fieldElements1, blob.coeffPolynomial.fieldElements) + + fieldElements2 := make([]fr.Element, len(blob.coeffPolynomial.fieldElements)) + copy(fieldElements2, blob.coeffPolynomial.fieldElements) + + // adding a 0 is fine + fieldElements1 = append(fieldElements1, fr.Element{}) + _, err = encodedPayloadFromElements(fieldElements1, blob.blobLength) + require.NoError(t, err) + + // adding a non-0 is non-fine + fieldElements2 = append(fieldElements2, fr.Element{0,0,0,1}) + _, err = encodedPayloadFromElements(fieldElements2, blob.blobLength) + require.Error(t, err) +} + +// TestEncodeWithFewerElements tests that having fewer bytes than expected doesn't throw an error +func TestEncodeWithFewerElements(t *testing.T) { + testRandom := random.NewTestRandom(t) + originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) + + blob, err := NewPayload(originalData).ToBlob(PolynomialFormCoeff) + require.NoError(t, err) + + fieldElements := make([]fr.Element, len(blob.coeffPolynomial.fieldElements) - 1) + // intentionally don't copy all the elements + copy(fieldElements, blob.coeffPolynomial.fieldElements[:len(blob.coeffPolynomial.fieldElements) - 1]) + + // even though the actual length will be less than the claimed length, we shouldn't see any error + encodedPayload, err := encodedPayloadFromElements(fieldElements, blob.blobLength) + require.NoError(t, err) + require.NotNil(t, encodedPayload) +} diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index a40a3dea98..79ee7a3111 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -42,8 +42,7 @@ func (p *evalPoly) toCoeffPoly() (*coeffPoly, error) { // toEncodedPayload converts an evalPoly into an encoded payload // -// This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating -// an evalPoly. -func (p *evalPoly) toEncodedPayload() (*encodedPayload, error) { - return encodedPayloadFromElements(p.fieldElements) +// blobLength is required, to be able to perform length checks on the encoded payload during construction +func (p *evalPoly) toEncodedPayload(blobLength uint32) (*encodedPayload, error) { + return encodedPayloadFromElements(p.fieldElements, blobLength) } diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 14116a1318..6df0e9488d 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -2,6 +2,8 @@ package codecs import ( "fmt" + + "github.com/Layr-Labs/eigenda/encoding" ) // Payload represents arbitrary user data, without any processing. @@ -47,7 +49,12 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("unknown polynomial form: %v", form) } - return BlobFromPolynomial(coeffPolynomial) + // it's possible that the number of field elements might already be a power of 2 + // in that case, calling NextPowerOf2 will just return the input value + // TODO: write a test to check this + blobLength := uint32(encoding.NextPowerOf2(len(coeffPolynomial.fieldElements))) + + return BlobFromPolynomial(coeffPolynomial, blobLength) } // GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index b1f8b39755..535c3fd867 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" ) // ConvertByPaddingEmptyByte takes bytes and insert an empty byte at the front of every 31 byte. @@ -77,11 +78,12 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { // PadPayload internally pads the input data by prepending a 0x00 to each chunk of 31 bytes. This guarantees that // the data will be a valid field element for the bn254 curve // -// Additionally, this function will add necessary padding to align the output to 32 bytes +// # Additionally, this function will add necessary padding to align the output to 32 bytes // // NOTE: this method is a reimplementation of ConvertByPaddingEmptyByte, with one meaningful difference: the alignment // of the output to encoding.BYTES_PER_SYMBOL. This alignment actually makes the padding logic simpler, and the // code that uses this function needs an aligned output anyway. +// TODO: test, especially lower bound func PadPayload(inputData []byte) []byte { // 31 bytes, for the bn254 curve bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) @@ -118,6 +120,7 @@ func PadPayload(inputData []byte) []byte { // NOTE: this method is a reimplementation of RemoveEmptyByteFromPaddedBytes, with one meaningful difference: this // function relies on the assumption that the input is aligned to encoding.BYTES_PER_SYMBOL, which makes the padding // removal logic simpler. +// TODO: test, especially lower bound func RemoveInternalPadding(paddedData []byte) ([]byte, error) { if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { return nil, fmt.Errorf( @@ -147,6 +150,7 @@ func RemoveInternalPadding(paddedData []byte) ([]byte, error) { // adding internal byte padding // // The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL +// TODO: test, especially lower bound func GetPaddedDataLength(inputLen uint32) uint32 { bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) chunkCount := inputLen / bytesPerChunk @@ -157,3 +161,50 @@ func GetPaddedDataLength(inputLen uint32) uint32 { return chunkCount * encoding.BYTES_PER_SYMBOL } + +// GetUnpaddedDataLength accepts the length of an array that has been padded with PadPayload +// +// It returns what the length of the output array would be, if you called RemoveInternalPadding on it. +// TODO: test, especially lower bound +func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { + if inputLen == 0 { + return 0, fmt.Errorf("input length is zero") + } + + if inputLen%encoding.BYTES_PER_SYMBOL != 0 { + return 0, fmt.Errorf( + "%d isn't a multiple of encoding.BYTES_PER_SYMBOL (%d)", + inputLen, encoding.BYTES_PER_SYMBOL) + } + + chunkCount := inputLen / encoding.BYTES_PER_SYMBOL + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + + unpaddedLength := chunkCount * bytesPerChunk + + return unpaddedLength, nil +} + +// GetMaxPermissiblePayloadLength accepts a blob length IN SYMBOLS, and returns the size IN BYTES of the largest payload +// that could fit inside the blob. +// TODO: test, especially lower bound +func GetMaxPermissiblePayloadLength(blobLength uint32) (uint32, error) { + if blobLength == 0 { + return 0, fmt.Errorf("input blob length is zero") + } + + // TODO (litt3): it's awkward to use a method defined in fft for this, but it's not trivial to move to a better + // location, due to the resulting cyclic imports, and I'd prefer not to reimplement. Ideally, a proper location + // would be found for this important utility function + if !fft.IsPowerOfTwo(uint64(blobLength)) { + return 0, fmt.Errorf("blobLength %d is not a power of two", blobLength) + } + + // subtract 32 from the blob length before doing the unpad operation, to account for the encoded payload header + maxPayloadLength, err := GetUnpaddedDataLength(blobLength*encoding.BYTES_PER_SYMBOL - 32) + if err != nil { + return 0, fmt.Errorf("get unpadded data length: %w", err) + } + + return maxPayloadLength, nil +} From b604c82b7d10774263f989189cf3aeac02974b68 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:35:12 -0500 Subject: [PATCH 16/35] Write test for power of 2 util Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/payload.go | 1 - encoding/utils_test.go | 35 +++++++++++++++++++++++++++++++++++ test/v2/client/test_client.go | 4 ++-- 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 encoding/utils_test.go diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 6df0e9488d..f825042c35 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -51,7 +51,6 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { // it's possible that the number of field elements might already be a power of 2 // in that case, calling NextPowerOf2 will just return the input value - // TODO: write a test to check this blobLength := uint32(encoding.NextPowerOf2(len(coeffPolynomial.fieldElements))) return BlobFromPolynomial(coeffPolynomial, blobLength) diff --git a/encoding/utils_test.go b/encoding/utils_test.go new file mode 100644 index 0000000000..5b63b4adc5 --- /dev/null +++ b/encoding/utils_test.go @@ -0,0 +1,35 @@ +package encoding + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNextPowerOf2(t *testing.T) { + testHeight := 65536 + + // 2 ^ 16 = 65536 + powersToGenerate := 17 + powers := make([]int, powersToGenerate) + for i := 0; i < powersToGenerate; i++ { + powers[i] = int(math.Pow(2, float64(i))) + } + + powerIndex := 0 + for i := 1; i <= testHeight; i++ { + nextPowerOf2 := NextPowerOf2(i) + require.Equal(t, nextPowerOf2, powers[powerIndex]) + + if i == powers[powerIndex] { + powerIndex++ + } + } + + // sanity check the test logic + require.Equal(t, powerIndex, len(powers)) + + // extra sanity check + require.Equal(t, 16, NextPowerOf2(16)) +} diff --git a/test/v2/client/test_client.go b/test/v2/client/test_client.go index 970b43842e..afc5132bd6 100644 --- a/test/v2/client/test_client.go +++ b/test/v2/client/test_client.go @@ -210,7 +210,7 @@ func NewTestClient( payloadClientConfig := clients.GetDefaultPayloadClientConfig() payloadClientConfig.EigenDACertVerifierAddr = config.EigenDACertVerifierAddress - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, codecs.DefaultBlobEncoding) + blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, codecs.PayloadEncodingVersion0) if err != nil { return nil, fmt.Errorf("failed to create blob codec: %w", err) } @@ -324,7 +324,7 @@ func (c *TestClient) GetPayloadDisperser(quorums []core.QuorumID) (*clients.Payl DisperseBlobTimeout: 1337 * time.Hour, // this suite enforces its own timeouts } - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.BlobEncodingVersion) + blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.PayloadEncodingVersion) if err != nil { return nil, fmt.Errorf("failed to create blob codec: %w", err) } From 2ef8e997c42cb0955866254bf096d263e8d32306 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 12 Feb 2025 08:41:06 -0500 Subject: [PATCH 17/35] Write more tests Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/encoded_payload_test.go | 66 +++++++++++----------- encoding/test_utils.go | 18 ++++++ encoding/utils/codec/codec.go | 4 -- encoding/utils/codec/codec_test.go | 31 +++++++++- encoding/utils_test.go | 11 ++-- 5 files changed, 85 insertions(+), 45 deletions(-) create mode 100644 encoding/test_utils.go diff --git a/api/clients/codecs/encoded_payload_test.go b/api/clients/codecs/encoded_payload_test.go index 0425b5065e..2dd81038fc 100644 --- a/api/clients/codecs/encoded_payload_test.go +++ b/api/clients/codecs/encoded_payload_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254/fr" "github.com/stretchr/testify/require" @@ -42,39 +43,40 @@ func TestDecodeLongBytes(t *testing.T) { // TestEncodeTooManyElements checks that encodedPayloadFromElements fails at the expect limit, relative to payload // length and blob length func TestEncodeTooManyElements(t *testing.T) { - // length in symbols - blobLength := uint32(16) - - maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) - require.NoError(t, err) - testRandom := random.NewTestRandom(t) - - almostTooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength)) - almostTooLongEncodedPayload, err := newEncodedPayload(NewPayload(almostTooLongData)) - require.NoError(t, err) - require.NotNil(t, almostTooLongEncodedPayload) - - almostTooLongCoeffPoly, err := almostTooLongEncodedPayload.toCoeffPoly() - require.NoError(t, err) - require.NotNil(t, almostTooLongCoeffPoly) - - // there are almost too many field elements for the defined blob length, but not quite - _, err = encodedPayloadFromElements(almostTooLongCoeffPoly.fieldElements, blobLength) - require.NoError(t, err) - - tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) - tooLongEncodedPayload, err := newEncodedPayload(NewPayload(tooLongData)) - require.NoError(t, err) - require.NotNil(t, tooLongEncodedPayload) - - tooLongCoeffPoly, err := tooLongEncodedPayload.toCoeffPoly() - require.NoError(t, err) - require.NotNil(t, tooLongCoeffPoly) - - // there is one too many field elements for the defined blob length - _, err = encodedPayloadFromElements(tooLongCoeffPoly.fieldElements, blobLength) - require.Error(t, err) + powersOf2 := encoding.GeneratePowersOfTwo(uint32(12)) + + for i := 0; i < len(powersOf2); i++ { + blobLength := powersOf2[i] + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) + require.NoError(t, err) + + almostTooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength)) + almostTooLongEncodedPayload, err := newEncodedPayload(NewPayload(almostTooLongData)) + require.NoError(t, err) + require.NotNil(t, almostTooLongEncodedPayload) + + almostTooLongCoeffPoly, err := almostTooLongEncodedPayload.toCoeffPoly() + require.NoError(t, err) + require.NotNil(t, almostTooLongCoeffPoly) + + // there are almost too many field elements for the defined blob length, but not quite + _, err = encodedPayloadFromElements(almostTooLongCoeffPoly.fieldElements, blobLength) + require.NoError(t, err) + + tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) + tooLongEncodedPayload, err := newEncodedPayload(NewPayload(tooLongData)) + require.NoError(t, err) + require.NotNil(t, tooLongEncodedPayload) + + tooLongCoeffPoly, err := tooLongEncodedPayload.toCoeffPoly() + require.NoError(t, err) + require.NotNil(t, tooLongCoeffPoly) + + // there is one too many field elements for the defined blob length + _, err = encodedPayloadFromElements(tooLongCoeffPoly.fieldElements, blobLength) + require.Error(t, err) + } } // TestTrailingNonZeros checks that any non-zero values that come after the end of the claimed payload length diff --git a/encoding/test_utils.go b/encoding/test_utils.go new file mode 100644 index 0000000000..8e924388f0 --- /dev/null +++ b/encoding/test_utils.go @@ -0,0 +1,18 @@ +package encoding + +import ( + "math" + + "golang.org/x/exp/constraints" +) + +// GeneratePowersOfTwo creates a slice of integers, containing powers of 2 (starting with element == 1), with +// powersToGenerate number of elements +func GeneratePowersOfTwo[T constraints.Integer](powersToGenerate T) []T { + powers := make([]T, powersToGenerate) + for i := T(0); i < powersToGenerate; i++ { + powers[i] = T(math.Pow(2, float64(i))) + } + + return powers +} diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 535c3fd867..a146f30416 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -167,10 +167,6 @@ func GetPaddedDataLength(inputLen uint32) uint32 { // It returns what the length of the output array would be, if you called RemoveInternalPadding on it. // TODO: test, especially lower bound func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { - if inputLen == 0 { - return 0, fmt.Errorf("input length is zero") - } - if inputLen%encoding.BYTES_PER_SYMBOL != 0 { return 0, fmt.Errorf( "%d isn't a multiple of encoding.BYTES_PER_SYMBOL (%d)", diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index a51b9ea06a..330e84e956 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "testing" + "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/stretchr/testify/require" @@ -50,8 +51,8 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { } } -// TestGetPaddedDataLength tests that GetPaddedDataLength is behaving as expected -func TestGetPaddedDataLength(t *testing.T) { +// TestGetPaddedDataLength tests that GetPaddedDataLength behaves relative to hardcoded expected results +func TestGetPaddedDataLengthAgainstKnowns(t *testing.T) { startLengths := []uint32{30, 31, 32, 33, 68} expectedResults := []uint32{32, 32, 64, 64, 96} @@ -59,3 +60,29 @@ func TestGetPaddedDataLength(t *testing.T) { require.Equal(t, codec.GetPaddedDataLength(startLengths[i]), expectedResults[i]) } } + +// TestPadUnpad makes sure that padding and unpadding doesn't corrupt underlying data +func TestPadUnpad(t *testing.T) { + testRandom := random.NewTestRandom(t) + testIterations := 1000 + + for i := 0; i < testIterations; i++ { + originalBytes := testRandom.Bytes(testRandom.Intn(1024) + 1) + + paddedBytes := codec.PadPayload(originalBytes) + require.Equal(t, len(paddedBytes)%32, 0) + + unpaddedBytes, err := codec.RemoveInternalPadding(paddedBytes) + require.Nil(t, err) + + expectedUnpaddedLength, err := codec.GetUnpaddedDataLength(uint32(len(paddedBytes))) + require.Nil(t, err) + require.Equal(t, expectedUnpaddedLength, uint32(len(unpaddedBytes))) + + // unpadded payload may have up to 31 extra trailing zeros, since RemoveInternalPadding doesn't consider these + require.Greater(t, len(originalBytes), len(unpaddedBytes)-32) + require.LessOrEqual(t, len(originalBytes), len(unpaddedBytes)) + + require.Equal(t, originalBytes, unpaddedBytes[:len(originalBytes)]) + } +} diff --git a/encoding/utils_test.go b/encoding/utils_test.go index 5b63b4adc5..5c193509c9 100644 --- a/encoding/utils_test.go +++ b/encoding/utils_test.go @@ -1,7 +1,6 @@ package encoding import ( - "math" "testing" "github.com/stretchr/testify/require" @@ -11,11 +10,8 @@ func TestNextPowerOf2(t *testing.T) { testHeight := 65536 // 2 ^ 16 = 65536 - powersToGenerate := 17 - powers := make([]int, powersToGenerate) - for i := 0; i < powersToGenerate; i++ { - powers[i] = int(math.Pow(2, float64(i))) - } + // i.e., the last element generated here == testHeight + powers := GeneratePowersOfTwo(17) powerIndex := 0 for i := 1; i <= testHeight; i++ { @@ -30,6 +26,7 @@ func TestNextPowerOf2(t *testing.T) { // sanity check the test logic require.Equal(t, powerIndex, len(powers)) - // extra sanity check + // extra sanity check, since we *really* rely on NextPowerOf2 returning + // the same value, if it's already a power of 2 require.Equal(t, 16, NextPowerOf2(16)) } From 13d16c764e4b6608050c1765c0067e3224a8d9e1 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:18:16 -0500 Subject: [PATCH 18/35] Finish utils tests Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- encoding/utils/codec/codec.go | 5 ----- encoding/utils/codec/codec_test.go | 23 ++++++++++++++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index a146f30416..1ea3685371 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -83,7 +83,6 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { // NOTE: this method is a reimplementation of ConvertByPaddingEmptyByte, with one meaningful difference: the alignment // of the output to encoding.BYTES_PER_SYMBOL. This alignment actually makes the padding logic simpler, and the // code that uses this function needs an aligned output anyway. -// TODO: test, especially lower bound func PadPayload(inputData []byte) []byte { // 31 bytes, for the bn254 curve bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) @@ -120,7 +119,6 @@ func PadPayload(inputData []byte) []byte { // NOTE: this method is a reimplementation of RemoveEmptyByteFromPaddedBytes, with one meaningful difference: this // function relies on the assumption that the input is aligned to encoding.BYTES_PER_SYMBOL, which makes the padding // removal logic simpler. -// TODO: test, especially lower bound func RemoveInternalPadding(paddedData []byte) ([]byte, error) { if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { return nil, fmt.Errorf( @@ -150,7 +148,6 @@ func RemoveInternalPadding(paddedData []byte) ([]byte, error) { // adding internal byte padding // // The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL -// TODO: test, especially lower bound func GetPaddedDataLength(inputLen uint32) uint32 { bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) chunkCount := inputLen / bytesPerChunk @@ -165,7 +162,6 @@ func GetPaddedDataLength(inputLen uint32) uint32 { // GetUnpaddedDataLength accepts the length of an array that has been padded with PadPayload // // It returns what the length of the output array would be, if you called RemoveInternalPadding on it. -// TODO: test, especially lower bound func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { if inputLen%encoding.BYTES_PER_SYMBOL != 0 { return 0, fmt.Errorf( @@ -183,7 +179,6 @@ func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { // GetMaxPermissiblePayloadLength accepts a blob length IN SYMBOLS, and returns the size IN BYTES of the largest payload // that could fit inside the blob. -// TODO: test, especially lower bound func GetMaxPermissiblePayloadLength(blobLength uint32) (uint32, error) { if blobLength == 0 { return 0, fmt.Errorf("input blob length is zero") diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 330e84e956..471edafb21 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -53,21 +53,38 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { // TestGetPaddedDataLength tests that GetPaddedDataLength behaves relative to hardcoded expected results func TestGetPaddedDataLengthAgainstKnowns(t *testing.T) { - startLengths := []uint32{30, 31, 32, 33, 68} - expectedResults := []uint32{32, 32, 64, 64, 96} + startLengths := []uint32{0, 30, 31, 32, 33, 68} + expectedResults := []uint32{0, 32, 32, 64, 64, 96} for i := range startLengths { require.Equal(t, codec.GetPaddedDataLength(startLengths[i]), expectedResults[i]) } } +// TestGetUnpaddedDataLengthAgainstKnowns tests that GetPaddedDataLength behaves relative to hardcoded expected results +func TestGetUnpaddedDataLengthAgainstKnowns(t *testing.T) { + startLengths := []uint32{0, 32, 64, 128} + expectedResults := []uint32{0, 31, 62, 124} + + for i := range startLengths { + unpaddedDataLength, err := codec.GetUnpaddedDataLength(startLengths[i]) + require.Nil(t, err) + + require.Equal(t, expectedResults[i], unpaddedDataLength) + } + + unpaddedDataLength, err := codec.GetUnpaddedDataLength(129) + require.Error(t, err) + require.Equal(t, uint32(0), unpaddedDataLength) +} + // TestPadUnpad makes sure that padding and unpadding doesn't corrupt underlying data func TestPadUnpad(t *testing.T) { testRandom := random.NewTestRandom(t) testIterations := 1000 for i := 0; i < testIterations; i++ { - originalBytes := testRandom.Bytes(testRandom.Intn(1024) + 1) + originalBytes := testRandom.Bytes(testRandom.Intn(1024)) paddedBytes := codec.PadPayload(originalBytes) require.Equal(t, len(paddedBytes)%32, 0) From 22e8134e858af661f931819bb21c6cc3a9247e1d Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:18:58 -0500 Subject: [PATCH 19/35] Do FFT work Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 15 ++++++++++----- api/clients/codecs/coeff_poly.go | 20 ++++++++++---------- api/clients/codecs/eval_poly.go | 22 ++++++++++------------ api/clients/codecs/payload.go | 12 +++++++----- encoding/fft/fft.go | 8 ++++++++ 5 files changed, 45 insertions(+), 32 deletions(-) diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 479a359341..3d4272ae92 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -10,11 +10,13 @@ import ( type Blob struct { coeffPolynomial *coeffPoly // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment - // This is the blob length in symbols, NOT in bytes + // This is the blob length IN SYMBOLS, not in bytes blobLength uint32 } -// BlobFromBytes initializes a Blob from bytes, and a blobLength in symbols +// BlobFromBytes initializes a Blob from bytes +// +// blobLength is the length of the blob IN SYMBOLS func BlobFromBytes(bytes []byte, blobLength uint32) (*Blob, error) { poly, err := coeffPolyFromBytes(bytes) if err != nil { @@ -24,11 +26,14 @@ func BlobFromBytes(bytes []byte, blobLength uint32) (*Blob, error) { return BlobFromPolynomial(poly, blobLength) } -// BlobFromPolynomial initializes a blob from a polynomial, and a blobLength in symbols +// BlobFromPolynomial initializes a blob from a polynomial +// +// blobLength is the length of the blob IN SYMBOLS func BlobFromPolynomial(coeffPolynomial *coeffPoly, blobLength uint32) (*Blob, error) { return &Blob{ coeffPolynomial: coeffPolynomial, - blobLength: blobLength}, nil + blobLength: blobLength, + }, nil } // GetBytes gets the raw bytes of the Blob @@ -52,7 +57,7 @@ func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { } case PolynomialFormEval: // the payload started off in evaluation form, so we first need to convert the blob's coeff poly into an eval poly - evalPoly, err := b.coeffPolynomial.toEvalPoly() + evalPoly, err := b.coeffPolynomial.toEvalPoly(b.blobLength) if err != nil { return nil, fmt.Errorf("coeff poly to eval poly: %w", err) } diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go index 1188936d31..54fecbe456 100644 --- a/api/clients/codecs/coeff_poly.go +++ b/api/clients/codecs/coeff_poly.go @@ -2,9 +2,7 @@ package codecs import ( "fmt" - "math" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/fft" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -34,14 +32,16 @@ func coeffPolyFromElements(elements []fr.Element) *coeffPoly { } // toEvalPoly converts a coeffPoly to an evalPoly, using the FFT operation -func (p *coeffPoly) toEvalPoly() (*evalPoly, error) { - // we need to pad to the next power of 2, to be able to take the FFT - paddedLength := encoding.NextPowerOf2(len(p.fieldElements)) - padding := make([]fr.Element, paddedLength-len(p.fieldElements)) - paddedElements := append(p.fieldElements, padding...) - - maxScale := uint8(math.Log2(float64(len(paddedElements)))) - fftedElements, err := fft.NewFFTSettings(maxScale).FFT(paddedElements, false) +// +// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT +func (p *coeffPoly) toEvalPoly(blobLength uint32) (*evalPoly, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLength(blobLength) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + fftedElements, err := fftSettings.FFT(p.fieldElements, false) if err != nil { return nil, fmt.Errorf("perform FFT: %w", err) } diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go index 79ee7a3111..5b03fe6932 100644 --- a/api/clients/codecs/eval_poly.go +++ b/api/clients/codecs/eval_poly.go @@ -2,9 +2,7 @@ package codecs import ( "fmt" - "math" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/fft" "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) @@ -13,8 +11,6 @@ import ( // // The underlying bytes represent 32 byte field elements, and the field elements represent the polynomial evaluation // at roots of unity. -// -// The number of field elements is always a power of 2. type evalPoly struct { fieldElements []fr.Element } @@ -25,14 +21,16 @@ func evalPolyFromElements(elements []fr.Element) *evalPoly { } // toCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation -func (p *evalPoly) toCoeffPoly() (*coeffPoly, error) { - // we need to pad to the next power of 2, to be able to take the FFT - paddedLength := encoding.NextPowerOf2(len(p.fieldElements)) - padding := make([]fr.Element, paddedLength-len(p.fieldElements)) - paddedElements := append(p.fieldElements, padding...) - - maxScale := uint8(math.Log2(float64(len(paddedElements)))) - ifftedElements, err := fft.NewFFTSettings(maxScale).FFT(paddedElements, true) +// +// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT +func (p *evalPoly) toCoeffPoly(blobLength uint32) (*coeffPoly, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLength(blobLength) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + ifftedElements, err := fftSettings.FFT(p.fieldElements, true) if err != nil { return nil, fmt.Errorf("perform IFFT: %w", err) } diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index f825042c35..adb3085c24 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -26,6 +26,7 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { } var coeffPolynomial *coeffPoly + var blobLength uint32 switch form { case PolynomialFormCoeff: // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in @@ -34,14 +35,18 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { if err != nil { return nil, fmt.Errorf("coeff poly from elements: %w", err) } + + blobLength = uint32(encoding.NextPowerOf2(len(coeffPolynomial.fieldElements))) case PolynomialFormEval: // the payload is in evaluation form, so we need to convert it to coeff form - evalPoly, err := encodedPayload.toEvalPoly() + evalPolynomial, err := encodedPayload.toEvalPoly() if err != nil { return nil, fmt.Errorf("eval poly from elements: %w", err) } - coeffPolynomial, err = evalPoly.toCoeffPoly() + blobLength = uint32(encoding.NextPowerOf2(len(evalPolynomial.fieldElements))) + + coeffPolynomial, err = evalPolynomial.toCoeffPoly(blobLength) if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } @@ -49,9 +54,6 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("unknown polynomial form: %v", form) } - // it's possible that the number of field elements might already be a power of 2 - // in that case, calling NextPowerOf2 will just return the input value - blobLength := uint32(encoding.NextPowerOf2(len(coeffPolynomial.fieldElements))) return BlobFromPolynomial(coeffPolynomial, blobLength) } diff --git a/encoding/fft/fft.go b/encoding/fft/fft.go index aec175fcca..c2199e04a8 100644 --- a/encoding/fft/fft.go +++ b/encoding/fft/fft.go @@ -27,6 +27,8 @@ package fft import ( + "math" + "github.com/Layr-Labs/eigenda/encoding" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -88,3 +90,9 @@ func NewFFTSettings(maxScale uint8) *FFTSettings { ReverseRootsOfUnity: rootzReverse, } } + +// FFTSettingsFromBlobLength accepts a blob length in symbols, and returns a new instance of FFT settings +func FFTSettingsFromBlobLength(blobLength uint32) *FFTSettings { + maxScale := uint8(math.Log2(float64(blobLength))) + return NewFFTSettings(maxScale) +} From 23b49c4fd892000b932c09cd326fe50da321a377 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 12 Feb 2025 13:32:26 -0500 Subject: [PATCH 20/35] Clean up encoded payload Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/encoded_payload.go | 16 ++--------- api/clients/codecs/encoded_payload_test.go | 32 ++++++++++------------ api/clients/codecs/payload.go | 24 ++++++---------- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index d22cdc74a8..6a6882ddfe 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -75,24 +75,14 @@ func (ep *encodedPayload) decode() (*Payload, error) { return NewPayload(unpaddedData[0:claimedLength]), nil } -// toEvalPoly converts the encoded payload to a polynomial in evaluation form -func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { +// toFieldElements converts the encoded payload to an array of field elements +func (ep *encodedPayload) toFieldElements() ([]fr.Element, error) { fieldElements, err := rs.ToFrArray(ep.bytes) if err != nil { return nil, fmt.Errorf("deserialize field elements: %w", err) } - return evalPolyFromElements(fieldElements), nil -} - -// toCoeffPoly converts the encoded payload to a polynomial in coefficient form -func (ep *encodedPayload) toCoeffPoly() (*coeffPoly, error) { - fieldElements, err := rs.ToFrArray(ep.bytes) - if err != nil { - return nil, fmt.Errorf("deserialize field elements: %w", err) - } - - return coeffPolyFromElements(fieldElements), nil + return fieldElements, nil } // encodedPayloadFromElements accepts an array of field elements, and converts them into an encoded payload diff --git a/api/clients/codecs/encoded_payload_test.go b/api/clients/codecs/encoded_payload_test.go index 2dd81038fc..5ee27e6be5 100644 --- a/api/clients/codecs/encoded_payload_test.go +++ b/api/clients/codecs/encoded_payload_test.go @@ -54,27 +54,19 @@ func TestEncodeTooManyElements(t *testing.T) { almostTooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength)) almostTooLongEncodedPayload, err := newEncodedPayload(NewPayload(almostTooLongData)) require.NoError(t, err) - require.NotNil(t, almostTooLongEncodedPayload) - - almostTooLongCoeffPoly, err := almostTooLongEncodedPayload.toCoeffPoly() + almostTooLongFieldElements, err := almostTooLongEncodedPayload.toFieldElements() require.NoError(t, err) - require.NotNil(t, almostTooLongCoeffPoly) - // there are almost too many field elements for the defined blob length, but not quite - _, err = encodedPayloadFromElements(almostTooLongCoeffPoly.fieldElements, blobLength) + _, err = encodedPayloadFromElements(almostTooLongFieldElements, blobLength) require.NoError(t, err) tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) tooLongEncodedPayload, err := newEncodedPayload(NewPayload(tooLongData)) require.NoError(t, err) - require.NotNil(t, tooLongEncodedPayload) - - tooLongCoeffPoly, err := tooLongEncodedPayload.toCoeffPoly() + tooLongFieldElements, err := tooLongEncodedPayload.toFieldElements() require.NoError(t, err) - require.NotNil(t, tooLongCoeffPoly) - // there is one too many field elements for the defined blob length - _, err = encodedPayloadFromElements(tooLongCoeffPoly.fieldElements, blobLength) + _, err = encodedPayloadFromElements(tooLongFieldElements, blobLength) require.Error(t, err) } } @@ -112,15 +104,21 @@ func TestEncodeWithFewerElements(t *testing.T) { testRandom := random.NewTestRandom(t) originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) - blob, err := NewPayload(originalData).ToBlob(PolynomialFormCoeff) + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + originalFieldElements, err := encodedPayload.toFieldElements() require.NoError(t, err) - fieldElements := make([]fr.Element, len(blob.coeffPolynomial.fieldElements) - 1) + truncatedFieldElements := make([]fr.Element, len(originalFieldElements)-1) // intentionally don't copy all the elements - copy(fieldElements, blob.coeffPolynomial.fieldElements[:len(blob.coeffPolynomial.fieldElements) - 1]) + copy(truncatedFieldElements, originalFieldElements[:len(originalFieldElements)-1]) + + // next power of 2 bytes, converted into symbol length + blobLength := encoding.NextPowerOf2(len(originalData)) / encoding.BYTES_PER_SYMBOL // even though the actual length will be less than the claimed length, we shouldn't see any error - encodedPayload, err := encodedPayloadFromElements(fieldElements, blob.blobLength) + reconstructedEncodedPayload, err := encodedPayloadFromElements(originalFieldElements, uint32(blobLength)) require.NoError(t, err) - require.NotNil(t, encodedPayload) + require.NotNil(t, reconstructedEncodedPayload) } diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index adb3085c24..5d77469e96 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -25,27 +25,22 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("encoding payload: %w", err) } + fieldElements, err := encodedPayload.toFieldElements() + if err != nil { + return nil, fmt.Errorf("encoded payload to field elements: %w", err) + } + + blobLength := uint32(encoding.NextPowerOf2(len(fieldElements))) + var coeffPolynomial *coeffPoly - var blobLength uint32 switch form { case PolynomialFormCoeff: // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in // coefficient form - coeffPolynomial, err = encodedPayload.toCoeffPoly() - if err != nil { - return nil, fmt.Errorf("coeff poly from elements: %w", err) - } - - blobLength = uint32(encoding.NextPowerOf2(len(coeffPolynomial.fieldElements))) + coeffPolynomial = coeffPolyFromElements(fieldElements) case PolynomialFormEval: // the payload is in evaluation form, so we need to convert it to coeff form - evalPolynomial, err := encodedPayload.toEvalPoly() - if err != nil { - return nil, fmt.Errorf("eval poly from elements: %w", err) - } - - blobLength = uint32(encoding.NextPowerOf2(len(evalPolynomial.fieldElements))) - + evalPolynomial := evalPolyFromElements(fieldElements) coeffPolynomial, err = evalPolynomial.toCoeffPoly(blobLength) if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) @@ -54,7 +49,6 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("unknown polynomial form: %v", form) } - return BlobFromPolynomial(coeffPolynomial, blobLength) } From e5363e48316d757523bbd7300e059084b7302e6d Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:10:39 -0500 Subject: [PATCH 21/35] Make encoding version uint8 instead of byte Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index 95b619de36..fbc21787cd 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -4,7 +4,7 @@ import ( "fmt" ) -type PayloadEncodingVersion byte +type PayloadEncodingVersion uint8 const ( // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] From 7801044891e39e9c9ed73b1404af20164c26a228 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:16:31 -0500 Subject: [PATCH 22/35] Add explanation for 0x00 padding Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_codec.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index fbc21787cd..3196c3b053 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -9,6 +9,8 @@ type PayloadEncodingVersion uint8 const ( // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] + // + // Each group of 32 bytes starts with a 0x00 byte so that they can be parsed as valid bn254 field elements. PayloadEncodingVersion0 PayloadEncodingVersion = 0x0 ) From c70fad8d2cb418706ae34cc9950887d9e1d826d7 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:21:44 -0500 Subject: [PATCH 23/35] Use fuzz testing Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_test.go | 19 ++++----- api/clients/codecs/conversion_test.go | 55 +++++++++------------------ 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index b1866bca96..250cc02258 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -1,23 +1,24 @@ package codecs import ( + "bytes" "testing" - "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/stretchr/testify/require" ) // TestBlobConversion checks that internal blob conversion methods produce consistent results -func TestBlobConversion(t *testing.T) { - testRandom := random.NewTestRandom(t) +func FuzzBlobConversion(f *testing.F) { + for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} { + f.Add(seed) + } - iterations := 1000 + f.Fuzz( + func(t *testing.T, originalData []byte) { + testBlobConversionForForm(t, originalData, PolynomialFormEval) + testBlobConversionForForm(t, originalData, PolynomialFormCoeff) + }) - for i := 0; i < iterations; i++ { - originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) - testBlobConversionForForm(t, originalData, PolynomialFormEval) - testBlobConversionForForm(t, originalData, PolynomialFormCoeff) - } } func testBlobConversionForForm(t *testing.T, payloadBytes []byte, form PolynomialForm) { diff --git a/api/clients/codecs/conversion_test.go b/api/clients/codecs/conversion_test.go index cd4142e0d2..6e82b8e0dd 100644 --- a/api/clients/codecs/conversion_test.go +++ b/api/clients/codecs/conversion_test.go @@ -4,49 +4,32 @@ import ( "bytes" "testing" - "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/stretchr/testify/require" ) -// TestConversionConsistency checks that data can be encoded and decoded repeatedly, always getting back the original data -// TODO: we should probably be using fuzzing instead of this kind of ad-hoc random search testing -func TestConversionConsistency(t *testing.T) { - testRandom := random.NewTestRandom(t) - - iterations := 100 - - for i := 0; i < iterations; i++ { - originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 - - payload := NewPayload(originalData) +// FuzzConversionConsistency checks that data can be encoded and decoded repeatedly, always getting back the original data +func FuzzConversionConsistency(f *testing.F) { + for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} { + f.Add(seed) + } - blob1, err := payload.ToBlob(PolynomialFormEval) - require.NoError(t, err) + f.Fuzz( + func(t *testing.T, originalData []byte) { + payload := NewPayload(originalData) - blob2, err := payload.ToBlob(PolynomialFormCoeff) - require.NoError(t, err) + blob1, err := payload.ToBlob(PolynomialFormEval) + require.NoError(t, err) - decodedPayload1, err := blob1.ToPayload(PolynomialFormEval) - require.NoError(t, err) + blob2, err := payload.ToBlob(PolynomialFormCoeff) + require.NoError(t, err) - decodedPayload2, err := blob2.ToPayload(PolynomialFormCoeff) - require.NoError(t, err) + decodedPayload1, err := blob1.ToPayload(PolynomialFormEval) + require.NoError(t, err) - // Compare the original data with the decoded data - if !bytes.Equal(originalData, decodedPayload1.GetBytes()) { - t.Fatalf( - "Iteration %d: original and data decoded from blob1 do not match\nOriginal: %v\nDecoded: %v", - i, - originalData, - decodedPayload1.GetBytes()) - } + decodedPayload2, err := blob2.ToPayload(PolynomialFormCoeff) + require.NoError(t, err) - if !bytes.Equal(originalData, decodedPayload2.GetBytes()) { - t.Fatalf( - "Iteration %d: original and data decoded from blob2 do not match\nOriginal: %v\nDecoded: %v", - i, - originalData, - decodedPayload1.GetBytes()) - } - } + require.Equal(t, originalData, decodedPayload1.GetBytes()) + require.Equal(t, originalData, decodedPayload2.GetBytes()) + }) } From 617faa7b638785ebbcfc71d02ebdc5e6d0795cf6 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:46:47 -0500 Subject: [PATCH 24/35] Simplify class structure Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 74 ++++++++++++++++------ api/clients/codecs/coeff_poly.go | 63 ------------------ api/clients/codecs/encoded_payload.go | 22 ++----- api/clients/codecs/encoded_payload_test.go | 32 +++++----- api/clients/codecs/eval_poly.go | 46 -------------- api/clients/codecs/payload.go | 27 ++++++-- 6 files changed, 99 insertions(+), 165 deletions(-) delete mode 100644 api/clients/codecs/coeff_poly.go delete mode 100644 api/clients/codecs/eval_poly.go diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 3d4272ae92..3c0174533e 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -2,13 +2,18 @@ package codecs import ( "fmt" + + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) // Blob is data that is dispersed on eigenDA. // // A Blob is represented under the hood by a coeff polynomial type Blob struct { - coeffPolynomial *coeffPoly + coeffPolynomial []fr.Element // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment // This is the blob length IN SYMBOLS, not in bytes blobLength uint32 @@ -18,18 +23,18 @@ type Blob struct { // // blobLength is the length of the blob IN SYMBOLS func BlobFromBytes(bytes []byte, blobLength uint32) (*Blob, error) { - poly, err := coeffPolyFromBytes(bytes) + coeffPolynomial, err := rs.ToFrArray(bytes) if err != nil { - return nil, fmt.Errorf("polynomial from bytes: %w", err) + return nil, fmt.Errorf("bytes to field elements: %w", err) } - return BlobFromPolynomial(poly, blobLength) + return BlobFromPolynomial(coeffPolynomial, blobLength) } // BlobFromPolynomial initializes a blob from a polynomial // // blobLength is the length of the blob IN SYMBOLS -func BlobFromPolynomial(coeffPolynomial *coeffPoly, blobLength uint32) (*Blob, error) { +func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLength uint32) (*Blob, error) { return &Blob{ coeffPolynomial: coeffPolynomial, blobLength: blobLength, @@ -38,7 +43,7 @@ func BlobFromPolynomial(coeffPolynomial *coeffPoly, blobLength uint32) (*Blob, e // GetBytes gets the raw bytes of the Blob func (b *Blob) GetBytes() []byte { - return b.coeffPolynomial.getBytes() + return rs.FieldElementsToBytes(b.coeffPolynomial) } // ToPayload converts the Blob into a Payload @@ -46,34 +51,61 @@ func (b *Blob) GetBytes() []byte { // The payloadStartingForm indicates how payloads are constructed by the dispersing client. Based on the starting form // of the payload, we can determine what operations must be done to the blob in order to reconstruct the original payload func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { - var encodedPayload *encodedPayload - var err error + encodedPayload, err := b.toEncodedPayload(payloadStartingForm) + if err != nil { + return nil, fmt.Errorf("to encoded payload: %w", err) + } + + payload, err := encodedPayload.decode() + if err != nil { + return nil, fmt.Errorf("decode payload: %w", err) + } + + return payload, nil +} + +// toEncodedPayload creates an encodedPayload from the blob +func (b *Blob) toEncodedPayload(payloadStartingForm PolynomialForm) (*encodedPayload, error) { + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLength) + if err != nil { + return nil, fmt.Errorf("get max permissible payload length: %w", err) + } + + var payloadElements []fr.Element switch payloadStartingForm { case PolynomialFormCoeff: // the payload started off in coefficient form, so no conversion needs to be done - encodedPayload, err = b.coeffPolynomial.toEncodedPayload(b.blobLength) - if err != nil { - return nil, fmt.Errorf("coeff poly to encoded payload: %w", err) - } + payloadElements = b.coeffPolynomial case PolynomialFormEval: // the payload started off in evaluation form, so we first need to convert the blob's coeff poly into an eval poly - evalPoly, err := b.coeffPolynomial.toEvalPoly(b.blobLength) + payloadElements, err = b.computeEvalPoly() if err != nil { return nil, fmt.Errorf("coeff poly to eval poly: %w", err) } - - encodedPayload, err = evalPoly.toEncodedPayload(b.blobLength) - if err != nil { - return nil, fmt.Errorf("eval poly to encoded payload: %w", err) - } default: return nil, fmt.Errorf("invalid polynomial form") } - payload, err := encodedPayload.decode() + encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength) if err != nil { - return nil, fmt.Errorf("decode payload: %w", err) + return nil, fmt.Errorf("encoded payload from elements %w", err) } - return payload, nil + return encodedPayload, nil } + +// computeEvalPoly converts a blob's coeffPoly to an evalPoly, using the FFT operation +func (b *Blob) computeEvalPoly() ([]fr.Element, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLength(b.blobLength) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + fftedElements, err := fftSettings.FFT(b.coeffPolynomial, false) + if err != nil { + return nil, fmt.Errorf("perform FFT: %w", err) + } + + return fftedElements, nil +} \ No newline at end of file diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go deleted file mode 100644 index 54fecbe456..0000000000 --- a/api/clients/codecs/coeff_poly.go +++ /dev/null @@ -1,63 +0,0 @@ -package codecs - -import ( - "fmt" - - "github.com/Layr-Labs/eigenda/encoding/fft" - "github.com/Layr-Labs/eigenda/encoding/rs" - "github.com/consensys/gnark-crypto/ecc/bn254/fr" -) - -// coeffPoly is a polynomial in coefficient form. -// -// The underlying bytes represent 32 byte field elements, and each field element represents a coefficient -type coeffPoly struct { - fieldElements []fr.Element -} - -// coeffPolyFromBytes creates a new polynomial from bytes. This function performs the necessary checks to guarantee that the -// bytes are well-formed, and returns a new object if they are -func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { - fieldElements, err := rs.ToFrArray(bytes) - if err != nil { - return nil, fmt.Errorf("deserialize field elements: %w", err) - } - - return &coeffPoly{fieldElements: fieldElements}, nil -} - -// coeffPolyFromElements creates a new coeffPoly from field elements. -func coeffPolyFromElements(elements []fr.Element) *coeffPoly { - return &coeffPoly{fieldElements: elements} -} - -// toEvalPoly converts a coeffPoly to an evalPoly, using the FFT operation -// -// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT -func (p *coeffPoly) toEvalPoly(blobLength uint32) (*evalPoly, error) { - // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, - // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves - // to present a computational burden, consider making this change. - fftSettings := fft.FFTSettingsFromBlobLength(blobLength) - - // the FFT method pads to the next power of 2, so we don't need to do that manually - fftedElements, err := fftSettings.FFT(p.fieldElements, false) - if err != nil { - return nil, fmt.Errorf("perform FFT: %w", err) - } - - return evalPolyFromElements(fftedElements), nil -} - -// GetBytes returns the bytes that underlie the polynomial -func (p *coeffPoly) getBytes() []byte { - return rs.FieldElementsToBytes(p.fieldElements) -} - -// toEncodedPayload converts a coeffPoly into an encoded payload -// -// blobLength is required, to be able to perform length checks on the encoded payload during construction. -// blobLength is in symbols, NOT bytes -func (p *coeffPoly) toEncodedPayload(blobLength uint32) (*encodedPayload, error) { - return encodedPayloadFromElements(p.fieldElements, blobLength) -} diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index 6a6882ddfe..ea6364bd45 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "fmt" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -14,7 +13,7 @@ import ( // // Example encoding: // -// Encoded Payload header (32 bytes total) Encoded Payload Data (len is multiple of 32) +// Encoded Payload header (32 bytes total) Encoded Payload Data (len is multiple of 32) // [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] type encodedPayload struct { // the size of these bytes is guaranteed to be a multiple of 32 @@ -87,23 +86,16 @@ func (ep *encodedPayload) toFieldElements() ([]fr.Element, error) { // encodedPayloadFromElements accepts an array of field elements, and converts them into an encoded payload // -// blobLength is the length of the blob IN SYMBOLS, as claimed by the blob commitment. This is needed to make sure -// that the claimed length in the encoded payload header is valid relative to the total blob length -func encodedPayloadFromElements(fieldElements []fr.Element, blobLength uint32) (*encodedPayload, error) { +// maxPayloadLength is the maximum length in bytes that the contained Payload is permitted to be +func encodedPayloadFromElements(fieldElements []fr.Element, maxPayloadLength uint32) (*encodedPayload, error) { polynomialBytes := rs.FieldElementsToBytes(fieldElements) // this is the payload length in bytes, as claimed by the encoded payload header payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) - maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) - if err != nil { - return nil, fmt.Errorf("get max permissible payload length: %w", err) - } - - if payloadLength > maxPermissiblePayloadLength { + if payloadLength > maxPayloadLength { return nil, fmt.Errorf( - `length claimed in encoded payload header (%d bytes) exceeds the max permissible length (%d bytes) - that could be contained in a blob of length %d symbols (%d bytes)`, - payloadLength, maxPermissiblePayloadLength, blobLength, blobLength*encoding.BYTES_PER_SYMBOL) + "payload length claimed in encoded payload header (%d bytes) is larger than the permitted maximum (%d bytes)", + payloadLength, maxPayloadLength) } // this is the length you would get if you padded a payload of the length claimed in the encoded payload header @@ -155,4 +147,4 @@ func checkTrailingZeros(inputBytes []byte, nonZeroLength uint32) error { } return nil -} \ No newline at end of file +} diff --git a/api/clients/codecs/encoded_payload_test.go b/api/clients/codecs/encoded_payload_test.go index 5ee27e6be5..241c0d7697 100644 --- a/api/clients/codecs/encoded_payload_test.go +++ b/api/clients/codecs/encoded_payload_test.go @@ -57,7 +57,7 @@ func TestEncodeTooManyElements(t *testing.T) { almostTooLongFieldElements, err := almostTooLongEncodedPayload.toFieldElements() require.NoError(t, err) // there are almost too many field elements for the defined blob length, but not quite - _, err = encodedPayloadFromElements(almostTooLongFieldElements, blobLength) + _, err = encodedPayloadFromElements(almostTooLongFieldElements, maxPermissiblePayloadLength) require.NoError(t, err) tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) @@ -66,7 +66,7 @@ func TestEncodeTooManyElements(t *testing.T) { tooLongFieldElements, err := tooLongEncodedPayload.toFieldElements() require.NoError(t, err) // there is one too many field elements for the defined blob length - _, err = encodedPayloadFromElements(tooLongFieldElements, blobLength) + _, err = encodedPayloadFromElements(tooLongFieldElements, maxPermissiblePayloadLength) require.Error(t, err) } } @@ -75,27 +75,28 @@ func TestEncodeTooManyElements(t *testing.T) { // cause an error to be returned. func TestTrailingNonZeros(t *testing.T) { testRandom := random.NewTestRandom(t) - // make original data 1025 bytes, so that there is plenty of wiggle room to tack on another couple bytes - // without having a payload that fails verification for being too long - originalData := testRandom.Bytes(1025) + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) - blob, err := NewPayload(originalData).ToBlob(PolynomialFormCoeff) + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) require.NoError(t, err) - fieldElements1 := make([]fr.Element, len(blob.coeffPolynomial.fieldElements)) - copy(fieldElements1, blob.coeffPolynomial.fieldElements) + originalElements, err := encodedPayload.toFieldElements() + require.NoError(t, err) - fieldElements2 := make([]fr.Element, len(blob.coeffPolynomial.fieldElements)) - copy(fieldElements2, blob.coeffPolynomial.fieldElements) + fieldElements1 := make([]fr.Element, len(originalElements)) + copy(fieldElements1, originalElements) + + fieldElements2 := make([]fr.Element, len(originalElements)) + copy(fieldElements2, originalElements) // adding a 0 is fine fieldElements1 = append(fieldElements1, fr.Element{}) - _, err = encodedPayloadFromElements(fieldElements1, blob.blobLength) + _, err = encodedPayloadFromElements(fieldElements1, uint32(len(fieldElements1)*encoding.BYTES_PER_SYMBOL)) require.NoError(t, err) // adding a non-0 is non-fine fieldElements2 = append(fieldElements2, fr.Element{0,0,0,1}) - _, err = encodedPayloadFromElements(fieldElements2, blob.blobLength) + _, err = encodedPayloadFromElements(fieldElements2, uint32(len(fieldElements2)*encoding.BYTES_PER_SYMBOL)) require.Error(t, err) } @@ -114,11 +115,10 @@ func TestEncodeWithFewerElements(t *testing.T) { // intentionally don't copy all the elements copy(truncatedFieldElements, originalFieldElements[:len(originalFieldElements)-1]) - // next power of 2 bytes, converted into symbol length - blobLength := encoding.NextPowerOf2(len(originalData)) / encoding.BYTES_PER_SYMBOL - // even though the actual length will be less than the claimed length, we shouldn't see any error - reconstructedEncodedPayload, err := encodedPayloadFromElements(originalFieldElements, uint32(blobLength)) + reconstructedEncodedPayload, err := encodedPayloadFromElements( + originalFieldElements, + uint32(len(originalFieldElements))*encoding.BYTES_PER_SYMBOL) require.NoError(t, err) require.NotNil(t, reconstructedEncodedPayload) } diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go deleted file mode 100644 index 5b03fe6932..0000000000 --- a/api/clients/codecs/eval_poly.go +++ /dev/null @@ -1,46 +0,0 @@ -package codecs - -import ( - "fmt" - - "github.com/Layr-Labs/eigenda/encoding/fft" - "github.com/consensys/gnark-crypto/ecc/bn254/fr" -) - -// evalPoly is a polynomial in evaluation form. -// -// The underlying bytes represent 32 byte field elements, and the field elements represent the polynomial evaluation -// at roots of unity. -type evalPoly struct { - fieldElements []fr.Element -} - -// evalPolyFromElements creates a new evalPoly from field elements. -func evalPolyFromElements(elements []fr.Element) *evalPoly { - return &evalPoly{fieldElements: elements} -} - -// toCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation -// -// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT -func (p *evalPoly) toCoeffPoly(blobLength uint32) (*coeffPoly, error) { - // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, - // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves - // to present a computational burden, consider making this change. - fftSettings := fft.FFTSettingsFromBlobLength(blobLength) - - // the FFT method pads to the next power of 2, so we don't need to do that manually - ifftedElements, err := fftSettings.FFT(p.fieldElements, true) - if err != nil { - return nil, fmt.Errorf("perform IFFT: %w", err) - } - - return coeffPolyFromElements(ifftedElements), nil -} - -// toEncodedPayload converts an evalPoly into an encoded payload -// -// blobLength is required, to be able to perform length checks on the encoded payload during construction -func (p *evalPoly) toEncodedPayload(blobLength uint32) (*encodedPayload, error) { - return encodedPayloadFromElements(p.fieldElements, blobLength) -} diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 5d77469e96..84ed1cde14 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) // Payload represents arbitrary user data, without any processing. @@ -32,16 +34,15 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { blobLength := uint32(encoding.NextPowerOf2(len(fieldElements))) - var coeffPolynomial *coeffPoly + var coeffPolynomial []fr.Element switch form { case PolynomialFormCoeff: // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in // coefficient form - coeffPolynomial = coeffPolyFromElements(fieldElements) + coeffPolynomial = fieldElements case PolynomialFormEval: // the payload is in evaluation form, so we need to convert it to coeff form - evalPolynomial := evalPolyFromElements(fieldElements) - coeffPolynomial, err = evalPolynomial.toCoeffPoly(blobLength) + coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLength) if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } @@ -56,3 +57,21 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { func (p *Payload) GetBytes() []byte { return p.bytes } + +// evalToCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation +// +// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT +func evalToCoeffPoly(evalPoly []fr.Element, blobLength uint32) ([]fr.Element, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLength(blobLength) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + ifftedElements, err := fftSettings.FFT(evalPoly, true) + if err != nil { + return nil, fmt.Errorf("perform IFFT: %w", err) + } + + return ifftedElements, nil +} From e22238b117a6e22727a3f6a1d021fba6a38b9607 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:18:49 -0500 Subject: [PATCH 25/35] Improve docs and naming Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 29 ++++++++++++++++++++--------- api/clients/codecs/payload.go | 11 +++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 3c0174533e..4d2e169ded 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -16,6 +16,12 @@ type Blob struct { coeffPolynomial []fr.Element // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment // This is the blob length IN SYMBOLS, not in bytes + // + // This value must be specified, rather than computed from the length of the coeffPolynomial, due to an edge case + // illustrated by the following example: imagine a user disperses a very small blob, only 64 bytes, and the last 40 + // bytes are trailing zeros. When a different user fetches the blob from a relay, it's possible that the relay could + // truncate the trailing zeros. If we were to say that blobLength = nextPowerOf2(len(coeffPolynomial)), then the + // user fetching and reconstructing this blob would determine that the blob length is 1 symbol, when it's actually 2. blobLength uint32 } @@ -48,10 +54,10 @@ func (b *Blob) GetBytes() []byte { // ToPayload converts the Blob into a Payload // -// The payloadStartingForm indicates how payloads are constructed by the dispersing client. Based on the starting form -// of the payload, we can determine what operations must be done to the blob in order to reconstruct the original payload -func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { - encodedPayload, err := b.toEncodedPayload(payloadStartingForm) +// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what +// conversion, if any, must be performed when creating a payload from the blob. +func (b *Blob) ToPayload(payloadForm PolynomialForm) (*Payload, error) { + encodedPayload, err := b.toEncodedPayload(payloadForm) if err != nil { return nil, fmt.Errorf("to encoded payload: %w", err) } @@ -65,22 +71,27 @@ func (b *Blob) ToPayload(payloadStartingForm PolynomialForm) (*Payload, error) { } // toEncodedPayload creates an encodedPayload from the blob -func (b *Blob) toEncodedPayload(payloadStartingForm PolynomialForm) (*encodedPayload, error) { +// +// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what +// conversion, if any, must be performed when creating an encoded payload from the blob. +func (b *Blob) toEncodedPayload(payloadForm PolynomialForm) (*encodedPayload, error) { maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLength) if err != nil { return nil, fmt.Errorf("get max permissible payload length: %w", err) } var payloadElements []fr.Element - switch payloadStartingForm { + switch payloadForm { case PolynomialFormCoeff: - // the payload started off in coefficient form, so no conversion needs to be done + // the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that + // eigenda also interprets blobs as coefficients payloadElements = b.coeffPolynomial case PolynomialFormEval: - // the payload started off in evaluation form, so we first need to convert the blob's coeff poly into an eval poly + // the payload is interpreted as evaluations of the polynomial, so the coefficient representation contained + // in the blob must be converted to the evaluation form payloadElements, err = b.computeEvalPoly() if err != nil { - return nil, fmt.Errorf("coeff poly to eval poly: %w", err) + return nil, fmt.Errorf("compute eval poly: %w", err) } default: return nil, fmt.Errorf("invalid polynomial form") diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 84ed1cde14..b0032d6b2a 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -21,7 +21,10 @@ func NewPayload(payloadBytes []byte) *Payload { } // ToBlob converts the Payload bytes into a Blob -func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { +// +// The payloadForm indicates how payloads are interpreted. The form of a payload dictates what conversion, if any, must +// be performed when creating a blob from the payload. +func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { encodedPayload, err := newEncodedPayload(p) if err != nil { return nil, fmt.Errorf("encoding payload: %w", err) @@ -35,19 +38,19 @@ func (p *Payload) ToBlob(form PolynomialForm) (*Blob, error) { blobLength := uint32(encoding.NextPowerOf2(len(fieldElements))) var coeffPolynomial []fr.Element - switch form { + switch payloadForm { case PolynomialFormCoeff: // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in // coefficient form coeffPolynomial = fieldElements case PolynomialFormEval: - // the payload is in evaluation form, so we need to convert it to coeff form + // the payload is in evaluation form, so we need to convert it to coeff form, since blobs are in coefficient form coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLength) if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } default: - return nil, fmt.Errorf("unknown polynomial form: %v", form) + return nil, fmt.Errorf("unknown polynomial form: %v", payloadForm) } return BlobFromPolynomial(coeffPolynomial, blobLength) From db456455267c5c09566182bc4e27d52463780c36 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:44:21 -0500 Subject: [PATCH 26/35] Improve test var name Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index 250cc02258..6749893ebb 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -21,17 +21,17 @@ func FuzzBlobConversion(f *testing.F) { } -func testBlobConversionForForm(t *testing.T, payloadBytes []byte, form PolynomialForm) { +func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm PolynomialForm) { payload := NewPayload(payloadBytes) - blob, err := payload.ToBlob(form) + blob, err := payload.ToBlob(payloadForm) require.NoError(t, err) blobBytes := blob.GetBytes() blobFromBytes, err := BlobFromBytes(blobBytes, blob.blobLength) require.NoError(t, err) - decodedPayload, err := blobFromBytes.ToPayload(form) + decodedPayload, err := blobFromBytes.ToPayload(payloadForm) require.NoError(t, err) decodedPayloadBytes := decodedPayload.GetBytes() From c70db76017c02bec9b9f6d58df17fa4cbf7ac21b Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:58:58 -0500 Subject: [PATCH 27/35] Use smarter power of 2 calc Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- encoding/test_utils.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/encoding/test_utils.go b/encoding/test_utils.go index 8e924388f0..3dc78dee0d 100644 --- a/encoding/test_utils.go +++ b/encoding/test_utils.go @@ -1,8 +1,6 @@ package encoding import ( - "math" - "golang.org/x/exp/constraints" ) @@ -11,7 +9,7 @@ import ( func GeneratePowersOfTwo[T constraints.Integer](powersToGenerate T) []T { powers := make([]T, powersToGenerate) for i := T(0); i < powersToGenerate; i++ { - powers[i] = T(math.Pow(2, float64(i))) + powers[i] = 1 << i } return powers From 3c239cc71310e1d36094a7f3579b18589d699953 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 12:29:30 -0500 Subject: [PATCH 28/35] Collapse fuzz tests Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 2 +- api/clients/codecs/blob_test.go | 15 ++++++------ api/clients/codecs/conversion_test.go | 35 --------------------------- 3 files changed, 8 insertions(+), 44 deletions(-) delete mode 100644 api/clients/codecs/conversion_test.go diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 4d2e169ded..eeb244f358 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -11,7 +11,7 @@ import ( // Blob is data that is dispersed on eigenDA. // -// A Blob is represented under the hood by a coeff polynomial +// A Blob is represented under the hood by an array of field elements, which represent a polynomial in coefficient form type Blob struct { coeffPolynomial []fr.Element // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index 6749893ebb..c9047d26d3 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -22,19 +22,18 @@ func FuzzBlobConversion(f *testing.F) { } func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm PolynomialForm) { - payload := NewPayload(payloadBytes) - - blob, err := payload.ToBlob(payloadForm) + blob, err := NewPayload(payloadBytes).ToBlob(payloadForm) require.NoError(t, err) - blobBytes := blob.GetBytes() - blobFromBytes, err := BlobFromBytes(blobBytes, blob.blobLength) + blobDeserialized, err := BlobFromBytes(blob.GetBytes(), blob.blobLength) require.NoError(t, err) - decodedPayload, err := blobFromBytes.ToPayload(payloadForm) + payloadFromBlob, err := blob.ToPayload(payloadForm) require.NoError(t, err) - decodedPayloadBytes := decodedPayload.GetBytes() + payloadFromDeserializedBlob, err := blobDeserialized.ToPayload(payloadForm) + require.NoError(t, err) - require.Equal(t, payloadBytes, decodedPayloadBytes) + require.Equal(t, payloadFromBlob.GetBytes(), payloadFromDeserializedBlob.GetBytes()) + require.Equal(t, payloadBytes, payloadFromBlob.GetBytes()) } diff --git a/api/clients/codecs/conversion_test.go b/api/clients/codecs/conversion_test.go deleted file mode 100644 index 6e82b8e0dd..0000000000 --- a/api/clients/codecs/conversion_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package codecs - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/require" -) - -// FuzzConversionConsistency checks that data can be encoded and decoded repeatedly, always getting back the original data -func FuzzConversionConsistency(f *testing.F) { - for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} { - f.Add(seed) - } - - f.Fuzz( - func(t *testing.T, originalData []byte) { - payload := NewPayload(originalData) - - blob1, err := payload.ToBlob(PolynomialFormEval) - require.NoError(t, err) - - blob2, err := payload.ToBlob(PolynomialFormCoeff) - require.NoError(t, err) - - decodedPayload1, err := blob1.ToPayload(PolynomialFormEval) - require.NoError(t, err) - - decodedPayload2, err := blob2.ToPayload(PolynomialFormCoeff) - require.NoError(t, err) - - require.Equal(t, originalData, decodedPayload1.GetBytes()) - require.Equal(t, originalData, decodedPayload2.GetBytes()) - }) -} From 15805a92efd8dd016e720263b8cf9e5ad2887e83 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 12:57:56 -0500 Subject: [PATCH 29/35] Fix borked alignment Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/eigenda_client_test.go | 20 ++++++++++---------- api/clients/v2/config.go | 2 +- encoding/utils/codec/codec.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/clients/eigenda_client_test.go b/api/clients/eigenda_client_test.go index cbe7457b05..318c99a8cb 100644 --- a/api/clients/eigenda_client_test.go +++ b/api/clients/eigenda_client_test.go @@ -65,7 +65,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -132,7 +132,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -202,7 +202,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: true, WaitForFinalization: true, }, @@ -234,7 +234,7 @@ func TestPutBlobFailDispersal(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -266,7 +266,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -298,7 +298,7 @@ func TestPutBlobFailureGeneral(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -330,7 +330,7 @@ func TestPutBlobFailureUnknown(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -364,7 +364,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -423,7 +423,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -485,7 +485,7 @@ func TestPutBlobTotalTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index a9a0df9b41..5922d6407d 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -114,7 +114,7 @@ type PayloadDisperserConfig struct { // NOTE: EigenDACertVerifierAddr does not have a defined default. It must always be specifically configured. func GetDefaultPayloadClientConfig() *PayloadClientConfig { return &PayloadClientConfig{ - PayloadEncodingVersion: codecs.PayloadEncodingVersion0, + PayloadEncodingVersion: codecs.PayloadEncodingVersion0, PayloadPolynomialForm: codecs.PolynomialFormEval, ContractCallTimeout: 5 * time.Second, BlockNumberPollInterval: 1 * time.Second, diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 1ea3685371..04527973d7 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -78,7 +78,7 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { // PadPayload internally pads the input data by prepending a 0x00 to each chunk of 31 bytes. This guarantees that // the data will be a valid field element for the bn254 curve // -// # Additionally, this function will add necessary padding to align the output to 32 bytes +// Additionally, this function will add necessary padding to align the output to 32 bytes // // NOTE: this method is a reimplementation of ConvertByPaddingEmptyByte, with one meaningful difference: the alignment // of the output to encoding.BYTES_PER_SYMBOL. This alignment actually makes the padding logic simpler, and the From c028c00d6bd07cf6cf1e05a625285d019b323de8 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:25:04 -0500 Subject: [PATCH 30/35] Rename blobLength to blobLengthSymbols Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 27 +++++++++++---------------- api/clients/codecs/payload.go | 12 ++++++------ encoding/fft/fft.go | 6 +++--- encoding/utils/codec/codec.go | 14 +++++++------- 4 files changed, 27 insertions(+), 32 deletions(-) diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index eeb244f358..9a3e65e76f 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -14,36 +14,31 @@ import ( // A Blob is represented under the hood by an array of field elements, which represent a polynomial in coefficient form type Blob struct { coeffPolynomial []fr.Element - // blobLength must be a power of 2, and should match the blobLength claimed in the BlobCommitment - // This is the blob length IN SYMBOLS, not in bytes + // blobLengthSymbols must be a power of 2, and should match the blobLength claimed in the BlobCommitment // // This value must be specified, rather than computed from the length of the coeffPolynomial, due to an edge case // illustrated by the following example: imagine a user disperses a very small blob, only 64 bytes, and the last 40 // bytes are trailing zeros. When a different user fetches the blob from a relay, it's possible that the relay could - // truncate the trailing zeros. If we were to say that blobLength = nextPowerOf2(len(coeffPolynomial)), then the + // truncate the trailing zeros. If we were to say that blobLengthSymbols = nextPowerOf2(len(coeffPolynomial)), then the // user fetching and reconstructing this blob would determine that the blob length is 1 symbol, when it's actually 2. - blobLength uint32 + blobLengthSymbols uint32 } // BlobFromBytes initializes a Blob from bytes -// -// blobLength is the length of the blob IN SYMBOLS -func BlobFromBytes(bytes []byte, blobLength uint32) (*Blob, error) { +func BlobFromBytes(bytes []byte, blobLengthSymbols uint32) (*Blob, error) { coeffPolynomial, err := rs.ToFrArray(bytes) if err != nil { return nil, fmt.Errorf("bytes to field elements: %w", err) } - return BlobFromPolynomial(coeffPolynomial, blobLength) + return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols) } // BlobFromPolynomial initializes a blob from a polynomial -// -// blobLength is the length of the blob IN SYMBOLS -func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLength uint32) (*Blob, error) { +func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLengthSymbols uint32) (*Blob, error) { return &Blob{ - coeffPolynomial: coeffPolynomial, - blobLength: blobLength, + coeffPolynomial: coeffPolynomial, + blobLengthSymbols: blobLengthSymbols, }, nil } @@ -75,7 +70,7 @@ func (b *Blob) ToPayload(payloadForm PolynomialForm) (*Payload, error) { // The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what // conversion, if any, must be performed when creating an encoded payload from the blob. func (b *Blob) toEncodedPayload(payloadForm PolynomialForm) (*encodedPayload, error) { - maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLength) + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols) if err != nil { return nil, fmt.Errorf("get max permissible payload length: %w", err) } @@ -110,7 +105,7 @@ func (b *Blob) computeEvalPoly() ([]fr.Element, error) { // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves // to present a computational burden, consider making this change. - fftSettings := fft.FFTSettingsFromBlobLength(b.blobLength) + fftSettings := fft.FFTSettingsFromBlobLengthSymbols(b.blobLengthSymbols) // the FFT method pads to the next power of 2, so we don't need to do that manually fftedElements, err := fftSettings.FFT(b.coeffPolynomial, false) @@ -119,4 +114,4 @@ func (b *Blob) computeEvalPoly() ([]fr.Element, error) { } return fftedElements, nil -} \ No newline at end of file +} diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index b0032d6b2a..9a182d8715 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -35,7 +35,7 @@ func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("encoded payload to field elements: %w", err) } - blobLength := uint32(encoding.NextPowerOf2(len(fieldElements))) + blobLengthSymbols := uint32(encoding.NextPowerOf2(len(fieldElements))) var coeffPolynomial []fr.Element switch payloadForm { @@ -45,7 +45,7 @@ func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { coeffPolynomial = fieldElements case PolynomialFormEval: // the payload is in evaluation form, so we need to convert it to coeff form, since blobs are in coefficient form - coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLength) + coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLengthSymbols) if err != nil { return nil, fmt.Errorf("eval poly to coeff poly: %w", err) } @@ -53,7 +53,7 @@ func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { return nil, fmt.Errorf("unknown polynomial form: %v", payloadForm) } - return BlobFromPolynomial(coeffPolynomial, blobLength) + return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols) } // GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data @@ -63,12 +63,12 @@ func (p *Payload) GetBytes() []byte { // evalToCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation // -// blobLength (in SYMBOLS) is required, to be able to choose the correct parameters when performing FFT -func evalToCoeffPoly(evalPoly []fr.Element, blobLength uint32) ([]fr.Element, error) { +// blobLengthSymbols is required, to be able to choose the correct parameters when performing FFT +func evalToCoeffPoly(evalPoly []fr.Element, blobLengthSymbols uint32) ([]fr.Element, error) { // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves // to present a computational burden, consider making this change. - fftSettings := fft.FFTSettingsFromBlobLength(blobLength) + fftSettings := fft.FFTSettingsFromBlobLengthSymbols(blobLengthSymbols) // the FFT method pads to the next power of 2, so we don't need to do that manually ifftedElements, err := fftSettings.FFT(evalPoly, true) diff --git a/encoding/fft/fft.go b/encoding/fft/fft.go index c2199e04a8..e9f7a1c5d0 100644 --- a/encoding/fft/fft.go +++ b/encoding/fft/fft.go @@ -91,8 +91,8 @@ func NewFFTSettings(maxScale uint8) *FFTSettings { } } -// FFTSettingsFromBlobLength accepts a blob length in symbols, and returns a new instance of FFT settings -func FFTSettingsFromBlobLength(blobLength uint32) *FFTSettings { - maxScale := uint8(math.Log2(float64(blobLength))) +// FFTSettingsFromBlobLengthSymbols accepts a blob length, and returns a new instance of FFT settings +func FFTSettingsFromBlobLengthSymbols(blobLengthSymbols uint32) *FFTSettings { + maxScale := uint8(math.Log2(float64(blobLengthSymbols))) return NewFFTSettings(maxScale) } diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 04527973d7..874453cb3d 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -177,22 +177,22 @@ func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { return unpaddedLength, nil } -// GetMaxPermissiblePayloadLength accepts a blob length IN SYMBOLS, and returns the size IN BYTES of the largest payload +// GetMaxPermissiblePayloadLength accepts a blob length, and returns the size IN BYTES of the largest payload // that could fit inside the blob. -func GetMaxPermissiblePayloadLength(blobLength uint32) (uint32, error) { - if blobLength == 0 { - return 0, fmt.Errorf("input blob length is zero") +func GetMaxPermissiblePayloadLength(blobLengthSymbols uint32) (uint32, error) { + if blobLengthSymbols == 0 { + return 0, fmt.Errorf("input blobLengthSymbols is zero") } // TODO (litt3): it's awkward to use a method defined in fft for this, but it's not trivial to move to a better // location, due to the resulting cyclic imports, and I'd prefer not to reimplement. Ideally, a proper location // would be found for this important utility function - if !fft.IsPowerOfTwo(uint64(blobLength)) { - return 0, fmt.Errorf("blobLength %d is not a power of two", blobLength) + if !fft.IsPowerOfTwo(uint64(blobLengthSymbols)) { + return 0, fmt.Errorf("blobLengthSymbols %d is not a power of two", blobLengthSymbols) } // subtract 32 from the blob length before doing the unpad operation, to account for the encoded payload header - maxPayloadLength, err := GetUnpaddedDataLength(blobLength*encoding.BYTES_PER_SYMBOL - 32) + maxPayloadLength, err := GetUnpaddedDataLength(blobLengthSymbols*encoding.BYTES_PER_SYMBOL - 32) if err != nil { return 0, fmt.Errorf("get unpadded data length: %w", err) } From 693d51365b33d9def49f72e7628ab8e64519c7f8 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:28:45 -0500 Subject: [PATCH 31/35] Fix test Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index c9047d26d3..0c879d395a 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -25,7 +25,7 @@ func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm Po blob, err := NewPayload(payloadBytes).ToBlob(payloadForm) require.NoError(t, err) - blobDeserialized, err := BlobFromBytes(blob.GetBytes(), blob.blobLength) + blobDeserialized, err := BlobFromBytes(blob.GetBytes(), blob.blobLengthSymbols) require.NoError(t, err) payloadFromBlob, err := blob.ToPayload(payloadForm) From f1a4e85f83ef84935aaba888cbb28dfb9e743612 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:35:53 -0500 Subject: [PATCH 32/35] Use serialze/deserialize nomenclature Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/blob.go | 10 +++++----- api/clients/codecs/blob_test.go | 6 +++--- api/clients/codecs/encoded_payload.go | 4 ++-- api/clients/codecs/payload.go | 4 ++-- encoding/rs/utils.go | 7 +++++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go index 9a3e65e76f..677d34000c 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/blob.go @@ -24,8 +24,8 @@ type Blob struct { blobLengthSymbols uint32 } -// BlobFromBytes initializes a Blob from bytes -func BlobFromBytes(bytes []byte, blobLengthSymbols uint32) (*Blob, error) { +// DeserializeBlob initializes a Blob from bytes +func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) { coeffPolynomial, err := rs.ToFrArray(bytes) if err != nil { return nil, fmt.Errorf("bytes to field elements: %w", err) @@ -42,9 +42,9 @@ func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLengthSymbols uint32) }, nil } -// GetBytes gets the raw bytes of the Blob -func (b *Blob) GetBytes() []byte { - return rs.FieldElementsToBytes(b.coeffPolynomial) +// Serialize gets the raw bytes of the Blob +func (b *Blob) Serialize() []byte { + return rs.SerializeFieldElements(b.coeffPolynomial) } // ToPayload converts the Blob into a Payload diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/blob_test.go index 0c879d395a..12f42ad794 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/blob_test.go @@ -25,7 +25,7 @@ func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm Po blob, err := NewPayload(payloadBytes).ToBlob(payloadForm) require.NoError(t, err) - blobDeserialized, err := BlobFromBytes(blob.GetBytes(), blob.blobLengthSymbols) + blobDeserialized, err := DeserializeBlob(blob.Serialize(), blob.blobLengthSymbols) require.NoError(t, err) payloadFromBlob, err := blob.ToPayload(payloadForm) @@ -34,6 +34,6 @@ func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm Po payloadFromDeserializedBlob, err := blobDeserialized.ToPayload(payloadForm) require.NoError(t, err) - require.Equal(t, payloadFromBlob.GetBytes(), payloadFromDeserializedBlob.GetBytes()) - require.Equal(t, payloadBytes, payloadFromBlob.GetBytes()) + require.Equal(t, payloadFromBlob.Serialize(), payloadFromDeserializedBlob.Serialize()) + require.Equal(t, payloadBytes, payloadFromBlob.Serialize()) } diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go index ea6364bd45..eea4a09ba7 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/encoded_payload.go @@ -26,7 +26,7 @@ func newEncodedPayload(payload *Payload) (*encodedPayload, error) { // first byte is always 0 to ensure the payloadHeader is a valid bn254 element encodedPayloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte - payloadBytes := payload.GetBytes() + payloadBytes := payload.Serialize() // encode payload length as uint32 binary.BigEndian.PutUint32( @@ -88,7 +88,7 @@ func (ep *encodedPayload) toFieldElements() ([]fr.Element, error) { // // maxPayloadLength is the maximum length in bytes that the contained Payload is permitted to be func encodedPayloadFromElements(fieldElements []fr.Element, maxPayloadLength uint32) (*encodedPayload, error) { - polynomialBytes := rs.FieldElementsToBytes(fieldElements) + polynomialBytes := rs.SerializeFieldElements(fieldElements) // this is the payload length in bytes, as claimed by the encoded payload header payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go index 9a182d8715..02a3272461 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/payload.go @@ -56,8 +56,8 @@ func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols) } -// GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data -func (p *Payload) GetBytes() []byte { +// Serialize returns the bytes that underlie the payload, i.e. the unprocessed user data +func (p *Payload) Serialize() []byte { return p.bytes } diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index 7d054fb9f4..dd2a8a1029 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -12,6 +12,9 @@ import ( ) // ToFrArray accept a byte array as an input, and converts it to an array of field elements +// +// TODO (litt3): it would be nice to rename this to "DeserializeFieldElements", as the counterpart to "SerializeFieldElements", +// but doing so would be a very large diff. I'm leaving this comment as a potential future cleanup. func ToFrArray(inputData []byte) ([]fr.Element, error) { bytes := padToBytesPerSymbol(inputData) @@ -30,8 +33,8 @@ func ToFrArray(inputData []byte) ([]fr.Element, error) { return outputElements, nil } -// FieldElementsToBytes accepts an array of field elements, and converts it to an array of bytes -func FieldElementsToBytes(fieldElements []fr.Element) []byte { +// SerializeFieldElements accepts an array of field elements, and serializes it to an array of bytes +func SerializeFieldElements(fieldElements []fr.Element) []byte { outputBytes := make([]byte, len(fieldElements)*encoding.BYTES_PER_SYMBOL) for i := 0; i < len(fieldElements); i++ { From 3db927475247142dbcd7e715a04cc5d199ea1fde Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:37:56 -0500 Subject: [PATCH 33/35] Separate v2 logic into new package Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/codecs/{ => v2}/blob.go | 11 ++++++----- api/clients/codecs/{ => v2}/blob_test.go | 9 +++++---- api/clients/codecs/{ => v2}/encoded_payload.go | 5 +++-- api/clients/codecs/{ => v2}/encoded_payload_test.go | 2 +- api/clients/codecs/{ => v2}/payload.go | 9 +++++---- 5 files changed, 20 insertions(+), 16 deletions(-) rename api/clients/codecs/{ => v2}/blob.go (93%) rename api/clients/codecs/{ => v2}/blob_test.go (80%) rename api/clients/codecs/{ => v2}/encoded_payload.go (97%) rename api/clients/codecs/{ => v2}/encoded_payload_test.go (99%) rename api/clients/codecs/{ => v2}/payload.go (92%) diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/v2/blob.go similarity index 93% rename from api/clients/codecs/blob.go rename to api/clients/codecs/v2/blob.go index 677d34000c..50a598afc1 100644 --- a/api/clients/codecs/blob.go +++ b/api/clients/codecs/v2/blob.go @@ -1,8 +1,9 @@ -package codecs +package v2 import ( "fmt" + "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/encoding/fft" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" @@ -51,7 +52,7 @@ func (b *Blob) Serialize() []byte { // // The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what // conversion, if any, must be performed when creating a payload from the blob. -func (b *Blob) ToPayload(payloadForm PolynomialForm) (*Payload, error) { +func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) { encodedPayload, err := b.toEncodedPayload(payloadForm) if err != nil { return nil, fmt.Errorf("to encoded payload: %w", err) @@ -69,7 +70,7 @@ func (b *Blob) ToPayload(payloadForm PolynomialForm) (*Payload, error) { // // The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what // conversion, if any, must be performed when creating an encoded payload from the blob. -func (b *Blob) toEncodedPayload(payloadForm PolynomialForm) (*encodedPayload, error) { +func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) { maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols) if err != nil { return nil, fmt.Errorf("get max permissible payload length: %w", err) @@ -77,11 +78,11 @@ func (b *Blob) toEncodedPayload(payloadForm PolynomialForm) (*encodedPayload, er var payloadElements []fr.Element switch payloadForm { - case PolynomialFormCoeff: + case codecs.PolynomialFormCoeff: // the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that // eigenda also interprets blobs as coefficients payloadElements = b.coeffPolynomial - case PolynomialFormEval: + case codecs.PolynomialFormEval: // the payload is interpreted as evaluations of the polynomial, so the coefficient representation contained // in the blob must be converted to the evaluation form payloadElements, err = b.computeEvalPoly() diff --git a/api/clients/codecs/blob_test.go b/api/clients/codecs/v2/blob_test.go similarity index 80% rename from api/clients/codecs/blob_test.go rename to api/clients/codecs/v2/blob_test.go index 12f42ad794..2c5a9fc531 100644 --- a/api/clients/codecs/blob_test.go +++ b/api/clients/codecs/v2/blob_test.go @@ -1,9 +1,10 @@ -package codecs +package v2 import ( "bytes" "testing" + "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/stretchr/testify/require" ) @@ -15,13 +16,13 @@ func FuzzBlobConversion(f *testing.F) { f.Fuzz( func(t *testing.T, originalData []byte) { - testBlobConversionForForm(t, originalData, PolynomialFormEval) - testBlobConversionForForm(t, originalData, PolynomialFormCoeff) + testBlobConversionForForm(t, originalData, codecs.PolynomialFormEval) + testBlobConversionForForm(t, originalData, codecs.PolynomialFormCoeff) }) } -func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm PolynomialForm) { +func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm codecs.PolynomialForm) { blob, err := NewPayload(payloadBytes).ToBlob(payloadForm) require.NoError(t, err) diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/v2/encoded_payload.go similarity index 97% rename from api/clients/codecs/encoded_payload.go rename to api/clients/codecs/v2/encoded_payload.go index eea4a09ba7..8b3b9d6fb5 100644 --- a/api/clients/codecs/encoded_payload.go +++ b/api/clients/codecs/v2/encoded_payload.go @@ -1,9 +1,10 @@ -package codecs +package v2 import ( "encoding/binary" "fmt" + "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -24,7 +25,7 @@ type encodedPayload struct { func newEncodedPayload(payload *Payload) (*encodedPayload, error) { encodedPayloadHeader := make([]byte, 32) // first byte is always 0 to ensure the payloadHeader is a valid bn254 element - encodedPayloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte + encodedPayloadHeader[1] = byte(codecs.PayloadEncodingVersion0) // encode version byte payloadBytes := payload.Serialize() diff --git a/api/clients/codecs/encoded_payload_test.go b/api/clients/codecs/v2/encoded_payload_test.go similarity index 99% rename from api/clients/codecs/encoded_payload_test.go rename to api/clients/codecs/v2/encoded_payload_test.go index 241c0d7697..27bd626b15 100644 --- a/api/clients/codecs/encoded_payload_test.go +++ b/api/clients/codecs/v2/encoded_payload_test.go @@ -1,4 +1,4 @@ -package codecs +package v2 import ( "testing" diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/v2/payload.go similarity index 92% rename from api/clients/codecs/payload.go rename to api/clients/codecs/v2/payload.go index 02a3272461..bced50eead 100644 --- a/api/clients/codecs/payload.go +++ b/api/clients/codecs/v2/payload.go @@ -1,8 +1,9 @@ -package codecs +package v2 import ( "fmt" + "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/fft" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -24,7 +25,7 @@ func NewPayload(payloadBytes []byte) *Payload { // // The payloadForm indicates how payloads are interpreted. The form of a payload dictates what conversion, if any, must // be performed when creating a blob from the payload. -func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { +func (p *Payload) ToBlob(payloadForm codecs.PolynomialForm) (*Blob, error) { encodedPayload, err := newEncodedPayload(p) if err != nil { return nil, fmt.Errorf("encoding payload: %w", err) @@ -39,11 +40,11 @@ func (p *Payload) ToBlob(payloadForm PolynomialForm) (*Blob, error) { var coeffPolynomial []fr.Element switch payloadForm { - case PolynomialFormCoeff: + case codecs.PolynomialFormCoeff: // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in // coefficient form coeffPolynomial = fieldElements - case PolynomialFormEval: + case codecs.PolynomialFormEval: // the payload is in evaluation form, so we need to convert it to coeff form, since blobs are in coefficient form coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLengthSymbols) if err != nil { From b8d43f59ba3c666d9c8946770c87b11cb9c0c04e Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:00:55 -0500 Subject: [PATCH 34/35] Use better package scheme Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/{codecs/v2 => v2/codecs}/blob.go | 2 +- api/clients/{codecs/v2 => v2/codecs}/blob_test.go | 2 +- api/clients/{codecs/v2 => v2/codecs}/encoded_payload.go | 2 +- api/clients/{codecs/v2 => v2/codecs}/encoded_payload_test.go | 2 +- api/clients/{codecs/v2 => v2/codecs}/payload.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename api/clients/{codecs/v2 => v2/codecs}/blob.go (99%) rename api/clients/{codecs/v2 => v2/codecs}/blob_test.go (98%) rename api/clients/{codecs/v2 => v2/codecs}/encoded_payload.go (99%) rename api/clients/{codecs/v2 => v2/codecs}/encoded_payload_test.go (99%) rename api/clients/{codecs/v2 => v2/codecs}/payload.go (99%) diff --git a/api/clients/codecs/v2/blob.go b/api/clients/v2/codecs/blob.go similarity index 99% rename from api/clients/codecs/v2/blob.go rename to api/clients/v2/codecs/blob.go index 50a598afc1..c52148f679 100644 --- a/api/clients/codecs/v2/blob.go +++ b/api/clients/v2/codecs/blob.go @@ -1,4 +1,4 @@ -package v2 +package codecs import ( "fmt" diff --git a/api/clients/codecs/v2/blob_test.go b/api/clients/v2/codecs/blob_test.go similarity index 98% rename from api/clients/codecs/v2/blob_test.go rename to api/clients/v2/codecs/blob_test.go index 2c5a9fc531..7fae932e47 100644 --- a/api/clients/codecs/v2/blob_test.go +++ b/api/clients/v2/codecs/blob_test.go @@ -1,4 +1,4 @@ -package v2 +package codecs import ( "bytes" diff --git a/api/clients/codecs/v2/encoded_payload.go b/api/clients/v2/codecs/encoded_payload.go similarity index 99% rename from api/clients/codecs/v2/encoded_payload.go rename to api/clients/v2/codecs/encoded_payload.go index 8b3b9d6fb5..a8eb972563 100644 --- a/api/clients/codecs/v2/encoded_payload.go +++ b/api/clients/v2/codecs/encoded_payload.go @@ -1,4 +1,4 @@ -package v2 +package codecs import ( "encoding/binary" diff --git a/api/clients/codecs/v2/encoded_payload_test.go b/api/clients/v2/codecs/encoded_payload_test.go similarity index 99% rename from api/clients/codecs/v2/encoded_payload_test.go rename to api/clients/v2/codecs/encoded_payload_test.go index 27bd626b15..241c0d7697 100644 --- a/api/clients/codecs/v2/encoded_payload_test.go +++ b/api/clients/v2/codecs/encoded_payload_test.go @@ -1,4 +1,4 @@ -package v2 +package codecs import ( "testing" diff --git a/api/clients/codecs/v2/payload.go b/api/clients/v2/codecs/payload.go similarity index 99% rename from api/clients/codecs/v2/payload.go rename to api/clients/v2/codecs/payload.go index bced50eead..df72ed6fed 100644 --- a/api/clients/codecs/v2/payload.go +++ b/api/clients/v2/codecs/payload.go @@ -1,4 +1,4 @@ -package v2 +package codecs import ( "fmt" From 781515cc44072fa508b38eeff06256ca9cf8ce1e Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:03:14 -0500 Subject: [PATCH 35/35] Fix merge Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/codecs/encoded_payload_test.go | 10 +++++----- encoding/utils/codec/codec_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/clients/v2/codecs/encoded_payload_test.go b/api/clients/v2/codecs/encoded_payload_test.go index 241c0d7697..0053715bc9 100644 --- a/api/clients/v2/codecs/encoded_payload_test.go +++ b/api/clients/v2/codecs/encoded_payload_test.go @@ -12,7 +12,7 @@ import ( // TestDecodeShortBytes checks that an encoded payload with a length less than claimed length fails at decode time func TestDecodeShortBytes(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) encodedPayload, err := newEncodedPayload(NewPayload(originalData)) @@ -28,7 +28,7 @@ func TestDecodeShortBytes(t *testing.T) { // TestDecodeLongBytes checks that an encoded payload with length too much greater than claimed fails at decode func TestDecodeLongBytes(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) encodedPayload, err := newEncodedPayload(NewPayload(originalData)) @@ -43,7 +43,7 @@ func TestDecodeLongBytes(t *testing.T) { // TestEncodeTooManyElements checks that encodedPayloadFromElements fails at the expect limit, relative to payload // length and blob length func TestEncodeTooManyElements(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() powersOf2 := encoding.GeneratePowersOfTwo(uint32(12)) for i := 0; i < len(powersOf2); i++ { @@ -74,7 +74,7 @@ func TestEncodeTooManyElements(t *testing.T) { // TestTrailingNonZeros checks that any non-zero values that come after the end of the claimed payload length // cause an error to be returned. func TestTrailingNonZeros(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) encodedPayload, err := newEncodedPayload(NewPayload(originalData)) @@ -102,7 +102,7 @@ func TestTrailingNonZeros(t *testing.T) { // TestEncodeWithFewerElements tests that having fewer bytes than expected doesn't throw an error func TestEncodeWithFewerElements(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) encodedPayload, err := newEncodedPayload(NewPayload(originalData)) diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 471edafb21..9af9c7ba84 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -80,7 +80,7 @@ func TestGetUnpaddedDataLengthAgainstKnowns(t *testing.T) { // TestPadUnpad makes sure that padding and unpadding doesn't corrupt underlying data func TestPadUnpad(t *testing.T) { - testRandom := random.NewTestRandom(t) + testRandom := random.NewTestRandom() testIterations := 1000 for i := 0; i < testIterations; i++ {