Skip to content

Commit

Permalink
feat: Use IPFS-compatible CIDs in CAS
Browse files Browse the repository at this point in the history 
The CIDs generated for the local CAS are now compatible with an IPFS node, assuming that it's running under default settings and that the data is less than 256KB.

In a future commit, the resolver and CAS will be updated to support both v0 and v1 CIDs.

Signed-off-by: Derek Trider <[email protected]>
  • Loading branch information
Derek Trider committed May 19, 2021
1 parent 4aca3a8 commit 1879506
Show file tree
Hide file tree
Showing 8 changed files with 797 additions and 37 deletions.
226 changes: 225 additions & 1 deletion cmd/orb-server/go.sum
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ require (
github.com/hyperledger/aries-framework-go/component/storageutil v0.0.0-20210422133815-2ef2d99cb692
github.com/hyperledger/aries-framework-go/spi v0.0.0-20210422144621-1355c6f90b44
github.com/igor-pavlenko/httpsignatures-go v0.0.21
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-ipfs-api v0.2.0
github.com/ipfs/go-merkledag v0.2.3
github.com/ipfs/go-unixfs v0.2.6
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multihash v0.0.14
github.com/ory/dockertest/v3 v3.6.3
github.com/piprate/json-gold v0.4.0
github.com/rs/cors v1.7.0
Expand Down
253 changes: 252 additions & 1 deletion go.sum
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pkg/anchor/handler/credential/handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ const sampleAnchorCredential = `{
}]
}`

const sampleAnchorCredentialCID = "QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ"
const sampleAnchorCredentialCID = "QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk"

func TestNew(t *testing.T) {
createNewAnchorCredentialHandler(t, createInMemoryCAS(t))
Expand Down Expand Up @@ -116,7 +116,7 @@ func TestAnchorCredentialHandler(t *testing.T) {
"failure while getting and storing data from the remote WebCAS endpoint: "+
"failed to retrieve data from")
require.Contains(t, err.Error(), "Response status code: 404. Response body: "+
"no content at QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ was found: content not found")
"no content at QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk was found: content not found")
})
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/cas/resolver/resolver_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ const sampleData = `{
}]
}`

const sampleDataCID = "QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ"
const sampleDataCID = "QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk"

func TestNew(t *testing.T) {
createNewResolver(t, createInMemoryCAS(t), createInMemoryCAS(t))
Expand Down Expand Up @@ -285,7 +285,7 @@ func TestResolver_Resolve(t *testing.T) {
data, err := resolver.Resolve(id, cid, []byte(sampleData))
require.EqualError(t, err, "failure while storing the data in the local CAS: "+
"successfully stored data into the local CAS, but the CID produced by the local CAS "+
"(QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ) does not match the CID from the original request "+
"(QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk) does not match the CID from the original request "+
"(bafkrwihwsnuregfeqh263vgdathcprnbvatyat6h6mu7ipjhhodcdbyhoy)")
require.Nil(t, data)
})
Expand Down Expand Up @@ -313,7 +313,7 @@ func TestResolver_Resolve(t *testing.T) {
require.Contains(t, err.Error(), "failure while getting and storing data from the remote "+
"WebCAS endpoint: failed to retrieve data from")
require.Contains(t, err.Error(), "Response status code: 404. Response body: "+
"no content at QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ was found: content not found")
"no content at QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk was found: content not found")
require.Nil(t, data)
})
t.Run("Fail to write to local CAS", func(t *testing.T) {
Expand Down Expand Up @@ -371,7 +371,7 @@ func TestResolver_Resolve(t *testing.T) {

data, err := resolver.Resolve(id, sampleDataCID, nil)
require.EqualError(t, err, "failed to get data stored at "+
"QmRQB1fQpB4ahvV1fsbjE3fKkT4U9oPjinRofjgS3B9ZEQ from the local CAS: "+
"QmW4LWKX9pD1ak6iZG3J7oC6xmp93476Dz1HCnQtMdPnNk from the local CAS: "+
"failed to get content from the underlying storage provider: get error")
require.Nil(t, data)
})
Expand Down
25 changes: 9 additions & 16 deletions pkg/store/cas/cas.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ import (
"fmt"

ariesstorage "github.com/hyperledger/aries-framework-go/spi/storage"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"github.com/ipfs/go-merkledag"
"github.com/ipfs/go-unixfs"
)

// ErrContentNotFound is used to indicate that content as a given address could not be found.
var ErrContentNotFound = errors.New("content not found")

// CAS represents a content-addressable storage provider.
// TODO (#344) Support writing and reading both v0 and v1 CIDs.
type CAS struct {
cas ariesstorage.Store
}
Expand All @@ -36,24 +37,16 @@ func New(provider ariesstorage.Provider) (*CAS, error) {
// Write writes the given content to the underlying storage provider.
// Returns the address of the content.
func (p *CAS) Write(content []byte) (string, error) {
// TODO #318 figure out why the CIDs produced here differ from the ones that IPFS generates.
prefix := cid.Prefix{
Version: 0,
MhType: mh.SHA2_256,
MhLength: -1, // default length
}
// The CID produced below is a v0 IPFS CID, assuming that:
// 1. The IPFS node is running with default settings, and
// 2. The size of the content passed in here is less than 256KB (the default chunk size).
cid := merkledag.NodeWithData(unixfs.FilePBData(content, uint64(len(content)))).Cid().String()

contentID, err := prefix.Sum(content)
if err != nil {
return "", fmt.Errorf("failed to generate CID: %w", err)
}

err = p.cas.Put(contentID.String(), content)
if err != nil {
if err := p.cas.Put(cid, content); err != nil {
return "", fmt.Errorf("failed to put content into underlying storage provider: %w", err)
}

return contentID.String(), nil
return cid, nil
}

// Read reads the content of the given address from the underlying storage provider.
Expand Down
148 changes: 139 additions & 9 deletions pkg/store/cas/cas_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,43 +9,98 @@ package cas_test
import (
"errors"
"testing"
"time"

"github.com/cenkalti/backoff/v4"
ariesmemstorage "github.com/hyperledger/aries-framework-go/component/storageutil/mem"
ariesmockstorage "github.com/hyperledger/aries-framework-go/component/storageutil/mock"
ariesstorage "github.com/hyperledger/aries-framework-go/spi/storage"
dctest "github.com/ory/dockertest/v3"
dc "github.com/ory/dockertest/v3/docker"
"github.com/stretchr/testify/require"

"github.com/trustbloc/orb/pkg/store/cas"
ipfscas "github.com/trustbloc/orb/pkg/cas/ipfs"
localcas "github.com/trustbloc/orb/pkg/store/cas"
)

const sampleAnchorCredential = `{
"@context": [
"https://www.w3.org/2018/credentials/v1",
"https://trustbloc.github.io/did-method-orb/contexts/anchor/v1",
"https://w3id.org/jws/v1"
],
"id": "http://sally.example.com/transactions/bafkreihwsnuregceqh263vgdathcprnbvatyat6h6mu7ipjhhodcdbyhoy",
"type": [
"VerifiableCredential",
"AnchorCredential"
],
"issuer": "https://sally.example.com/services/orb",
"issuanceDate": "2021-01-27T09:30:10Z",
"credentialSubject": {
"operationCount": 1,
"coreIndex": "bafkreihwsnuregceqh263vgdathcprnbvatyat6h6mu7ipjhhodcdbyhoy",
"namespace": "did:orb",
"version": "1",
"previousAnchors": {
"EiA329wd6Aj36YRmp7NGkeB5ADnVt8ARdMZMPzfXsjwTJA": "bafkreibmrmenuxhgaomod4m26ds5ztdujxzhjobgvpsyl2v2ndcskq2iay",
"EiABk7KK58BVLHMataxgYZjTNbsHgtD8BtjF0tOWFV29rw": "bafkreibh3whnisud76knkv7z7ucbf3k2rs6knhvajernrdabdbfaomakli"
},
"type": "Anchor"
},
"proof": [{
"type": "JsonWebSignature2020",
"proofPurpose": "assertionMethod",
"created": "2021-01-27T09:30:00Z",
"verificationMethod": "did:example:abcd#key",
"domain": "sally.example.com",
"jws": "eyJ..."
},
{
"type": "JsonWebSignature2020",
"proofPurpose": "assertionMethod",
"created": "2021-01-27T09:30:05Z",
"verificationMethod": "did:example:abcd#key",
"domain": "https://witness1.example.com/ledgers/maple2021",
"jws": "eyJ..."
},
{
"type": "JsonWebSignature2020",
"proofPurpose": "assertionMethod",
"created": "2021-01-27T09:30:06Z",
"verificationMethod": "did:example:efgh#key",
"domain": "https://witness2.example.com/ledgers/spruce2021",
"jws": "eyJ..."
}]
}`

func TestNew(t *testing.T) {
t.Run("Success", func(t *testing.T) {
provider, err := cas.New(ariesmemstorage.NewProvider())
provider, err := localcas.New(ariesmemstorage.NewProvider())
require.NoError(t, err)
require.NotNil(t, provider)
})
t.Run("Fail to store in underlying storage provider", func(t *testing.T) {
provider, err := cas.New(&ariesmockstorage.Provider{ErrOpenStore: errors.New("open store error")})
provider, err := localcas.New(&ariesmockstorage.Provider{ErrOpenStore: errors.New("open store error")})
require.EqualError(t, err, "failed to open store in underlying storage provider: open store error")
require.Nil(t, provider)
})
}

func TestProvider_Write_Read(t *testing.T) {
t.Run("Success", func(t *testing.T) {
provider, err := cas.New(ariesmemstorage.NewProvider())
provider, err := localcas.New(ariesmemstorage.NewProvider())
require.NoError(t, err)

address, err := provider.Write([]byte("content"))
require.NoError(t, err)
require.Equal(t, "QmeKWPxUJP9M3WJgBuj8ykLtGU37iqur5gZ8cDCi49WJVG", address)
require.Equal(t, "QmbSnCcHziqhjNRyaunfcCvxPiV3fNL3fWL8nUrp5yqwD5", address)

content, err := provider.Read(address)
require.NoError(t, err)
require.Equal(t, "content", string(content))
})
t.Run("Fail to put content bytes into underlying storage provider", func(t *testing.T) {
provider, err := cas.New(&ariesmockstorage.Provider{
provider, err := localcas.New(&ariesmockstorage.Provider{
OpenStoreReturn: &ariesmockstorage.Store{
ErrPut: errors.New("put error"),
},
Expand All @@ -58,19 +113,19 @@ func TestProvider_Write_Read(t *testing.T) {
})
t.Run("Fail to get content bytes from underlying storage provider", func(t *testing.T) {
t.Run("Data not found", func(t *testing.T) {
provider, err := cas.New(&ariesmockstorage.Provider{
provider, err := localcas.New(&ariesmockstorage.Provider{
OpenStoreReturn: &ariesmockstorage.Store{
ErrGet: ariesstorage.ErrDataNotFound,
},
})
require.NoError(t, err)

content, err := provider.Read("AVUSIO1wArQ56ayEXyI1fYIrrBREcw-9tgFtPslDIpe57J9z")
require.Equal(t, err, cas.ErrContentNotFound)
require.Equal(t, err, localcas.ErrContentNotFound)
require.Nil(t, content)
})
t.Run("Other error", func(t *testing.T) {
provider, err := cas.New(&ariesmockstorage.Provider{
provider, err := localcas.New(&ariesmockstorage.Provider{
OpenStoreReturn: &ariesmockstorage.Store{
ErrGet: errors.New("get error"),
},
Expand All @@ -83,3 +138,78 @@ func TestProvider_Write_Read(t *testing.T) {
})
})
}

func TestEnsureLocalCASAndIPFSProduceSameCIDs(t *testing.T) {
pool, ipfsResource := startIPFSDockerContainer(t)

defer func() {
require.NoError(t, pool.Purge(ipfsResource), "failed to purge IPFS resource")
}()

smallSimpleData := []byte("content")
cidFromIPFS := addDataToIPFS(t, smallSimpleData)
cidFromLocalCAS := addDataToLocalCAS(t, smallSimpleData)

require.Equal(t, cidFromIPFS, cidFromLocalCAS)

sampleAnchorCredentialBytes := []byte(sampleAnchorCredential)

cidFromIPFS = addDataToIPFS(t, sampleAnchorCredentialBytes)
cidFromLocalCAS = addDataToLocalCAS(t, sampleAnchorCredentialBytes)

require.Equal(t, cidFromIPFS, cidFromLocalCAS)
}

func startIPFSDockerContainer(t *testing.T) (*dctest.Pool, *dctest.Resource) {
t.Helper()

pool, err := dctest.NewPool("")
require.NoError(t, err, "failed to create pool")

ipfsResource, err := pool.RunWithOptions(&dctest.RunOptions{
Repository: "ipfs/go-ipfs",
Tag: "master-2021-04-22-eea198f",
PortBindings: map[dc.Port][]dc.PortBinding{
"5001/tcp": {{HostIP: "", HostPort: "5001"}},
},
})
if err != nil {
require.FailNow(t, "Failed to start IPFS Docker image."+
" This can happen if there is an IPFS container still running from a previous unit test run."+
` Try "docker ps" from the command line and kill the old container if it's still running.`)
}

return pool, ipfsResource
}

func addDataToIPFS(t *testing.T, data []byte) string {
t.Helper()

ipfsCASClient := ipfscas.New("localhost:5001")

// IPFS will need some time to start up, hence the need for retries.
var cidFromIPFS string

err := backoff.Retry(func() error {
cid, errWrite := ipfsCASClient.Write(data)

cidFromIPFS = cid

return errWrite
}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Millisecond*500), 10))
require.NoError(t, err)

return cidFromIPFS
}

func addDataToLocalCAS(t *testing.T, data []byte) string {
t.Helper()

provider, err := localcas.New(ariesmemstorage.NewProvider())
require.NoError(t, err)

cid, err := provider.Write(data)
require.NoError(t, err)

return cid
}
Loading

0 comments on commit 1879506

Please sign in to comment.