Merge pull request #29 from eigerco/feat/25/car-implementation

CARv2 implementation

.rustfmt.toml

@@ -1,3 +1,5 @@
 edition = "2021"
 group_imports = "StdExternalCrate"
+hard_tabs = false
 imports_granularity = "Crate"
+tab_spaces = 4

Cargo.toml

@@ -6,7 +6,7 @@ license-file = "LICENSE"
 repository = "https://github.com/eigerco/polka-storage"
 
 [workspace]
-members = ["node", "runtime", "storage/polka-index"]
+members = ["node", "runtime", "storage/mater", "storage/polka-index"]
 resolver = "2"
 
 # FIXME(#@jmg-duarte,#7,14/5/24): remove the patch once something >1.11.0 is released
@@ -28,30 +28,47 @@ panic = 'abort'         # Use abort on panic to reduce binary size
 substrate-build-script-utils = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" }
 substrate-wasm-builder = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-v1.11.0" }
 
+async-stream = "0.3.5"
+bitflags = "2.5.0"
+byteorder = "1.5.0"
+bytes = "1.6.0"
 ciborium = "0.2.2"
 cid = { version = "0.11.1" }
 clap = { version = "4.5.3" }
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
 color-print = "0.3.4"
+digest = "0.10.7"
 futures = "0.3.28"
 hex-literal = { version = "0.4.1" }
+indexmap = "2.2.6"
+integer-encoding = "4.0.0"
+ipld-core = "0.4.1"
+ipld-dagpb = "0.2.1"
+itertools = "0.13.0"
 jsonrpsee = { version = "0.22" }
 log = { version = "0.4.21", default-features = false }
 polkavm = "0.9.3"
 polkavm-derive = "0.9.1"
 polkavm-linker = "0.9.2"
+quick-protobuf = "0.8.1"
 quote = { version = "1.0.33" }
+rand = "0.8.5"
 rocksdb = { version = "0.21" }
 scale-info = { version = "2.11.1", default-features = false }
 serde = { version = "1.0.197", default-features = false }
 serde-big-array = { version = "0.3.2" }
 serde_derive = { version = "1.0.117" }
+serde_ipld_dagcbor = "0.6.1"
 serde_json = { version = "1.0.114", default-features = false }
 serde_yaml = { version = "0.9" }
+sha2 = "0.10.8"
 smallvec = "1.11.0"
 syn = { version = "2.0.53" }
 tempfile = "3.10.1"
 thiserror = { version = "1.0.48" }
+tokio = "1.37.0"
+tokio-stream = "0.1.15"
+tokio-util = "0.7.11"
 tracing-subscriber = { version = "0.3.18" }
 
 # Local

storage/mater/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license-file.workspace = true
+name = "mater"                # name WIP
+repository.workspace = true
+version = "0.1.0"
+
+[dependencies]
+async-stream.workspace = true
+bitflags.workspace = true
+byteorder = { workspace = true, features = ["i128"] }
+bytes.workspace = true
+digest.workspace = true
+futures.workspace = true
+indexmap.workspace = true
+integer-encoding = { workspace = true, features = ["tokio_async"] }
+ipld-core = { workspace = true, features = ["serde"] }
+ipld-dagpb.workspace = true
+quick-protobuf.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde_ipld_dagcbor.workspace = true
+sha2.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["fs", "macros", "rt"] }
+tokio-stream.workspace = true
+tokio-util = { workspace = true, features = ["io"] }
+
+[dev-dependencies]
+rand.workspace = true
+
+
+[lints]
+workspace = true

storage/mater/README.md

@@ -0,0 +1,29 @@
+# Mater
+
+A Rust library to read and write CAR files.
+
+This library is based on [beetle](https://github.com/n0-computer/beetle).
+
+## Specifications
+
+CARv1 specification: https://ipld.io/specs/transport/car/carv1/
+CARv2 specification: https://ipld.io/specs/transport/car/carv2/
+UnixFS specification: https://github.com/ipfs/specs/blob/e4e5754ad4a4bfbb2ebe63f4c27631f573703de0/UNIXFS.md
+
+## Developing
+
+### Overview
+
+This crate is composed of three main modules:
+
+- `unixfs/` — which covers the main UnixFS abstractions
+- `v1/` — which contains the CARv1 implementation and respective abstractions
+- `v2/` — which contains the CARv2 implementation and respective abstractions
+
+### Further notes
+
+The [`unixfs_pb.rs`](src/unixfs/unixfs_pb.rs) was automatically generated using
+[`pb-rs`](https://github.com/tafia/quick-protobuf/tree/master/pb-rs).
+The file was generated and checked-in instead of making `pb-rs` part of the build
+because the definition file ([`unixfs.proto`](src/unixfs/unixfs.proto)) does not
+change frequently, hence, there is no need to add complexity to the build process.