Skip to content

Commit

Permalink
Merge pull request #324 from benjamin-747/main
Browse files Browse the repository at this point in the history
Refactor: Added two modules as basic git object and storage provider
  • Loading branch information
genedna authored Feb 22, 2024
2 parents 37cf837 + 8ec1b47 commit 7b35e23
Show file tree
Hide file tree
Showing 66 changed files with 2,635 additions and 150 deletions.
11 changes: 5 additions & 6 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ members = [
"kvcache",
"delta",
"mercury",
"jupiter",
"jupiter/entity",
"venus",
]
exclude = ["mda", "craft", "fuse"]

Expand Down Expand Up @@ -74,16 +77,12 @@ russh = "0.42.0"
russh-keys = "0.42.0"
axum = "0.7.4"
hex = "0.4.3"
sea-orm = { version = "0.12.14", features = [
"sqlx-postgres",
"sqlx-mysql",
"runtime-tokio-rustls",
"macros",
] }
sea-orm = "0.12.14"
redis = "0.24.0"
flate2 = "1.0.28"
bstr = "1.9.0"
colored = "2.1.0"
idgenerator = "2.0.0"

[build-dependencies]
shadow-rs = "0.26.0"
4 changes: 4 additions & 0 deletions b_link.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
version https://gitmega.dev/spec/v1
{{objectType}} {{sha1}}
storage_type {{type}}
storage_locaton {{location}}
1 change: 1 addition & 0 deletions common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ anyhow = { workspace = true }
sea-orm = { workspace = true }
thiserror = { workspace = true }
clap = { workspace = true, features = ["derive"] }
idgenerator = { workspace = true }
15 changes: 14 additions & 1 deletion common/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,20 @@
//!
//!
use idgenerator::IdInstance;

pub const ZERO_ID: &str = match std::str::from_utf8(&[b'0'; 40]) {
Ok(s) => s,
Err(_) => panic!("can't get ZERO_ID"),
};
};

pub fn generate_id() -> i64 {
let mut new_id: i64 = 0;
let mut times = 100;
while times > 0 {
// Call `next_id` to generate a new unique id.
new_id = IdInstance::next_id();
times -= 1;
}
new_id
}
134 changes: 66 additions & 68 deletions docs/database.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@ Similar to the 'tree' in Git, Mega maintains relationships between files and fil
The purpose of the B-link file is to store file index information, serving as a replacement for blobs in Git. The design of this structure is inspired by the specification of Git LFS, as follows:

```bash
version https://mega.com/directory/spec/v1
version https://gitmega.dev/spec/v1
blob 3a739f77180d81aa45d9bd11eb6be7098bf1991f
storage_type local_fs
storage_location /tmp/.mega/{{reponame}}/.objects/3a/73/9f77180d81aa45d9bd11eb6be7098bf1991f
```

It includes the following records:
Expand Down Expand Up @@ -58,46 +60,42 @@ It includes the following records:
### ER Diagram


```mermaid
```mermaid
erDiagram
msnap["MEGA-SNAPSHOT"] mc["MEGA-COMMITS"] mt["MEGA-TREE"] mb["MEGA-BLOB"] mtag["MEGA-TAG"] mmr["MEGA-MR"]
grp["GIT-REPO"] grf["GIT-REFS"] gc["GIT-COMMIT"] gt["GIT-TREE"] gb["GIT-BLOB"] gtag["GIT-TAG"] gp["GIT-PR"] gi["GIT-ISSUE"]
raw["RAW-OBJETCS"]
lo["LFS-OBJECTS"] lk["LFS-LOCKS"]
msnap |o--|{ mc : "belong to"
msnap |o--|{ gt : contains
mc ||--|| mt : points
mc ||--|| raw : points
mc ||--|| mmr : "belong to"
mt }|--o{ mb : points
mt ||--|| raw : points
mt }|--|| mmr : "belong to"
mt }o..o{ gt : points
mb ||--|| raw : points
mb }|--|| mmr : "belong to"
mtag |o--o| mc : points
mtag ||--|| raw : points
raw ||--o| lo : points
lo ||--o| lk : points
gp }o--|| grp : "belong to"
gi }o--|| grp : "belong to"
grf ||--|| gc : points
grf ||--|| gtag : points
grf }|--|| grp : "belong to"
gc ||--|| gt : has
gc ||--|| raw : has
gc }|--|| grp : "belong to"
gt ||--o{ gb : has
gt ||--|| raw : points
gt }|--|| grp : "belong to"
gb ||--|| raw : points
gb }|--|| grp : "belong to"
gtag }o--|| grp : "belong to"
gtag |o--o| gc : points
gtag ||--|| raw : points
```
MEGA-SNAPSHOT |o--|{ MEGA-COMMITS : "belong to"
MEGA-SNAPSHOT |o--|{ GIT-TREE : contains
MEGA-COMMITS ||--|| MEGA-TREE : points
MEGA-COMMITS ||--|| RAW-OBJETCS : points
MEGA-COMMITS ||--|| MEGA-MR : "belong to"
MEGA-TREE }|--o{ MEGA-BLOB : points
MEGA-TREE ||--|| RAW-OBJETCS : points
MEGA-TREE }|--|| MEGA-MR : "belong to"
MEGA-TREE }o..o{ GIT-TREE : points
MEGA-BLOB ||--|| RAW-OBJETCS : points
MEGA-BLOB }|--|| MEGA-MR : "belong to"
MEGA-TAG |o--o| MEGA-COMMITS : points
MEGA-TAG ||--|| RAW-OBJETCS : points
RAW-OBJETCS ||--o| LFS-OBJECTS : points
LFS-OBJECTS ||--o| LFS-LOCKS : points
GIT-PR }o--|| GIT-REPO : "belong to"
GIT-ISSUE }o--|| GIT-REPO : "belong to"
GIT-REFS ||--|| GIT-COMMIT : points
GIT-REFS ||--|| GIT-TAG : points
GIT-REFS }|--|| GIT-REPO : "belong to"
GIT-COMMIT ||--|| GIT-TREE : has
GIT-COMMIT ||--|| RAW-OBJETCS : has
GIT-COMMIT }|--|| GIT-REPO : "belong to"
GIT-TREE ||--o{ GIT-BLOB : has
GIT-TREE ||--|| RAW-OBJETCS : points
GIT-TREE }|--|| GIT-REPO : "belong to"
GIT-BLOB ||--|| RAW-OBJETCS : points
GIT-BLOB }|--|| GIT-REPO : "belong to"
GIT-TAG }o--|| GIT-REPO : "belong to"
GIT-TAG |o--o| GIT-COMMIT : points
GIT-TAG ||--|| RAW-OBJETCS : points
```

### Table Details

Expand All @@ -119,21 +117,21 @@ erDiagram

#### mega_commit

| Column | Type | Constraints | Description |
| ---------- | ----------- | ----------- | ----------------------------------------------- |
| id | BIGINT | PRIMARY KEY | |
| commit_id | VARCHAR(40) | NOT NULL | |
| tree | VARCHAR(40) | NOT NULL | |
| parents_id | TEXT[] | | |
| author | TEXT | | |
| committer | TEXT | | |
| content | TEXT | | |
| mr_id | VARCHAR(20) | | |
| status | VARCHAR(20) | NOT NULL | mr satus, might be 'Open','Merged' and 'Closed' |
| size | INT | NOT NULL | used for magic sort in pack process |
| full_path | TEXT | NOT NULL | used for magic sort in pack process |
| created_at | TIMESTAMP | NOT NULL | |
| updated_at | TIMESTAMP | NOT NULL | |
| Column | Type | Constraints | Description |
| ---------- | ----------- | ----------- | ---------------------------------------------- |
| id | BIGINT | PRIMARY KEY | |
| commit_id | VARCHAR(40) | NOT NULL | |
| tree | VARCHAR(40) | NOT NULL | |
| parents_id | TEXT[] | NOT NULL | |
| author | TEXT | | |
| committer | TEXT | | |
| content | TEXT | | |
| mr_id | VARCHAR(20) | | |
| status | VARCHAR(20) | NOT NULL | mr satus, can be 'Open', 'Merged' and 'Closed' |
| size | INT | NOT NULL | used for magic sort in pack process |
| full_path | TEXT | NOT NULL | used for magic sort in pack process |
| created_at | TIMESTAMP | NOT NULL | |
| updated_at | TIMESTAMP | NOT NULL | |


#### mega_tree
Expand Down Expand Up @@ -188,7 +186,7 @@ erDiagram
| id | BIGINT | PRIMARY KEY | |
| mr_link | VARCHAR(40) | NOT NULL | A MR identifier with a length of 6-8 characters. |
| mr_msg | VARCHAR(255) | NOT NULL | |
| merge_date | TIMESTAMP | NOT NULL | |
| merge_date | TIMESTAMP | | |
| status | VARCHAR(20) | NOT NULL | |
| created_at | TIMESTAMP | NOT NULL | |
| updated_at | TIMESTAMP | NOT NULL | |
Expand All @@ -215,7 +213,7 @@ erDiagram
| repo_id | BIGINT | NOT NULL | |
| ref_name | TEXT | NOT NULL | reference name, can be branch and tag |
| ref_git_id | VARCHAR(40) | NOT NULL | point to the commit or tag object |
| is_commit | BOOLEAN | NOT NULL | set true if point to a commit |
| ref_type | VARCHAR(20) | NOT NULL | ref_type: can be 'tag' or 'branch' |
| created_at | TIMESTAMP | NOT NULL | |
| updated_at | TIMESTAMP | NOT NULL | |

Expand All @@ -237,7 +235,7 @@ erDiagram
| repo_id | BIGINT | NOT NULL |
| commit_id | VARCHAR(40) | NOT NULL |
| tree | VARCHAR(40) | NOT NULL |
| pid | TEXT[] | |
| parents_id | TEXT[] | NOT NULL |
| author | TEXT | |
| committer | TEXT | |
| content | TEXT | |
Expand Down Expand Up @@ -290,15 +288,15 @@ erDiagram

#### raw_objects

| Column | Type | Constraints | Description |
| ------------------ | ----------- | ----------- | ----------------------------------------------------------------------- |
| id | BIGINT | PRIMARY KEY | |
| sha1 | VARCHAR(40) | NOT NULL | git object's sha1 hash |
| object_type | VARCHAR(20) | NOT NULL | |
| storage_type | INT | NOT NULL | data storage type, can be 0-database; 1-local file system; 2-remote url |
| data | BYTEA | | |
| local_storage_path | TEXT | | |
| remote_url | TEXT | | |
| Column | Type | Constraints | Description |
| ------------------ | ----------- | ----------- | ----------------------------------------------------------------- |
| id | BIGINT | PRIMARY KEY | |
| sha1 | VARCHAR(40) | NOT NULL | git object's sha1 hash |
| object_type | VARCHAR(20) | NOT NULL | |
| storage_type | INT | NOT NULL | data storage type, can be 'database', 'local-fs' and 'remote_url' |
| data | BYTEA | | |
| local_storage_path | TEXT | | |
| remote_url | TEXT | | |


#### git_pr
Expand Down Expand Up @@ -488,4 +486,4 @@ erDiagram
- Generating entities:
Entities can be generated from the database table structure with the following command
`sea-orm-cli generate entity -u "postgres://${DB_USERNAME}:${DB_SECRET}@${DB_HOST}/mega" -o database/entity/src`
`sea-orm-cli generate entity -u "postgres://postgres:$postgres@localhost/mega_re" -o jupiter/entity/src`
34 changes: 34 additions & 0 deletions jupiter/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
[package]
name = "jupiter"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[lib]
name = "jupiter"
path = "src/lib.rs"


[dependencies]
common = { path = "../common" }
db_entity = { path = "./entity" }
venus = { path = "../venus" }

sea-orm = { workspace = true, features = [
"sqlx-postgres",
"sqlx-mysql",
"runtime-tokio-rustls",
"macros",
] }
tracing = { workspace = true }
bytes = { workspace = true }
chrono = { workspace = true }
async-trait = { workspace = true }
futures = { workspace = true }
serde_json ={ workspace = true }

handlebars = "5.1.0"

[dev-dependencies]
tokio = { workspace = true, features = ["macros"] }
14 changes: 14 additions & 0 deletions jupiter/entity/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[package]
name = "db_entity"
version = "0.1.0"
edition = "2021"
publish = false

[lib]
name = "db_entity"
path = "src/lib.rs"

[dependencies]
serde = { workspace = true, features = ["derive"] }
chrono = { workspace = true }
sea-orm = { workspace = true, features = ["sqlx-postgres", "sqlx-mysql"] }
45 changes: 45 additions & 0 deletions jupiter/entity/src/db_enums.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
use sea_orm::{DeriveActiveEnum, EnumIter};

#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
#[sea_orm(rs_type = "String", db_type = "String(Some(1))")]
pub enum StorageType {
#[sea_orm(string_value = "database")]
Database,
#[sea_orm(string_value = "local_fs")]
LocalFs,
#[sea_orm(string_value = "remote_url")]
RemoteUrl,
}

impl ToString for StorageType {
fn to_string(&self) -> String {
match self {
StorageType::Database => String::from("database"),
StorageType::LocalFs => String::from("local_fs"),
StorageType::RemoteUrl => String::from("remote_url"),
}
}
}

#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
#[sea_orm(rs_type = "String", db_type = "String(Some(1))")]
pub enum MergeStatus {
#[sea_orm(string_value = "open")]
Open,
#[sea_orm(string_value = "merged")]
Merged,
#[sea_orm(string_value = "closed")]
Closed,
}



#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
#[sea_orm(rs_type = "String", db_type = "String(Some(1))")]
pub enum RefType {
#[sea_orm(string_value = "branch")]
Branch,
#[sea_orm(string_value = "tag")]
Tag,
}

26 changes: 26 additions & 0 deletions jupiter/entity/src/git_blob.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
use sea_orm::entity::prelude::*;

#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "git_blob")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: i64,
pub repo_id: i64,
pub blob_id: String,
pub name: Option<String>,
pub size: i32,
#[sea_orm(column_type = "Text")]
pub full_path: String,
#[sea_orm(column_type = "Text")]
pub content: String,
pub content_type: Option<String>,
pub commit_id: String,
pub created_at: DateTime,
}

#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}

impl ActiveModelBehavior for ActiveModel {}
Loading

1 comment on commit 7b35e23

@vercel
Copy link

@vercel vercel bot commented on 7b35e23 Feb 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

mega – ./

mega-gitmono.vercel.app
gitmega.dev
mega-git-main-gitmono.vercel.app
www.gitmega.dev

Please sign in to comment.