From b7ac67dc244e5344d29b67c8dd576253499140c1 Mon Sep 17 00:00:00 2001 From: Sofi O'Hobby Date: Wed, 29 Jan 2025 13:51:35 -0500 Subject: [PATCH 1/4] patch including slow wallet info --- src/extract_snapshot.rs | 10 ++++++---- src/schema_account_state.rs | 34 ++++++++++++++++++++++++++-------- tests/test_extract_state.rs | 4 ++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/extract_snapshot.rs b/src/extract_snapshot.rs index f028b61..9618cb9 100644 --- a/src/extract_snapshot.rs +++ b/src/extract_snapshot.rs @@ -59,8 +59,9 @@ pub async fn extract_v5_snapshot(archive_path: &Path) -> Result() { - s.slow_wallet_unlocked = sw.unlocked; - s.slow_wallet_transferred = sw.transferred; + s.slow_wallet_acc = true; + s.slow_wallet_unlocked = Some(sw.unlocked); + s.slow_wallet_transferred = Some(sw.transferred); } if let Ok(tower) = acc.get_resource::() { @@ -117,8 +118,9 @@ pub async fn extract_current_snapshot(archive_path: &Path) -> Result()? { - s.slow_wallet_unlocked = sw.unlocked; - s.slow_wallet_transferred = sw.transferred; + s.slow_wallet_acc = true; + s.slow_wallet_unlocked = Some(sw.unlocked); + s.slow_wallet_transferred = Some(sw.transferred); } // Infer if it is a donor voice account diff --git a/src/schema_account_state.rs b/src/schema_account_state.rs index f8fb0cb..2c7f0b1 100644 --- a/src/schema_account_state.rs +++ b/src/schema_account_state.rs @@ -17,8 +17,9 @@ pub struct WarehouseAccState { pub time: WarehouseTime, pub sequence_num: u64, pub balance: u64, - pub slow_wallet_unlocked: u64, - pub slow_wallet_transferred: u64, + pub slow_wallet_unlocked: Option, + pub slow_wallet_transferred: Option, + pub slow_wallet_acc: bool, pub donor_voice_acc: bool, pub miner_height: Option, } @@ -29,8 +30,9 @@ impl Default for WarehouseAccState { address: AccountAddress::ZERO, sequence_num: 0, balance: 0, - slow_wallet_unlocked: 0, - slow_wallet_transferred: 0, + slow_wallet_unlocked: None, + slow_wallet_transferred: None, + slow_wallet_acc: false, donor_voice_acc: false, miner_height: None, time: WarehouseTime::default(), @@ -56,18 +58,33 @@ impl WarehouseAccState { /// creates one transaction record in the cypher query map format /// Note original data was in an RFC rfc3339 with Z for UTC, Cypher seems to prefer with offsets +00000 pub fn to_cypher_object_template(&self) -> String { + let slow_wallet_unlocked_literal = match self.slow_wallet_unlocked { + Some(n) => n.to_string(), + None => "NULL".to_string(), + }; + let slow_wallet_transferred_literal = match self.slow_wallet_transferred { + Some(n) => n.to_string(), + None => "NULL".to_string(), + }; + + let miner_height_literal = match self.miner_height { + Some(n) => n.to_string(), + None => "NULL".to_string(), + }; + format!( - r#"{{address: "{}", balance: {}, version: {}, epoch: {},sequence_num: {}, slow_unlocked: {}, slow_transfer: {}, framework_version: "{}", donor_voice: {}, miner_height: {}}}"#, + r#"{{address: "{}", balance: {}, version: {}, epoch: {},sequence_num: {}, slow_unlocked: {}, slow_transfer: {}, framework_version: "{}", slow_wallet: {}, donor_voice: {}, miner_height: {}}}"#, self.address.to_hex_literal(), self.balance, self.time.version, self.time.epoch, self.sequence_num, - self.slow_wallet_unlocked, - self.slow_wallet_transferred, + slow_wallet_unlocked_literal, + slow_wallet_transferred_literal, self.time.framework_version, + self.slow_wallet_acc, self.donor_voice_acc, - self.miner_height.unwrap_or(0) + miner_height_literal ) } @@ -99,6 +116,7 @@ MERGE (snap:Snapshot {{ sequence_num: tx.sequence_num, slow_unlocked: tx.slow_unlocked, slow_transfer: tx.slow_transfer, + slow_wallet: tx.slow_wallet, donor_voice: tx.donor_voice, miner_height: coalesce(tx.miner_height, 0) }}) diff --git a/tests/test_extract_state.rs b/tests/test_extract_state.rs index 30152b4..035cfc3 100644 --- a/tests/test_extract_state.rs +++ b/tests/test_extract_state.rs @@ -15,8 +15,8 @@ async fn test_extract_v5_manifest() -> Result<()> { assert!(&first.address.to_hex_literal() == "0x407d4d486fdc4e796504135e545be77"); assert!(first.balance == 100135989588); - assert!(first.slow_wallet_unlocked == 140001000000); - assert!(first.slow_wallet_transferred == 15999000000); + assert!(first.slow_wallet_unlocked == Some(140001000000)); + assert!(first.slow_wallet_transferred == Some(15999000000)); assert!(first.sequence_num == 7); Ok(()) From 8a723f732499b5cc5313bf255c59654d0d4abcdd Mon Sep 17 00:00:00 2001 From: Nella Fitz Forte Date: Wed, 29 Jan 2025 14:16:25 -0500 Subject: [PATCH 2/4] refator snapshot insert query --- src/load_tx_cypher.rs | 1 - src/neo4j_init.rs | 3 +++ src/schema_account_state.rs | 37 ++++++++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/load_tx_cypher.rs b/src/load_tx_cypher.rs index e03af44..6cf1402 100644 --- a/src/load_tx_cypher.rs +++ b/src/load_tx_cypher.rs @@ -96,7 +96,6 @@ pub async fn impl_batch_tx_insert( // cypher queries makes it annoying to do a single insert of users and // txs let cypher_string = write_batch_user_create(&list_str); - // dbg!(format!("{:#}",cypher_string)); // Execute the query let cypher_query = query(&cypher_string); diff --git a/src/neo4j_init.rs b/src/neo4j_init.rs index e7f43c0..176b6b0 100644 --- a/src/neo4j_init.rs +++ b/src/neo4j_init.rs @@ -49,6 +49,8 @@ pub static INDEX_EXCHANGE_LINK_LEDGER: &str = " pub static INDEX_LIFETIME: &str = " CREATE INDEX link_ledger IF NOT EXISTS FOR ()-[r:Lifetime]->() ON (r.amount) "; + +pub static INDEX_SNAPSHOT: &str = "CREATE INDEX snapshot_account_id IF NOT EXISTS FOR (n:Snapshot) ON (n.address, n.epoch, n.version)"; /// get the testing neo4j connection pub async fn get_neo4j_localhost_pool(port: u16) -> Result { let uri = format!("127.0.0.1:{port}"); @@ -85,6 +87,7 @@ pub async fn maybe_create_indexes(graph: &Graph) -> Result<()> { INDEX_EXCHANGE_LEDGER, INDEX_EXCHANGE_LINK_LEDGER, INDEX_LIFETIME, + INDEX_SNAPSHOT, ]) .await?; txn.commit().await?; diff --git a/src/schema_account_state.rs b/src/schema_account_state.rs index 2c7f0b1..4f6dc17 100644 --- a/src/schema_account_state.rs +++ b/src/schema_account_state.rs @@ -57,7 +57,7 @@ impl WarehouseAccState { impl WarehouseAccState { /// creates one transaction record in the cypher query map format /// Note original data was in an RFC rfc3339 with Z for UTC, Cypher seems to prefer with offsets +00000 - pub fn to_cypher_object_template(&self) -> String { + pub fn acc_state_to_cypher_map(&self) -> String { let slow_wallet_unlocked_literal = match self.slow_wallet_unlocked { Some(n) => n.to_string(), None => "NULL".to_string(), @@ -92,7 +92,7 @@ impl WarehouseAccState { pub fn to_cypher_map(list: &[Self]) -> String { let mut list_literal = "".to_owned(); for el in list { - let s = el.to_cypher_object_template(); + let s = el.acc_state_to_cypher_map(); list_literal.push_str(&s); list_literal.push(','); } @@ -109,17 +109,32 @@ UNWIND tx_data AS tx MERGE (addr:Account {{address: tx.address}}) MERGE (snap:Snapshot {{ address: tx.address, - balance: tx.balance, epoch: tx.epoch, - framework_version: tx.framework_version, - version: tx.version, - sequence_num: tx.sequence_num, - slow_unlocked: tx.slow_unlocked, - slow_transfer: tx.slow_transfer, - slow_wallet: tx.slow_wallet, - donor_voice: tx.donor_voice, - miner_height: coalesce(tx.miner_height, 0) + version: tx.version }}) + +SET + snap.balance = tx.balance, + snap.framework_version = tx.framework_version, + snap.sequence_num = tx.sequence_num, + snap.slow_wallet = tx.slow_wallet, + snap.donor_voice = tx.donor_voice + +// Conditionally add `tx.miner_height` if it exists +FOREACH (_ IN CASE WHEN tx.miner_height IS NOT NULL THEN [1] ELSE [] END | + SET snap.miner_height = tx.miner_height +) + +// Conditionally add `tx.slow_unlocked` if it exists +FOREACH (_ IN CASE WHEN tx.slow_unlocked IS NOT NULL THEN [1] ELSE [] END | + SET snap.slow_unlocked = tx.slow_unlocked +) + +// Conditionally add `tx.slow_transfer` if it exists +FOREACH (_ IN CASE WHEN tx.slow_transfer IS NOT NULL THEN [1] ELSE [] END | + SET snap.slow_transfer = tx.slow_transfer +) + MERGE (addr)-[rel:State {{version: tx.version}}]->(snap) RETURN COUNT(snap) AS merged_snapshots From c3cb7d1fc02519b906e353bd211c3e02e32a21df Mon Sep 17 00:00:00 2001 From: Vale McPresto Date: Wed, 29 Jan 2025 15:23:13 -0500 Subject: [PATCH 3/4] update libra-framework --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d5404aa..a9ae316 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,10 @@ diem-crypto = { git = "https://github.com/0LNetworkCommunity/diem.git", branch = env_logger = "^0.11" flate2 = "^1.0" glob = "^0.3" -libra-backwards-compatibility = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "patch-compat-5.2.0" } -libra-cached-packages = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "patch-compat-5.2.0" } -libra-storage = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "patch-compat-5.2.0" } -libra-types = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "patch-compat-5.2.0" } +libra-backwards-compatibility = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "main" } +libra-cached-packages = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "main" } +libra-storage = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "main" } +libra-types = { git = "https://github.com/0LNetworkCommunity/libra-framework.git", branch = "main" } log = "^0.4" neo4rs = "0.8.0" once_cell = "^1.2" From 75e99b5ca6c48d7abe0de97aa2c30b3466202632 Mon Sep 17 00:00:00 2001 From: Nella Beaver Date: Wed, 29 Jan 2025 16:19:42 -0500 Subject: [PATCH 4/4] make account state display decimals --- src/extract_snapshot.rs | 16 ++++++++++------ src/json_rescue_v5_extract.rs | 6 ++---- src/schema_account_state.rs | 8 ++++---- src/schema_transaction.rs | 11 +++-------- src/util.rs | 7 +++++++ tests/test_extract_state.rs | 8 ++++---- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/extract_snapshot.rs b/src/extract_snapshot.rs index 9618cb9..998fca9 100644 --- a/src/extract_snapshot.rs +++ b/src/extract_snapshot.rs @@ -21,6 +21,7 @@ use log::{error, info, warn}; use crate::{ scan::FrameworkVersion, schema_account_state::{WarehouseAccState, WarehouseTime}, + util::COIN_DECIMAL_PRECISION, }; // uses libra-compatibility to parse the v5 manifest files, and decode v5 format bytecode into current version data structures (v6+); @@ -56,12 +57,14 @@ pub async fn extract_v5_snapshot(archive_path: &Path) -> Result() { - s.balance = b.coin() + s.balance = b.coin() as f64 / COIN_DECIMAL_PRECISION as f64; } if let Ok(sw) = acc.get_resource::() { s.slow_wallet_acc = true; - s.slow_wallet_unlocked = Some(sw.unlocked); - s.slow_wallet_transferred = Some(sw.transferred); + s.slow_wallet_unlocked = + Some(sw.unlocked as f64 / COIN_DECIMAL_PRECISION as f64); + s.slow_wallet_transferred = + Some(sw.transferred as f64 / COIN_DECIMAL_PRECISION as f64); } if let Ok(tower) = acc.get_resource::() { @@ -114,13 +117,14 @@ pub async fn extract_current_snapshot(archive_path: &Path) -> Result()? { - s.balance = b.coin(); + s.balance = b.coin() as f64 / COIN_DECIMAL_PRECISION as f64; } if let Some(sw) = el.get_resource::()? { s.slow_wallet_acc = true; - s.slow_wallet_unlocked = Some(sw.unlocked); - s.slow_wallet_transferred = Some(sw.transferred); + s.slow_wallet_unlocked = Some(sw.unlocked as f64 / COIN_DECIMAL_PRECISION as f64); + s.slow_wallet_transferred = + Some(sw.transferred as f64 / COIN_DECIMAL_PRECISION as f64); } // Infer if it is a donor voice account diff --git a/src/json_rescue_v5_extract.rs b/src/json_rescue_v5_extract.rs index 15be6d1..763ca37 100644 --- a/src/json_rescue_v5_extract.rs +++ b/src/json_rescue_v5_extract.rs @@ -1,10 +1,8 @@ use crate::{ scan::FrameworkVersion, - schema_transaction::{ - EntryFunctionArgs, RelationLabel, WarehouseEvent, WarehouseTxMaster, - COIN_DECIMAL_PRECISION, LEGACY_REBASE_MULTIPLIER, - }, + schema_transaction::{EntryFunctionArgs, RelationLabel, WarehouseEvent, WarehouseTxMaster}, unzip_temp::decompress_tar_archive, + util::{COIN_DECIMAL_PRECISION, LEGACY_REBASE_MULTIPLIER}, }; use chrono::DateTime; use diem_crypto::HashValue; diff --git a/src/schema_account_state.rs b/src/schema_account_state.rs index 4f6dc17..f2b65e9 100644 --- a/src/schema_account_state.rs +++ b/src/schema_account_state.rs @@ -16,9 +16,9 @@ pub struct WarehouseAccState { pub address: AccountAddress, pub time: WarehouseTime, pub sequence_num: u64, - pub balance: u64, - pub slow_wallet_unlocked: Option, - pub slow_wallet_transferred: Option, + pub balance: f64, + pub slow_wallet_unlocked: Option, + pub slow_wallet_transferred: Option, pub slow_wallet_acc: bool, pub donor_voice_acc: bool, pub miner_height: Option, @@ -29,7 +29,7 @@ impl Default for WarehouseAccState { Self { address: AccountAddress::ZERO, sequence_num: 0, - balance: 0, + balance: 0.0, slow_wallet_unlocked: None, slow_wallet_transferred: None, slow_wallet_acc: false, diff --git a/src/schema_transaction.rs b/src/schema_transaction.rs index 768b33e..82a3187 100644 --- a/src/schema_transaction.rs +++ b/src/schema_transaction.rs @@ -1,4 +1,6 @@ -use crate::{cypher_templates::to_cypher_object, scan::FrameworkVersion}; +use crate::{ + cypher_templates::to_cypher_object, scan::FrameworkVersion, util::COIN_DECIMAL_PRECISION, +}; use chrono::{DateTime, Utc}; use diem_crypto::HashValue; @@ -12,13 +14,6 @@ use libra_backwards_compatibility::sdk::{ use libra_types::{exports::AccountAddress, move_resource::coin_register_event::CoinRegisterEvent}; use serde::{Deserialize, Serialize}; -// TODO check decimal precision -/// Conversion of coins from V5 to V6 -pub const LEGACY_REBASE_MULTIPLIER: u64 = 35; -/// Decimal precision -// TODO: duplication, this is probably defined in libra-framework somewhere -pub const COIN_DECIMAL_PRECISION: u64 = 1000000; - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum RelationLabel { Unknown, // undefined tx diff --git a/src/util.rs b/src/util.rs index fd223aa..8750eca 100644 --- a/src/util.rs +++ b/src/util.rs @@ -3,6 +3,13 @@ use diem_types::account_address::AccountAddress; use log::error; use serde::{Deserialize, Deserializer}; +// TODO check decimal precision +/// Conversion of coins from V5 to V6 +pub const LEGACY_REBASE_MULTIPLIER: u64 = 35; +/// Decimal precision +// TODO: duplication, this is probably defined in libra-framework somewhere +pub const COIN_DECIMAL_PRECISION: u64 = 1000000; + /// Helper function to parse "YYYY-MM-DD" into `DateTime` pub fn parse_date(date_str: &str) -> DateTime { let datetime_str = format!("{date_str}T00:00:00Z"); // Append time and UTC offset diff --git a/tests/test_extract_state.rs b/tests/test_extract_state.rs index 035cfc3..9b5c66d 100644 --- a/tests/test_extract_state.rs +++ b/tests/test_extract_state.rs @@ -5,7 +5,7 @@ use libra_forensic_db::extract_snapshot::{extract_current_snapshot, extract_v5_s use support::fixtures::{v5_state_manifest_fixtures_path, v7_state_manifest_fixtures_path}; #[tokio::test] -async fn test_extract_v5_manifest() -> Result<()> { +async fn test_extract_v5_from_manifest() -> Result<()> { let archive_path = v5_state_manifest_fixtures_path(); assert!(archive_path.exists()); let s = extract_v5_snapshot(&archive_path).await?; @@ -14,9 +14,9 @@ async fn test_extract_v5_manifest() -> Result<()> { let first = s.first().unwrap(); assert!(&first.address.to_hex_literal() == "0x407d4d486fdc4e796504135e545be77"); - assert!(first.balance == 100135989588); - assert!(first.slow_wallet_unlocked == Some(140001000000)); - assert!(first.slow_wallet_transferred == Some(15999000000)); + assert!(first.balance == 100135.989588); + assert!(first.slow_wallet_unlocked == Some(140001.000000)); + assert!(first.slow_wallet_transferred == Some(15999.000000)); assert!(first.sequence_num == 7); Ok(())