From 89d2809b973b9748a72fe9f7e06668e892fb426c Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:43:34 +0530 Subject: [PATCH 1/7] feat: shifted init script from dbgen to scripts --- .gitignore | 2 ++ scripts/init_db.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 scripts/init_db.sh diff --git a/.gitignore b/.gitignore index 687f218..d68ff75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ pkgs/* !pkgs/.gitkeep +db/* +!db/.gitkeep target/ *.db \ No newline at end of file diff --git a/scripts/init_db.sh b/scripts/init_db.sh new file mode 100755 index 0000000..c21031b --- /dev/null +++ b/scripts/init_db.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +echo $(date +"%s") > db/starttime + +echo "Initializing local databases" + +mkdir -p db +[ -f db/local.db ] && rm db/local.db +touch db/local.db + +sqlite3 db/local.db < db/endtime \ No newline at end of file From f3510e8f09203dabea0de2b04e2cf6eabeef7f80 Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:44:24 +0530 Subject: [PATCH 2/7] feat: shifted init script from dbgen to scripts --- db/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 db/.gitkeep diff --git a/db/.gitkeep b/db/.gitkeep new file mode 100644 index 0000000..e69de29 From 660cd8e112ec9aff6d71108caabc11ff83430dbe Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:45:54 +0530 Subject: [PATCH 3/7] feat: arch repo parser --- packages/archlinux-repo-parser/Cargo.toml | 7 + packages/archlinux-repo-parser/src/de.rs | 906 ++++++++++++++++++++ packages/archlinux-repo-parser/src/error.rs | 53 ++ packages/archlinux-repo-parser/src/lib.rs | 7 + packages/archlinux-repo-parser/src/ser.rs | 332 +++++++ 5 files changed, 1305 insertions(+) create mode 100644 packages/archlinux-repo-parser/Cargo.toml create mode 100644 packages/archlinux-repo-parser/src/de.rs create mode 100644 packages/archlinux-repo-parser/src/error.rs create mode 100644 packages/archlinux-repo-parser/src/lib.rs create mode 100644 packages/archlinux-repo-parser/src/ser.rs diff --git a/packages/archlinux-repo-parser/Cargo.toml b/packages/archlinux-repo-parser/Cargo.toml new file mode 100644 index 0000000..0b4241a --- /dev/null +++ b/packages/archlinux-repo-parser/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "archlinux-repo-parser" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0.127", features = ["derive"] } \ No newline at end of file diff --git a/packages/archlinux-repo-parser/src/de.rs b/packages/archlinux-repo-parser/src/de.rs new file mode 100644 index 0000000..735711f --- /dev/null +++ b/packages/archlinux-repo-parser/src/de.rs @@ -0,0 +1,906 @@ +use std::ops::{AddAssign, MulAssign, Neg}; + +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde::Deserialize; + +use crate::error::{Error, Result}; +use std::str::FromStr; + +pub struct Deserializer<'de> { + input: &'de str, +} + +impl<'de> Deserializer<'de> { + #[allow(clippy::should_implement_trait)] + pub fn from_str(input: &'de str) -> Self { + Deserializer { input } + } +} + +pub fn from_str<'a, T>(s: &'a str) -> Result +where + T: Deserialize<'a>, +{ + let mut de = Deserializer::from_str(s); + let mut deserializer = TopDeserializer::new(&mut de); + let t = T::deserialize(&mut deserializer)?; + if de.input.is_empty() { + Ok(t) + } else { + println!("{}", &de.input); + Err(Error::TrailingCharacters) + } +} + +impl<'de> Deserializer<'de> { + fn parse_line(&mut self) -> Result<&'de str> { + match self.input.find('\n') { + Some(len) => { + let s = &self.input[..len]; + self.input = &self.input[len + 1..]; + Ok(s) + } + None => { + if self.input.is_empty() { + Err(Error::Eof) + } else { + let s = self.input; + self.input = ""; + Ok(s) + } + } + } + } + + fn peek_delimiter(&mut self) -> bool { + match self.input.find('\n') { + Some(len) => len == 0, + None => self.input.is_empty(), + } + } + + fn parse_field_name(&mut self) -> Result<&'de str> { + let line = self.parse_line()?; + if line.starts_with('%') && line.ends_with('%') { + Ok(&line[1..line.len() - 1]) + } else { + Err(Error::FieldNameUnexpectedWrapper(String::from(line))) + } + } + + fn parse_string(&mut self) -> Result<&'de str> { + let line = self.parse_line()?; + if line.is_empty() { + Err(Error::DelimiterNotExpected) + } else { + Ok(line) + } + } + + fn parse_char(&mut self) -> Result { + let line = self.parse_line()?; + if line.is_empty() { + Err(Error::DelimiterNotExpected) + } else if line.len() != 1 { + Err(Error::CharOverflow) + } else { + Ok(line.chars().next().unwrap()) + } + } + + fn parse_delimiter(&mut self) -> Result<()> { + match self.input.find('\n') { + Some(len) => { + let s = &self.input[..len]; + self.input = &self.input[len + 1..]; + if s.is_empty() { + Ok(()) + } else { + Err(Error::DelimiterExpected) + } + } + None => Ok(()), + } + } + + fn parse_unsigned(&mut self) -> Result + where + T: AddAssign + MulAssign + FromStr, + { + let line = self.parse_line()?; + match line.parse() { + Ok(n) => Ok(n), + Err(_) => Err(Error::IntegerError), + } + } + + fn parse_signed(&mut self) -> Result + where + T: Neg + AddAssign + MulAssign + FromStr, + { + let line = self.parse_line()?; + match line.parse() { + Ok(n) => Ok(n), + Err(_) => Err(Error::IntegerError), + } + } +} + +impl<'de, 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { + type Error = Error; + + fn deserialize_any(self, _: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_bool(self, _: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_i8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i8(self.parse_signed()?) + } + + fn deserialize_i16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i16(self.parse_signed()?) + } + + fn deserialize_i32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i32(self.parse_signed()?) + } + + fn deserialize_i64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i64(self.parse_signed()?) + } + + fn deserialize_u8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u8(self.parse_unsigned()?) + } + + fn deserialize_u16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u16(self.parse_unsigned()?) + } + + fn deserialize_u32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u32(self.parse_unsigned()?) + } + + fn deserialize_u64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u64(self.parse_unsigned()?) + } + + fn deserialize_f32(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_f64(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_char(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_char(self.parse_char()?) + } + + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_borrowed_str(self.parse_string()?) + } + + fn deserialize_string(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_str(visitor) + } + + // The `Serializer` implementation on the previous page serialized byte + // arrays as JSON arrays of bytes. Handle that representation here. + fn deserialize_bytes(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_byte_buf(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.peek_delimiter() { + self.parse_delimiter()?; + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.peek_delimiter() { + self.parse_delimiter()?; + visitor.visit_unit() + } else { + Err(Error::DelimiterExpected) + } + } + + fn deserialize_unit_struct(self, _name: &'static str, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_unit(visitor) + } + + fn deserialize_newtype_struct(self, _name: &'static str, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq(mut self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let value = visitor.visit_seq(NewlineSeparated::new(&mut self))?; + Ok(value) + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_map(mut self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let value = visitor.visit_map(NewlineSeparated::new(&mut self))?; + Ok(value) + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_map(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + // _visitor.visit_enum(self.parse_string()?.into_deserializer()) + Err(Error::NotSupported) + } + + fn deserialize_identifier(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_borrowed_str(self.parse_field_name()?) + } + + fn deserialize_ignored_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_any(visitor) + } +} + +struct TopDeserializer<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, +} + +impl<'a, 'de> TopDeserializer<'a, 'de> { + fn new(de: &'a mut Deserializer<'de>) -> Self { + TopDeserializer { de } + } +} + +impl<'de, 'a> de::Deserializer<'de> for &'a mut TopDeserializer<'a, 'de> { + type Error = Error; + + fn deserialize_any(self, _: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_bool(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_i8(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_i16(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_i32(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_i64(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_u8(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_u16(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_u32(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_u64(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_f32(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_f64(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_char(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_str(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_string(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_bytes(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_byte_buf(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_option(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_unit(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + _visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + _visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_seq(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_tuple(self, _len: usize, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + _visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_map(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_map(visitor) + } + + fn deserialize_struct( + self, + name: &'static str, + fields: &'static [&'static str], + visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_struct(name, fields, visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + _visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_identifier(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::StructExpected) + } + + fn deserialize_ignored_any(self, _visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } +} + +struct ValueDeserializer<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, + allow_arrays: bool, +} + +impl<'a, 'de> ValueDeserializer<'a, 'de> { + fn new(de: &'a mut Deserializer<'de>, allow_arrays: bool) -> Self { + ValueDeserializer { de, allow_arrays } + } +} + +impl<'de, 'a> de::Deserializer<'de> for &'a mut ValueDeserializer<'a, 'de> { + type Error = Error; + + fn deserialize_any(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_any(visitor) + } + + fn deserialize_bool(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_bool(visitor) + } + + fn deserialize_i8(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_i8(visitor) + } + + fn deserialize_i16(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_i16(visitor) + } + + fn deserialize_i32(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_i32(visitor) + } + + fn deserialize_i64(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_i64(visitor) + } + + fn deserialize_u8(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_u8(visitor) + } + + fn deserialize_u16(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_u16(visitor) + } + + fn deserialize_u32(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_u32(visitor) + } + + fn deserialize_u64(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_u64(visitor) + } + + fn deserialize_f32(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_f32(visitor) + } + + fn deserialize_f64(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_f64(visitor) + } + + fn deserialize_char(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_char(visitor) + } + + fn deserialize_str(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_str(visitor) + } + + fn deserialize_string(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_string(visitor) + } + + fn deserialize_bytes(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_bytes(visitor) + } + + fn deserialize_byte_buf(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_byte_buf(visitor) + } + + fn deserialize_option(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_option(visitor) + } + + fn deserialize_unit(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_unit(visitor) + } + + fn deserialize_unit_struct( + self, + name: &'static str, + visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_unit_struct(name, visitor) + } + + fn deserialize_newtype_struct( + self, + name: &'static str, + visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_newtype_struct(name, visitor) + } + + fn deserialize_seq(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + if self.allow_arrays { + self.de.deserialize_seq(visitor) + } else { + Err(Error::NotSupported) + } + } + + fn deserialize_tuple(self, size: usize, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + if self.allow_arrays { + self.de.deserialize_tuple(size, visitor) + } else { + Err(Error::NotSupported) + } + } + + fn deserialize_tuple_struct( + self, + name: &'static str, + size: usize, + visitor: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + if self.allow_arrays { + self.de.deserialize_tuple_struct(name, size, visitor) + } else { + Err(Error::NotSupported) + } + } + + fn deserialize_map(self, _: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_struct( + self, + _: &'static str, + _: &'static [&'static str], + _: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_enum( + self, + _: &'static str, + _: &'static [&'static str], + _: V, + ) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_identifier(self, _: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + Err(Error::NotSupported) + } + + fn deserialize_ignored_any(self, visitor: V) -> Result<>::Value> + where + V: Visitor<'de>, + { + self.de.deserialize_ignored_any(visitor) + } +} + +struct NewlineSeparated<'a, 'de: 'a> { + de: &'a mut Deserializer<'de>, +} + +impl<'a, 'de> NewlineSeparated<'a, 'de> { + fn new(de: &'a mut Deserializer<'de>) -> Self { + NewlineSeparated { de } + } +} + +impl<'de, 'a> SeqAccess<'de> for NewlineSeparated<'a, 'de> { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + if self.de.peek_delimiter() { + return Ok(None); + } + let mut deserializer = ValueDeserializer::new(self.de, false); + seed.deserialize(&mut deserializer).map(Some) + } +} + +impl<'de, 'a> MapAccess<'de> for NewlineSeparated<'a, 'de> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + if self.de.peek_delimiter() { + return Ok(None); + } + seed.deserialize(&mut *self.de).map(Some) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + let mut deserializer = ValueDeserializer::new(self.de, true); + let result = seed.deserialize(&mut deserializer)?; + self.de.parse_delimiter()?; + Ok(result) + } +} diff --git a/packages/archlinux-repo-parser/src/error.rs b/packages/archlinux-repo-parser/src/error.rs new file mode 100644 index 0000000..2aeeef3 --- /dev/null +++ b/packages/archlinux-repo-parser/src/error.rs @@ -0,0 +1,53 @@ +use std::fmt::{self, Display}; + +use serde::{de, ser}; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug, PartialEq)] +pub enum Error { + Message(String), + Eof, + FieldNameUnexpectedWrapper(String), + DelimiterExpected, + IntegerError, + NotSupported, + DelimiterNotExpected, + CharOverflow, + TrailingCharacters, + StructExpected, +} + +impl ser::Error for Error { + fn custom(msg: T) -> Self { + Error::Message(msg.to_string()) + } +} + +impl de::Error for Error { + fn custom(msg: T) -> Self { + Error::Message(msg.to_string()) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::Message(msg) => formatter.write_str(msg), + Error::Eof => formatter.write_str("unexpected end of input"), + Error::FieldNameUnexpectedWrapper(line) => formatter.write_str(&format!( + "unexpected field wrapper at line {}. Field should start and end with '%' symbol", + line + )), + Error::DelimiterExpected => formatter.write_str("delimiter expected"), + Error::IntegerError => formatter.write_str("cannot parse integer"), + Error::NotSupported => formatter.write_str("action not supported"), + Error::DelimiterNotExpected => formatter.write_str("delimiter not expected"), + Error::CharOverflow => formatter.write_str("char field must have only one letter"), + Error::TrailingCharacters => formatter.write_str("unexpected trailing characters"), + Error::StructExpected => formatter.write_str("expected struct type"), + } + } +} + +impl std::error::Error for Error {} \ No newline at end of file diff --git a/packages/archlinux-repo-parser/src/lib.rs b/packages/archlinux-repo-parser/src/lib.rs new file mode 100644 index 0000000..f544814 --- /dev/null +++ b/packages/archlinux-repo-parser/src/lib.rs @@ -0,0 +1,7 @@ +mod de; +mod error; +mod ser; + +pub use de::{from_str, Deserializer}; +pub use error::{Error, Result}; +pub use ser::{to_string, Serializer}; \ No newline at end of file diff --git a/packages/archlinux-repo-parser/src/ser.rs b/packages/archlinux-repo-parser/src/ser.rs new file mode 100644 index 0000000..afdd119 --- /dev/null +++ b/packages/archlinux-repo-parser/src/ser.rs @@ -0,0 +1,332 @@ +use serde::{ser, Serialize}; + +use crate::error::{Error, Result}; + +pub struct Serializer { + output: String, +} + +pub fn to_string(value: &T) -> Result +where + T: Serialize, +{ + let mut serializer = Serializer { + output: String::new(), + }; + value.serialize(&mut serializer)?; + Ok(serializer.output) +} + +impl<'a> ser::Serializer for &'a mut Serializer { + type Ok = (); + type Error = Error; + + type SerializeSeq = Self; + type SerializeTuple = Self; + type SerializeTupleStruct = Self; + type SerializeTupleVariant = Self; + type SerializeMap = Self; + type SerializeStruct = Self; + type SerializeStructVariant = Self; + + fn serialize_bool(self, _: bool) -> Result<()> { + Err(Error::NotSupported) + } + + fn serialize_i8(self, v: i8) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i16(self, v: i16) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i32(self, v: i32) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i64(self, v: i64) -> Result<()> { + self.output += &v.to_string(); + Ok(()) + } + + fn serialize_u8(self, v: u8) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u16(self, v: u16) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u32(self, v: u32) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u64(self, v: u64) -> Result<()> { + self.output += &v.to_string(); + Ok(()) + } + + fn serialize_f32(self, v: f32) -> Result<()> { + self.serialize_f64(f64::from(v)) + } + + fn serialize_f64(self, _v: f64) -> Result<()> { + Err(Error::NotSupported) + } + + fn serialize_char(self, v: char) -> Result<()> { + self.serialize_str(&v.to_string()) + } + + fn serialize_str(self, v: &str) -> Result<()> { + self.output += v; + Ok(()) + } + + fn serialize_bytes(self, _v: &[u8]) -> Result<()> { + Err(Error::NotSupported) + } + + fn serialize_none(self) -> Result<()> { + self.serialize_unit() + } + + fn serialize_some(self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(self) + } + + fn serialize_unit(self) -> Result<()> { + self.output += "\n"; + Ok(()) + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + self.serialize_unit() + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + self.serialize_str(variant) + } + + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(self) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> + where + T: ?Sized + Serialize, + { + self.output += "%"; + variant.serialize(&mut *self)?; + self.output += "%"; + value.serialize(&mut *self)?; + self.output += "\n"; + Ok(()) + } + + fn serialize_seq(self, _len: Option) -> Result { + Ok(self) + } + + fn serialize_tuple(self, len: usize) -> Result { + self.serialize_seq(Some(len)) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.serialize_seq(Some(len)) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(Error::NotSupported) + } + + fn serialize_map(self, _len: Option) -> Result { + Ok(self) + } + + fn serialize_struct(self, _name: &'static str, len: usize) -> Result { + self.serialize_map(Some(len)) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(Error::NotSupported) + } +} + +impl<'a> ser::SerializeSeq for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self)?; + self.output += "\n"; + Ok(()) + } + + fn end(self) -> Result<()> { + self.output = self.output[..self.output.len() - 1].to_owned(); // Cut last \n + Ok(()) + } +} + +impl<'a> ser::SerializeTuple for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self)?; + self.output += "\n"; + Ok(()) + } + + fn end(self) -> Result<()> { + self.output = self.output[..self.output.len() - 1].to_owned(); // Cut last \n + Ok(()) + } +} + +impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self)?; + self.output += "\n"; + Ok(()) + } + + fn end(self) -> Result<()> { + self.output = self.output[..self.output.len() - 1].to_owned(); // Cut last \n + Ok(()) + } +} + +impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, _: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + Err(Error::NotSupported) + } + + fn end(self) -> Result<()> { + Err(Error::NotSupported) + } +} + +impl<'a> ser::SerializeMap for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.output += "%"; + key.serialize(&mut **self)?; + self.output += "%"; + Ok(()) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.output += "\n"; + value.serialize(&mut **self)?; + self.output += "\n"; + self.output += "\n"; + Ok(()) + } + + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.output += "%"; + key.serialize(&mut **self)?; + self.output += "%"; + self.output += "\n"; + value.serialize(&mut **self)?; + self.output += "\n"; + self.output += "\n"; + Ok(()) + } + + fn end(self) -> Result<()> { + Ok(()) + } +} + +// Similar to `SerializeTupleVariant`, here the `end` method is responsible for +// closing both of the curly braces opened by `serialize_struct_variant`. +impl<'a> ser::SerializeStructVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, _key: &'static str, _value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + Err(Error::NotSupported) + } + + fn end(self) -> Result<()> { + Err(Error::NotSupported) + } +} \ No newline at end of file From a86fea4191009ce4503e47d8985b56f068acb047 Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:46:40 +0530 Subject: [PATCH 4/7] patch: switch to use local arch-repo-parser --- Cargo.lock | 4 +--- Cargo.toml | 4 +++- packages/dbgen/Cargo.toml | 2 +- packages/dbgen/src/main.rs | 39 -------------------------------------- 4 files changed, 5 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c186a7..a426121 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,9 +78,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "archlinux-repo-parser" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a793d159890f0f75c60275cdc838de33f8441a439bcc158148fcc4a1bb9403" +version = "0.1.0" dependencies = [ "serde", ] diff --git a/Cargo.toml b/Cargo.toml index d6feb88..b3446c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ "packages/aurparser", - "packages/dbgen", "packages/repoparser", + "packages/dbgen", + "packages/repoparser", + "packages/archlinux-repo-parser", ] diff --git a/packages/dbgen/Cargo.toml b/packages/dbgen/Cargo.toml index 9c034b8..2594e9f 100644 --- a/packages/dbgen/Cargo.toml +++ b/packages/dbgen/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] serde = { version = "1.0.127", features = ["derive"] } -archlinux-repo-parser = "0.1.6" +archlinux-repo-parser = { path = "../archlinux-repo-parser" } libsql = "0.4.0" tokio = { version = "1.38.0", features = ["full"] } repoparser = { path = "../repoparser" } \ No newline at end of file diff --git a/packages/dbgen/src/main.rs b/packages/dbgen/src/main.rs index 3af87d2..447ec76 100644 --- a/packages/dbgen/src/main.rs +++ b/packages/dbgen/src/main.rs @@ -18,45 +18,6 @@ async fn main() { let db = db_result.expect("Failed to build the database"); let client = db.connect().expect("Failed to connect to the database"); - client - .execute( - "CREATE TABLE repos ( - name TEXT PRIMARY KEY - )", - () - ) - .await - .unwrap(); - client - .execute( - "CREATE TABLE packages ( - name TEXT NOT NULL, - file_name TEXT NOT NULL, - base TEXT, - version TEXT NOT NULL, - description TEXT, - groups TEXT, - compressed_size INTEGER, - installed_size INTEGER, - md5_sum TEXT, - sha256_sum TEXT, - pgp_signature TEXT, - home_url TEXT, - license TEXT, - arch TEXT, - build_date DATE, - packager TEXT, - replaces TEXT, - conflicts TEXT, - provides TEXT, - repo TEXT NOT NULL, - FOREIGN KEY(repo) REFERENCES repos(name) - )", - () - ) - .await - .unwrap(); - let root_path = Path::new("./pkgs"); let entries = fs::read_dir(root_path).unwrap(); let mut pkg_databases: Vec = Vec::new(); From 037304c8ef98e5b45b6931d003190e5ac5962aeb Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:47:34 +0530 Subject: [PATCH 5/7] patch: minor changes --- scripts/get_pkgs.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/get_pkgs.sh b/scripts/get_pkgs.sh index 085dff8..bb49ed1 100755 --- a/scripts/get_pkgs.sh +++ b/scripts/get_pkgs.sh @@ -57,7 +57,6 @@ mkdir -p pkgs/multilib mkdir -p pkgs/chaotic-aur mkdir -p pkgs/blackarch mkdir -p pkgs/mingw64 -mkdir -p db mkdir -p pkgs/visual-studio-code-insiders mkdir -p pkgs/build.kilabit.info mkdir -p pkgs/dx37essentials From c57590d674ca32397193e01fb3f1d2ff71b9b8bc Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:48:21 +0530 Subject: [PATCH 6/7] ci: minor changes --- .github/workflows/test-run.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-run.yml b/.github/workflows/test-run.yml index 163e279..9d7bf98 100644 --- a/.github/workflows/test-run.yml +++ b/.github/workflows/test-run.yml @@ -16,6 +16,8 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Download PKG Databases run: ./scripts/get_pkgs.sh + - name: Init Local Database + run: ./scripts/init_db.sh - name: Building with Cargo uses: actions-rs/cargo@v1 with: From 1d33b5eb0962fde0be8459312fce1afeb27bc284 Mon Sep 17 00:00:00 2001 From: "Jyotirmoy Bandyopadhyaya [Bravo68]" Date: Thu, 13 Jun 2024 02:48:57 +0530 Subject: [PATCH 7/7] ci: added sync script github action --- .github/workflows/sync-run.yml | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/sync-run.yml diff --git a/.github/workflows/sync-run.yml b/.github/workflows/sync-run.yml new file mode 100644 index 0000000..bd56d58 --- /dev/null +++ b/.github/workflows/sync-run.yml @@ -0,0 +1,39 @@ +name: "Project Test Run" + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Installing Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + - uses: ankane/setup-postgres@v1 + with: + postgres-version: 15 + - name: Download PKG Databases + run: ./scripts/get_pkgs.sh + - name: Init Local Database + run: ./scripts/init_db.sh + - name: Building with Cargo + uses: actions-rs/cargo@v1 + with: + command: build + args: --release + - name: Run DBGen Parse + run: cargo run --release --bin dbgen + - name: Run AUR Parser + run: cargo run --release --bin aurparser + - name: Run Additional Scripts + run: | + curl https://github.com/Roxedus/Pgloader-bin/releases/download/ec8f41d/pgloader -L -o pgloader + chmod +x pgloader + ./pgloader sqlite://db/local.db ${{ secrets.PG_CONNECTION_STRING }} + - uses: actions/upload-artifact@v4 + with: + name: localdbs + path: | + **/local.db + db/local.db \ No newline at end of file