Skip to content

Commit

Permalink
Add database support
Browse files Browse the repository at this point in the history
  • Loading branch information
@cholcombe973
cholcombe973 committed Jan 8, 2019
1 parent f98cfe5 commit 8e5c5e3
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 94 deletions.
30 changes: 28 additions & 2 deletions src/dbschema/bynar_stats.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ $BODY$

DECLARE
new_row INTEGER;
new_rev INTEGER := 3;
new_rev INTEGER := 4;
current_revision INTEGER;
BEGIN

Expand Down Expand Up @@ -185,8 +185,34 @@ BEGIN
UPDATE TABLE operation_types SET op_name='waiting_for_replacement' WHERE op_name='waitingforreplacement';
END IF;

IF (current_revision < 4)
THEN
-- Captures the detail of each hardware replacement operation
-- for the server
CREATE TABLE IF NOT EXISTS hardware (
device_id SERIAL NOT NULL UNIQUE,
detail_id INTEGER REFERENCES storage_details(detail_id) ON DELETE CASCADE,
device_name VARCHAR NOT NULL,
device_location VARCHAR, --some devices have location data
state VARCHAR, -- refers to device state in the state machine
serial_number VARCHAR, -- some hardware devices have serial numbers
UNIQUE (device_name, detail_id)
);
CREATE TABLE IF NOT EXISTS hardware_types (
hardware_id SERIAL NOT NULL UNIQUE,
hardware_type VARCHAR (256) PRIMARY KEY NOT NULL
);
INSERT INTO hardware_types (hardware_type) VALUES ('array_controller');
INSERT INTO hardware_types (hardware_type) VALUES ('disk');
INSERT INTO hardware_types (hardware_type) VALUES ('fan');
INSERT INTO hardware_types (hardware_type) VALUES ('ilo_manager');
INSERT INTO hardware_types (hardware_type) VALUES ('power_supply');
INSERT INTO hardware_types (hardware_type) VALUES ('storage_controller');
END IF;


-- Add next revision here
-- IF (current_revision < 4)
-- IF (current_revision < 5)
-- THEN
-- SQL statements
-- END IF;
Expand Down
35 changes: 35 additions & 0 deletions src/in_progress.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1085,3 +1085,38 @@ pub fn is_disk_waiting_repair(
Ok(true)
}
}

pub fn is_hardware_waiting_repair(
pool: &Pool<ConnectionManager>,
storage_detail_id: u32,
device_name: &str,
serial_number: Option<&str>,
) -> BynarResult<bool> {
let conn = get_connection_from_pool(pool)?;
// is there is any operation for this hardware that is waiting for replacement
let mut stmt = "SELECT status FROM operation_details
JOIN operations USING (operation_id)
JOIN hardware USING (device_id)
WHERE device_name=$1 AND
detail_id=$2 AND
type_id = (SELECT type_id FROM operation_types WHERE op_name=$3) AND
state=$4"
.to_string();
let detail_id = storage_detail_id as i32;
let operation_type = OperationType::WaitingForReplacement.to_string();
let state_type = State::WaitingForReplacement.to_string();
let mut params: Vec<&postgres::types::ToSql> =
vec![&device_name, &detail_id, &operation_type, &state_type];
// Add the serial_number to the query if given
if let Some(ref serial) = serial_number {
stmt.push_str(" and serial_number=$5");
params.push(serial);
}

let stmt_query = conn.query(&stmt, &params)?;
if stmt_query.is_empty() {
Ok(false)
} else {
Ok(true)
}
}
114 changes: 65 additions & 49 deletions src/lib/error.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ pub enum BynarError {
BlockUtilsError(BlockUtilsError),
Error(String),
GojiError(GojiError),
HardwareError {
error: String,
name: String,
location: Option<String>,
location_format: Option<String>,
serial_number: Option<String>,
},
IoError(IOError),
LvmError(LvmError),
NixError(NixError),
Expand All @@ -48,58 +55,48 @@ pub enum BynarError {

impl fmt::Display for BynarError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.description())
}
}

impl err for BynarError {
fn description(&self) -> &str {
match *self {
BynarError::BlockUtilsError(ref e) => e.description(),
BynarError::Error(ref e) => &e,
BynarError::GojiError(ref e) => e.description(),
BynarError::IoError(ref e) => e.description(),
BynarError::LvmError(ref e) => e.description(),
BynarError::NixError(ref e) => e.description(),
BynarError::ParseIntError(ref e) => e.description(),
BynarError::PostgresError(ref e) => e.description(),
BynarError::ProtobufError(ref e) => e.description(),
BynarError::BlockUtilsError(ref e) => write!(f, "{}", e.description()),
BynarError::Error(ref e) => write!(f, "{}", e),
BynarError::GojiError(ref e) => write!(f, "{}", e),
BynarError::HardwareError {
ref name,
ref location,
ref location_format,
ref error,
ref serial_number,
} => {
let mut err = format!("Error: {}. {}", error, name);
if let Some(serial) = serial_number {
err.push_str(&format!(" with serial {}", serial));
}
if let Some(lo) = location {
err.push_str(&format!(" at location {} ", lo));
}
if let Some(lo_fmt) = location_format {
err.push_str(&format!(" with format {} ", lo_fmt));
}
write!(f, "{}", err)
}
BynarError::IoError(ref e) => write!(f, "{}", e),
BynarError::LvmError(ref e) => write!(f, "{}", e),
BynarError::NixError(ref e) => write!(f, "{}", e),
BynarError::ParseIntError(ref e) => write!(f, "{}", e),
BynarError::PostgresError(ref e) => write!(f, "{}", e),
BynarError::ProtobufError(ref e) => write!(f, "{}", e),
BynarError::PwdError(ref e) => match e {
PwdError::StringConvError(s) => &s,
PwdError::NullPtr => "nullptr",
PwdError::StringConvError(s) => write!(f, "{}", s),
PwdError::NullPtr => write!(f, "Null pointer err"),
},
BynarError::R2d2Error(ref e) => e.description(),
BynarError::RadosError(ref e) => e.description(),
BynarError::ReqwestError(ref e) => e.description(),
BynarError::SerdeJsonError(ref e) => e.description(),
BynarError::SlackError(ref e) => e.description(),
BynarError::SqliteError(ref e) => e.description(),
BynarError::UuidError(ref e) => e.description(),
BynarError::VaultError(ref e) => e.description(),
BynarError::ZmqError(ref e) => e.description(),
}
}
fn cause(&self) -> Option<&dyn err> {
match *self {
BynarError::BlockUtilsError(ref e) => e.cause(),
BynarError::Error(_) => None,
BynarError::GojiError(ref e) => e.cause(),
BynarError::IoError(ref e) => e.cause(),
BynarError::LvmError(ref e) => e.cause(),
BynarError::NixError(ref e) => e.cause(),
BynarError::ParseIntError(ref e) => e.cause(),
BynarError::PostgresError(ref e) => e.cause(),
BynarError::ProtobufError(ref e) => e.cause(),
BynarError::PwdError(_) => None,
BynarError::R2d2Error(ref e) => e.cause(),
BynarError::RadosError(ref e) => e.cause(),
BynarError::ReqwestError(ref e) => e.cause(),
BynarError::SerdeJsonError(ref e) => e.cause(),
BynarError::SlackError(ref e) => e.cause(),
BynarError::SqliteError(ref e) => e.cause(),
BynarError::UuidError(ref e) => e.cause(),
BynarError::VaultError(ref e) => e.cause(),
BynarError::ZmqError(ref e) => e.cause(),
BynarError::R2d2Error(ref e) => write!(f, "{}", e),
BynarError::RadosError(ref e) => write!(f, "{}", e),
BynarError::ReqwestError(ref e) => write!(f, "{}", e),
BynarError::SerdeJsonError(ref e) => write!(f, "{}", e),
BynarError::SlackError(ref e) => write!(f, "{}", e),
BynarError::SqliteError(ref e) => write!(f, "{}", e),
BynarError::UuidError(ref e) => write!(f, "{}", e),
BynarError::VaultError(ref e) => write!(f, "{}", e),
BynarError::ZmqError(ref e) => write!(f, "{}", e),
}
}
}
Expand All @@ -116,6 +113,25 @@ impl BynarError {
BynarError::BlockUtilsError(ref err) => err.to_string(),
BynarError::Error(ref err) => err.to_string(),
BynarError::GojiError(ref err) => err.to_string(),
BynarError::HardwareError {
ref name,
ref location,
ref location_format,
ref error,
ref serial_number,
} => {
let mut err = format!("Error: {}. {}", error, name);
if let Some(serial) = serial_number {
err.push_str(&format!(" with serial {}", serial));
}
if let Some(lo) = location {
err.push_str(&format!(" at location {} ", lo));
}
if let Some(lo_fmt) = location_format {
err.push_str(&format!(" with format {} ", lo_fmt));
}
err
}
BynarError::IoError(ref err) => err.to_string(),
BynarError::LvmError(ref err) => err.to_string(),
BynarError::NixError(ref err) => err.to_string(),
Expand Down
80 changes: 63 additions & 17 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,15 +262,57 @@ fn check_for_failed_disks(
Ok(())
}

fn evaluate(results: Vec<BynarResult<()>>, error_string: &mut String) -> bool {
let mut error_found = false;
fn evaluate(
results: Vec<BynarResult<()>>,
config: &ConfigSettings,
pool: &Pool<ConnectionManager>,
host_mapping: &HostDetailsMapping,
) -> BynarResult<()> {
for result in results {
if let Err(e) = result {
error_string.push_str(&format!("{}\n", e));
error_found = true;
match e {
// This is the error we're after
BynarError::HardwareError { ref name, ref serial_number, .. } => {
let serial = serial_number.as_ref().map(|s| &**s);
let in_progress = in_progress::is_hardware_waiting_repair(
pool,
host_mapping.storage_detail_id,
name,
serial,
)?;
if !in_progress {
//file a ticket
debug!("Creating support ticket");
let mut op_info = OperationInfo::new(host_mapping.entry_id, 0);
add_or_update_operation(pool, &mut op_info)?;
let ticket_id = create_support_ticket(
config,
"Bynar: Hardware Failure",
&format!("{}", e),
)?;
let op_id = match op_info.operation_id {
None => {
error!("Operation not recorded for {}", "",);
0
}
Some(i) => i,
};
debug!("Recording ticket id {} in database", ticket_id);
let mut operation_detail =
OperationDetail::new(op_id, OperationType::WaitingForReplacement);
operation_detail.set_tracking_id(ticket_id);
add_or_update_operation_detail(pool, &mut operation_detail)?;
}
}
_ => {
//Ignore other error types?
error!("evaluate error: {:?}", e);
return Err(e);
}
};
}
}
error_found
Ok(())
}

fn check_for_failed_hardware(
Expand All @@ -290,19 +332,23 @@ fn check_for_failed_hardware(
host_info.machine_architecture,
host_info.kernel,
));
let results = test_hardware::check_hardware(&config, &host_info, &pool, &host_mapping)?;
let results = test_hardware::check_hardware(&config)?;
if !simulate {
// Check if evaluate found any errors
if evaluate(results.disk_drives, &mut description)
|| evaluate(results.manager, &mut description)
|| evaluate(results.power, &mut description)
|| evaluate(results.storage_enclosures, &mut description)
|| evaluate(results.thermals, &mut description)
{
//file a ticket
debug!("Creating support ticket");
let ticket_id = create_support_ticket(config, "Bynar: Hardware Failure", &description)?;
debug!("Recording ticket id {} in database", ticket_id);
// Check if evaluate found any errors and log anything other then hardware errors
if let Err(e) = evaluate(results.disk_drives, config, pool, host_mapping) {
error!("Disk drive evaluation error: {:?}", e);
}
if let Err(e) = evaluate(results.manager, config, pool, host_mapping) {
error!("Hardware manager evaluation error: {:?}", e);
}
if let Err(e) = evaluate(results.power, config, pool, host_mapping) {
error!("Power supply evaluation error: {:?}", e);
}
if let Err(e) = evaluate(results.storage_enclosures, config, pool, host_mapping) {
error!("Storage enclosures evaluation error: {:?}", e);
}
if let Err(e) = evaluate(results.thermals, config, pool, host_mapping) {
error!("Thermal evaluation error: {:?}", e);
}
}

Expand Down
Loading

0 comments on commit 8e5c5e3

Please sign in to comment.