Skip to content

Commit

Permalink
Merge branch 'master' into enhancement/180-command-manager-configuration
Browse files Browse the repository at this point in the history
Signed-off-by: Álex Ruiz <[email protected]>
  • Loading branch information
AlexRuiz7 authored Jan 22, 2025
2 parents 0f69aac + 73e9dd8 commit f4d0712
Show file tree
Hide file tree
Showing 6 changed files with 239 additions and 16 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @wazuh/devel-indexer
* @wazuh/devel-xdrsiem-indexer
6 changes: 5 additions & 1 deletion plugins/command-manager/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,10 @@ testClusters.integTest {

run {
useCluster testClusters.integTest
exec {
// Generate data for the .agents index, it is useful for the agent_groups commands development and testing.
commandLine 'bash', './scripts/populate-agents-index.sh', '-n', '10', '-o', 'build/tmp/logs/'
}
}

// updateVersion: Task to auto update version to the next development iteration
Expand All @@ -162,4 +166,4 @@ task updateVersion {
// String tokenization to support -SNAPSHOT
ant.replaceregexp(file: 'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags: 'g', byline: true)
}
}
}
201 changes: 201 additions & 0 deletions plugins/command-manager/scripts/populate-agents-index.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
#!/bin/bash

# Constants and Configuration
INDEX_NAME=".agents"
USERNAME="admin"
PASSWORD="admin"
IP="127.0.0.1"
PORT="9200"

# Default number of documents to index
number=0
log_dir="tmp/logs"
log_file="$log_dir/populate-agents-index.log"

# Function to check if URL is up
function wait_for_cluster() {
local max_retries=12
local sleep_interval=5 # seconds
local url="http://$IP:$PORT/_cluster/health"

for ((i = 1; i <= max_retries; i++)); do
response=$(curl -s -o /dev/null -w "%{http_code}" -u $USERNAME:$PASSWORD $url)
if [[ $response -eq 200 ]]; then
echo "Cluster is up and running."
return 0
else
echo "Cluster not available yet. Waiting..."
sleep $sleep_interval
fi
done

echo "Failed to connect to the cluster after $max_retries retries."
return 1
}

# Function to generate random date
function generate_random_date() {
local start_date
local end_date
local random_date

start_date=$(date -u +%s)
end_date=$((start_date - 864000))
random_date=$((start_date - RANDOM % (start_date - end_date)))

date -u -r "$random_date" '+%Y-%m-%dT%H:%M:%S.%3NZ'
}

# Function to generate random groups
function generate_random_groups() {
local groups=()
for ((i = 1; i <= $((RANDOM % 5 + 1)); i++)); do
groups+=("\"group00$((RANDOM % 6))\"")
done
printf '[%s]' "$(
IFS=,
echo "${groups[*]}"
)"
}

# Function to generate random agent
function generate_random_agent() {
local agent
agent=$(
cat <<EOF
{
"agent": {
"id": "agent$((RANDOM % 100))",
"name": "Agent$((RANDOM % 100))",
"type": "$(shuf -e windows linux macos -n 1)",
"version": "v$((RANDOM % 10))-stable",
"status": "$(shuf -e active inactive -n 1)",
"last_login": "$(generate_random_date)",
"groups": $(generate_random_groups),
"key": "key$((RANDOM % 1000))",
"host": $(generate_random_host)
}
}
EOF
)
echo "$agent"
}

# Function to generate random host
function generate_random_host() {
local family
family=$(shuf -e debian ubuntu macos ios android RHEL -n 1)
local version
version="$((RANDOM % 100)).$((RANDOM % 100))"
local host
host=$(
cat <<EOF
{
"architecture": "$(shuf -e x86_64 arm64 -n 1)",
"boot": {"id": "boot$((RANDOM % 10000))"},
"cpu": {"usage": $(echo "scale=2; $RANDOM % 100" | bc)},
"disk": {"read": {"bytes": $((RANDOM % 1000001))}, "write": {"bytes": $((RANDOM % 1000001))}},
"domain": "domain$((RANDOM % 1000))",
"geo": {
"city_name": "$(shuf -e 'San Francisco' 'New York' Berlin Tokyo -n 1)",
"continent_code": "$(shuf -e NA EU AS -n 1)",
"continent_name": "$(shuf -e 'North America' Europe Asia -n 1)",
"country_iso_code": "$(shuf -e US DE JP -n 1)",
"country_name": "$(shuf -e 'United States' Germany Japan -n 1)",
"location": {"lat": $(echo "scale=6; $RANDOM % 180 - 90" | bc), "lon": $(echo "scale=6; $RANDOM % 360 - 180" | bc)},
"name": "geo$((RANDOM % 1000))",
"postal_code": "$((10000 + RANDOM % 90000))",
"region_iso_code": "region$((RANDOM % 1000))",
"region_name": "Region $((RANDOM % 1000))",
"timezone": "$(shuf -e PST EST CET JST -n 1)"
},
"hostname": "host$((RANDOM % 10000))",
"id": "hostid$((RANDOM % 10000))",
"ip": "$((RANDOM % 256)).$((RANDOM % 256)).$((RANDOM % 256)).$((RANDOM % 256))",
"mac": "$(printf '%02x:%02x:%02x:%02x:%02x:%02x' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)))",
"name": "hostname$((RANDOM % 10000))",
"network": {"egress": {"bytes": $((RANDOM % 1000001)), "packets": $((RANDOM % 1000001))}, "ingress": {"bytes": $((RANDOM % 1000001)), "packets": $((RANDOM % 1000001))}},
"os": {"family": "$family", "full": "$family $version", "kernel": "kernel$((RANDOM % 1000))", "name": "$family", "platform": "$(shuf -e linux windows macos -n 1)", "type": "$family", "version": "$version"},
"pid_ns_ino": "$((1000000 + RANDOM % 9000000))",
"risk": {"calculated_level": "$(shuf -e low medium high -n 1)", "calculated_score": $(echo "scale=2; $RANDOM % 100" | bc), "calculated_score_norm": $(echo "scale=2; $RANDOM % 100 / 100" | bc), "static_level": "$(shuf -e low medium high -n 1)", "static_score": $(echo "scale=2; $RANDOM % 100" | bc), "static_score_norm": $(echo "scale=2; $RANDOM % 100 / 100" | bc)},
"uptime": $((RANDOM % 1000001))
}
EOF
)
echo "$host"
}

# Function to inject documents to agents index
function index_documents() {
local data=$1
url="http://$IP:$PORT/$INDEX_NAME/_doc"
response=$(curl -s -o /dev/null -w "%{http_code}" -u $USERNAME:$PASSWORD -H 'Content-Type: application/json' -d "$data" -X POST $url)
if [[ $response -ne 201 ]]; then
echo "Error: $response"
fi
}

function parse_args() {
while getopts ":n:o:h" opt; do
case ${opt} in
h)
echo "Usage: $0 [options]"
echo "Options:"
echo " -n <number> Number of documents to generate. If not provided, the script will prompt for the number of docs to generate."
echo " -o <log_output> (Optional) Directory to store the output log. Default: 'tmp/logs/'"
echo " -h (Optional) Display this help message"
echo "Example: $0 -n 100"
echo
exit 0
;;
n)
number=$OPTARG
;;
o)
log_dir=$OPTARG
log_file="$log_dir/populate-agents-index.log"
;;
\?)
echo "Invalid option: $OPTARG" 1>&2
exit 1
;;
esac
done
}

# Main function
function populate_index() {
if [[ $number -lt 1 ]]; then
echo -n "How many docs do you want to generate? "
read -r number
if ! [[ "$number" =~ ^[0-9]+$ ]]; then
echo "Invalid input. Please enter a valid number."
return
fi
fi

echo "Waiting for the cluster to be up and running..."
if ! wait_for_cluster; then
echo "Cluster did not start in time. Exiting."
exit 1
fi

echo "Generating and indexing $number docs..."

for ((i = 1; i <= number; i++)); do
doc=$(generate_random_agent)
echo "$doc"
index_documents "$doc"
done

echo "Data generation completed."
}

parse_args "$@"

if [[ ! -d "$log_dir" ]]; then
mkdir -p "$log_dir"
fi

# Run the populate_index function in the background and redirect output to log file
(populate_index) >"$log_file" 2>&1 &
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearchOpenxception;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.action.search.CreatePitRequest;
import org.opensearch.action.search.CreatePitResponse;
Expand All @@ -41,6 +42,7 @@
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import com.wazuh.commandmanager.CommandManagerPlugin;
import com.wazuh.commandmanager.model.*;
Expand Down Expand Up @@ -101,9 +103,11 @@ public static <T> T getNestedObject(Map<String, Object> map, String key, Class<T
* delivery timestamps are earlier than the current time
*
* @param searchResponse The search results page
* @throws IllegalStateException Rethrown from setSentStatus()
* @throws IllegalStateException from setFailureStatus()
* @throws OpenSearchTimeoutException from setFailureStatus()
*/
public void handlePage(SearchResponse searchResponse) throws IllegalStateException {
public void handlePage(SearchResponse searchResponse)
throws IllegalStateException, OpenSearchTimeoutException {
SearchHits searchHits = searchResponse.getHits();

final ZonedDateTime current_time = DateUtils.nowWithMillisResolution();
Expand All @@ -123,9 +127,11 @@ public void handlePage(SearchResponse searchResponse) throws IllegalStateExcepti
*
* @param hit The page's result we are to update.
* @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}.
* @throws OpenSearchTimeoutException Raised by {@link ActionFuture#actionGet(long)}.
*/
@SuppressWarnings("unchecked")
private void setFailureStatus(SearchHit hit) throws IllegalStateException {
private void setFailureStatus(SearchHit hit)
throws IllegalStateException, OpenSearchTimeoutException {
final Map<String, Object> commandMap =
getNestedObject(
hit.getSourceAsMap(),
Expand All @@ -143,7 +149,7 @@ private void setFailureStatus(SearchHit hit) throws IllegalStateException {
.id(hit.getId());
this.client
.index(indexRequest)
.actionGet(PluginSettings.getInstance().getTimeout() * 1000);
.actionGet(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
}

Expand All @@ -156,8 +162,8 @@ private void setFailureStatus(SearchHit hit) throws IllegalStateException {
* @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}.
*/
public SearchResponse pitQuery(PointInTimeBuilder pointInTimeBuilder, Object[] searchAfter)
throws IllegalStateException {
final SearchRequest searchRequest = new SearchRequest(PluginSettings.getIndexName());
throws IllegalStateException, OpenSearchTimeoutException {
final SearchRequest searchRequest = new SearchRequest(CommandManagerPlugin.INDEX_NAME);
final TermQueryBuilder termQueryBuilder =
QueryBuilders.termQuery(SearchThread.COMMAND_STATUS_FIELD, Status.PENDING);
final TimeValue timeout =
Expand Down Expand Up @@ -206,6 +212,8 @@ public void run() {
log.error("ArrayIndexOutOfBoundsException retrieving page: {}", e.getMessage());
} catch (IllegalStateException e) {
log.error("IllegalStateException retrieving page: {}", e.getMessage());
} catch (OpenSearchTimeoutException e) {
log.error("Query timed out: {}", e.getMessage());
} catch (Exception e) {
log.error("Generic exception retrieving page: {}", e.getMessage());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@

import java.io.IOException;

import reactor.util.annotation.NonNull;
import reactor.util.annotation.Nullable;

/** Command's action fields. */
public class Action implements ToXContentObject {
public static final String ACTION = "action";
Expand All @@ -38,7 +41,7 @@ public class Action implements ToXContentObject {
* @param args actual command.
* @param version version of the action.
*/
public Action(String name, Args args, String version) {
public Action(@NonNull String name, @Nullable Args args, String version) {
this.name = name;
this.args = args;
this.version = version;
Expand All @@ -53,7 +56,7 @@ public Action(String name, Args args, String version) {
*/
public static Action parse(XContentParser parser) throws IOException {
String name = "";
Args args = null;
Args args = new Args();
String version = "";

while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
Expand Down Expand Up @@ -82,7 +85,9 @@ public static Action parse(XContentParser parser) throws IOException {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(ACTION);
builder.field(NAME, this.name);
this.args.toXContent(builder, ToXContentObject.EMPTY_PARAMS);
if (this.args != null) {
this.args.toXContent(builder, ToXContentObject.EMPTY_PARAMS);
}
builder.field(VERSION, this.version);
return builder.endObject();
}
Expand All @@ -91,12 +96,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
public String toString() {
return "Action{"
+ "name='"
+ name
+ this.name
+ '\''
+ ", args="
+ args
+ this.args
+ ", version='"
+ version
+ this.version
+ '\''
+ '}';
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,18 @@ public class Args implements ToXContentObject {
public static final String ARGS = "args";
private final Map<String, Object> args;

/** Parameterless constructor. */
public Args() {
this.args = new HashMap<>();
}

/**
* Constructor method
* Constructor with parameters.
*
* @param args Initializes the args object
*/
public Args(Map<String, Object> args) {
this.args = args;
this.args = new HashMap<>(args);
}

/**
Expand Down

0 comments on commit f4d0712

Please sign in to comment.