From 505f32e86d2251c31ab24956d0f81e97bf7671a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Tue, 2 Feb 2021 14:00:16 +0100 Subject: [PATCH 01/51] README.md: add Unattendend all-in-one section --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 32ccb1d..3ae028f 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,10 @@ This repository contains CloudFormation templates and provision scripts to deplo * Kibana server seats behind an internet facing load balancer, that optionally loads an SSL Certificate for HTTPS * A Splunk Indexer instance with a Splunk app for Wazuh installed on it. * Six Wazuh agents installed on different operating systems: Red Hat 7, CentOS 7, Ubuntu, Debian, Amazon Linux and Windows. + + +## Unattendend all-in-one + +* Install scipt following [Wazuh Unattended installation](https://documentation.wazuh.com/current/installation-guide/open-distro/all-in-one-deployment/unattended-installation.html) +* A single node with Amazon Linux + From ba66ef311fd9d3c4c6147ddb0d375d628042e2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Tue, 2 Feb 2021 14:00:54 +0100 Subject: [PATCH 02/51] Create template.yml --- all-in-one/unattended/template.yml | 336 +++++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 all-in-one/unattended/template.yml diff --git a/all-in-one/unattended/template.yml b/all-in-one/unattended/template.yml new file mode 100644 index 0000000..18579e7 --- /dev/null +++ b/all-in-one/unattended/template.yml @@ -0,0 +1,336 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Provides an unattended all-in-one Wazuh installation +Parameters: + InstanceType: + AllowedValues: + - t1.micro + - t2.nano + - t2.micro + - t2.small + - t2.medium + - t2.large + - m1.small + - m1.medium + - m1.large + - m1.xlarge + - m2.xlarge + - m2.2xlarge + - m2.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - c1.medium + - c1.xlarge + - c3.large + - c3.xlarge + - c3.2xlarge + - c3.4xlarge + - c3.8xlarge + - c4.large + - c4.xlarge + - c4.2xlarge + - c4.4xlarge + - c4.8xlarge + - g2.2xlarge + - g2.8xlarge + - r3.large + - r3.xlarge + - r3.2xlarge + - r3.4xlarge + - r3.8xlarge + - i2.xlarge + - i2.2xlarge + - i2.4xlarge + - i2.8xlarge + - d2.xlarge + - d2.2xlarge + - d2.4xlarge + - d2.8xlarge + - hi1.4xlarge + - hs1.8xlarge + - cr1.8xlarge + - cc2.8xlarge + - cg1.4xlarge + ConstraintDescription: must be a valid EC2 instance type. + Default: t2.small + Description: EC2 instance type + Type: String + KeyName: + ConstraintDescription: Can contain only ASCII characters. + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Type: AWS::EC2::KeyPair::KeyName + SSHLocation: + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x + Default: 0.0.0.0/0 + Description: The IP address range that can be used to SSH to the EC2 instances + MaxLength: '18' + MinLength: '9' + Type: String + +Mappings: +Mappings: + AWSInstanceType2Arch: + c1.medium: + Arch: HVM64 + c1.xlarge: + Arch: HVM64 + c3.2xlarge: + Arch: HVM64 + c3.4xlarge: + Arch: HVM64 + c3.8xlarge: + Arch: HVM64 + c3.large: + Arch: HVM64 + c3.xlarge: + Arch: HVM64 + c4.2xlarge: + Arch: HVM64 + c4.4xlarge: + Arch: HVM64 + c4.8xlarge: + Arch: HVM64 + c4.large: + Arch: HVM64 + c4.xlarge: + Arch: HVM64 + cc2.8xlarge: + Arch: HVM64 + cr1.8xlarge: + Arch: HVM64 + d2.2xlarge: + Arch: HVM64 + d2.4xlarge: + Arch: HVM64 + d2.8xlarge: + Arch: HVM64 + d2.xlarge: + Arch: HVM64 + g2.2xlarge: + Arch: HVMG2 + g2.8xlarge: + Arch: HVMG2 + hi1.4xlarge: + Arch: HVM64 + hs1.8xlarge: + Arch: HVM64 + i2.2xlarge: + Arch: HVM64 + i2.4xlarge: + Arch: HVM64 + i2.8xlarge: + Arch: HVM64 + i2.xlarge: + Arch: HVM64 + m1.large: + Arch: HVM64 + m1.medium: + Arch: HVM64 + m1.small: + Arch: HVM64 + m1.xlarge: + Arch: HVM64 + m2.2xlarge: + Arch: HVM64 + m2.4xlarge: + Arch: HVM64 + m2.xlarge: + Arch: HVM64 + m3.2xlarge: + Arch: HVM64 + m3.large: + Arch: HVM64 + m3.medium: + Arch: HVM64 + m3.xlarge: + Arch: HVM64 + m4.10xlarge: + Arch: HVM64 + m4.2xlarge: + Arch: HVM64 + m4.4xlarge: + Arch: HVM64 + m4.large: + Arch: HVM64 + m4.xlarge: + Arch: HVM64 + r3.2xlarge: + Arch: HVM64 + r3.4xlarge: + Arch: HVM64 + r3.8xlarge: + Arch: HVM64 + r3.large: + Arch: HVM64 + r3.xlarge: + Arch: HVM64 + t1.micro: + Arch: HVM64 + t2.large: + Arch: HVM64 + t2.medium: + Arch: HVM64 + t2.micro: + Arch: HVM64 + t2.nano: + Arch: HVM64 + t2.small: + Arch: HVM64 + + AWSRegionArch2AMI: + us-east-1: + HVM64: ami-0c6b1d09930fac512 + HVMCENTOS7: ami-02eac2c0129f6376b + HVMUBUNTU64: ami-024a64a6685d05041 + HVMREDHAT7: ami-6871a115 + HVMDEBIAN: ami-0357081a1383dc76b + HVMWINDOWS: ami-0a9ca0496f746e6e0 + us-east-2: + HVM64: ami-0ebbf2179e615c338 + HVMCENTOS7: ami-0f2b4fc905b0bd1f1 + HVMUBUNTU64: ami-097ebb39620d8d54b + HVMREDHAT7: ami-03291866 + HVMDEBIAN: ami-09c10a66337c79669 + HVMWINDOWS: ami-0087a83ed4a60d1e9 + us-west-1: + HVM64: ami-015954d5e5548d13b + HVMUBUNTU64: ami-040dfc3ebf1bfc4f6 + HVMCENTOS7: ami-074e2d6769f445be5 + HVMREDHAT7: ami-18726478 + HVMDEBIAN: ami-0adbaf2e0ce044437 + HVMWINDOWS: ami-05bf35c67c02cd868 + us-west-2: + HVM64: ami-0cb72367e98845d43 + HVMUBUNTU64: ami-0196ce5c34425a906 + HVMCENTOS7: ami-01ed306a12b7d1c96 + HVMREDHAT7: ami-28e07e50 + HVMDEBIAN: ami-05a3ef6744aa96514 + HVMWINDOWS: ami-04ad37d2932b886c0 + ca-central-1: + HVM64: ami-08a9b721ecc5b0a53 + HVMUBUNTU64: ami-0380eb76ff3ad603f + HVMCENTOS7: ami-033e6106180a626d0 + HVMREDHAT7: ami-49f0762d + HVMDEBIAN: ami-04413a263a7d94982 + HVMWINDOWS: ami-020e569ea1f3a4e1c + eu-west-1: + HVM64: ami-030dbca661d402413 + HVMUBUNTU64: ami-0b2a4d260c54e8d3d + HVMCENTOS7: ami-0ff760d16d9497662 + HVMREDHAT7: ami-7c491f05 + HVMDEBIAN: ami-0968f6a31fc6cffc0 + HVMWINDOWS: ami-03838ccd5cfb84782 + eu-west-2: + HVM64: ami-0009a33f033d8b7b6 + HVMUBUNTU64: ami-09dd110e91f421069 + HVMCENTOS7: ami-0eab3a90fc693af19 + HVMREDHAT7: ami-7c1bfd1b + HVMDEBIAN: ami-0faa9c9b5399088fd + HVMWINDOWS: ami-0ebf422d2a92724ec + eu-west-3: + HVM64: ami-0ebb3a801d5fb8b9b + HVMUBUNTU64: ami-00e557eb4a269bf1c + HVMCENTOS7: ami-0e1ab783dc9489f34 + HVMREDHAT7: ami-5026902d + HVMDEBIAN: ami-0cd23820af84edc85 + HVMWINDOWS: ami-022cfeccb4b72d6b8 + ap-northeast-1: + HVM64: ami-00d101850e971728d + HVMUBUNTU64: ami-0b5a5c971fc30e5d1 + HVMCENTOS7: ami-045f38c93733dd48d + HVMREDHAT7: ami-6b0d5f0d + HVMDEBIAN: ami-09fbcd30452841cb9 + HVMWINDOWS: ami-02192102f14f0a10a + ap-northeast-2: + HVM64: ami-08ab3f7e72215fe91 + HVMUBUNTU64: ami-06af4ace0697354bf + HVMCENTOS7: ami-06cf2a72dadf92410 + HVMREDHAT7: ami-3eee4150 + HVMDEBIAN: ami-08363ccce96df1fff + HVMWINDOWS: ami-0708a3b845edea89c + ap-southeast-1: + HVM64: ami-0b5a47f8865280111 + HVMUBUNTU64: ami-0355471dc9f264631 + HVMCENTOS7: ami-0b4dd9d65556cac22 + HVMREDHAT7: ami-76144b0a + HVMDEBIAN: ami-0555b1a5444087dd4 + HVMWINDOWS: ami-0afce41e503676765 + ap-southeast-2: + HVM64: ami-0fb7513bcdc525c3b + HVMUBUNTU64: ami-0065540df76a93885 + HVMCENTOS7: ami-08bd00d7713a39e7d + HVMREDHAT7: ami-67589505 + HVMDEBIAN: ami-029c54f988446691a + HVMWINDOWS: ami-0628ef1f10e34307d + ap-south-1: + HVM64: ami-00e782930f1c3dbc7 + HVMUBUNTU64: ami-076b389b9989430c2 + HVMCENTOS7: ami-02e60be79e78fef21 + HVMREDHAT7: ami-5b673c34 + HVMDEBIAN: ami-0dc98cbb0d0e49162 + HVMWINDOWS: ami-0e719217acb64308e + sa-east-1: + HVM64: ami-058141e091292ecf0 + HVMUBUNTU64: ami-009e4c831385f8901 + HVMCENTOS7: ami-0b8d86d4bf91850af + HVMREDHAT7: ami-b0b7e3dc + HVMDEBIAN: ami-030580e61468e54bd + HVMWINDOWS: ami-07df12f3c5005cd1f + + Subnet2CIDR: + WazuhVpc: + CIDR: 10.0.0.0/16 + SubnetWazuh: + CIDR: 10.0.0.0/24 + SubnetAgents: + CIDR: 10.0.1.0/24 + SubnetElasticsearch: + CIDR: 10.0.2.0/24 + +Resources: + ServerInstance: + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + SecurityGroups: + - Ref: SecurityGroup + UserData: + Fn::Base64: !Sub | + #!/bin/bash + echo "Downloading script..." + curl -so ~/all-in-one-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/4.0/resources/open-distro/unattended-installation/all-in-one-installation.sh + echo "Installing script..." + bash ~/all-in-one-installation.sh + Type: AWS::EC2::Instance + + SecurityGroup: + Properties: + GroupDescription: Enable HTTPS access via port 443 + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: '443' + IpProtocol: tcp + ToPort: '443' + - CidrIp: + Ref: SSHLocation + FromPort: '22' + IpProtocol: tcp + ToPort: '22' + Type: AWS::EC2::SecurityGroup From 7278c8b68c9a289ba0026d1e5e1b419eb8b54e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 4 Feb 2021 15:11:44 +0100 Subject: [PATCH 03/51] Add config_cluster.yml template --- .../unattended/templates/config_cluster.yml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 distributed/unattended/templates/config_cluster.yml diff --git a/distributed/unattended/templates/config_cluster.yml b/distributed/unattended/templates/config_cluster.yml new file mode 100644 index 0000000..4598cd4 --- /dev/null +++ b/distributed/unattended/templates/config_cluster.yml @@ -0,0 +1,32 @@ +## Multi-node configuration + +## Elasticsearch configuration + +cluster.name: {{elastic_cluster}} + +cluster.initial_master_nodes: + - {{master_node_1}} + - {{master_node_2}} + - {{master_node_3}} + +discovery.seed_hosts: + - {{elasticsearch_ip_node1}} + - {{elasticsearch_ip_node2}} + - {{elasticsearch_ip_node3}} + +## Certificates creation + +# Clients certificates +clients: + - name: admin + dn: CN=admin,OU=Docu,O=Wazuh,L=California,C=US + admin: true + - name: filebeat + dn: CN=filebeat,OU=Docu,O=Wazuh,L=California,C=US + + +# Kibana-instance +- {{kibana_ip}} + +# Wazuh-master-configuration +- {{wazuh_master_server_IP}} From 2393b72b0109243117efd6845b991109736127fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 4 Feb 2021 16:46:44 +0100 Subject: [PATCH 04/51] config_cluster.yml: Fix identation --- .../unattended/templates/config_cluster.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/distributed/unattended/templates/config_cluster.yml b/distributed/unattended/templates/config_cluster.yml index 4598cd4..482d420 100644 --- a/distributed/unattended/templates/config_cluster.yml +++ b/distributed/unattended/templates/config_cluster.yml @@ -2,17 +2,17 @@ ## Elasticsearch configuration -cluster.name: {{elastic_cluster}} +cluster.name: {{cluster_name}} cluster.initial_master_nodes: - - {{master_node_1}} - - {{master_node_2}} - - {{master_node_3}} + - {{master_node_1}} + - {{master_node_2}} + - {{master_node_3}} discovery.seed_hosts: - - {{elasticsearch_ip_node1}} - - {{elasticsearch_ip_node2}} - - {{elasticsearch_ip_node3}} + - {{elasticsearch_ip_node1}} + - {{elasticsearch_ip_node2}} + - {{elasticsearch_ip_node3}} ## Certificates creation From 5027d55a87cd2edfa844c10a0f4ef4c5b7f676d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 5 Feb 2021 10:32:26 +0100 Subject: [PATCH 05/51] config_cluster.yml: add master and worker node names --- distributed/unattended/templates/config_cluster.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/distributed/unattended/templates/config_cluster.yml b/distributed/unattended/templates/config_cluster.yml index 482d420..def0312 100644 --- a/distributed/unattended/templates/config_cluster.yml +++ b/distributed/unattended/templates/config_cluster.yml @@ -21,9 +21,10 @@ clients: - name: admin dn: CN=admin,OU=Docu,O=Wazuh,L=California,C=US admin: true - - name: filebeat - dn: CN=filebeat,OU=Docu,O=Wazuh,L=California,C=US - + - name: {{wazuh_master_name}} + dn: CN={{wazuh_master_name}},OU=Docu,O=Wazuh,L=California,C=US + - name: {{wazuh_worker_name}} + dn: CN={{wazuh_worker_name}},OU=Docu,O=Wazuh,L=California,C=US # Kibana-instance - {{kibana_ip}} From 870f687e9ffea4871657fd3230fac91ecba9d013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 5 Feb 2021 15:02:28 +0100 Subject: [PATCH 06/51] Add all-in-one distribuited template --- distributed/unattended/template.yml | 326 ++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 distributed/unattended/template.yml diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml new file mode 100644 index 0000000..0698c2d --- /dev/null +++ b/distributed/unattended/template.yml @@ -0,0 +1,326 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Provides an unattended all-in-one Wazuh installation +Parameters: + ## AWS + AvailabilityZone: + Description: Select an availability zone for the VPC + Type: 'List' + InstanceType: + AllowedValues: + - t2.small + - t2.medium + - t2.large + - t2.xlarge + ConstraintDescription: must be a valid EC2 instance type. + Default: t2.large + Description: EC2 instance type + Type: String + KeyName: + ConstraintDescription: Can contain only ASCII characters. + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Type: AWS::EC2::KeyPair::KeyName + SSHLocation: + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x + Default: 0.0.0.0/0 + Description: The IP address range that can be used to SSH to the EC2 instances + MaxLength: '18' + MinLength: '9' + Type: String + + ## Version + WazuhVersion: + AllowedValues: + - "4.0.1" + - "4.0.2" + - "4.0.3" + - "4.0.4" + Description: Wazuh version + Default: "4.0.4" + Type: String + +Mappings: + Node2NameAndIP: + MasterNode: + Name: master-node + IP: 10.0.0.100 + WorkerNode: + Name: worker-node + IP: 10.0.0.200 + Elastic1: + Name: elastic-node-1 + IP: 10.0.1.101 + Elastic2: + Name: elastic-node-2 + IP: 10.0.1.102 + Elastic3: + Name: elastic-node-3 + IP: 10.0.1.103 + + AWSInstanceType2Arch: + t2.large: + Arch: HVM64 + t2.medium: + Arch: HVM64 + t2.micro: + Arch: HVM64 + t2.xlarge: + Arch: HVM64 + t2.small: + Arch: HVM64 + + AWSRegionArch2AMI: + us-east-1: + HVM64: ami-0c6b1d09930fac512 + HVMCENTOS7: ami-02eac2c0129f6376b + HVMUBUNTU64: ami-024a64a6685d05041 + HVMREDHAT7: ami-6871a115 + HVMDEBIAN: ami-0357081a1383dc76b + HVMWINDOWS: ami-0a9ca0496f746e6e0 + us-east-2: + HVM64: ami-0ebbf2179e615c338 + HVMCENTOS7: ami-0f2b4fc905b0bd1f1 + HVMUBUNTU64: ami-097ebb39620d8d54b + HVMREDHAT7: ami-03291866 + HVMDEBIAN: ami-09c10a66337c79669 + HVMWINDOWS: ami-0087a83ed4a60d1e9 + us-west-1: + HVM64: ami-015954d5e5548d13b + HVMUBUNTU64: ami-040dfc3ebf1bfc4f6 + HVMCENTOS7: ami-074e2d6769f445be5 + HVMREDHAT7: ami-18726478 + HVMDEBIAN: ami-0adbaf2e0ce044437 + HVMWINDOWS: ami-05bf35c67c02cd868 + us-west-2: + HVM64: ami-0cb72367e98845d43 + HVMUBUNTU64: ami-0196ce5c34425a906 + HVMCENTOS7: ami-01ed306a12b7d1c96 + HVMREDHAT7: ami-28e07e50 + HVMDEBIAN: ami-05a3ef6744aa96514 + HVMWINDOWS: ami-04ad37d2932b886c0 + + Subnet2CIDR: + WazuhVpc: + CIDR: 10.0.0.0/16 + SubnetWazuh: + CIDR: 10.0.0.0/24 + SubnetElasticsearch: + CIDR: 10.0.1.0/24 + +Resources: + # Network resources + WazuhVpc: + Type: 'AWS::EC2::VPC' + Properties: + CidrBlock: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + EnableDnsSupport: 'true' + EnableDnsHostnames: 'true' + Tags: + - Key: Application + Value: !Ref 'AWS::StackId' + - Key: Name + Value: !Ref 'AWS::StackName' + + SubnetWazuh: + Type: 'AWS::EC2::Subnet' + Properties: + VpcId: !Ref WazuhVpc + AvailabilityZone: !Select [ "0", !Ref AvailabilityZone ] + CidrBlock: !FindInMap + - Subnet2CIDR + - SubnetWazuh + - CIDR + Tags: + - Key: Application + Value: !Ref 'AWS::StackId' + - Key: Name + Value: !Sub '${AWS::StackName}-SubnetWazuh' + + SubnetElasticsearch: + Type: 'AWS::EC2::Subnet' + Properties: + VpcId: !Ref WazuhVpc + AvailabilityZone: !Select [ "0", !Ref AvailabilityZone ] + CidrBlock: !FindInMap + - Subnet2CIDR + - SubnetElasticsearch + - CIDR + Tags: + - Key: Application + Value: !Ref 'AWS::StackId' + - Key: Name + Value: !Sub '${AWS::StackName}-SubnetElasticsearch' + + # Internet access and routing + InternetGateway: + Type: 'AWS::EC2::InternetGateway' + Properties: + Tags: + - Key: Application + Value: !Ref 'AWS::StackId' + - Key: Name + Value: !Sub '${AWS::StackName}-InternetGateway' + + GatewayToInternet: + Type: 'AWS::EC2::VPCGatewayAttachment' + Properties: + VpcId: !Ref WazuhVpc + InternetGatewayId: !Ref InternetGateway + + PublicRouteTable: + Type: 'AWS::EC2::RouteTable' + Properties: + VpcId: !Ref WazuhVpc + Tags: + - Key: Application + Value: !Ref 'AWS::StackId' + - Key: Name + Value: !Sub '${AWS::StackName}-PublicRouteTable' + + PublicRoute: + Type: 'AWS::EC2::Route' + DependsOn: GatewayToInternet + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + SubnetWazuhPublicRouteTable: + Type: 'AWS::EC2::SubnetRouteTableAssociation' + Properties: + SubnetId: !Ref SubnetWazuh + RouteTableId: !Ref PublicRouteTable + + SubnetElasticPublicRouteTable: + Type: 'AWS::EC2::SubnetRouteTableAssociation' + Properties: + SubnetId: !Ref SubnetElasticsearch + RouteTableId: !Ref PublicRouteTable + + # Security groups + WazuhSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Wazuh security group + VpcId: !Ref WazuhVpc + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-WazuhSecurityGroup' + + ElasticSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Elasticsearch security group + VpcId: !Ref WazuhVpc + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + - IpProtocol: tcp + FromPort: 5601 + ToPort: 5601 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-ElasticSecurityGroup' + + # Instances + Elastic1: + Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + Elastic1InstallationSet: + - DownloadFile + - ConfigFile + - InstallELK + DownloadFile: + commands: + download-elk-installation-script: + command: "curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" + env: + WazuhVersion: !Ref WazuhVersion + cwd: "~" + test: "test ! -e ~/elastic-stack-installation.sh" + ConfigFile: + files: + ~/config.yml: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/config_cluster.yml + mode: '000700' + owner: root + group: root + context: + cluster_name: "elastic_cluster" + master_node_1: !FindInMap [Node2NameAndIP, Elastic1, Name] + master_node_2: !FindInMap [Node2NameAndIP, Elastic2, Name] + master_node_3: !FindInMap [Node2NameAndIP, Elastic3, Name] + elasticsearch_ip_node1: !FindInMap [Node2NameAndIP, Elastic1, IP] + elasticsearch_ip_node2: !FindInMap [Node2NameAndIP, Elastic2, IP] + elasticsearch_ip_node3: !FindInMap [Node2NameAndIP, Elastic3, IP] + wazuh_master_name: !FindInMap [Node2NameAndIP, MasterNode, Name] + wazuh_worker_name: !FindInMap [Node2NameAndIP, WorkerNode, Name] + kibana_ip: !FindInMap [Node2NameAndIP, Elastic3, IP] + wazuh_master_server_IP: !FindInMap [Node2NameAndIP, MasterNode, IP] + InstallELK: + commands: + install-elk: + command: "bash ~/elastic-stack-installation.sh -e -k -c -n $NodeName" + env: + NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] + cwd: "~" + test: "test ! -e ~/certs.tar" + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - Ref: "ElasticSecurityGroup" + SubnetId: + Ref: "SubnetElasticsearch" + PrivateIpAddress: !FindInMap [Node2NameAndIP, Elastic1, IP] + Tags: + - Key: Name + Value: ElasticInitialNode + UserData: + Fn::Base64: !Sub | + #!/bin/bash + /opt/aws/bin/cfn-init --stack ${AWS::StackName} \ + --resource Amazon \ + --configsets Elastic1InstallationSet + --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? + --stack ${AWS::StackName} \ + --resource Amazon \ + --region ${AWS::Region} + DependsOn: GatewayToInternet From 74c024d042e77bc1329adeb3d5ea4012babc5021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 5 Feb 2021 15:03:20 +0100 Subject: [PATCH 07/51] Remove some instances types --- all-in-one/unattended/template.yml | 223 +---------------------------- 1 file changed, 3 insertions(+), 220 deletions(-) diff --git a/all-in-one/unattended/template.yml b/all-in-one/unattended/template.yml index 18579e7..befb351 100644 --- a/all-in-one/unattended/template.yml +++ b/all-in-one/unattended/template.yml @@ -3,62 +3,12 @@ Description: Provides an unattended all-in-one Wazuh installation Parameters: InstanceType: AllowedValues: - - t1.micro - - t2.nano - - t2.micro - t2.small - t2.medium - t2.large - - m1.small - - m1.medium - - m1.large - - m1.xlarge - - m2.xlarge - - m2.2xlarge - - m2.4xlarge - - m3.medium - - m3.large - - m3.xlarge - - m3.2xlarge - - m4.large - - m4.xlarge - - m4.2xlarge - - m4.4xlarge - - m4.10xlarge - - c1.medium - - c1.xlarge - - c3.large - - c3.xlarge - - c3.2xlarge - - c3.4xlarge - - c3.8xlarge - - c4.large - - c4.xlarge - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - g2.2xlarge - - g2.8xlarge - - r3.large - - r3.xlarge - - r3.2xlarge - - r3.4xlarge - - r3.8xlarge - - i2.xlarge - - i2.2xlarge - - i2.4xlarge - - i2.8xlarge - - d2.xlarge - - d2.2xlarge - - d2.4xlarge - - d2.8xlarge - - hi1.4xlarge - - hs1.8xlarge - - cr1.8xlarge - - cc2.8xlarge - - cg1.4xlarge + - t2.xlarge ConstraintDescription: must be a valid EC2 instance type. - Default: t2.small + Default: t2.large Description: EC2 instance type Type: String KeyName: @@ -74,112 +24,15 @@ Parameters: MinLength: '9' Type: String -Mappings: Mappings: AWSInstanceType2Arch: - c1.medium: - Arch: HVM64 - c1.xlarge: - Arch: HVM64 - c3.2xlarge: - Arch: HVM64 - c3.4xlarge: - Arch: HVM64 - c3.8xlarge: - Arch: HVM64 - c3.large: - Arch: HVM64 - c3.xlarge: - Arch: HVM64 - c4.2xlarge: - Arch: HVM64 - c4.4xlarge: - Arch: HVM64 - c4.8xlarge: - Arch: HVM64 - c4.large: - Arch: HVM64 - c4.xlarge: - Arch: HVM64 - cc2.8xlarge: - Arch: HVM64 - cr1.8xlarge: - Arch: HVM64 - d2.2xlarge: - Arch: HVM64 - d2.4xlarge: - Arch: HVM64 - d2.8xlarge: - Arch: HVM64 - d2.xlarge: - Arch: HVM64 - g2.2xlarge: - Arch: HVMG2 - g2.8xlarge: - Arch: HVMG2 - hi1.4xlarge: - Arch: HVM64 - hs1.8xlarge: - Arch: HVM64 - i2.2xlarge: - Arch: HVM64 - i2.4xlarge: - Arch: HVM64 - i2.8xlarge: - Arch: HVM64 - i2.xlarge: - Arch: HVM64 - m1.large: - Arch: HVM64 - m1.medium: - Arch: HVM64 - m1.small: - Arch: HVM64 - m1.xlarge: - Arch: HVM64 - m2.2xlarge: - Arch: HVM64 - m2.4xlarge: - Arch: HVM64 - m2.xlarge: - Arch: HVM64 - m3.2xlarge: - Arch: HVM64 - m3.large: - Arch: HVM64 - m3.medium: - Arch: HVM64 - m3.xlarge: - Arch: HVM64 - m4.10xlarge: - Arch: HVM64 - m4.2xlarge: - Arch: HVM64 - m4.4xlarge: - Arch: HVM64 - m4.large: - Arch: HVM64 - m4.xlarge: - Arch: HVM64 - r3.2xlarge: - Arch: HVM64 - r3.4xlarge: - Arch: HVM64 - r3.8xlarge: - Arch: HVM64 - r3.large: - Arch: HVM64 - r3.xlarge: - Arch: HVM64 - t1.micro: - Arch: HVM64 t2.large: Arch: HVM64 t2.medium: Arch: HVM64 t2.micro: Arch: HVM64 - t2.nano: + t2.xlarge: Arch: HVM64 t2.small: Arch: HVM64 @@ -213,76 +66,6 @@ Mappings: HVMREDHAT7: ami-28e07e50 HVMDEBIAN: ami-05a3ef6744aa96514 HVMWINDOWS: ami-04ad37d2932b886c0 - ca-central-1: - HVM64: ami-08a9b721ecc5b0a53 - HVMUBUNTU64: ami-0380eb76ff3ad603f - HVMCENTOS7: ami-033e6106180a626d0 - HVMREDHAT7: ami-49f0762d - HVMDEBIAN: ami-04413a263a7d94982 - HVMWINDOWS: ami-020e569ea1f3a4e1c - eu-west-1: - HVM64: ami-030dbca661d402413 - HVMUBUNTU64: ami-0b2a4d260c54e8d3d - HVMCENTOS7: ami-0ff760d16d9497662 - HVMREDHAT7: ami-7c491f05 - HVMDEBIAN: ami-0968f6a31fc6cffc0 - HVMWINDOWS: ami-03838ccd5cfb84782 - eu-west-2: - HVM64: ami-0009a33f033d8b7b6 - HVMUBUNTU64: ami-09dd110e91f421069 - HVMCENTOS7: ami-0eab3a90fc693af19 - HVMREDHAT7: ami-7c1bfd1b - HVMDEBIAN: ami-0faa9c9b5399088fd - HVMWINDOWS: ami-0ebf422d2a92724ec - eu-west-3: - HVM64: ami-0ebb3a801d5fb8b9b - HVMUBUNTU64: ami-00e557eb4a269bf1c - HVMCENTOS7: ami-0e1ab783dc9489f34 - HVMREDHAT7: ami-5026902d - HVMDEBIAN: ami-0cd23820af84edc85 - HVMWINDOWS: ami-022cfeccb4b72d6b8 - ap-northeast-1: - HVM64: ami-00d101850e971728d - HVMUBUNTU64: ami-0b5a5c971fc30e5d1 - HVMCENTOS7: ami-045f38c93733dd48d - HVMREDHAT7: ami-6b0d5f0d - HVMDEBIAN: ami-09fbcd30452841cb9 - HVMWINDOWS: ami-02192102f14f0a10a - ap-northeast-2: - HVM64: ami-08ab3f7e72215fe91 - HVMUBUNTU64: ami-06af4ace0697354bf - HVMCENTOS7: ami-06cf2a72dadf92410 - HVMREDHAT7: ami-3eee4150 - HVMDEBIAN: ami-08363ccce96df1fff - HVMWINDOWS: ami-0708a3b845edea89c - ap-southeast-1: - HVM64: ami-0b5a47f8865280111 - HVMUBUNTU64: ami-0355471dc9f264631 - HVMCENTOS7: ami-0b4dd9d65556cac22 - HVMREDHAT7: ami-76144b0a - HVMDEBIAN: ami-0555b1a5444087dd4 - HVMWINDOWS: ami-0afce41e503676765 - ap-southeast-2: - HVM64: ami-0fb7513bcdc525c3b - HVMUBUNTU64: ami-0065540df76a93885 - HVMCENTOS7: ami-08bd00d7713a39e7d - HVMREDHAT7: ami-67589505 - HVMDEBIAN: ami-029c54f988446691a - HVMWINDOWS: ami-0628ef1f10e34307d - ap-south-1: - HVM64: ami-00e782930f1c3dbc7 - HVMUBUNTU64: ami-076b389b9989430c2 - HVMCENTOS7: ami-02e60be79e78fef21 - HVMREDHAT7: ami-5b673c34 - HVMDEBIAN: ami-0dc98cbb0d0e49162 - HVMWINDOWS: ami-0e719217acb64308e - sa-east-1: - HVM64: ami-058141e091292ecf0 - HVMUBUNTU64: ami-009e4c831385f8901 - HVMCENTOS7: ami-0b8d86d4bf91850af - HVMREDHAT7: ami-b0b7e3dc - HVMDEBIAN: ami-030580e61468e54bd - HVMWINDOWS: ami-07df12f3c5005cd1f Subnet2CIDR: WazuhVpc: From e96c385fbca101f980217372bc7c4d4b750c3fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Mon, 8 Feb 2021 15:19:41 +0100 Subject: [PATCH 08/51] Upgrade template.yml --- distributed/unattended/template.yml | 116 +++++++++++++++++++++------- 1 file changed, 87 insertions(+), 29 deletions(-) diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml index 0698c2d..dc1f08d 100644 --- a/distributed/unattended/template.yml +++ b/distributed/unattended/template.yml @@ -27,7 +27,17 @@ Parameters: MaxLength: '18' MinLength: '9' Type: String - + Ec2User: + AllowedValues: + - ec2-user + Default: ec2-user + Description: Username in EC2 instance + Type: String + SshPubKey: + Description: Elastic1 ssh pub key + Type: String + NoEcho: 'True' + ## Version WazuhVersion: AllowedValues: @@ -210,10 +220,7 @@ Resources: - IpProtocol: tcp FromPort: 22 ToPort: 22 - CidrIp: !FindInMap - - Subnet2CIDR - - WazuhVpc - - CIDR + CidrIp: 0.0.0.0/0 Tags: - Key: Name Value: !Sub '${AWS::StackName}-WazuhSecurityGroup' @@ -227,17 +234,11 @@ Resources: - IpProtocol: tcp FromPort: 22 ToPort: 22 - CidrIp: !FindInMap - - Subnet2CIDR - - WazuhVpc - - CIDR + CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 5601 - ToPort: 5601 - CidrIp: !FindInMap - - Subnet2CIDR - - WazuhVpc - - CIDR + FromPort: 443 + ToPort: 443 + CidrIp: 0.0.0.0/0 Tags: - Key: Name Value: !Sub '${AWS::StackName}-ElasticSecurityGroup' @@ -248,10 +249,22 @@ Resources: Metadata: 'AWS::CloudFormation::Init': configSets: + SSHConfigSet: + - SshKeys Elastic1InstallationSet: - DownloadFile - ConfigFile - InstallELK + SshKeys: + commands: + create-keys: + command: "sudo -u $Ec2User ssh-keygen -q -t rsa -N '' -f /home/$Ec2User/.ssh/id_rsa <<- https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/config_cluster.yml mode: '000700' @@ -278,16 +291,21 @@ Resources: elasticsearch_ip_node3: !FindInMap [Node2NameAndIP, Elastic3, IP] wazuh_master_name: !FindInMap [Node2NameAndIP, MasterNode, Name] wazuh_worker_name: !FindInMap [Node2NameAndIP, WorkerNode, Name] - kibana_ip: !FindInMap [Node2NameAndIP, Elastic3, IP] + kibana_ip: !FindInMap [Node2NameAndIP, Elastic1, IP] wazuh_master_server_IP: !FindInMap [Node2NameAndIP, MasterNode, IP] InstallELK: commands: - install-elk: - command: "bash ~/elastic-stack-installation.sh -e -k -c -n $NodeName" + install-elastic: + command: "bash ~/elastic-stack-installation.sh -e -c -n $NodeName" env: NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] cwd: "~" test: "test ! -e ~/certs.tar" + install-kibana: + command: "bash ~/elastic-stack-installation.sh -k -c -n $NodeName" + env: + NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] + cwd: "~" Properties: ImageId: Fn::FindInMap: @@ -311,16 +329,56 @@ Resources: PrivateIpAddress: !FindInMap [Node2NameAndIP, Elastic1, IP] Tags: - Key: Name - Value: ElasticInitialNode + Value: Elastic1 + UserData: + Fn::Base64: !Sub | + #!/bin/bash -xe + scp -i ${AWS:} + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets SSHConfigSet,Elastic1InstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic1 --region ${AWS::Region} + DependsOn: GatewayToInternet + + Elastic2: + Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + SSHConfigSet: + - SshKey + SshKey: + commands: + paste-ssh-pub-key: + command: "cat $PubKey >> /home/$Ec2User/.ssh/authorized_keys" + env: + PubKey: !Ref SshPubKey + Ec2User: !Ref Ec2User + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - Ref: "ElasticSecurityGroup" + SubnetId: + Ref: "SubnetElasticsearch" + PrivateIpAddress: !FindInMap [Node2NameAndIP, Elastic2, IP] + Tags: + - Key: Name + Value: Elastic2 UserData: Fn::Base64: !Sub | - #!/bin/bash - /opt/aws/bin/cfn-init --stack ${AWS::StackName} \ - --resource Amazon \ - --configsets Elastic1InstallationSet - --region ${AWS::Region} - /opt/aws/bin/cfn-signal -e $? - --stack ${AWS::StackName} \ - --resource Amazon \ - --region ${AWS::Region} + #!/bin/bash -xe + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic2 --configsets SSHConfigSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic2 --region ${AWS::Region} DependsOn: GatewayToInternet From 1c044b85264e47a9dedb3fe3235c8dd2358e238c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Tue, 9 Feb 2021 14:19:37 +0100 Subject: [PATCH 09/51] Upgrade template.yml --- distributed/unattended/template.yml | 99 ++++++++++++++++++----------- 1 file changed, 63 insertions(+), 36 deletions(-) diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml index dc1f08d..9bc5ac4 100644 --- a/distributed/unattended/template.yml +++ b/distributed/unattended/template.yml @@ -33,11 +33,17 @@ Parameters: Default: ec2-user Description: Username in EC2 instance Type: String - SshPubKey: - Description: Elastic1 ssh pub key + SshKey: + Description: AWS Cloudformation Stack SSH Private Key Type: String NoEcho: 'True' - + + # Log + LogFile: + Description: Log file path to keep track of actions + Type: String + Default: /var/log/wazuh-cloudformation.log + ## Version WazuhVersion: AllowedValues: @@ -249,27 +255,40 @@ Resources: Metadata: 'AWS::CloudFormation::Init': configSets: - SSHConfigSet: - - SshKeys Elastic1InstallationSet: + - SshConfig - DownloadFile - ConfigFile + - ScpFiles - InstallELK - SshKeys: + SshConfig: + files: + /tmp/ssh.key: + content: !Sub | + -----BEGIN RSA PRIVATE KEY----- + ${SshKey} + -----END RSA PRIVATE KEY----- + mode: '000600' + owner: root + group: root commands: - create-keys: - command: "sudo -u $Ec2User ssh-keygen -q -t rsa -N '' -f /home/$Ec2User/.ssh/id_rsa <<> $LogFile" env: - Ec2User: !Ref Ec2User + LogFile: !Ref LogFile + test: "test -e ~/.ssh/ssh.key" DownloadFile: commands: download-elk-installation-script: - command: "curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" + command: + "logger -s \"Downloading elastic-stack-installation.sh... \" 2>> $LogFile && \ + curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" env: + LogFile: !Ref LogFile WazuhVersion: !Ref WazuhVersion cwd: "~" test: "test ! -e ~/elastic-stack-installation.sh" @@ -293,19 +312,45 @@ Resources: wazuh_worker_name: !FindInMap [Node2NameAndIP, WorkerNode, Name] kibana_ip: !FindInMap [Node2NameAndIP, Elastic1, IP] wazuh_master_server_IP: !FindInMap [Node2NameAndIP, MasterNode, IP] + commands: + write-log: + command: "logger -s \"Creating custom config.yml... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + test: "test -e ~/config.yml" + ScpFiles: + commands: + scp-elastic-2: + command: + "logger -s \"Copying files to $IP (Elastic2)...\" 2>> $LogFile && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/elastic-stack-installation.sh $Ec2User@$IP:~/elastic-stack-installation.sh && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/config.yml $Ec2User@$IP:~/config.yml" + env: + Ec2User: !Ref Ec2User + IP: !FindInMap [Node2NameAndIP, Elastic2, IP] + LogFile: !Ref LogFile InstallELK: commands: install-elastic: - command: "bash ~/elastic-stack-installation.sh -e -c -n $NodeName" + command: + "logger -s \"Installing elastic... \" 2>> $LogFile && \ + bash ~/elastic-stack-installation.sh -e -c -n $NodeName" env: + LogFile: !Ref LogFile NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] cwd: "~" - test: "test ! -e ~/certs.tar" + test: + "test ! -e ~/certs.tar && \ + test ! -e /etc/elasticsearch.yml" install-kibana: - command: "bash ~/elastic-stack-installation.sh -k -c -n $NodeName" + command: + "logger -s \"Installing kibana... \" 2>> $LogFile && \ + bash ~/elastic-stack-installation.sh -k -n $NodeName" env: + LogFile: !Ref LogFile NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] cwd: "~" + test: "test ! -e /etc/kibana/kibana.yml" Properties: ImageId: Fn::FindInMap: @@ -333,25 +378,12 @@ Resources: UserData: Fn::Base64: !Sub | #!/bin/bash -xe - scp -i ${AWS:} - /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets SSHConfigSet,Elastic1InstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic1 --region ${AWS::Region} DependsOn: GatewayToInternet Elastic2: Type: AWS::EC2::Instance - Metadata: - 'AWS::CloudFormation::Init': - configSets: - SSHConfigSet: - - SshKey - SshKey: - commands: - paste-ssh-pub-key: - command: "cat $PubKey >> /home/$Ec2User/.ssh/authorized_keys" - env: - PubKey: !Ref SshPubKey - Ec2User: !Ref Ec2User Properties: ImageId: Fn::FindInMap: @@ -376,9 +408,4 @@ Resources: Tags: - Key: Name Value: Elastic2 - UserData: - Fn::Base64: !Sub | - #!/bin/bash -xe - /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic2 --configsets SSHConfigSet --region ${AWS::Region} - /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic2 --region ${AWS::Region} DependsOn: GatewayToInternet From 304a60bbb778aff54e0119dd03aa7d23e7a1645b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Wed, 10 Feb 2021 18:22:36 +0100 Subject: [PATCH 10/51] Add test_file.sh --- distributed/unattended/templates/test_files.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 distributed/unattended/templates/test_files.sh diff --git a/distributed/unattended/templates/test_files.sh b/distributed/unattended/templates/test_files.sh new file mode 100644 index 0000000..7b23675 --- /dev/null +++ b/distributed/unattended/templates/test_files.sh @@ -0,0 +1,18 @@ +#!/bin/bash +CONFIG_FILE={{src}}/config.yml +ELASTIC_INSTALL_SCRIPT={{src}}/elastic-stack-installation.sh + +find=0 +while [ $find -eq 0 ] +do + if [[ -f "$CONFIG_FILE" ]] && [[ -f "$ELASTIC_INSTALL_SCRIPT" ]] + then + find=1 + logger -s "Files found. Moving them to /root..." 2>> {{log_file}} + mv $CONFIG_FILE {{dst}}/config.yml + mv $ELASTIC_INSTALL_SCRIPT {{dst}}/elastic-stack-installation.sh + else + logger -s "Files not found. Sleeping 10 seconds..." 2>> {{log_file}} + sleep 10 + fi +done From ce65f990149c511e9ae7ade7e38a650340079cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Wed, 10 Feb 2021 18:23:55 +0100 Subject: [PATCH 11/51] Fix dst log --- distributed/unattended/templates/test_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/unattended/templates/test_files.sh b/distributed/unattended/templates/test_files.sh index 7b23675..1894b2c 100644 --- a/distributed/unattended/templates/test_files.sh +++ b/distributed/unattended/templates/test_files.sh @@ -8,7 +8,7 @@ do if [[ -f "$CONFIG_FILE" ]] && [[ -f "$ELASTIC_INSTALL_SCRIPT" ]] then find=1 - logger -s "Files found. Moving them to /root..." 2>> {{log_file}} + logger -s "Files found. Moving them to {{dst}}..." 2>> {{log_file}} mv $CONFIG_FILE {{dst}}/config.yml mv $ELASTIC_INSTALL_SCRIPT {{dst}}/elastic-stack-installation.sh else From babec26c599e0889da2e491b31bb24c95ab747b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Wed, 10 Feb 2021 18:36:36 +0100 Subject: [PATCH 12/51] Fix file type --- distributed/unattended/templates/test_files.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distributed/unattended/templates/test_files.sh b/distributed/unattended/templates/test_files.sh index 1894b2c..8a06c2c 100644 --- a/distributed/unattended/templates/test_files.sh +++ b/distributed/unattended/templates/test_files.sh @@ -1,15 +1,15 @@ #!/bin/bash -CONFIG_FILE={{src}}/config.yml +CERTS_FILE={{src}}/certs.tar ELASTIC_INSTALL_SCRIPT={{src}}/elastic-stack-installation.sh find=0 while [ $find -eq 0 ] do - if [[ -f "$CONFIG_FILE" ]] && [[ -f "$ELASTIC_INSTALL_SCRIPT" ]] + if [[ -f "$CERTS_FILE" ]] && [[ -f "$ELASTIC_INSTALL_SCRIPT" ]] then find=1 logger -s "Files found. Moving them to {{dst}}..." 2>> {{log_file}} - mv $CONFIG_FILE {{dst}}/config.yml + mv $CERTS_FILE {{dst}}/certs.tar mv $ELASTIC_INSTALL_SCRIPT {{dst}}/elastic-stack-installation.sh else logger -s "Files not found. Sleeping 10 seconds..." 2>> {{log_file}} From 6eaafc052545a2aeb7279facb904d4e1d77185bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Wed, 10 Feb 2021 19:20:47 +0100 Subject: [PATCH 13/51] template.yml: upgrade Elastic nodes installation --- distributed/unattended/template.yml | 193 +++++++++++++++++++++------- 1 file changed, 150 insertions(+), 43 deletions(-) diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml index 9bc5ac4..80f375a 100644 --- a/distributed/unattended/template.yml +++ b/distributed/unattended/template.yml @@ -37,6 +37,7 @@ Parameters: Description: AWS Cloudformation Stack SSH Private Key Type: String NoEcho: 'True' + ConstraintDescription: Do not copy "BEGIN RSA PRIVATE KEY" and "END RSA PRIVATE KEY" lines # Log LogFile: @@ -259,8 +260,10 @@ Resources: - SshConfig - DownloadFile - ConfigFile - - ScpFiles - - InstallELK + - InstallElastic + - InstallKibana + DeployFiles: + - ScpFiles SshConfig: files: /tmp/ssh.key: @@ -272,27 +275,30 @@ Resources: owner: root group: root commands: - move-ssh-key: - command: - "mv /tmp/ssh.key ~/.ssh" - test: "test ! -e ~/.ssh/ssh.key" write-logs: - command: "logger -s \"Created ssh.key in /root/.ssh... \" 2>> $LogFile" + command: "logger -s \"Creating ssh.key in ~/.ssh... \" 2>> $LogFile" env: LogFile: !Ref LogFile - test: "test -e ~/.ssh/ssh.key" + move-ssh-key: + command: + "mv /tmp/ssh.key ~/.ssh" DownloadFile: commands: + write-log: + command: "logger -s \"Downloading elastic-stack-installation.sh... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile download-elk-installation-script: - command: - "logger -s \"Downloading elastic-stack-installation.sh... \" 2>> $LogFile && \ - curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" + command: "curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" env: - LogFile: !Ref LogFile WazuhVersion: !Ref WazuhVersion cwd: "~" - test: "test ! -e ~/elastic-stack-installation.sh" ConfigFile: + commands: + write-log: + command: "logger -s \"Creating custom config.yml... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile files: /root/config.yml: source: !Sub >- @@ -312,45 +318,44 @@ Resources: wazuh_worker_name: !FindInMap [Node2NameAndIP, WorkerNode, Name] kibana_ip: !FindInMap [Node2NameAndIP, Elastic1, IP] wazuh_master_server_IP: !FindInMap [Node2NameAndIP, MasterNode, IP] - commands: + InstallElastic: + commands: write-log: - command: "logger -s \"Creating custom config.yml... \" 2>> $LogFile" + command: "logger -s \"Installing elastic... \" 2>> $LogFile" env: LogFile: !Ref LogFile - test: "test -e ~/config.yml" - ScpFiles: - commands: - scp-elastic-2: - command: - "logger -s \"Copying files to $IP (Elastic2)...\" 2>> $LogFile && \ - scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/elastic-stack-installation.sh $Ec2User@$IP:~/elastic-stack-installation.sh && \ - scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/config.yml $Ec2User@$IP:~/config.yml" - env: - Ec2User: !Ref Ec2User - IP: !FindInMap [Node2NameAndIP, Elastic2, IP] - LogFile: !Ref LogFile - InstallELK: - commands: install-elastic: - command: - "logger -s \"Installing elastic... \" 2>> $LogFile && \ - bash ~/elastic-stack-installation.sh -e -c -n $NodeName" + command: "bash ~/elastic-stack-installation.sh -e -c -n $NodeName" env: - LogFile: !Ref LogFile NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] cwd: "~" - test: - "test ! -e ~/certs.tar && \ - test ! -e /etc/elasticsearch.yml" - install-kibana: - command: - "logger -s \"Installing kibana... \" 2>> $LogFile && \ - bash ~/elastic-stack-installation.sh -k -n $NodeName" + ignoreErrors: "true" + InstallKibana: + commands: + write-log: + command: "logger -s \"Installing kibana... \" 2>> $LogFile" env: LogFile: !Ref LogFile + install-kibana: + command: "bash ~/elastic-stack-installation.sh -k -n $NodeName" + env: NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] cwd: "~" - test: "test ! -e /etc/kibana/kibana.yml" + ignoreErrors: "true" + ScpFiles: + commands: + write-log: + command: "logger -s \"Copying files to $IP (Elastic2)...\" 2>> $LogFile" + env: + LogFile: !Ref LogFile + IP: !FindInMap [Node2NameAndIP, Elastic2, IP] + scp-elastic-2: + command: + "scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/elastic-stack-installation.sh $Ec2User@$IP:~/elastic-stack-installation.sh && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$IP:~/certs.tar" + env: + Ec2User: !Ref Ec2User + IP: !FindInMap [Node2NameAndIP, Elastic2, IP] Properties: ImageId: Fn::FindInMap: @@ -378,12 +383,45 @@ Resources: UserData: Fn::Base64: !Sub | #!/bin/bash -xe - /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet,DeployFiles --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic1 --region ${AWS::Region} DependsOn: GatewayToInternet Elastic2: Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + Elastic2InstallationSet: + - TestFiles + - InstallElastic + TestFiles: + files: + /root/test_files.sh: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/test_files.sh + mode: '000700' + owner: root + group: root + context: + src: !Sub /home/${Ec2User} + dst: /root + log_file: !Ref LogFile + commands: + move_files: + command: "bash ~/test_files.sh" + InstallElastic: + commands: + write-log: + command: "logger -s \"Installing elastic... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + install-elastic: + command: "bash ~/elastic-stack-installation.sh -e -n $NodeName" + env: + NodeName: !FindInMap [Node2NameAndIP, Elastic2, Name] + cwd: "~" + ignoreErrors: "true" Properties: ImageId: Fn::FindInMap: @@ -408,4 +446,73 @@ Resources: Tags: - Key: Name Value: Elastic2 - DependsOn: GatewayToInternet + UserData: + Fn::Base64: !Sub | + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic2 --configsets Elastic2InstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic2 --region ${AWS::Region} + DependsOn: Elastic1 + + Elastic3: + Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + Elastic3InstallationSet: + - TestFiles + - InstallElastic + TestFiles: + files: + /root/test_files.sh: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/test_files.sh + mode: '000700' + owner: root + group: root + context: + src: !Sub /home/${Ec2User} + dst: /root + log_file: !Ref LogFile + commands: + move_files: + command: "bash ~/test_files.sh" + InstallElastic: + commands: + write-log: + command: "logger -s \"Installing elastic... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + install-elastic: + command: "bash ~/elastic-stack-installation.sh -e -n $NodeName" + env: + NodeName: !FindInMap [Node2NameAndIP, Elastic3, Name] + cwd: "~" + ignoreErrors: "true" + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - Ref: "ElasticSecurityGroup" + SubnetId: + Ref: "SubnetElasticsearch" + PrivateIpAddress: !FindInMap [Node2NameAndIP, Elastic3, IP] + Tags: + - Key: Name + Value: Elastic3 + UserData: + Fn::Base64: !Sub | + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic3 --configsets Elastic3InstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic3 --region ${AWS::Region} + DependsOn: Elastic1 From d3e29649c34ce2adc04b2d8364b3588546088808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 11 Feb 2021 12:54:46 +0100 Subject: [PATCH 14/51] Change script name and only check certs.tar --- .../unattended/templates/check_certs.sh | 16 ++++++++++++++++ distributed/unattended/templates/test_files.sh | 18 ------------------ 2 files changed, 16 insertions(+), 18 deletions(-) create mode 100644 distributed/unattended/templates/check_certs.sh delete mode 100644 distributed/unattended/templates/test_files.sh diff --git a/distributed/unattended/templates/check_certs.sh b/distributed/unattended/templates/check_certs.sh new file mode 100644 index 0000000..1730edb --- /dev/null +++ b/distributed/unattended/templates/check_certs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +CERTS_FILE={{src}}/certs.tar +find=0 +while [ $find -eq 0 ] +do + if [[ -f "$CERTS_FILE" ]] + then + find=1 + logger -s "Cert files found. Moving them to {{dst}}..." 2>> {{log_file}} + mv $CERTS_FILE {{dst}}/certs.tar + else + logger -s "Cert files not found. Sleeping 10 seconds..." 2>> {{log_file}} + sleep 10 + fi +done diff --git a/distributed/unattended/templates/test_files.sh b/distributed/unattended/templates/test_files.sh deleted file mode 100644 index 8a06c2c..0000000 --- a/distributed/unattended/templates/test_files.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -CERTS_FILE={{src}}/certs.tar -ELASTIC_INSTALL_SCRIPT={{src}}/elastic-stack-installation.sh - -find=0 -while [ $find -eq 0 ] -do - if [[ -f "$CERTS_FILE" ]] && [[ -f "$ELASTIC_INSTALL_SCRIPT" ]] - then - find=1 - logger -s "Files found. Moving them to {{dst}}..." 2>> {{log_file}} - mv $CERTS_FILE {{dst}}/certs.tar - mv $ELASTIC_INSTALL_SCRIPT {{dst}}/elastic-stack-installation.sh - else - logger -s "Files not found. Sleeping 10 seconds..." 2>> {{log_file}} - sleep 10 - fi -done From 15762f29e6018c3c0c2a2a482294c5894d41382b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 11 Feb 2021 13:46:39 +0100 Subject: [PATCH 15/51] remove log file --- distributed/unattended/templates/check_certs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributed/unattended/templates/check_certs.sh b/distributed/unattended/templates/check_certs.sh index 1730edb..a371557 100644 --- a/distributed/unattended/templates/check_certs.sh +++ b/distributed/unattended/templates/check_certs.sh @@ -7,10 +7,10 @@ do if [[ -f "$CERTS_FILE" ]] then find=1 - logger -s "Cert files found. Moving them to {{dst}}..." 2>> {{log_file}} + echo "Cert files found. Moving them to {{dst}}..." mv $CERTS_FILE {{dst}}/certs.tar else - logger -s "Cert files not found. Sleeping 10 seconds..." 2>> {{log_file}} + echo "Cert files not found. Sleeping 10 seconds..." sleep 10 fi done From a677aa4ac058ecb58641072180ddaa2147a1ee9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 11 Feb 2021 14:15:11 +0100 Subject: [PATCH 16/51] Check whether file already in /root --- distributed/unattended/templates/check_certs.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/distributed/unattended/templates/check_certs.sh b/distributed/unattended/templates/check_certs.sh index a371557..f922e90 100644 --- a/distributed/unattended/templates/check_certs.sh +++ b/distributed/unattended/templates/check_certs.sh @@ -4,11 +4,15 @@ CERTS_FILE={{src}}/certs.tar find=0 while [ $find -eq 0 ] do - if [[ -f "$CERTS_FILE" ]] + if [[ -f "{{dst}}/certs.tar" ]] + then + find=1 + echo "Cert files already in /root" + elif [[ -f "$CERTS_FILE" ]] then find=1 echo "Cert files found. Moving them to {{dst}}..." - mv $CERTS_FILE {{dst}}/certs.tar + mv $CERTS_FILE {{dst}} else echo "Cert files not found. Sleeping 10 seconds..." sleep 10 From c6501423d92f055161ad3f7fa382c6341cf0b9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Thu, 11 Feb 2021 14:52:28 +0100 Subject: [PATCH 17/51] Upgrade template.yml --- distributed/unattended/template.yml | 357 +++++++++++++++++++++++++--- 1 file changed, 318 insertions(+), 39 deletions(-) diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml index 80f375a..7799733 100644 --- a/distributed/unattended/template.yml +++ b/distributed/unattended/template.yml @@ -275,7 +275,7 @@ Resources: owner: root group: root commands: - write-logs: + write-log: command: "logger -s \"Creating ssh.key in ~/.ssh... \" 2>> $LogFile" env: LogFile: !Ref LogFile @@ -320,42 +320,42 @@ Resources: wazuh_master_server_IP: !FindInMap [Node2NameAndIP, MasterNode, IP] InstallElastic: commands: - write-log: - command: "logger -s \"Installing elastic... \" 2>> $LogFile" - env: - LogFile: !Ref LogFile install-elastic: - command: "bash ~/elastic-stack-installation.sh -e -c -n $NodeName" + command: "bash ~/elastic-stack-installation.sh -d -e -c -n $NodeName | logger -s 2>> $LogFile" env: NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] + LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" InstallKibana: commands: - write-log: - command: "logger -s \"Installing kibana... \" 2>> $LogFile" - env: - LogFile: !Ref LogFile install-kibana: - command: "bash ~/elastic-stack-installation.sh -k -n $NodeName" + command: + "bash ~/elastic-stack-installation.sh -d -k -n $NodeName >> | logger -s 2>> $LogFile" env: NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] + LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" ScpFiles: commands: write-log: - command: "logger -s \"Copying files to $IP (Elastic2)...\" 2>> $LogFile" + command: "logger -s \"Copying files to all nodes...\" 2>> $LogFile" env: LogFile: !Ref LogFile IP: !FindInMap [Node2NameAndIP, Elastic2, IP] - scp-elastic-2: + scp-certs: command: - "scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/elastic-stack-installation.sh $Ec2User@$IP:~/elastic-stack-installation.sh && \ - scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$IP:~/certs.tar" + "scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$Elastic2IP:~/certs.tar && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$Elastic3IP:~/certs.tar && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$WazuhMasterIP:~/certs.tar && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$WazuhWorkerIP:~/certs.tar" env: Ec2User: !Ref Ec2User - IP: !FindInMap [Node2NameAndIP, Elastic2, IP] + Elastic2IP: !FindInMap [Node2NameAndIP, Elastic2, IP] + Elastic3IP: !FindInMap [Node2NameAndIP, Elastic3, IP] + WazuhMasterIP: !FindInMap [Node2NameAndIP, MasterNode, IP] + WazuhWorkerIP: !FindInMap [Node2NameAndIP, WorkerNode, IP] Properties: ImageId: Fn::FindInMap: @@ -382,9 +382,29 @@ Resources: Value: Elastic1 UserData: Fn::Base64: !Sub | + Content-Type: multipart/mixed; boundary="//" + MIME-Version: 1.0 + + --// + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="cloud-config.txt" + + #cloud-config + cloud_final_modules: + - [scripts-user, always] + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="userdata.txt" + #!/bin/bash -xe /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet,DeployFiles --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic1 --region ${AWS::Region} + --// DependsOn: GatewayToInternet Elastic2: @@ -393,13 +413,25 @@ Resources: 'AWS::CloudFormation::Init': configSets: Elastic2InstallationSet: - - TestFiles + - DownloadFile + - CheckCerts - InstallElastic - TestFiles: + DownloadFile: + commands: + write-log: + command: "logger -s \"Downloading elastic-stack-installation.sh... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + download-elk-installation-script: + command: "curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" + env: + WazuhVersion: !Ref WazuhVersion + cwd: "~" + CheckCerts: files: - /root/test_files.sh: + /root/check_certs.sh: source: !Sub >- - https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/test_files.sh + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/check_certs.sh mode: '000700' owner: root group: root @@ -409,17 +441,15 @@ Resources: log_file: !Ref LogFile commands: move_files: - command: "bash ~/test_files.sh" + command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" + test: "test ! -e ~/certs.tar" InstallElastic: commands: - write-log: - command: "logger -s \"Installing elastic... \" 2>> $LogFile" - env: - LogFile: !Ref LogFile install-elastic: - command: "bash ~/elastic-stack-installation.sh -e -n $NodeName" + command: "bash ~/elastic-stack-installation.sh -e -n $NodeName | logger -s 2>> $LogFile" env: NodeName: !FindInMap [Node2NameAndIP, Elastic2, Name] + LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" Properties: @@ -448,9 +478,30 @@ Resources: Value: Elastic2 UserData: Fn::Base64: !Sub | + Content-Type: multipart/mixed; boundary="//" + MIME-Version: 1.0 + + --// + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="cloud-config.txt" + + #cloud-config + cloud_final_modules: + - [scripts-user, always] + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="userdata.txt" + + #!/bin/bash -xe /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic2 --configsets Elastic2InstallationSet --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic2 --region ${AWS::Region} - DependsOn: Elastic1 + --// + DependsOn: GatewayToInternet Elastic3: Type: AWS::EC2::Instance @@ -458,33 +509,44 @@ Resources: 'AWS::CloudFormation::Init': configSets: Elastic3InstallationSet: - - TestFiles + - DownloadFile + - CheckCerts - InstallElastic - TestFiles: + DownloadFile: + commands: + write-log: + command: "logger -s \"Downloading elastic-stack-installation.sh... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + download-elk-installation-script: + command: "curl -so ~/elastic-stack-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/elastic-stack-installation.sh" + env: + WazuhVersion: !Ref WazuhVersion + cwd: "~" + CheckCerts: files: - /root/test_files.sh: + /root/check_certs.sh: source: !Sub >- - https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/test_files.sh + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/check_certs.sh mode: '000700' owner: root group: root context: src: !Sub /home/${Ec2User} dst: /root - log_file: !Ref LogFile commands: move_files: - command: "bash ~/test_files.sh" - InstallElastic: - commands: - write-log: - command: "logger -s \"Installing elastic... \" 2>> $LogFile" + command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" env: LogFile: !Ref LogFile + test: "test ! -e ~/certs.tar" + InstallElastic: + commands: install-elastic: - command: "bash ~/elastic-stack-installation.sh -e -n $NodeName" + command: "bash ~/elastic-stack-installation.sh -e -n $NodeName | logger -s 2>> $LogFile" env: NodeName: !FindInMap [Node2NameAndIP, Elastic3, Name] + LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" Properties: @@ -513,6 +575,223 @@ Resources: Value: Elastic3 UserData: Fn::Base64: !Sub | + Content-Type: multipart/mixed; boundary="//" + MIME-Version: 1.0 + + --// + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="cloud-config.txt" + + #cloud-config + cloud_final_modules: + - [scripts-user, always] + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="userdata.txt" + + #!/bin/bash -xe /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic3 --configsets Elastic3InstallationSet --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic3 --region ${AWS::Region} - DependsOn: Elastic1 + --// + DependsOn: GatewayToInternet + + WazuhMaster: + Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + WazuhMasterInstallationSet: + - DownloadFile + - CheckCerts + - InstallWazuh + DownloadFile: + commands: + write-log: + command: "logger -s \"Downloading wazuh-server-installation.sh... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + download-elk-installation-script: + command: "curl -so ~/wazuh-server-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/wazuh-server-installation.sh" + env: + WazuhVersion: !Ref WazuhVersion + cwd: "~" + CheckCerts: + files: + /root/check_certs.sh: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/check_certs.sh + mode: '000700' + owner: root + group: root + context: + src: !Sub /home/${Ec2User} + dst: /root + commands: + move_files: + command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" + env: + LogFile: !Ref LogFile + test: "test ! -e ~/certs.tar" + InstallWazuh: + commands: + install-wazuh: + command: "bash ~/wazuh-server-installation.sh -n $NodeName >> $LogFile" + env: + NodeName: !FindInMap [Node2NameAndIP, MasterNode, Name] + LogFile: !Ref LogFile + cwd: "~" + ignoreErrors: "true" + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - Ref: "WazuhSecurityGroup" + SubnetId: + Ref: "SubnetWazuh" + PrivateIpAddress: !FindInMap [Node2NameAndIP, MasterNode, IP] + Tags: + - Key: Name + Value: WazuhMaster + UserData: + Fn::Base64: !Sub | + Content-Type: multipart/mixed; boundary="//" + MIME-Version: 1.0 + + --// + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="cloud-config.txt" + + #cloud-config + cloud_final_modules: + - [scripts-user, always] + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="userdata.txt" + + #!/bin/bash -xe + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource WazuhMaster --configsets WazuhMasterInstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource WazuhMaster --region ${AWS::Region} + --// + DependsOn: GatewayToInternet + + WazuhWorker: + Type: AWS::EC2::Instance + Metadata: + 'AWS::CloudFormation::Init': + configSets: + WazuhWorkerInstallationSet: + - DownloadFile + - CheckCerts + - InstallWazuh + DownloadFile: + commands: + write-log: + command: "logger -s \"Downloading wazuh-server-installation.sh... \" 2>> $LogFile" + env: + LogFile: !Ref LogFile + download-elk-installation-script: + command: "curl -so ~/wazuh-server-installation.sh https://raw.githubusercontent.com/wazuh/wazuh-documentation/v$WazuhVersion/resources/open-distro/unattended-installation/distributed/wazuh-server-installation.sh" + env: + WazuhVersion: !Ref WazuhVersion + cwd: "~" + CheckCerts: + files: + /root/check_certs.sh: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/check_certs.sh + mode: '000700' + owner: root + group: root + context: + src: !Sub /home/${Ec2User} + dst: /root + log_file: !Ref LogFile + commands: + move_files: + command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" + env: + LogFile: !Ref LogFile + test: "test ! -e ~/certs.tar" + InstallWazuh: + commands: + install-wazuh: + command: "bash ~/wazuh-server-installation.sh -n $NodeName | logger -s 2>> $LogFile" + env: + NodeName: !FindInMap [Node2NameAndIP, WorkerNode, Name] + LogFile: !Ref LogFile + cwd: "~" + ignoreErrors: "true" + Properties: + ImageId: + Fn::FindInMap: + - AWSRegionArch2AMI + - Ref: AWS::Region + - Fn::FindInMap: + - AWSInstanceType2Arch + - Ref: InstanceType + - Arch + InstanceType: + Ref: InstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - Ref: "WazuhSecurityGroup" + SubnetId: + Ref: "SubnetWazuh" + PrivateIpAddress: !FindInMap [Node2NameAndIP, WorkerNode, IP] + Tags: + - Key: Name + Value: WazuhWorker + UserData: + Fn::Base64: !Sub | + Content-Type: multipart/mixed; boundary="//" + MIME-Version: 1.0 + + --// + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="cloud-config.txt" + + #cloud-config + cloud_final_modules: + - [scripts-user, always] + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="userdata.txt" + + #!/bin/bash -xe + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource WazuhWorker --configsets WazuhWorkerInstallationSet --region ${AWS::Region} + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource WazuhWorker --region ${AWS::Region} + --// + DependsOn: GatewayToInternet + From 7afab6f62ef9d338b2613ac145b18925f3978668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 12 Feb 2021 12:39:51 +0100 Subject: [PATCH 18/51] Add check_ports.sh --- .../unattended/templates/check_ports.sh | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 distributed/unattended/templates/check_ports.sh diff --git a/distributed/unattended/templates/check_ports.sh b/distributed/unattended/templates/check_ports.sh new file mode 100644 index 0000000..e8ae92d --- /dev/null +++ b/distributed/unattended/templates/check_ports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +USER=$1 +IP=$2 +PORT=$3 + +open=0 +while [ $open -eq 0 ] +do + ssh -i ~/.ssh/ssh.key $USER@$IP "netstat -tulnp" | grep $PORT + if [[ $? -eq 0 ]] # Open + then + open=1 + echo "Port $PORT open in $IP" + else # Close + echo "Port $PORT close in $IP. Sleeping 10 seconds..." + sleep 10 + fi +done + From e2582a7fd812359afb79ab4d43b54888dab9203a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 12 Feb 2021 12:43:26 +0100 Subject: [PATCH 19/51] Redirect output to /dev/null --- distributed/unattended/templates/check_ports.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distributed/unattended/templates/check_ports.sh b/distributed/unattended/templates/check_ports.sh index e8ae92d..3a68d92 100644 --- a/distributed/unattended/templates/check_ports.sh +++ b/distributed/unattended/templates/check_ports.sh @@ -7,7 +7,7 @@ PORT=$3 open=0 while [ $open -eq 0 ] do - ssh -i ~/.ssh/ssh.key $USER@$IP "netstat -tulnp" | grep $PORT + ssh -i ~/.ssh/ssh.key $USER@$IP "sudo netstat -tulnp" | grep $PORT &> /dev/null if [[ $? -eq 0 ]] # Open then open=1 @@ -18,3 +18,4 @@ do fi done + From 7600bac12eda4f4cec4dce35f032f5fb7a6cc36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa?= Date: Fri, 12 Feb 2021 18:21:55 +0100 Subject: [PATCH 20/51] Upgrade template.yml --- distributed/unattended/template.yml | 142 ++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 19 deletions(-) diff --git a/distributed/unattended/template.yml b/distributed/unattended/template.yml index 7799733..a7f0f54 100644 --- a/distributed/unattended/template.yml +++ b/distributed/unattended/template.yml @@ -55,6 +55,13 @@ Parameters: Description: Wazuh version Default: "4.0.4" Type: String + + # WazuhCluster + WazuhClusterKey: + Description: Key that will be used to encrypt communication between Wazuh cluster nodes + Type: String + NoEcho: 'True' + ConstraintDescription: Run "openssl rand -hex 16" Mappings: Node2NameAndIP: @@ -228,6 +235,24 @@ Resources: FromPort: 22 ToPort: 22 CidrIp: 0.0.0.0/0 + - IpProtocol: icmp + FromPort: -1 + ToPort: -1 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 1514 + ToPort: 1516 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + - IpProtocol: tcp + FromPort: 55000 + ToPort: 55000 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR Tags: - Key: Name Value: !Sub '${AWS::StackName}-WazuhSecurityGroup' @@ -242,10 +267,28 @@ Resources: FromPort: 22 ToPort: 22 CidrIp: 0.0.0.0/0 + - IpProtocol: icmp + FromPort: -1 + ToPort: -1 + CidrIp: 0.0.0.0/0 - IpProtocol: tcp FromPort: 443 ToPort: 443 CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 9200 + ToPort: 9400 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR + - IpProtocol: tcp + FromPort: 5000 + ToPort: 5000 + CidrIp: !FindInMap + - Subnet2CIDR + - WazuhVpc + - CIDR Tags: - Key: Name Value: !Sub '${AWS::StackName}-ElasticSecurityGroup' @@ -262,8 +305,10 @@ Resources: - ConfigFile - InstallElastic - InstallKibana - DeployFiles: - - ScpFiles + DeployAndCheck: + - ScpAndNetstat + Cluster: + - SecurityAdmin SshConfig: files: /tmp/ssh.key: @@ -330,32 +375,54 @@ Resources: InstallKibana: commands: install-kibana: - command: - "bash ~/elastic-stack-installation.sh -d -k -n $NodeName >> | logger -s 2>> $LogFile" + command: "bash ~/elastic-stack-installation.sh -d -k -n $NodeName >> | logger -s 2>> $LogFile" env: NodeName: !FindInMap [Node2NameAndIP, Elastic1, Name] LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" - ScpFiles: + ScpAndNetstat: + files: + /root/check_ports.sh: + source: !Sub >- + https://raw.githubusercontent.com/wazuh/wazuh-cloudformation/feature-unattended-installation/distributed/unattended/templates/check_ports.sh + mode: '000700' + owner: root + group: root commands: - write-log: - command: "logger -s \"Copying files to all nodes...\" 2>> $LogFile" - env: - LogFile: !Ref LogFile - IP: !FindInMap [Node2NameAndIP, Elastic2, IP] - scp-certs: + run-commands: command: - "scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$Elastic2IP:~/certs.tar && \ + "logger -s \"Copying files to all nodes...\" 2>> $LogFile && \ + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$Elastic2IP:~/certs.tar && \ scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$Elastic3IP:~/certs.tar && \ scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$WazuhMasterIP:~/certs.tar && \ - scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$WazuhWorkerIP:~/certs.tar" + scp -i ~/.ssh/ssh.key -o StrictHostKeyChecking=no ~/certs.tar $Ec2User@$WazuhWorkerIP:~/certs.tar && \ + bash ~/check_ports.sh $Ec2User $Elastic2IP 9200 | logger -s 2>> $LogFile && \ + bash ~/check_ports.sh $Ec2User $Elastic3IP 9200 | logger -s 2>> $LogFile && \ + bash ~/check_ports.sh $Ec2User $WazuhMasterIP 1514 | logger -s 2>> $LogFile && \ + bash ~/check_ports.sh $Ec2User $WazuhWorkerIP 1514 | logger -s 2>> $LogFile 1514" env: + LogFile: !Ref LogFile Ec2User: !Ref Ec2User Elastic2IP: !FindInMap [Node2NameAndIP, Elastic2, IP] Elastic3IP: !FindInMap [Node2NameAndIP, Elastic3, IP] WazuhMasterIP: !FindInMap [Node2NameAndIP, MasterNode, IP] WazuhWorkerIP: !FindInMap [Node2NameAndIP, WorkerNode, IP] + SecurityAdmin: + commands: + run-script: + command: + "/usr/share/elasticsearch/plugins/opendistro_security/tools/securityadmin.sh \ + -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ \ + -icl \ + -nhnv \ + -cacert /etc/elasticsearch/certs/root-ca.pem \ + -cert /etc/elasticsearch/certs/admin.pem \ + -key /etc/elasticsearch/certs/admin.key \ + -h $ElasticIP" + env: + ElasticIP: !FindInMap [Node2NameAndIP, Elastic1, IP] + Properties: ImageId: Fn::FindInMap: @@ -402,7 +469,7 @@ Resources: Content-Disposition: attachment; filename="userdata.txt" #!/bin/bash -xe - /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet,DeployFiles --region ${AWS::Region} + /opt/aws/bin/cfn-init --stack ${AWS::StackName} --resource Elastic1 --configsets Elastic1InstallationSet,DeployAndCheck --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Elastic1 --region ${AWS::Region} --// DependsOn: GatewayToInternet @@ -441,8 +508,9 @@ Resources: log_file: !Ref LogFile commands: move_files: - command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" - test: "test ! -e ~/certs.tar" + command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" + env: + LogFile: !Ref LogFile InstallElastic: commands: install-elastic: @@ -539,7 +607,6 @@ Resources: command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" env: LogFile: !Ref LogFile - test: "test ! -e ~/certs.tar" InstallElastic: commands: install-elastic: @@ -609,6 +676,7 @@ Resources: - DownloadFile - CheckCerts - InstallWazuh + - ConfigureMaster DownloadFile: commands: write-log: @@ -636,7 +704,6 @@ Resources: command: "bash ~/check_certs.sh | logger -s 2>> $LogFile" env: LogFile: !Ref LogFile - test: "test ! -e ~/certs.tar" InstallWazuh: commands: install-wazuh: @@ -646,6 +713,44 @@ Resources: LogFile: !Ref LogFile cwd: "~" ignoreErrors: "true" + ConfigureMaster: + commands: + edit-ossec-conf: + command: + "xmllint --shell $OssecConf <