From 8b580286ba0cde54b187d87b30f023981b73bf6c Mon Sep 17 00:00:00 2001 From: Shane Peters Date: Tue, 15 Jan 2019 13:59:01 -0500 Subject: [PATCH] misc changes --- containers/elasticsearch | 4 ---- containers/fouroneone | 2 +- containers/ids | 3 +-- containers/kafka | 4 ++-- deploy | 15 ++++++++++----- limits => limits.dev | 0 limits.prod | 30 ++++++++++++++++++++++++++++++ 7 files changed, 44 insertions(+), 14 deletions(-) rename limits => limits.dev (100%) create mode 100644 limits.prod diff --git a/containers/elasticsearch b/containers/elasticsearch index 36a000e..67b534d 100755 --- a/containers/elasticsearch +++ b/containers/elasticsearch @@ -28,12 +28,8 @@ pip install elasticsearch-curator sed -i 's/#cluster.name: my-application/cluster.name: odin/g' /etc/elasticsearch/elasticsearch.yml sed -i 's/#node.name: node-1/node.name: node-1/g' /etc/elasticsearch/elasticsearch.yml -sed -i 's/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/g' /etc/elasticsearch/elasticsearch.yml sed -i "s/#network.host: 192.168.0.1/network.host: ${IP}/g" /etc/elasticsearch/elasticsearch.yml -sed -i 's/-Xms2g/-Xms8g/g' /etc/elasticsearch/jvm.options -sed -i 's/-Xmx2g/-Xmx8g/g' /etc/elasticsearch/jvm.options - mkdir /etc/curator/ cat >/etc/curator/delete_indices.yml </etc/hostname echo -e "${IP}\tfouroneone" >> /etc/hosts diff --git a/containers/ids b/containers/ids index 18a0abb..0fb5053 100755 --- a/containers/ids +++ b/containers/ids @@ -224,8 +224,7 @@ else critical-stack-intel pull fi -add-apt-repository -y -u ppa:oisf/suricata-stable -apt-get install -y prometheus-node-exporter suricata +apt-get install -y prometheus-node-exporter systemctl enable bro systemctl start bro diff --git a/containers/kafka b/containers/kafka index 170c256..b38573e 100755 --- a/containers/kafka +++ b/containers/kafka @@ -8,7 +8,7 @@ log() { echo -e "\t\e[96m[*]${1}\e[93m" } -export KAFKA_URL='http://apache.claz.org/kafka/0.11.0.0/kafka_2.11-0.11.0.0.tgz' +export KAFKA_URL='http://apache.claz.org/kafka/2.1.0/kafka_2.12-2.1.0.tgz' export IP=$(ip route | awk '/src/{print $9}') echo "kafka" >/etc/hostname echo "${IP}\tkafka" >> /etc/hosts @@ -22,7 +22,7 @@ apt-get install -y htop wget default-jre zookeeperd prometheus-node-exporter useradd -r -d /opt/kafka -s /bin/true kafka mkdir /var/lib/kafka && chown kafka /var/lib/kafka -wget -O /opt/kafka.tgz ${KAFKA_URL} +wget -O /opt/kafka.tgz ${KAFKA_URL} || (echo "COULDN'T DOWNLOAD KAFKA" && exit 1) tar -xzf /opt/kafka.tgz -C /opt rm /opt/kafka.tgz mv /opt/kafka_* /opt/kafka diff --git a/deploy b/deploy index c5d6a41..8be231d 100755 --- a/deploy +++ b/deploy @@ -9,16 +9,23 @@ log() { } if [ "$#" -lt 2 ]; then - log "usage: sudo ${0} ZFS_DATASET TAP_INTERFACE MGMT_INTERFACE" + log "usage: sudo ${0} " exit 1 fi +set -x + export ZPOOL=${1} export TAP=${2} export MGMT=${3} +export PROD=${4} export MGMT_IP=$(ip -o -4 a show ${MGMT} | awk '{print $4}' |cut -d '/' -f 1) export LXC='/snap/bin/lxc' -source limits +if [ ! -z $PROD ]; then + source limits.prod +else + source limits.dev +fi need_zfs() { log "ZFS dataset \"${1}\" wasn't found. I suggest you create it and restart the deploy." @@ -54,15 +61,13 @@ setup_lxd() { ZPOOL=${1} log "Deploying lxd on ${ZPOOL}." lxd init --auto --storage-backend=zfs --storage-pool="${ZPOOL}" - ${LXC} network create odinbr0 dns.domain="odin" ipv4.address="10.13.37.1/24" ipv4.nat=true ipv6.address=none - ${LXC} network attach-profile odinbr0 default eth0 chown -R ${SUDO_USER}:${SUDO_USER} ${HOME}/.config/lxc } setup_containers() { export BROFACE=${1} - # Order is important - start the pipeline (kafka) first, fsf is before bro because it bro submits files to it, etc... + # Order is important - start the pipeline (kafka) first, fsf is before ids because it bro submits files to it, etc... export CONTAINERS="kafka elasticsearch logstash kibana fsf ids rita prometheus fouroneone" for CON in ${CONTAINERS}; do diff --git a/limits b/limits.dev similarity index 100% rename from limits rename to limits.dev diff --git a/limits.prod b/limits.prod new file mode 100644 index 0000000..549d8aa --- /dev/null +++ b/limits.prod @@ -0,0 +1,30 @@ +export CPU_ids="6" +export CPU_kafka="2" +export CPU_elasticsearch="4" +export CPU_logstash="1" +export CPU_kibana="2" +export CPU_fouroneone="2" +export CPU_rita="4" +export CPU_fsf="4" +export CPU_prometheus="2" + +export MEM_ids="8192MB" +export MEM_kafka="1024MB" +export MEM_elasticsearch="16384MB" +export MEM_logstash="1024MB" +export MEM_kibana="1024MB" +export MEM_fouroneone="2048MB" +export MEM_rita="8192MB" +export MEM_fsf="4096MB" +export MEM_prometheus="2048MB" + +export DISK_ids="20GB" +export DISK_kafka="20GB" +export DISK_elasticsearch="250GB" +export DISK_logstash="10GB" +export DISK_kibana="10GB" +export DISK_fouroneone="10GB" +export DISK_rita="120GB" +export DISK_fsf="80GB" +export DISK_prometheus="60GB" +