From b89ba1ad5a012e6492a07d6b4313f871eeff7756 Mon Sep 17 00:00:00 2001 From: Shane Peters Date: Fri, 11 Jan 2019 10:45:03 -0500 Subject: [PATCH] initial commit --- README.md | 2 + containers/elasticsearch | 97 +++ containers/fouroneone | 129 ++++ containers/fsf | 119 ++++ containers/graylog | 22 + containers/ids | 232 +++++++ containers/kafka | 71 +++ containers/kibana | 36 ++ containers/logstash | 235 +++++++ containers/prometheus | 380 +++++++++++ containers/rita | 50 ++ deploy | 155 +++++ destroy | 46 ++ extra/bro_api.py | 18 + extra/grafana_dashboards.json | 1117 +++++++++++++++++++++++++++++++++ extra/kibana_dashboards.json | 550 ++++++++++++++++ extra/squid.conf | 113 ++++ limits | 36 ++ 18 files changed, 3408 insertions(+) create mode 100644 README.md create mode 100644 containers/elasticsearch create mode 100644 containers/fouroneone create mode 100644 containers/fsf create mode 100644 containers/graylog create mode 100644 containers/ids create mode 100644 containers/kafka create mode 100644 containers/kibana create mode 100644 containers/logstash create mode 100644 containers/prometheus create mode 100644 containers/rita create mode 100755 deploy create mode 100755 destroy create mode 100644 extra/bro_api.py create mode 100644 extra/grafana_dashboards.json create mode 100644 extra/kibana_dashboards.json create mode 100644 extra/squid.conf create mode 100644 limits diff --git a/README.md b/README.md new file mode 100644 index 0000000..97b7271 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# What is ist +Odin is an all-in-one NSM diff --git a/containers/elasticsearch b/containers/elasticsearch new file mode 100644 index 0000000..36a000e --- /dev/null +++ b/containers/elasticsearch @@ -0,0 +1,97 @@ +#!/bin/bash +# Configure Elasticsearch for Odin +# + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export IP=$(ip route | awk '/src/{print $9}') +echo "elasticsearch" >/etc/hostname +echo -e "${IP}\telasticsearch" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget default-jre python-pip + +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" >/etc/apt/sources.list.d/elastic-5.x.list +apt-get update +apt-get install elasticsearch -y +apt-get clean +pip install elasticsearch-curator + +sed -i 's/#cluster.name: my-application/cluster.name: odin/g' /etc/elasticsearch/elasticsearch.yml +sed -i 's/#node.name: node-1/node.name: node-1/g' /etc/elasticsearch/elasticsearch.yml +sed -i 's/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/g' /etc/elasticsearch/elasticsearch.yml +sed -i "s/#network.host: 192.168.0.1/network.host: ${IP}/g" /etc/elasticsearch/elasticsearch.yml + +sed -i 's/-Xms2g/-Xms8g/g' /etc/elasticsearch/jvm.options +sed -i 's/-Xmx2g/-Xmx8g/g' /etc/elasticsearch/jvm.options + +mkdir /etc/curator/ +cat >/etc/curator/delete_indices.yml <- + Delete indices older than 30 days for odin- prefixed indices. + options: + ignore_empty_list: True + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: odin- + exclude: + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: 30 + exclude: +EOF + +cat >/etc/curator/curator.yml< /etc/cron.daily/curator </etc/hostname +echo -e "${IP}\tfouroneone" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove -y snapd lxd +apt-get upgrade -y +apt-get install -y htop wget unzip apache2 libapache2-mod-php php-xml php7.0-mbstring php7.0-sqlite php7.0-curl sqlite3 +apt-get clean + +curl -sS https://getcomposer.org/installer | sudo php -- --install-dir=/usr/local/bin --filename=composer +a2enmod rewrite headers ssl +wget -O /tmp/411.conf ${CONF_411} +sed -i 's/HOSTNAME/fouroneone/g' /tmp/411.conf +mv /tmp/411.conf /etc/apache2/sites-available/411.conf + +cd /tmp +openssl genrsa -des3 -passout pass:x -out server.pass.key 2048 +openssl rsa -passin pass:x -in server.pass.key -out server.key +rm server.pass.key +openssl req -new -key server.key -out server.csr \ + -subj "/C=US/ST=Ohio/L=Lima/O=SecOps/OU=Odin/CN=fouroneone" +openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt +mv server.crt /etc/ssl/certs/411.pem +mv server.key /etc/ssl/private/411.key + +a2dissite 000-default +a2ensite 411 +systemctl restart apache2 + +wget -O /tmp/411.tgz ${FOUR11_URL} +mkdir /var/www/411; sudo tar -xzf /tmp/411.tgz -C /var/www/411 +chown -R www-data:www-data /var/www/411 +cd /var/www/411/ +sudo -u www-data composer install --no-dev --optimize-autoloader +apt-get install -y prometheus-node-exporter + +cat >config.php < [ + 'enabled' => false, + 'header' => null, + 'auto_create' => false, + 'domain' =>null, + ], + 'api' => [ + 'enabled' => true + ] +]; + +\$config['db'] = [ + 'dsn' => 'sqlite:' . realpath(__DIR__ . '/data.db'), + 'user' => 'root', + 'pass' => null, +]; + +\$config['elasticsearch'] = [ + 'alerts' => [ + 'hosts' => ['http://elasticsearch'], + 'index_hosts' => [], + 'ssl_cert' => null, + 'index' => 411, + 'date_based' => false, + 'date_interval' => null, + 'date_field' => 'alert_date', + 'date_type' => null, + 'src_url' => null, + ], + 'odin' => [ + 'hosts' => ['http://elasticsearch'], + 'index_hosts' => [], + 'ssl_cert' => null, + 'index'=> '[odin-]Y.m.d', + 'date_based' => true, + 'date_interval' => 'd', + 'date_field' => '@timestamp', + 'date_type' => null, + 'src_url' => null, + ], +]; + +\$config['graphite'] = [ + 'graphite' => [ + 'host' => null, + ], +]; + +\$config['threatexchange'] = [ + 'api_token' => null, + 'api_secret' => null, +]; + +\$config['jira'] = [ + 'host' => null, + 'user' => null, + 'pass' => null, +]; + +\$config['slack'] = [ + 'webhook_url' => null +]; +EOF + +sudo -u www-data sqlite3 data.db < db.sql +sudo -u www-data bin/migration.php +chown -R www-data:www-data /var/www/411 +cat >/etc/cron.d/411 < /dev/null 2>&1 && /var/www/411/bin/worker.php > /dev/null 2>&1 +EOF +systemctl restart cron +sudo -u www-data /var/www/411/bin/create_site.php && sudo -u www-data /var/www/411/bin/create_user.php + +echo -e "\e[0m" diff --git a/containers/fsf b/containers/fsf new file mode 100644 index 0000000..d5fd644 --- /dev/null +++ b/containers/fsf @@ -0,0 +1,119 @@ +#!/bin/bash +# Configure FSF for Odin + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export VT_KEY=99dfd41c7ff9cd406982f801f2393907678f562fb149a8e538d0680c14e0060a +export FSF_URL="https://github.com/EmersonElectricCo/fsf/archive/master.zip" +export IP=$(ip route | awk '/src/{print $9}') +echo "fsf" >/etc/hostname +echo -e "${IP}\tfsf" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y jq htop wget python-pip + +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" >/etc/apt/sources.list.d/elastic-5.x.list +apt-get update +apt-get install -y filebeat + +apt-get install -y python-yara autoconf dh-autoreconf python-dev \ + libpython2.7-stdlib python-pip libffi-dev ssdeep python-ssdeep upx unrar \ + libfuzzy-dev unzip libssl-dev net-tools cabextract python-pefile \ + python-cffi yara python-yara + +pip install czipfile hachoir-parser hachoir-core hachoir-regex \ + hachoir-metadata hachoir-subfile ConcurrentLogHandler pypdf2 xmltodict \ + rarfile pylzma oletools pyasn1_modules pyasn1 pyelftools javatools \ + requests git+https://github.com/aaronst/macholibre.git + +useradd -r -c "File Scanner" -m -d /opt/fsf -s /bin/true fsf +cd /opt/fsf +wget -O fsf.zip ${FSF_URL} +unzip fsf.zip && rm fsf.zip +touch scan.log +mv fsf-master bin && chown -R fsf:fsf bin scan.log +cd bin + +sed -i 's/FULL\/PATH\/TO\/fsf/opt\/fsf\/bin/g' /opt/fsf/bin/fsf-server/conf/config.py +sed -i 's/tmp/opt\/fsf/g' /opt/fsf/bin/fsf-server/conf/config.py + +if [ ! -z ${VT_KEY} ]; then + echo " [*] Enabling VirusTotal Checks for PE and ELF files." + sed -i "s/YOUR API KEY HERE/${VT_KEY}/g" /opt/fsf/bin/fsf-server/modules/META_VT_INSPECT.py + sed -i "s/META_PE'/META_PE', 'META_VT_INSPECT'/g" /opt/fsf/bin/fsf-server/conf/disposition.py + sed -i "s/META_ELF'/META_ELF', 'META_VT_INSPECT'/g" /opt/fsf/bin/fsf-server/conf/disposition.py +fi + +cat >/etc/logrotate.d/scanner </etc/systemd/system/fsf.service < /etc/logrotate.d/scanner < /etc/filebeat/filebeat.yml </etc/hostname +echo -e "${IP}\tlogstash" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget openjdk-8-jre-headless uuid-runtime pwgen mongodb-server prometheus-node-exporter + +echo -e "\e[0m" diff --git a/containers/ids b/containers/ids new file mode 100644 index 0000000..18a0abb --- /dev/null +++ b/containers/ids @@ -0,0 +1,232 @@ +#!/bin/bash +# Configure Elasticsearech for Odin +# + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export CRITSTACK_KEY=f9bc6af4-5cc6-4fa5-623b-b0906572d703 +export BRO_FACE=eth1 +export BRO_URL=https://www.bro.org/downloads/bro-2.5.2.tar.gz +export BROPKG_URL=https://github.com/bro/package-manager/archive/master.zip +export LIBKAFKA_URL=https://github.com/edenhill/librdkafka/archive/master.zip +export FSF_URL=https://github.com/EmersonElectricCo/fsf/archive/master.zip +export NCPU=$(grep processor /proc/cpuinfo |tail -1 |awk '/:/{print $3}') && let NCPU=$NCPU+1 +export IP=$(ip route | awk '/src/{print $9}') +echo "bro" >/etc/hostname +echo -e "${IP}\tbro" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd mdadm -y +apt-get upgrade -y +apt-get clean +apt-get install -y htop wget cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev unzip python-pip linux-headers-`uname -r` + +useradd -r -c "Bro IDS" -m -d /opt/bro -s /bin/bash bro + +cd /opt +wget -O fsf.zip ${FSF_URL} +unzip fsf.zip && rm fsf.zip +mv fsf-master/fsf-client /opt/fsf +rm -rf /opt/fsf-master /opt/master.zip +sed -i 's/127.0.0.1/fsf/g' /opt/fsf/conf/config.py +chown -R bro:bro /opt/fsf + +cd /usr/local/src +wget -O bro.tgz ${BRO_URL} +tar xzf bro.tgz +rm bro.tgz +mv bro-2* bro +cd bro +./configure --prefix=/opt/bro +make -j${NCPU} +make install + +cd ../ +wget -O librdkafka.zip ${LIBKAFKA_URL} +unzip librdkafka.zip +rm librdkafka.zip +mv librdkafka-* librdkafka +cd librdkafka +./configure && make && make install + +cd /usr/local/src/bro/aux/plugins/kafka +./configure && make && make install + +chown -R bro:bro /opt/bro +#pip install bro-pkg -- currently out of date? +cd /usr/local/src +wget -O bropkg.zip ${BROPKG_URL} +unzip bropkg.zip +rm bropkg.zip +mv package-manager* bro-pkg +cd bro-pkg +python setup.py install + +echo 'PATH="/opt/bro/bin:$PATH"' >> /etc/profile +export PATH="/opt/bro/bin:$PATH" +sudo -i -u bro env PATH=/opt/bro/bin:$PATH bro-pkg --verbose autoconfig +sudo -i -u bro env PATH=/opt/bro/bin:$PATH bro-pkg --verbose refresh --aggregate +sudo -i -u bro env PATH=/opt/bro/bin:$PATH bro-pkg --verbose install bro-af_packet-plugin --force + +sed -i 's/MailConnectionSummary = 1/MailConnectionSummary = 0/g' /opt/bro/etc/broctl.cfg +sed -i 's/MinDiskSpace = 5/MinDiskSpace = 0/g' /opt/bro/etc/broctl.cfg +sed -i 's/MailHostUpDown = 1/MailHostUpDown = 0/g' /opt/bro/etc/broctl.cfg +sed -i 's/LogRotationInterval = 3600/LogRotationInterval = 86400/g' /opt/bro/etc/broctl.cfg +sed -i 's/LogExpireInterval = 0/LogExpireInterval = 60/g' /opt/bro/etc/broctl.cfg +sed -i 's/StatsLogExpireInterval = 0/StatsLogExpireInterval = 1/g' /opt/bro/etc/broctl.cfg + +cat >> /opt/bro/share/bro/site/local.bro < /opt/bro/etc/node.cfg < /etc/network/interfaces.d/60-ids.cfg </dev/null; done + post-up echo 1 > /proc/sys/net/ipv6/conf/${BRO_FACE}/disable_ipv6 +EOF +ifup eth1 + +cat >/etc/systemd/system/bro.service </etc/hostname +echo "${IP}\tkafka" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget default-jre zookeeperd prometheus-node-exporter + +useradd -r -d /opt/kafka -s /bin/true kafka +mkdir /var/lib/kafka && chown kafka /var/lib/kafka +wget -O /opt/kafka.tgz ${KAFKA_URL} +tar -xzf /opt/kafka.tgz -C /opt +rm /opt/kafka.tgz +mv /opt/kafka_* /opt/kafka +chown -R kafka /opt/kafka + +cat >/etc/systemd/system/kafka.service<> /etc/cron.weekly/kafka_cleanup</etc/hostname +echo -e "${IP}\tkibana" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget default-jre + +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" >/etc/apt/sources.list.d/elastic-5.x.list +apt-get update +apt-get install kibana -y +apt-get clean + +sed -i 's/#server.host: "localhost"/server.host: "kibana"/g' /etc/kibana/kibana.yml +sed -i 's/#server.name: "your-hostname"/server.name: "kibana"/g' /etc/kibana/kibana.yml +sed -i 's/#elasticsearch.url: "http:\/\/localhost:9200"/elasticsearch.url: "http:\/\/elasticsearch:9200"/g' /etc/kibana/kibana.yml + +apt-get install -y prometheus-node-exporter + +systemctl enable kibana +systemctl start kibana +echo -e "\e[0m" diff --git a/containers/logstash b/containers/logstash new file mode 100644 index 0000000..a306862 --- /dev/null +++ b/containers/logstash @@ -0,0 +1,235 @@ +#!/bin/bash +# Configure Logstash for Odin +# + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export IP=$(ip route | awk '/src/{print $9}') +echo "logstash" >/etc/hostname +echo -e "${IP}\tlogstash" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget default-jre prometheus-node-exporter + +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" >/etc/apt/sources.list.d/elastic-5.x.list +apt-get update +apt-get install logstash -y +apt-get clean + +cat >/etc/logstash/conf.d/odin.conf < "kafka:9092" + topics => ["conn", "dhcp", "dns", "files", "http", "intel", "notice", "software", "ssh", "ssl", "weird", "x509", "stats", "smtp", "fsf"] + consumer_threads => 4 + decorate_events => true + codec => "json" + type => "odin" + } +} + +filter{ + mutate { + remove_field => ["[kafka][key]", "[beat]"] + } + if [id.orig_h] { + geoip { + source => "id.orig_h" + target => "geoip" + } + if ![geoip.ip] { + if [id.resp_h] { + geoip { + source => "id.resp_h" + target => "geoip" + } + } + } + } + mutate { + remove_tag => ["_geoip_lookup_failure"] + } +} + +output{ + if [type] == "odin" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "odin-%{+YYYY.MM.dd}" + template => "/etc/logstash/odin_mapping.json" + template_name => "odin-*" + template_overwrite => true + } + } + + if "fsf" in [tags] { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "fsf-%{+YYYY.MM.dd}" + template => "/etc/logstash/fsf_mapping.json" + template_name => "fsf-*" + template_overwrite => true + } + } + +} +EOF + +cat >> /etc/logstash/odinmap.gz.b64 < /etc/logstash/odin_mapping.json.gz +gunzip /etc/logstash/odin_mapping.json.gz + +cat > /etc/logstash/fsfmap.gz.b64 < /etc/logstash/fsf_mapping.json.gz +gunzip /etc/logstash/fsf_mapping.json.gz + +rm /etc/logstash/odinmap.gz.b64 +rm /etc/logstash/fsfmap.gz.b64 + +apt-get install -y prometheus-node-exporter + +systemctl enable logstash +systemctl start logstash +echo -e "\e[0m" diff --git a/containers/prometheus b/containers/prometheus new file mode 100644 index 0000000..28d96ec --- /dev/null +++ b/containers/prometheus @@ -0,0 +1,380 @@ +#!/bin/bash +# Configure FSF for Odin +# + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export IP=$(ip route | awk '/src/{print $9}') +echo "prometheus" >/etc/hostname +echo -e "${IP}\tprometheus" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget libfreetype6 fontconfig-config prometheus + +#ubuntu ships with very old version of grafana (2.x) +wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.4.1_amd64.deb +dpkg -i grafana_4.4.1_amd64.deb +apt-get -f install -y + +cat > /etc/prometheus/prometheus.yml < /var/lib/grafana/t.gz.b64 < /var/lib/grafana/grafana.db.gz +gunzip /var/lib/grafana/grafana.db.gz +rm /var/lib/grafana/t.gz.b64 +chown -R grafana:grafana /var/lib/grafana + +systemctl daemon-reload +systemctl start grafana-server +systemctl enable grafana-server +systemctl enable prometheus +systemctl restart prometheus +echo -e "\e[0m" diff --git a/containers/rita b/containers/rita new file mode 100644 index 0000000..230d8de --- /dev/null +++ b/containers/rita @@ -0,0 +1,50 @@ +#!/bin/bash +# Configure BHIS RITA for Odin +# + +echo -e "\e[93m" + +log() { + echo -e "\t\e[96m[*]${1}\e[93m" +} + +export IP=$(ip route | awk '/src/{print $9}') +echo "rita" >/etc/hostname +echo -e "${IP}\trita" >> /etc/hosts + +deluser -q --remove-home ubuntu + +apt-get update || exit 1 +apt-get --purge remove snapd lxd -y +apt-get upgrade -y +apt-get install -y htop wget unzip + +wget https://github.com/ocmdev/rita/archive/master.zip +unzip master.zip +cd rita-master +bash install.sh +cd ../ +rm -rf rita-master master.zip +systemctl enable mongod +systemctl start mongod + +sed -i 's/\/path\/to\/top\/level\/directory\//\/opt\/bro\/logs/g' /root/.rita/config.yaml +sed -i 's/PrefixForDatabase/rita-/g' /root/.rita/config.yaml +sed -i 's/UseDates: false/UseDates: true/g' /root/.rita/config.yaml +sed -i 's/#DefaultDatabase: DefaultDatabaseName/DefaultDatabase: odin/g' /root/.rita/config.yaml +sed -i 's/DirectoryMap/#DirectoryMap/g' /root/.rita/config.yaml +sed -i 's/UniqueDir/#UniqueDir/g' /root/.rita/config.yaml +sed -i 's/UniqueDir2/#UniqueDir2/g' /root/.rita/config.yaml + +cat >/etc/cron.daily/rita < >(tee -i odin_log.txt) + +log() { + echo -en "\t\e[96m[*] ${1}\e[0m\n" +} + +if [ "$#" -lt 2 ]; then + log "If you want deploy SSL-inspection proxy, provide ICAP_INTERFACE." + log "Make sure your switch is configured to mirror traffic from this port " + log "to the TAP_INTERFACE so Bro can see and utilize it's icap analyzer." + log " ** MUST USE SUDO ** " + log "usage: sudo ${0} ZFS_DATASET TAP_INTERFACE MGMT_INTERFACE [ICAP_INTERFACE]" + exit 1 +fi + +export ZPOOL=${1} +export TAP=${2} +export MGMT=${3} +export ICAP=${4} +export MGMT_IP=$(ip -o -4 a show ${MGMT} | awk '{print $4}' |cut -d '/' -f 1) + +source limits +zfs get all ${ZPOOL} >/dev/null || need_zfs ${ZPOOL} + +need_zfs() { + log "ZFS dataset \"${1}\" wasn't found. I suggest you create it and restart the deploy." + log "Here are the available disks. *** DON'T OVERWRITE YOUR OS! *** :" + ls -l /dev/disk/by-path/ |egrep -v "total|ata|part" |awk '{print $9, " ===> ", $11}' + exit 1 +} + + +setup_packages() { + apt-get remove -y snapd + apt-add-repository -y ppa:ubuntu-lxc/stable + apt-get update + apt-get upgrade -y + apt-get install -y lxd zfsutils-linux htop unzip vim jq httpie prometheus-node-exporter +} + + +setup_limits_on_profile() { + cCPU=CPU_${1} + cMEM=MEM_${1} + cDISK=DISK_${1} + + lxc profile set ${1} limits.cpu ${!cCPU} + lxc profile set ${1} limits.memory ${!cMEM} + lxc profile device set ${1} root size ${!cDISK} + return +} + + +setup_lxd() { + ZPOOL=${1} + log "Deploying lxd on ${ZPOOL}." + sed -i 's/Restart=on-failure/Restart=on-failure\nLimitMEMLOCK=infinity/g' /lib/systemd/system/lxd.service + systemctl daemon-reload + systemctl restart lxd.service + + lxd init --auto --storage-backend=zfs --storage-pool="${ZPOOL}" + lxc network create odinbr0 dns.domain="odin" ipv4.address="10.13.37.1/24" ipv4.nat=true ipv6.address=none + lxc network attach-profile odinbr0 default eth0 + chown -R ${SUDO_USER}:${SUDO_USER} ${HOME}/.config/lxc +} + + +setup_containers() { + export BROFACE=${1} + export ICAPFACE=${2} + # Order is important - start the pipeline (kafka) first, fsf is before bro because it bro submits files to it, etc... + export CONTAINERS="kafka elasticsearch graylog fsf ids rita prometheus fouroneone" + + for CON in ${CONTAINERS}; do + lxc profile copy default ${CON} + + if [[ "${CON}" == "ids" ]]; then + lxc profile device add ${CON} eth1 nic nictype=physical parent=${BROFACE} + lxc profile set ${CON} security.privileged true + elif [[ "${CON}" == "rita" ]]; then + mkdir -p /var/lib/lxd/storage-pools/default/containers/ids/rootfs/opt/bro/logs + lxc profile device add ${CON} brologs disk source=/var/lib/lxd/storage-pools/default/containers/ids/rootfs/opt/bro/logs path=/opt/bro/logs + fi + + setup_limits_on_profile ${CON} + lxc launch ubuntu:xenial ${CON} -p ${CON} + lxc file push containers/${CON} ${CON}/ + + done + + sleep 10 #startup and DHCP + + for CON in ${CONTAINERS}; do + log "Installing ${CON}" + lxc exec ${CON} -- /${CON} + done + + lxc list -c 4n |egrep -v "NAME|\+" | awk '{print $2, $5}' |tr ' ' '\t' >> /etc/hosts +} + +setup_firewall() { + kibana_ip=$(lxc list -c n4 |awk '/kibana/{print $4}') + fouroneone_ip=$(lxc list -c n4 |awk '/fouroneone/{print $4}') + prometheus_ip=$(lxc list -c n4 |awk '/prometheus/{print $4}') + + iptables -t nat -A PREROUTING -i ${MGMT} -p tcp -m tcp --dport 443 -j DNAT --to-destination ${fouroneone_ip} + iptables -t nat -A PREROUTING -i ${MGMT} -p tcp -m tcp --dport 9090 -j DNAT --to-destination ${prometheus_ip} + iptables -t nat -A PREROUTING -i ${MGMT} -p tcp -m tcp --dport 3000 -j DNAT --to-destination ${prometheus_ip} + iptables -t nat -A PREROUTING -i ${MGMT} -p tcp -m tcp --dport 5601 -j DNAT --to-destination ${kibana_ip} + iptables -t nat -A INPUT -i ${MGMT} -p tcp -m tcp --dport 22 -j ACCEPT + iptables-save > /etc/network/iptables.up.rules + cat > /etc/network/if-pre-up.d/iptablesload <> /etc/sysctl.conf <> /etc/security/limits.conf < /etc/network/iptables.up.rules + +export CONTAINERS=" bro kafka elasticsearch logstash kibana fouroneone icap squid rita fsf prometheus" +for CONT in ${CONTAINERS}; do + echo "Destroying ${CONT}" + lxc stop ${CONT} 2>/dev/null + lxc delete ${CONT} 2>/dev/null +done + +sudo kill -9 `ps ax |awk '/odinbr0/{print $1}' | head -1` +ip link del dev odinbr0 +systemctl stop lxd.socket +systemctl stop lxd.service + +zfs destroy -r tank/lxd +rm -rf /var/lib/lxd + +sed -i 's/LimitMEMLOCK=infinity//g' /lib/systemd/system/lxd.service +systemctl daemon-reload + +echo -e "\e[0m" diff --git a/extra/bro_api.py b/extra/bro_api.py new file mode 100644 index 0000000..b5568ad --- /dev/null +++ b/extra/bro_api.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import sys +import base64 +from subprocess import check_output +from flask import Flask + +BRO_CONFIG='/opt/bro/share/bro/site/local.bro' + +@app.route('/config') +def config_get(): + cmd = ['cat', BRO_CONFIG] + res = check_output(cmd) + res = base64.b64encode(res) + data = {'acknowledged':'true', 'config':str(res)} + return data + +app = Flask(__name__) diff --git a/extra/grafana_dashboards.json b/extra/grafana_dashboards.json new file mode 100644 index 0000000..c97e2d7 --- /dev/null +++ b/extra/grafana_dashboards.json @@ -0,0 +1,1117 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "id": null, + "title": "Node Exporter Server Metrics", + "description": "Dashboard to view multiple servers", + "tags": [ + "prometheus" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "25px", + "panels": [ + { + "content": "", + "editable": true, + "error": false, + "id": 11, + "minSpan": 2, + "mode": "html", + "repeat": "node", + "span": 12, + "style": {}, + "title": "$node", + "type": "text" + } + ], + "title": "Title" + }, + { + "collapse": false, + "editable": true, + "height": "25px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 20, + "interval": null, + "links": [], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "repeat": "node", + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "grid": { + "threshold1": 0, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (mode)(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mode}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='user',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='nice',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='iowait',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='steal',instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='idle',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='irq',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='softirq',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='guest',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "CPU" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Memory" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Load" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Disk Used" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Network Traffic" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [ + { + "allFormat": "glob", + "current": { + "tags": [], + "text": null, + "value": [] + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "query": "label_values(node_boot_time, instance)", + "refresh": 1, + "sort": 1, + "type": "query", + "options": [] + } + ] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 12, + "version": 30, + "links": [], + "gnetId": 405 +} diff --git a/extra/kibana_dashboards.json b/extra/kibana_dashboards.json new file mode 100644 index 0000000..3893d62 --- /dev/null +++ b/extra/kibana_dashboards.json @@ -0,0 +1,550 @@ +[ + { + "_id": "SOFTWARE", + "_type": "dashboard", + "_source": { + "title": "SOFTWARE", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":1,\"id\":\"Software-Software-List\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":5,\"type\":\"visualization\"},{\"col\":5,\"id\":\"Software-Unique\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":8,\"id\":\"Software-Top-Types\",\"panelIndex\":2,\"row\":1,\"size_x\":5,\"size_y\":5,\"type\":\"visualization\"},{\"id\":\"Software-Search\",\"type\":\"search\",\"panelIndex\":4,\"size_x\":12,\"size_y\":9,\"col\":1,\"row\":6,\"columns\":[\"host\",\"name\",\"unparsed_version\",\"software_type\"],\"sort\":[\"@timestamp\",\"desc\"]}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-24h", + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + }, + { + "_id": "FILES", + "_type": "dashboard", + "_source": { + "title": "FILES", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":1,\"id\":\"Files-Top-Mime-Types\",\"panelIndex\":1,\"row\":3,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Files-Analyzers\",\"panelIndex\":3,\"row\":3,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"rx_hosts\",\"tx_hosts\",\"mime_type\",\"seen_bytes\"],\"id\":\"Files\",\"panelIndex\":2,\"row\":7,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"id\":\"Files-Events\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":9,\"size_y\":2,\"col\":1,\"row\":1},{\"id\":\"File-Total-Events\",\"type\":\"visualization\",\"panelIndex\":5,\"size_x\":3,\"size_y\":2,\"col\":10,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-24h", + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + }, + { + "_id": "DNS", + "_type": "dashboard", + "_source": { + "title": "DNS", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":1,\"id\":\"DNS-Top-Queries\",\"panelIndex\":2,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":5,\"id\":\"DNS-Top-Query-Types\",\"panelIndex\":4,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"DNS-Top-Answers\",\"panelIndex\":3,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"srcip\",\"dstip\",\"query\",\"answers\",\"qtype_name\",\"rcode_name\",\"rtt\",\"geoip.region_name\",\"qclass_name\"],\"id\":\"DNS-Connections\",\"panelIndex\":1,\"row\":8,\"size_x\":12,\"size_y\":10,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"id\":\"DNS-Events\",\"type\":\"visualization\",\"panelIndex\":5,\"size_x\":8,\"size_y\":3,\"col\":1,\"row\":1},{\"id\":\"DNS-Total-Events\",\"type\":\"visualization\",\"panelIndex\":6,\"size_x\":4,\"size_y\":3,\"col\":9,\"row\":1}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{\"P-5\":{\"vis\":{\"legendOpen\":false}}}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-24h", + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + }, + { + "_id": "STATS", + "_type": "dashboard", + "_source": { + "title": "STATS", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":1,\"id\":\"Stats-Totals\",\"panelIndex\":6,\"row\":1,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Stats-Bytes-Received\",\"panelIndex\":8,\"row\":3,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Stats-Pkts-Processed-Per-Worker\",\"panelIndex\":7,\"row\":5,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Stats-Mem-Usage\",\"panelIndex\":2,\"row\":5,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Stats-Packet-Lag\",\"panelIndex\":1,\"row\":9,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Stats-Events-Processed\",\"panelIndex\":3,\"row\":9,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Stats-Total-Bytes\",\"panelIndex\":9,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":false}},\"P-2\":{\"vis\":{\"legendOpen\":false}},\"P-3\":{\"vis\":{\"legendOpen\":false}},\"P-7\":{\"vis\":{\"legendOpen\":false}},\"P-6\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-24h", + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } + } + }, + { + "_id": "DNS-Connections", + "_type": "search", + "_source": { + "title": "DNS Connections", + "description": "", + "hits": 0, + "columns": [ + "srcip", + "dstip", + "query", + "answers", + "qtype_name", + "rcode_name", + "rtt", + "geoip.region_name", + "qclass_name" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"dns\\\"\",\"analyze_wildcard\":true}}}" + } + } + }, + { + "_id": "Bro-Stats", + "_type": "search", + "_source": { + "title": "Bro Stats", + "description": "", + "hits": 0, + "columns": [ + "mem", + "peer", + "pkt_lag", + "tcp_conns", + "dns_requests", + "events_queued", + "events_proc", + "udp_conns" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\"\",\"analyze_wildcard\":true}}}" + } + } + }, + { + "_id": "Notice", + "_type": "search", + "_source": { + "title": "Notice", + "description": "", + "hits": 0, + "columns": [ + "srcip", + "dstip", + "p", + "note", + "geoip.postal_code", + "geoip.region_name", + "geoip.country_code2" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"notice\\\"\",\"analyze_wildcard\":true}}}" + } + } + }, + { + "_id": "Files", + "_type": "search", + "_source": { + "title": "Files", + "description": "", + "hits": 0, + "columns": [ + "rx_hosts", + "tx_hosts", + "mime_type", + "seen_bytes", + "analyzers", + "md5", + "timedout" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"kafka.topic:\\\"files\\\"\"}}}" + } + } + }, + { + "_id": "Software-Search", + "_type": "search", + "_source": { + "title": "Software Search", + "description": "", + "hits": 0, + "columns": [ + "host", + "name", + "unparsed_version", + "software_type" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"software\\\"\",\"analyze_wildcard\":true}}}" + } + } + }, + { + "_id": "Stats-Packet-Lag", + "_type": "visualization", + "_source": { + "title": "Stats - Packet Lag", + "visState": "{\"title\":\"Stats - Packet Lag\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":false,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"legendPosition\":\"top\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"pkt_lag\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\" AND peer:\\\"odin-*\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Mem-Usage", + "_type": "visualization", + "_source": { + "title": "Stats - Mem Usage", + "visState": "{\"title\":\"Stats - Mem Usage\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{\"max\":0.2}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mem\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"peer.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Events-Processed", + "_type": "visualization", + "_source": { + "title": "Stats - Events Processed", + "visState": "{\"title\":\"Stats - Events Processed\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"mode\":\"silhouette\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"events_proc\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"events_queued\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"kafka.topic:\\\"stats\\\"\"}},\"filter\":[]}" + } + } + }, + { + "_id": "Conns-Top-Dest-Ports", + "_type": "visualization", + "_source": { + "title": "Conns - Top Dest Ports", + "visState": "{\"title\":\"Conns - Top Dest Ports\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"id.resp_p\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Top Dest Ports\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"conn\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Packets-Dropped", + "_type": "visualization", + "_source": { + "title": "Stats - Packets Dropped", + "visState": "{\"title\":\"Stats - Packets Dropped\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"pkts_dropped\",\"customLabel\":\"Packets Dropped\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\" AND peer:\\\"odin-*\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "DNS-Events", + "_type": "visualization", + "_source": { + "title": "DNS - Events", + "visState": "{\"title\":\"DNS - Events\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"dns\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "DNS-Top-Query-Types", + "_type": "visualization", + "_source": { + "title": "DNS - Top Query Types", + "visState": "{\"title\":\"DNS - Top Query Types\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"qtype_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"dns\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "DNS-Top-Queries", + "_type": "visualization", + "_source": { + "title": "DNS - Top Queries", + "visState": "{\"title\":\"DNS - Top Queries\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"query.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"dns\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Files-Analyzers", + "_type": "visualization", + "_source": { + "title": "Files - Analyzers", + "visState": "{\"title\":\"Files - Analyzers\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"analyzers.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Analyzers Used\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"files\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Files-Events", + "_type": "visualization", + "_source": { + "title": "Files - Events", + "visState": "{\"title\":\"Files - Events\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"File Events\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"files\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Notice-Events", + "_type": "visualization", + "_source": { + "title": "Notice - Events", + "visState": "{\"title\":\"Notice - Events\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Events\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"notice\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "File-Total-Events", + "_type": "visualization", + "_source": { + "title": "File - Total Events", + "visState": "{\"title\":\"File - Total Events\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"File Events\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"files\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Bytes-Received", + "_type": "visualization", + "_source": { + "title": "Stats - Bytes Received", + "visState": "{\"title\":\"Stats - Bytes Received\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"bytes_recv\",\"customLabel\":\"Bytes Received\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Totals", + "_type": "visualization", + "_source": { + "title": "Stats - Totals", + "visState": "{\"title\":\"Stats - Totals\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":\"32\"},\"aggs\":[{\"id\":\"4\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"tcp_conns\",\"customLabel\":\"TCP\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"udp_conns\",\"customLabel\":\"UDP\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"icmp_conns\",\"customLabel\":\"ICMP\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\" AND peer:\\\"odin-*\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Total-Bytes", + "_type": "visualization", + "_source": { + "title": "Stats - Total Bytes", + "visState": "{\"title\":\"Stats - Total Bytes\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":\"32\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"bytes_recv\",\"customLabel\":\"Total Bytes\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "DNS-Top-Answers", + "_type": "visualization", + "_source": { + "title": "DNS - Top Answers", + "visState": "{\"title\":\"DNS - Top Answers\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"right\",\"shareYAxis\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"answers.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"kafka.topic:\\\"dns\\\"\"}},\"filter\":[]}" + } + } + }, + { + "_id": "DNS-Total-Events", + "_type": "visualization", + "_source": { + "title": "DNS - Total Events", + "visState": "{\"title\":\"DNS - Total Events\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":\"72\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total Events\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"dns\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Files-Top-Mime-Types", + "_type": "visualization", + "_source": { + "title": "Files - Top Mime Types", + "visState": "{\"title\":\"Files - Top Mime Types\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"mime_type.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"files\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Software-Software-List", + "_type": "visualization", + "_source": { + "title": "Software - Software List", + "visState": "{\"title\":\"Software - Software List\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Frequent Software\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"software\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Software-Top-Types", + "_type": "visualization", + "_source": { + "title": "Software - Top Types", + "visState": "{\"title\":\"Software - Top Types\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"legendPosition\":\"right\",\"shareYAxis\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"software_type.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"kafka.topic:\\\"software\\\"\"}},\"filter\":[]}" + } + } + }, + { + "_id": "Software-Unique", + "_type": "visualization", + "_source": { + "title": "Software - Unique", + "visState": "{\"title\":\"Software - Unique\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"name.keyword\",\"customLabel\":\"Unique Softwares\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"software\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Notice-Types", + "_type": "visualization", + "_source": { + "title": "Notice - Types", + "visState": "{\"title\":\"Notice - Types\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"note.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Top Notice Types\"}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"notice\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Stats-Pkts-Processed-Per-Worker", + "_type": "visualization", + "_source": { + "title": "Stats - Pkts Processed Per Worker", + "visState": "{\"title\":\"Stats - Pkts Processed Per Worker\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"defaultYExtents\":false,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"mode\":\"stacked\",\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"shareYAxis\":true,\"smoothLines\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"pkts_proc\",\"customLabel\":\"Processed Packets\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"peer.keyword\",\"size\":12,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Peer\",\"row\":true}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"odin-*\",\"query\":{\"query_string\":{\"query\":\"kafka.topic:\\\"stats\\\" AND peer:\\\"odin-*\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + } +] \ No newline at end of file diff --git a/extra/squid.conf b/extra/squid.conf new file mode 100644 index 0000000..b6980e4 --- /dev/null +++ b/extra/squid.conf @@ -0,0 +1,113 @@ +http_port 0.0.0.0:3128 ssl-bump generate-host-certificates=on dynamic_cert_mem_cache_size=10MB cert=/etc/squid/serverkey.pem capath=/usr/share/certs/ cipher=EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS tls-dh=prime256v1:/etc/dh-parameters.2048 options=NO_SSLv2,NO_SSLv3,NO_TLSv1,SINGLE_DH_USE,SINGLE_ECDH_USE + +icp_port 0 +digest_generation off +dns_v4_first off +pid_filename /var/run/squid/squid.pid +cache_effective_user squid +cache_effective_group proxy +error_default_language en +icon_directory /usr/local/etc/squid/icons +visible_hostname odin +cache_mgr odin@infosec +access_log /var/squid/logs/access.log +cache_log /var/squid/logs/cache.log +cache_store_log none +netdb_filename /var/squid/logs/netdb.state +pinger_enable on +pinger_program /usr/local/libexec/squid/pinger +sslcrtd_program /usr/local/libexec/squid/ssl_crtd -s /var/squid/lib/ssl_db -M 4MB -b 2048 +sslcrtd_children 5 +sslproxy_capath /usr/local/share/certs/ +sslproxy_options NO_SSLv2,NO_SSLv3,NO_TLSv1,SINGLE_DH_USE,SINGLE_ECDH_USE +sslproxy_cipher EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS +sslproxy_cert_error allow all +sslproxy_cert_adapt setValidAfter all +sslproxy_cert_adapt setValidBefore all + +logfile_rotate 10 +debug_options rotate=10 +shutdown_lifetime 3 seconds +# Allow local network(s) on interface(s) +acl localnet src 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12 +forwarded_for transparent +via off +httpd_suppress_version_string on +uri_whitespace encode + +acl dynamic urlpath_regex cgi-bin \? +cache deny dynamic + +cache_mem 4096 MB +maximum_object_size_in_memory 102400 KB +memory_replacement_policy heap GDSF +cache_replacement_policy heap LFUDA +minimum_object_size 0 KB +maximum_object_size 51200 MB +cache_dir aufs /var/squid/cache 150000 16 256 +offline_mode off +cache_swap_low 90 +cache_swap_high 95 +cache allow all +# Add any of your own refresh_pattern entries above these. +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 + +# Setup some default acls +# ACLs all, manager, localhost, and to_localhost are predefined. +acl allsrc src all +acl safeports port 21 70 80 210 280 443 488 563 591 631 777 901 3128 3129 1025-65535 +acl sslports port 443 563 + +acl purge method PURGE +acl connect method CONNECT + +# Define protocols used for redirects +acl HTTP proto HTTP +acl HTTPS proto HTTPS + +# SslBump Peek and Splice +# http://wiki.squid-cache.org/Features/SslPeekAndSplice +# http://wiki.squid-cache.org/ConfigExamples/Intercept/SslBumpExplicit +# Match against the current step during ssl_bump evaluation [fast] +# Never matches and should not be used outside the ssl_bump context. +# +# At each SslBump step, Squid evaluates ssl_bump directives to find +# the next bumping action (e.g., peek or splice). Valid SslBump step +# values and the corresponding ssl_bump evaluation moments are: +# SslBump1: After getting TCP-level and HTTP CONNECT info. +# SslBump2: After getting TLS Client Hello info. +# SslBump3: After getting TLS Server Hello info. +# These ACLs exist even when 'SSL/MITM Mode' is set to 'Custom' so that +# they can be used there for custom configuration. +acl step1 at_step SslBump1 +acl step2 at_step SslBump2 +acl step3 at_step SslBump3 +http_access allow manager localhost + +http_access deny manager +http_access allow purge localhost +http_access deny purge +http_access deny !safeports +http_access deny CONNECT !sslports + +# Always allow localhost connections +http_access allow localhost + +request_body_max_size 0 KB +delay_pools 1 +delay_class 1 2 +delay_parameters 1 -1/-1 -1/-1 +delay_initial_bucket_level 100 +delay_access 1 allow allsrc + + +ssl_bump peek step1 +ssl_bump bump all +# Setup allowed ACLs +# Allow local network(s) on interface(s) +http_access allow localnet +# Default block all to be sure +http_access deny allsrc diff --git a/limits b/limits new file mode 100644 index 0000000..8d18d15 --- /dev/null +++ b/limits @@ -0,0 +1,36 @@ +export CPU_bro="6" +export CPU_kafka="2" +export CPU_elasticsearch="4" +export CPU_logstash="1" +export CPU_kibana="2" +export CPU_fouroneone="2" +export CPU_icap="2" +export CPU_squid="2" +export CPU_rita="4" +export CPU_fsf="4" +export CPU_prometheus="2" + +export MEM_bro="8192MB" +export MEM_kafka="1024MB" +export MEM_elasticsearch="16384MB" +export MEM_logstash="1024MB" +export MEM_kibana="1024MB" +export MEM_fouroneone="2048MB" +export MEM_icap="512MB" +export MEM_squid="4096MB" +export MEM_rita="8192MB" +export MEM_fsf="4096MB" +export MEM_prometheus="2048MB" + +export DISK_bro="20GB" +export DISK_kafka="20GB" +export DISK_elasticsearch="250GB" +export DISK_logstash="10GB" +export DISK_kibana="10GB" +export DISK_fouroneone="10GB" +export DISK_icap="10GB" +export DISK_squid="20GB" +export DISK_rita="120GB" +export DISK_fsf="80GB" +export DISK_prometheus="60GB" +