shane/threatline
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Shane Peters
2018-11-10 13:26:49 -05:00
parent 57cbadbe8a
commit 525ecff602
21 changed files with 3049 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
doc/APACHE_KAFKA.tgz Normal file
View File

Binary file not shown.

2117
doc/es_mapping.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
doc/kafka_topics.sh Normal file
View File

@@ -0,0 +1,11 @@
#!/bin/sh
KBIN=/usr/local/share/java/kafka/bin
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLINGEST-lodi --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLINGEST-wyomic --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLINGEST-qwf --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLNORMALIZED --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLENRICHED --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLALERTS --partitions=3 --replication-factor=1
${KBIN}/kafka-topics.sh --zookeeper 10.15.0.38:2181 --create --topic TLARCHIVE --partitions=3 --replication-factor=1

101
doc/setup_pfsense.sh Normal file
View File

@@ -0,0 +1,101 @@
# FreeBSD 11.1-RELEASE
# Install dependencies
pkg install -y bash git flex bison cmake libpcap librdkafka python py27-sqlite3 caf swig
# Compile Bro (no install)
# Needs compiled because build/src/bifcl is needed to compile plugins
mkdir /usr/local/src; cd /usr/local/src/
git clone https://github.com/bro/bro
cd bro; ./configure && make -j2
# Compile kafka plugin (no install)
# This will generate APACHE_KAFKA.tar.gz
cd /usr/local/src/
git clone https://github.com/apache/metron-bro-plugin-kafka.git
./configure --bro-dist=/usr/local/src/bro
make
# Copy APACHE_KAFKA.tgz to pfsense
# Login into pfsense and enable FreeBSD repos (temporarily)
sed -i '' 's/FreeBSD: { enabled: no/FreeBSD: { enabled: yes/g' /usr/local/share/pfSense/pkg/repos/pfSense-repo.conf
sed -i '' 's/FreeBSD: { enabled: no/FreeBSD: { enabled: yes/g' /usr/local/etc/pkg/repos/FreeBSD.conf
pkg install -y bro librdkafka
sed -i '' 's/FreeBSD: { enabled: yes/FreeBSD: { enabled: no/g' /usr/local/share/pfSense/pkg/repos/pfSense-repo.conf
sed -i '' 's/FreeBSD: { enabled: yes/FreeBSD: { enabled: no/g' /usr/local/etc/pkg/repos/FreeBSD.conf
pkg update
# Extract plugin and enable it
tar xzf APACHE_KAFKA.tgz -C /usr/local/lib/bro/plugins
cat > /usr/local/share/bro/site/local.bro <<EOF
@load misc/loaded-scripts
@load tuning/defaults
@load misc/capture-loss
@load misc/stats
@load misc/scan
@load frameworks/software/vulnerable
@load frameworks/software/version-changes
@load-sigs frameworks/signatures/detect-windows-shells
@load protocols/ftp/software
@load protocols/smtp/software
@load protocols/ssh/software
@load protocols/http/software
@load protocols/ftp/detect
@load protocols/conn/known-hosts
@load protocols/conn/known-services
@load protocols/ssl/known-certs
@load protocols/ssl/validate-certs
@load protocols/ssl/log-hostcerts-only
@load protocols/ssh/geo-data
@load protocols/ssh/detect-bruteforcing
@load protocols/ssh/interesting-hostnames
@load frameworks/files/hash-all-files
@load frameworks/files/detect-MHR
@load policy/protocols/conn/vlan-logging
@load policy/protocols/conn/mac-logging
@load policy/protocols/smb
@load Apache/Kafka/logs-to-kafka.bro
redef Kafka::topic_name = "TLINGEST-CLIENTNAME";
redef Kafka::tag_json = T;
redef Kafka::logs_to_send = set(CaptureLoss::LOG, PacketFilter::LOG, Stats::LOG, Conn::LOG, DHCP::LOG, DNS::LOG, FTP::LOG, HTTP::LOG, IRC::LOG, KRB::LOG, NTLM::LOG, RADIUS::LOG, RDP::LOG, SIP::LOG, SMB::CMD_LOG, SMB::FILES_LOG, SMB::MAPPING_LOG, SMTP::LOG, SNMP::LOG, SOCKS::LOG, SSH::LOG, SSL::LOG, Syslog::LOG, Tunnel::LOG, Files::LOG, PE::LOG, X509::LOG, Intel::LOG, Notice::LOG, Software::LOG, Weird::LOG, CaptureLoss::LOG);
redef Kafka::kafka_conf = table(["client.id"] = "setme.client"
, ["compression.codec"] = "lz4"
, ["request.required.acks"] = "0"
, ["metadata.broker.list"] = "10.15.0.40:9092,10.15.0.41:9092,10.15.0.42:9092"
);
EOF
cat > /usr/local/etc/node.cfg <<EOF
[logger]
type=logger
host=localhost
[manager]
type=manager
host=localhost
[proxy-1]
type=proxy
host=localhost
[worker-1]
type=worker
host=localhost
interface=igb1
EOF
cat > /usr/local/etc/broctl.cfg <<EOF
MailTo = root@localhost
MailConnectionSummary = 0
MinDiskSpace = 0
MailHostUpDown = 0
LogRotationInterval = 3600
LogExpireInterval = 0
StatsLogEnable = 1
StatsLogExpireInterval = 1
StatusCmdShowAll = 0
CrashExpireInterval = 1
SitePolicyScripts = local.bro
LogDir = /usr/local/logs
SpoolDir = /usr/local/spool
CfgDir = /usr/local/etc
EOF
broctl check

51
doc/threatline Normal file
View File

@@ -0,0 +1,51 @@
# PROVIDE: threatline
# REQUIRE: LOGIN
# KEYWORD: shutdown
# Add the following lines to /etc/rc.conf to enable threatline:
#
# threatline_enable="YES"
#
#
. /etc/rc.subr
name="threatline"
rcvar=threatline_enable
# read configuration and set defaults
load_rc_config "$name"
: ${threatline_enable="NO"}
: ${threatline_agents="normalize enrich check archive"}
: ${threatline_pidfile="/tmp/threatline.pid"}
: ${threatline_path="/usr/local/threatline/threatline/threatline.py"}
start_cmd="threatline_start"
stop_cmd="threatline_stop"
daemon_head=/usr/sbin/daemon
python_path=/usr/local/bin/python2.7
threatline_start()
{
if checkyesno ${rcvar}; then
echo "* starting threatline... "
$daemon_head -p $threatline_pidfile $python_path $threatline_path $threatline_agents
fi
}
threatline_stop()
{
if checkyesno ${rcvar}; then
echo "* stopping threatline... "
#pkill python
kill `ps ax | awk '/threatline/{print $1}'` 2>/dev/null
fi
}
threatline_restart()
{
threatline_stop
threatline_start
}
run_rc_command "$1"