Upgrading ELK to 5.4.3

Upgrading to latest (5.4.3) ELK stack. Adding new fields for files.log
in latest Bro version for logstash parsing.

README.md

@@ -20,9 +20,9 @@ New Functionality:
 
 Fixes:
   * Optimized Logstash Config
-  * Updated Bro IDS to 2.5
-  * Updated Logstash to version 5.3.0
-  * Updated Elasticsearch to version 5.3.0
-  * Update kibana to version 5.3.0
+  * Updated Bro IDS to 2.5.1
+  * Updated Logstash to version 5.4.3
+  * Updated Elasticsearch to version 5.4.3
+  * Update kibana to version 5.4.3
 
 

install/elasticSearch.py

@@ -8,7 +8,7 @@
 import hashCheck
 
 def install():
-	elasticLatest='5.3.0'
+	elasticLatest='5.4.3'
 	#Install Elasticsearch
 	elasticInstalled=False
 	if os.path.isfile('/etc/elasticsearch/elasticsearch.yml'):
@@ -29,16 +29,16 @@ def install():
 			elasticInstalled=True
 	if elasticInstalled == False:
 		print "Installing Elasticsearch"
-		print "  Downloading Elasticsearch 5.3.0"
-		os.popen('sudo wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.deb 2>&1').read()
-		if not os.path.isfile('elasticsearch-5.3.0.deb'):
+		print "  Downloading Elasticsearch 5.4.3"
+		os.popen('sudo wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.3.deb 2>&1').read()
+		if not os.path.isfile('elasticsearch-5.4.3.deb'):
 			sys.exit('Error downloading elasticsearch')
-		if not hashCheck.checkHash('elasticsearch-5.3.0.deb'):
+		if not hashCheck.checkHash('elasticsearch-5.4.3.deb'):
 			sys.exit('Error downloading elasticsearch, mismatched file hashes')
 		print "  Installing Elasticsearch"
-		os.popen('sudo dpkg -i elasticsearch-5.3.0.deb').read()
+		os.popen('sudo dpkg -i elasticsearch-5.4.3.deb').read()
 		print "  Cleaning Up Installation Files"
-		os.remove('elasticsearch-5.3.0.deb')
+		os.remove('elasticsearch-5.4.3.deb')
 		os.popen('sudo update-rc.d elasticsearch defaults').read()
 		#Change heap size to 500m (1/2 of phyical memory)
 		shutil.move('/etc/elasticsearch/jvm.options','/etc/elasticsearch/jvm.orig')

install/hashCheck.py

@@ -2,17 +2,16 @@
 
 
 def getHash(fileName):
-    if fileName == 'elasticsearch-5.3.0.deb':
-        return 'dab27ca0f49463a0f2e194780186653d22327660'
-    elif fileName == 'kibana-5.3.0-linux-x86_64.tar.gz':
-        return '4e9daf275f8ef749fba931c1f5c35f85662efd53'
-    elif fileName == 'kibana-5.3.0-linux-x86.tar.gz':
-        return '6323e46abff74fd1af37a040539664d30f672cd8'
-    elif fileName == 'logstash-5.3.0.deb':
-        return '12f1a8c3f6de535d8a9b723e6bc396523e706f15'
+    if fileName == 'elasticsearch-5.4.3.deb':
+        return '294ac7ada78a3944cf05f3d43b62d4df4dd55c8f'
+    elif fileName == 'kibana-5.4.3-linux-x86_64.tar.gz':
+        return 'a5892ec6fd8271d508206ee8319473a06d2a5ac6'
+    elif fileName == 'kibana-5.4.3-linux-x86.tar.gz':
+        return '29e4a8903ebfc6cbe75b2ef5800f83893d076318'
+    elif fileName == 'logstash-5.4.3.deb':
+        return '1ed81009deea11b0cc5e747bf07c1af76d9eb12d'
     elif fileName == 'bro-2.5.1.tar.gz':
         return '9c133dd3a075be1084f9bf53d79c42ddcf23633c'
-        #return '12c6dc0c38e7515dbac530ba0890a0bce6066fa3' <-- bro 2.5
     return ''
 
 def checkHash(fileName):

install/kibana.py

@@ -2,7 +2,7 @@
 import hashCheck
 
 def install(chosenInterfaceIP):
-	kibanaLatest='5.3.0'
+	kibanaLatest='5.4.3'
 
 	cpuArch=os.uname()[4]
 	cwd=os.getcwd()
@@ -15,31 +15,31 @@ def install(chosenInterfaceIP):
 			kibanaInstalled=True
 	if kibanaInstalled == False:
 		print "Installing Kibana"
-		print "  Downloading Kibana 5.3.0"
+		print "  Downloading Kibana 5.4.3"
 		if cpuArch == 'x86_64':
-			os.popen('sudo wget https://artifacts.elastic.co/downloads/kibana/kibana-5.3.0-linux-x86_64.tar.gz 2>&1').read()
-			if not os.path.isfile('kibana-5.3.0-linux-x86_64.tar.gz'):
+			os.popen('sudo wget https://artifacts.elastic.co/downloads/kibana/kibana-5.4.3-linux-x86_64.tar.gz 2>&1').read()
+			if not os.path.isfile('kibana-5.4.3-linux-x86_64.tar.gz'):
 				sys.exit('Error downloading Kibana')
-			if not hashCheck.checkHash('kibana-5.3.0-linux-x86_64.tar.gz'):
+			if not hashCheck.checkHash('kibana-5.4.3-linux-x86_64.tar.gz'):
 				sys.exit('Error downloading kibana, mismatched file hashes')
 			print "  Installing Kibana"
-			os.popen('sudo tar -xzf kibana-5.3.0-linux-x86_64.tar.gz').read()
-			shutil.copytree('kibana-5.3.0-linux-x86_64/','/opt/kibana')
+			os.popen('sudo tar -xzf kibana-5.4.3-linux-x86_64.tar.gz').read()
+			shutil.copytree('kibana-5.4.3-linux-x86_64/','/opt/kibana')
 			print "  Cleaning Up Installation Files"
-			os.remove('kibana-5.3.0-linux-x86_64.tar.gz')
-			shutil.rmtree("kibana-5.3.0-linux-x86_64/")
+			os.remove('kibana-5.4.3-linux-x86_64.tar.gz')
+			shutil.rmtree("kibana-5.4.3-linux-x86_64/")
 		else:
-			os.popen('sudo wget https://artifacts.elastic.co/downloads/kibana/kibana-5.3.0-linux-x86.tar.gz 2>&1').read()
-			if not os.path.isfile('kibana-5.3.0-linux-x86.tar.gz'):
+			os.popen('sudo wget https://artifacts.elastic.co/downloads/kibana/kibana-5.4.3-linux-x86.tar.gz 2>&1').read()
+			if not os.path.isfile('kibana-5.4.3-linux-x86.tar.gz'):
 				sys.exit('Error downloading Kibana')
-			if not hashCheck.checkHash('kibana-5.3.0-linux-x86.tar.gz'):
+			if not hashCheck.checkHash('kibana-5.4.3-linux-x86.tar.gz'):
 				sys.exit('Error downloading kibana, mismatched file hashes')
 			print "  Installing Kibana"
-			os.popen('sudo tar -xzf kibana-5.3.0-linux-x86.tar.gz').read()
-			shutil.copytree('kibana-5.3.0-linux-x86/','/opt/kibana')
+			os.popen('sudo tar -xzf kibana-5.4.3-linux-x86.tar.gz').read()
+			shutil.copytree('kibana-5.4.3-linux-x86/','/opt/kibana')
 			print "  Cleaning Up Installation Files"
-			os.remove('kibana-5.3.0-linux-x86.tar.gz')
-			shutil.rmtree("kibana-5.3.0-linux-x86/")
+			os.remove('kibana-5.4.3-linux-x86.tar.gz')
+			shutil.rmtree("kibana-5.4.3-linux-x86/")
 
 		#Custom stuff for ARM
 		if not cpuArch.startswith('x86'):

install/logstash.py

@@ -11,7 +11,7 @@ def get_user_input(input_string):
 
 def install(esServer,esUser,esPass):
 	print "Installing Logstash"
-	logstashLatest='5.3.0'
+	logstashLatest='5.4.3'
 
 	cpuArch=os.uname()[4]
 	cwd=os.getcwd()
@@ -54,16 +54,16 @@ def install(esServer,esUser,esPass):
 			smtpPort = get_user_input("    \033[1mEnter SMTP Port (ex: 587)\033[0m: ")
 			smtpUser = get_user_input("    \033[1mEnter Email Address (ex: [email protected])\033[0m: ")
 			smtpPass = getpass.getpass("    \033[1mEnter Email Password (ex: P@55word)\033[0m: ")
-		print "  Downloading Logstash 5.3.0"
-		os.popen('sudo wget https://artifacts.elastic.co/downloads/logstash/logstash-5.3.0.deb 2>&1').read()
-		if not os.path.isfile('logstash-5.3.0.deb'):
+		print "  Downloading Logstash 5.4.3"
+		os.popen('sudo wget https://artifacts.elastic.co/downloads/logstash/logstash-5.4.3.deb 2>&1').read()
+		if not os.path.isfile('logstash-5.4.3.deb'):
 			sys.exit('Error downloading logstash')
-		if not hashCheck.checkHash('logstash-5.3.0.deb'):
+		if not hashCheck.checkHash('logstash-5.4.3.deb'):
 			sys.exit('Error downloading logstash, mismatched file hashes')
 		print "  Installing Logstash"
-		os.popen('sudo dpkg -i logstash-5.3.0.deb').read()
+		os.popen('sudo dpkg -i logstash-5.4.3.deb').read()
 		print "  Cleaning Up Logstash Installation Files"
-		os.remove('logstash-5.3.0.deb')
+		os.remove('logstash-5.4.3.deb')
 		os.popen('sudo systemctl enable logstash.service').read()
 
 		if not cpuArch.startswith('x86'):

logstash/rules/bro.rule

@@ -4,7 +4,7 @@ connLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t
 dhcpLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<mac>[^\t]+)\t(?<assigned_ip>[^\t]+)\t(?<lease_time>[^\t]+)\t(?<trans_id>.*)
 dnsLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<proto>[^\t]+)\t(?<trans_id>[^\t]+)\t(?<rtt>[^\t]+)\t(?<query>[^\t]+)\t(?<qclass>[^\t]+)\t(?<qclass_name>[^\t]+)\t(?<qtype>[^\t]+)\t(?<qtype_name>[^\t]+)\t(?<rcode>[^\t]+)\t(?<rcode_name>[^\t]+)\t(?<AA>[^\t]+)\t(?<TC>[^\t]+)\t(?<RD>[^\t]+)\t(?<RA>[^\t]+)\t(?<Z>[^\t]+)\t(?<answers>[^\t]+)\t(?<TTLs>[^\t]+)\t(?<rejected>.*)
 dpdLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<proto>[^\t]+)\t(?<analyzer>[^\t]+)\t(?<failure_reason>.*)
-filesLog (?<ts>\d+\.\d+)\t(?<fuid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<conn_uid>[^\t]+)\t(?<source>[^\t]+)\t(?<depth>[^\t]+)\t(?<analyzers>[^\t]+)\t(?<mime_type>[^\t]+)\t(?<filename>[^\t]+)\t(?<duration>[^\t]+)\t(?<local_orig>[^\t]+)\t(?<is_orig>[^\t]+)\t(?<seen_bytes>[^\t]+)\t(?<total_bytes>[^\t]+)\t(?<missing_bytes>[^\t]+)\t(?<overflow_bytes>[^\t]+)\t(?<timed_out>[^\t]+)\t(?<parent_fuid>[^\t]+)\t(?<md5>[^\t]+)\t(?<sha1>[^\t]+)\t(?<sha256>[^\t]+)\t(?<extracted>.*)
+filesLog (?<ts>\d+\.\d+)\t(?<fuid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<conn_uid>[^\t]+)\t(?<source>[^\t]+)\t(?<depth>[^\t]+)\t(?<analyzers>[^\t]+)\t(?<mime_type>[^\t]+)\t(?<filename>[^\t]+)\t(?<duration>[^\t]+)\t(?<local_orig>[^\t]+)\t(?<is_orig>[^\t]+)\t(?<seen_bytes>[^\t]+)\t(?<total_bytes>[^\t]+)\t(?<missing_bytes>[^\t]+)\t(?<overflow_bytes>[^\t]+)\t(?<timed_out>[^\t]+)\t(?<parent_fuid>[^\t]+)\t(?<md5>[^\t]+)\t(?<sha1>[^\t]+)\t(?<sha256>[^\t]+)\t(?<extracted>[^\t]+)\t(?<extracted_cutoff>[^\t]+)\t(?<extracted_size>.*)
 ftpLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<user>[^\t]+)\t(?<password>[^\t]+)\t(?<command>[^\t]+)\t(?<arg>[^\t]+)\t(?<mime_type>[^\t]+)\t(?<file_size>[^\t]+)\t(?<reply_code>[^\t]+)\t(?<reply_msg>[^\t]+)\t(?<data_channel_passive>[^\t]+)\t(?<data_channel_orig_h>[^\t]+)\t(?<data_channel_resp_h>[^\t]+)\t(?<data_channel_resp_p>[^\t]+)\t(?<fuid>.*)
 httpLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<trans_depth>[^\t]+)\t(?<method>[^\t]+)\t(?<server_name>[^\t]+)\t(?<uri>[^\t]+)\t(?<referrer>[^\t]+)\t(?<version>[^\t]+)\t(?<user_agent>[^\t]+)\t(?<request_body_len>[^\t]+)\t(?<response_body_len>[^\t]+)\t(?<status_code>[^\t]+)\t(?<status_msg>[^\t]+)\t(?<info_code>[^\t]+)\t(?<info_msg>[^\t]+)\t(?<tags>[^\t]+)\t(?<username>[^\t]+)\t(?<password>[^\t]+)\t(?<proxied>[^\t]+)\t(?<orig_fuids>[^\t]+)\t(?<orig_filenames>[^\t]+)\t(?<orig_mime_types>[^\t]+)\t(?<resp_fuids>[^\t]+)\t(?<resp_filenames>[^\t]+)\t(?<resp_mime_types>.*)
 intelLog (?<ts>\d+\.\d+)\t(?<uid>(\w+|-))\t(?<orig_h>[^\t]+)\t(?<orig_p>[^\t]+)\t(?<resp_h>[^\t]+)\t(?<resp_p>[^\t]+)\t(?<fuid>[^\t]+)\t(?<file_mime_type>[^\t]+)\t(?<file_desc>[^\t]+)\t(?<seen_indicator>[^\t]+)\t(?<seen_indicator_type>[^\t]+)\t(?<seen_where>[^\t]+)\t(?<seen_node>[^\t]+)\t(?<sources>.*)