Skip to content

Commit

Permalink
3.2.0: Splunk HEC stats (metrics) and notifications (events), bug fix…
Browse files Browse the repository at this point in the history 
…es (#73)

* WIP: fixes certain problems with baseurl not being '/'

* fix webserver formatting

* update go deps

* adds splunk HEC for stats

* add Splunk HTTP Event Collector to notifications

* add example blocks for Splunk HEC in stats and notifications

* adding baseURL type and enforcing trailing slash FromConfig (#71)

* adding baseURL type and enforcing trailing slash FromConfig

* fix api routes in react code. fix minor issue in webserver.go for embeded fs pathing.

---------

Co-authored-by: Dustin Essington <[email protected]>

* make Splunk HEC calls async to improve execution time

---------

Co-authored-by: Shane Hughes <[email protected]>
  • Loading branch information
@aetaric @shanehughes1990
aetaric and shanehughes1990 authored Oct 5, 2023
1 parent 85747c9 commit d53b071
Show file tree
Hide file tree
Showing 13 changed files with 657 additions and 217 deletions.
11 changes: 11 additions & 0 deletions checkrr.yaml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,14 @@ notifications:
- unknowndetected
- startrun
- endrun
splunk:
url: ""
token: ""
notificationtypes:
- reacquire
- unknowndetected
- startrun
- endrun
stats: # These will slow down the runtime substantually, but... DATA
influxdb1:
url: ""
Expand All @@ -131,6 +139,9 @@ stats: # These will slow down the runtime substantually, but... DATA
token: ""
org: "default"
bucket: "checkrr"
splunk:
address: "https://127.0.0.1:8088/services/collector" # You must have HTTP Event Collection enabled and configured
token: "HEC Token" # this is the HEC token for your input. Configure it to force ingestion into a metrics index
webserver:
port: 8585
baseurl: "/"
Expand Down
62 changes: 61 additions & 1 deletion features/stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"time"

influxdb2 "github.com/influxdata/influxdb-client-go/v2"
Expand All @@ -22,6 +24,8 @@ type Stats struct {
writeAPI2 api.WriteAPIBlocking `json:"-"`
config viper.Viper `json:"-"`
Log log.Logger `json:"-"`
splunk Splunk `json:"-"`
splunkConfigured bool `json:"-"`
SonarrSubmissions uint64 `json:"sonarrSubmissions"`
RadarrSubmissions uint64 `json:"radarrSubmissions"`
LidarrSubmissions uint64 `json:"lidarrSubmissions"`
Expand All @@ -39,6 +43,30 @@ type Stats struct {
DB *bolt.DB `json:"-"`
}

type SplunkStats struct {
Fields *SplunkFields `json:"fields"`
Time int64 `json:"time"`
Event string `json:"event"`
}

type SplunkFields struct {
SonarrSubmissions uint64 `json:"metric_name:checkrr.sonarrSubmissions"`
RadarrSubmissions uint64 `json:"metric_name:checkrr.radarrSubmissions"`
LidarrSubmissions uint64 `json:"metric_name:checkrr.lidarrSubmissions"`
FilesChecked uint64 `json:"metric_name:checkrr.filesChecked"`
HashMatches uint64 `json:"metric_name:checkrr.hashMatches"`
HashMismatches uint64 `json:"metric_name:checkrr.hashMismatches"`
VideoFiles uint64 `json:"metric_name:checkrr.videoFiles"`
AudioFiles uint64 `json:"metric_name:checkrr.audioFiles"`
UnknownFileCount uint64 `json:"metric_name:checkrr.unknownFileCount"`
NonVideo uint64 `json:"metric_name:checkrr.nonVideo"`
}

type Splunk struct {
address string
token string
}

func (s *Stats) FromConfig(config viper.Viper) {
s.config = config
if config.Sub("influxdb1") != nil {
Expand All @@ -63,6 +91,12 @@ func (s *Stats) FromConfig(config viper.Viper) {
s.writeAPI2.EnableBatching()
s.Log.WithFields(log.Fields{"startup": true, "influxdb": "enabled"}).Info("Sending data to InfluxDB 2.x")
}
if config.Sub("splunk") != nil {
splunk := config.Sub("splunk")
s.splunk = Splunk{address: splunk.GetString("address"), token: splunk.GetString("token")}
s.splunkConfigured = true
s.Log.WithFields(log.Fields{"startup": true, "splunk stats": "enabled"}).Info("Sending stats data to Splunk")
}
}

func (s *Stats) Start() {
Expand Down Expand Up @@ -134,7 +168,7 @@ func (s *Stats) Render() {
t.Render()
}

func (s Stats) Write(field string, count uint64) {
func (s *Stats) Write(field string, count uint64) {
// Send to influxdb if enabled
if s.writeAPI1 != nil {
p := influxdb2.NewPointWithMeasurement("checkrr").
Expand All @@ -151,6 +185,32 @@ func (s Stats) Write(field string, count uint64) {
SetTime(time.Now())
s.writeAPI2.WritePoint(context.Background(), p)
}
// Send to splunk if configured
if s.splunkConfigured {
t := time.Now().Unix()
splunkfields := SplunkFields{FilesChecked: s.FilesChecked, HashMatches: s.HashMatches, HashMismatches: s.HashMismatches,
SonarrSubmissions: s.SonarrSubmissions, RadarrSubmissions: s.RadarrSubmissions, LidarrSubmissions: s.LidarrSubmissions,
VideoFiles: s.VideoFiles, NonVideo: s.NonVideo, AudioFiles: s.AudioFiles, UnknownFileCount: s.UnknownFileCount}
splunkstats := SplunkStats{Event: "metric", Time: t, Fields: &splunkfields}
go func(splunkstats SplunkStats) {
client := &http.Client{}
j, _ := json.Marshal(splunkstats)
var data = strings.NewReader(string(j))
req, err := http.NewRequest("POST", s.splunk.address, data)
if err != nil {
log.Warn(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Splunk %s", s.splunk.token))
resp, err := client.Do(req)
if err != nil {
log.Warn(err)
}
if resp.StatusCode != 200 {
log.Warnf("Recieved %d status code from Splunk", resp.StatusCode)
}
defer resp.Body.Close()
}(splunkstats)
}
// Update stats DB
err := s.DB.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("Checkrr-stats"))
Expand Down
98 changes: 64 additions & 34 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,47 @@ go 1.19

require (
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
github.com/disgoorg/disgo v0.16.7
github.com/disgoorg/disgo v0.16.11
github.com/disgoorg/snowflake/v2 v2.0.1
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.9.1
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
github.com/gregdel/pushover v1.2.0
github.com/gregdel/pushover v1.3.0
github.com/h2non/filetype v1.1.3
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/jedib0t/go-pretty/v6 v6.4.6
github.com/jedib0t/go-pretty/v6 v6.4.7
github.com/kalafut/imohash v1.0.2
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.3
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.16.0
github.com/xconstruct/go-pushbullet v0.0.0-20171206132031-67759df45fbb
go.etcd.io/bbolt v1.3.7
golift.io/starr v0.14.1-0.20221014065949-c78a10375d37
golift.io/starr v1.0.0
gopkg.in/vansante/go-ffprobe.v2 v2.1.1
)

require (
github.com/bytedance/sonic v1.10.0-rc // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/labstack/echo/v4 v4.11.1 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
golang.org/x/arch v0.4.0 // indirect
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
)

require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
github.com/CloudyKit/jet/v6 v6.2.0 // indirect
github.com/Joker/jade v1.1.3 // indirect
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bytedance/sonic v1.10.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deepmap/oapi-codegen v1.13.0 // indirect
github.com/deepmap/oapi-codegen v1.15.0 // indirect
github.com/disgoorg/json v1.1.0 // indirect
github.com/disgoorg/log v1.2.1 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/flosch/pongo2/v4 v4.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand All @@ -64,43 +60,77 @@ require (
github.com/go-openapi/validate v0.22.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.14.1 // indirect
github.com/go-playground/validator/v10 v10.15.5 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gotify/go-api-client/v2 v2.0.4
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kataras/blocks v0.0.8 // indirect
github.com/kataras/golog v0.1.9 // indirect
github.com/kataras/iris/v12 v12.2.7 // indirect
github.com/kataras/pio v0.0.12 // indirect
github.com/kataras/sitemap v0.0.6 // indirect
github.com/kataras/tunnel v0.0.4 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/knz/go-libedit v1.10.1 // indirect
github.com/labstack/echo/v4 v4.11.1 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailgun/raymond/v2 v2.0.48 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/microcosm-cc/bluemonday v1.0.25 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.9 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tdewolff/minify/v2 v2.12.9 // indirect
github.com/tdewolff/parse/v2 v2.6.8 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
go.mongodb.org/mongo-driver v1.12.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yosssi/ace v0.0.5 // indirect
go.mongodb.org/mongo-driver v1.12.1 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/arch v0.5.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading

0 comments on commit d53b071

Please sign in to comment.