Services: support storagegateway metrics (prometheus-community#632)

* Services: support storagegateway metrics

* Support tags for GatewayName only storagegateway metrics

Such as SmbV3Sessions

README.md

@@ -86,6 +86,7 @@ We will contact you as soon as possible.
   * ses (AWS/SES) - Simple Email Service
   * shield (AWS/DDoSProtection) - Distributed Denial of Service (DDoS) protection service
   * sqs (AWS/SQS) - Simple Queue Service
+  * storagegateway (AWS/StorageGateway) - On-premises access to cloud storage
   * tgw (AWS/TransitGateway) - Transit Gateway
   * vpn (AWS/VPN) - VPN connection
   * asg (AWS/AutoScaling) - Auto Scaling Group

pkg/abstract.go

@@ -49,12 +49,13 @@ func scrapeAwsData(
 					}
 
 					clientTag := tagsInterface{
-						client:           cache.GetTagging(&region, role),
-						apiGatewayClient: cache.GetAPIGateway(&region, role),
-						asgClient:        cache.GetASG(&region, role),
-						dmsClient:        cache.GetDMS(&region, role),
-						ec2Client:        cache.GetEC2(&region, role),
-						logger:           jobLogger,
+						client:               cache.GetTagging(&region, role),
+						apiGatewayClient:     cache.GetAPIGateway(&region, role),
+						asgClient:            cache.GetASG(&region, role),
+						dmsClient:            cache.GetDMS(&region, role),
+						ec2Client:            cache.GetEC2(&region, role),
+						storagegatewayClient: cache.GetStorageGateway(&region, role),
+						logger:               jobLogger,
 					}
 
 					resources, metrics := scrapeDiscoveryJobUsingMetricData(ctx, discoveryJob, region, result.Account, config.Discovery.ExportedTagsOnMetrics, clientTag, clientCloudwatch, metricsPerQuery, discoveryJob.RoundingPeriod, tagSemaphore, jobLogger)

pkg/aws_tags.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
+	"github.com/aws/aws-sdk-go/service/storagegateway/storagegatewayiface"
 )
 
 // taggedResource is an AWS resource with tags
@@ -77,12 +78,13 @@ func (r taggedResource) metricTags(tagsOnMetrics exportedTagsOnMetrics) []Tag {
 
 // https://docs.aws.amazon.com/sdk-for-go/api/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface/
 type tagsInterface struct {
-	client           resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
-	asgClient        autoscalingiface.AutoScalingAPI
-	apiGatewayClient apigatewayiface.APIGatewayAPI
-	ec2Client        ec2iface.EC2API
-	dmsClient        databasemigrationserviceiface.DatabaseMigrationServiceAPI
-	logger           Logger
+	client               resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
+	asgClient            autoscalingiface.AutoScalingAPI
+	apiGatewayClient     apigatewayiface.APIGatewayAPI
+	ec2Client            ec2iface.EC2API
+	dmsClient            databasemigrationserviceiface.DatabaseMigrationServiceAPI
+	storagegatewayClient storagegatewayiface.StorageGatewayAPI
+	logger               Logger
 }
 
 func (iface tagsInterface) get(ctx context.Context, job *Job, region string) ([]*taggedResource, error) {

pkg/prometheus.go

@@ -45,6 +45,10 @@ var (
 		Name: "yace_cloudwatch_ec2api_requests_total",
 		Help: "Help is not implemented yet.",
 	})
+	storagegatewayAPICounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "yace_cloudwatch_storagegatewayapi_requests_total",
+		Help: "Help is not implemented yet.",
+	})
 	dmsAPICounter = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "yace_cloudwatch_dmsapi_requests_total",
 		Help: "Help is not implemented yet.",

pkg/services.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/databasemigrationservice"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/storagegateway"
 )
 
 type ResourceFunc func(context.Context, tagsInterface, *Job, string) ([]*taggedResource, error)
@@ -654,6 +655,50 @@ var (
 			DimensionRegexps: []*string{
 				aws.String("(?P<QueueName>[^:]+)$"),
 			},
+		}, {
+			Namespace: "AWS/StorageGateway",
+			Alias:     "storagegateway",
+			ResourceFilters: []*string{
+				aws.String("storagegateway"),
+			},
+			DimensionRegexps: []*string{
+				aws.String(":gateway/(?P<GatewayId>[^:]+)$"),
+				aws.String(":share/(?P<ShareId>[^:]+)$"),
+				aws.String("^(?P<GatewayId>[^:/]+)/(?P<GatewayName>[^:]+)$"),
+			},
+			ResourceFunc: func(ctx context.Context, iface tagsInterface, job *Job, region string) (resources []*taggedResource, err error) {
+				pageNum := 0
+				return resources, iface.storagegatewayClient.ListGatewaysPagesWithContext(ctx, &storagegateway.ListGatewaysInput{},
+					func(page *storagegateway.ListGatewaysOutput, more bool) bool {
+						pageNum++
+						storagegatewayAPICounter.Inc()
+
+						for _, gwa := range page.Gateways {
+							resource := taggedResource{
+								ARN:       fmt.Sprintf("%s/%s", *gwa.GatewayId, *gwa.GatewayName),
+								Namespace: job.Type,
+								Region:    region,
+							}
+
+							tagsRequest := &storagegateway.ListTagsForResourceInput{
+								ResourceARN: gwa.GatewayARN,
+							}
+							tagsResponse, _ := iface.storagegatewayClient.ListTagsForResource(tagsRequest)
+							storagegatewayAPICounter.Inc()
+
+							for _, t := range tagsResponse.Tags {
+								resource.Tags = append(resource.Tags, Tag{Key: *t.Key, Value: *t.Value})
+							}
+
+							if resource.filterThroughTags(job.SearchTags) {
+								resources = append(resources, &resource)
+							}
+						}
+
+						return pageNum < 100
+					},
+				)
+			},
 		}, {
 			Namespace: "AWS/TransitGateway",
 			Alias:     "tgw",

pkg/sessions.go

@@ -23,6 +23,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	r "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
+	"github.com/aws/aws-sdk-go/service/storagegateway"
+	"github.com/aws/aws-sdk-go/service/storagegateway/storagegatewayiface"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/aws/aws-sdk-go/service/sts/stsiface"
 )
@@ -38,6 +40,7 @@ type SessionCache interface {
 	GetEC2(*string, Role) ec2iface.EC2API
 	GetDMS(*string, Role) databasemigrationserviceiface.DatabaseMigrationServiceAPI
 	GetAPIGateway(*string, Role) apigatewayiface.APIGatewayAPI
+	GetStorageGateway(*string, Role) storagegatewayiface.StorageGatewayAPI
 	Refresh()
 	Clear()
 }
@@ -59,13 +62,14 @@ type clientCache struct {
 	// if we know that this job is only used for static
 	// then we don't have to construct as many cached connections
 	// later on
-	onlyStatic bool
-	cloudwatch cloudwatchiface.CloudWatchAPI
-	tagging    resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
-	asg        autoscalingiface.AutoScalingAPI
-	ec2        ec2iface.EC2API
-	dms        databasemigrationserviceiface.DatabaseMigrationServiceAPI
-	apiGateway apigatewayiface.APIGatewayAPI
+	onlyStatic     bool
+	cloudwatch     cloudwatchiface.CloudWatchAPI
+	tagging        resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
+	asg            autoscalingiface.AutoScalingAPI
+	ec2            ec2iface.EC2API
+	dms            databasemigrationserviceiface.DatabaseMigrationServiceAPI
+	apiGateway     apigatewayiface.APIGatewayAPI
+	storageGateway storagegatewayiface.StorageGatewayAPI
 }
 
 // NewSessionCache creates a new session cache to use when fetching data from
@@ -153,6 +157,7 @@ func (s *sessionCache) Clear() {
 			s.clients[role][region].ec2 = nil
 			s.clients[role][region].dms = nil
 			s.clients[role][region].apiGateway = nil
+			s.clients[role][region].storageGateway = nil
 		}
 	}
 	s.cleared = true
@@ -189,6 +194,7 @@ func (s *sessionCache) Refresh() {
 			s.clients[role][region].ec2 = createEC2Session(s.session, &region, role, s.fips, s.logger.IsDebugEnabled())
 			s.clients[role][region].dms = createDMSSession(s.session, &region, role, s.fips, s.logger.IsDebugEnabled())
 			s.clients[role][region].apiGateway = createAPIGatewaySession(s.session, &region, role, s.fips, s.logger.IsDebugEnabled())
+			s.clients[role][region].storageGateway = createStorageGatewaySession(s.session, &region, role, s.fips, s.logger.IsDebugEnabled())
 		}
 	}
 
@@ -293,6 +299,21 @@ func (s *sessionCache) GetAPIGateway(region *string, role Role) apigatewayiface.
 
 }
 
+func (s *sessionCache) GetStorageGateway(region *string, role Role) storagegatewayiface.StorageGatewayAPI {
+	// if we have not refreshed then we need to lock in case we are accessing concurrently
+	if !s.refreshed {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+	}
+	if sess, ok := s.clients[role][*region]; ok && sess.storageGateway != nil {
+		return sess.storageGateway
+	}
+
+	s.clients[role][*region].storageGateway = createStorageGatewaySession(s.session, region, role, s.fips, s.logger.IsDebugEnabled())
+	return s.clients[role][*region].storageGateway
+
+}
+
 func setExternalID(ID string) func(p *stscreds.AssumeRoleProvider) {
 	return func(p *stscreds.AssumeRoleProvider) {
 		if ID != "" {
@@ -403,6 +424,23 @@ func createASGSession(sess *session.Session, region *string, role Role, isDebugE
 	return autoscaling.New(sess, setSTSCreds(sess, config, role))
 }
 
+func createStorageGatewaySession(sess *session.Session, region *string, role Role, fips bool, isDebugEnabled bool) storagegatewayiface.StorageGatewayAPI {
+	maxStorageGatewayAPIRetries := 5
+	config := &aws.Config{Region: region, MaxRetries: &maxStorageGatewayAPIRetries}
+
+	if fips {
+		// https://aws.amazon.com/compliance/fips/
+		endpoint := fmt.Sprintf("https://storagegateway-fips.%s.amazonaws.com", *region)
+		config.Endpoint = aws.String(endpoint)
+	}
+
+	if isDebugEnabled {
+		config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
+	}
+
+	return storagegateway.New(sess, setSTSCreds(sess, config, role))
+}
+
 func createEC2Session(sess *session.Session, region *string, role Role, fips bool, isDebugEnabled bool) ec2iface.EC2API {
 	maxEC2APIRetries := 10
 	config := &aws.Config{Region: region, MaxRetries: &maxEC2APIRetries}