Run ListMetrics calls with goroutines.

Execute ListMetrics calls in separate goroutines (one for each metric),
in a similar way to how GetMetricData requests are handled.

This change removes semaphore usage (it's not used in GetMetricData as
well) for the sake of making it easier to reason about the code. It can
easily be added back in case we hit any issue around e.g. rate limits.

In local tests, the speedup seems to be particularly effective when
requesting 4 or more metrics in parallel (e.g. especially with EC2/EBS).

pkg/abstract.go

@@ -183,26 +183,37 @@ func getMetricDataForQueries(
 	logger log.Logger) []cloudwatchData {
 	var getMetricDatas []cloudwatchData
 
+	mux := &sync.Mutex{}
+	var wg sync.WaitGroup
+
 	// For every metric of the job
 	for _, metric := range discoveryJob.Metrics {
-		// Get the full list of metrics
-		// This includes, for this metric the possible combinations
-		// of dimensions and value of dimensions with data
-		tagSemaphore <- struct{}{}
+		wg.Add(1)
+		go func(m *Metric) {
+			defer wg.Done()
+			// Get the full list of metrics
+			// This includes, for this metric the possible combinations
+			// of dimensions and value of dimensions with data
+			metricsList, err := getFullMetricsList(ctx, svc.Namespace, m, clientCloudwatch)
+
+			if err != nil {
+				level.Error(logger).Log("msg", "Failed to get full metric list", "err", err, "metric_name", m.Name, "namespace", svc.Namespace)
+				return
+			}
 
-		metricsList, err := getFullMetricsList(ctx, svc.Namespace, metric, clientCloudwatch)
-		<-tagSemaphore
+			if len(resources) == 0 {
+				level.Debug(logger).Log("msg", "No resources for metric", "err", err, "metric_name", m.Name, "namespace", svc.Namespace)
+			}
 
-		if err != nil {
-			level.Error(logger).Log("msg", "Failed to get full metric list", "err", err, "metric_name", metric.Name, "namespace", svc.Namespace)
-			continue
-		}
+			metricDatas := getFilteredMetricDatas(region, accountId, discoveryJob.Type, discoveryJob.CustomTags, tagsOnMetrics, svc.DimensionRegexps, resources, metricsList.Metrics, m)
 
-		if len(resources) == 0 {
-			level.Debug(logger).Log("msg", "No resources for metric", "err", err, "metric_name", metric.Name, "namespace", svc.Namespace)
-		}
-		getMetricDatas = append(getMetricDatas, getFilteredMetricDatas(region, accountId, discoveryJob.Type, discoveryJob.CustomTags, tagsOnMetrics, svc.DimensionRegexps, resources, metricsList.Metrics, metric)...)
+			mux.Lock()
+			getMetricDatas = append(getMetricDatas, metricDatas...)
+			mux.Unlock()
+		}(metric)
 	}
+
+	wg.Wait()
 	return getMetricDatas
 }