Skip to content

Add support for EKS Pod Identity in ECS Prometheus discovery #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vSharique wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add support for EKS Pod Identity in ECS Prometheus discovery #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18776e4
feat(main): adding support for using eks pod identity
Apr 17, 2025
b626548
Updating README.md
Apr 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ Then, run it as follows:
passed in via the `--config.role-arn` option. This option also
allows for cross-account access, depending on which account
the role is defined in.
* When running in an EKS cluster, you can use pod identity for authentication by setting the `--config.use-eks-pod-identity`
flag to `true`. This will use the pod's IAM role for authentication instead of environment variables or instance profile credentials.
* Start the program, using the command line option
`-config.write-to` to point the program to the specific
folder that your Prometheus master can read from.
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/teralytics/prometheus-ecs-discovery
go 1.15

require (
github.com/aws/aws-sdk-go-v2 v1.3.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.1.4
github.com/aws/aws-sdk-go-v2/credentials v1.1.4
github.com/aws/aws-sdk-go-v2/service/ec2 v1.3.0
Expand Down
176 changes: 131 additions & 45 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ import (
"fmt"
"io/ioutil"
"log"
"regexp"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand All @@ -37,17 +39,18 @@ import (
)

type labels struct {
TaskArn string `yaml:"task_arn"`
TaskName string `yaml:"task_name"`
JobName string `yaml:"job,omitempty"`
TaskRevision string `yaml:"task_revision"`
TaskGroup string `yaml:"task_group"`
ClusterArn string `yaml:"cluster_arn"`
ContainerName string `yaml:"container_name"`
ContainerArn string `yaml:"container_arn"`
DockerImage string `yaml:"docker_image"`
MetricsPath string `yaml:"__metrics_path__,omitempty"`
Scheme string `yaml:"__scheme__,omitempty"`
TaskArn string `yaml:"task_arn"`
TaskName string `yaml:"task_name"`
JobName string `yaml:"job,omitempty"`
TaskRevision string `yaml:"task_revision"`
TaskGroup string `yaml:"task_group"`
ClusterArn string `yaml:"cluster_arn"`
ContainerName string `yaml:"container_name"`
ContainerArn string `yaml:"container_arn"`
DockerImage string `yaml:"docker_image"`
MetricsPath string `yaml:"__metrics_path__,omitempty"`
Scheme string `yaml:"__scheme__,omitempty"`
Tags map[string]string `yaml:",inline"`
}

// Docker label for enabling dynamic port detection
Expand All @@ -58,14 +61,31 @@ var outFile = flag.String("config.write-to", "ecs_file_sd.yml", "path of file to
var interval = flag.Duration("config.scrape-interval", 60*time.Second, "interval at which to scrape the AWS API for ECS service discovery information")
var times = flag.Int("config.scrape-times", 0, "how many times to scrape before exiting (0 = infinite)")
var roleArn = flag.String("config.role-arn", "", "ARN of the role to assume when scraping the AWS API (optional)")
var useEksPodIdentity = flag.Bool("config.use-eks-pod-identity", false, "Use EKS pod identity for authentication")
var filterECSTags = flag.String("config.filter-ecs-tags", "", "Specify CSV list of ECS task tags to filter and add as labels (defaults to all tags)")
var prometheusPortLabel = flag.String("config.port-label", "PROMETHEUS_EXPORTER_PORT", "Docker label to define the scrape port of the application (if missing an application won't be scraped)")
var prometheusPathLabel = flag.String("config.path-label", "PROMETHEUS_EXPORTER_PATH", "Docker label to define the scrape path of the application")
var prometheusSchemeLabel= flag.String("config.scheme-label", "PROMETHEUS_EXPORTER_SCHEME", "Docker label to define the scheme of the target application")
var prometheusSchemeLabel = flag.String("config.scheme-label", "PROMETHEUS_EXPORTER_SCHEME", "Docker label to define the scheme of the target application")
var prometheusFilterLabel = flag.String("config.filter-label", "", "Docker label (and optionally value) to require to scrape the application")
var prometheusServerNameLabel = flag.String("config.server-name-label", "PROMETHEUS_EXPORTER_SERVER_NAME", "Docker label to define the server name")
var prometheusJobNameLabel = flag.String("config.job-name-label", "PROMETHEUS_EXPORTER_JOB_NAME", "Docker label to define the job name")
var prometheusDynamicPortDetection = flag.Bool("config.dynamic-port-detection", false, fmt.Sprintf("If true, only tasks with the Docker label %s=1 will be scraped", dynamicPortLabel))

// filterECSTagsEnabled tracks if ECS tags needs to be filtered
var filterECSTagsEnabled bool = false

// isFlagPassed returns true if the flag has been used on the command line
func isFlagPassed(name string) bool {
found := false
flag.Visit(func(f *flag.Flag) {
if f.Name == name {
found = true
}
})

return found
}

// logError is a convenience function that decodes all possible ECS
// errors and displays them to standard error.
func logError(err error) {
Expand Down Expand Up @@ -124,32 +144,45 @@ type PrometheusTaskInfo struct {
Labels labels `yaml:"labels"`
}

// Regex used for normalizing label names
var labelRE = regexp.MustCompile("[^a-zA-Z0-9_]")

// normalizeLabelName normalizes label name passed as argument
// according to Prometheus' label convention
func normalizeLabelName(label string) string {
return labelRE.ReplaceAllString(label, "_")
}

// Filtered ECS Tags that will be propagated as labels
var filteredECSTags map[string]interface{}

// ExporterInformation returns a list of []*PrometheusTaskInfo
// enumerating the IPs, ports that the task's containers exports
// to Prometheus (one per container), so long as the Docker
// labels in its corresponding container definition for the
// container in the task has a PROMETHEUS_EXPORTER_PORT
//
// Example:
// ...
// "Name": "apache",
// "DockerLabels": {
// "PROMETHEUS_EXPORTER_PORT": "1234"
// },
// ...
// "PortMappings": [
// {
// "ContainerPort": 1883,
// "HostPort": 0,
// "Protocol": "tcp"
// },
// {
// "ContainerPort": 1234,
// "HostPort": 0,
// "Protocol": "tcp"
// }
// ],
// ...
//
// ...
// "Name": "apache",
// "DockerLabels": {
// "PROMETHEUS_EXPORTER_PORT": "1234"
// },
// ...
// "PortMappings": [
// {
// "ContainerPort": 1883,
// "HostPort": 0,
// "Protocol": "tcp"
// },
// {
// "ContainerPort": 1234,
// "HostPort": 0,
// "Protocol": "tcp"
// }
// ],
// ...
func (t *AugmentedTask) ExporterInformation() []*PrometheusTaskInfo {
ret := []*PrometheusTaskInfo{}
var host string
Expand Down Expand Up @@ -182,6 +215,21 @@ func (t *AugmentedTask) ExporterInformation() []*PrometheusTaskInfo {
filter = strings.Split(*prometheusFilterLabel, "=")
}

ecsTags := make(map[string]string)

// Add ECS task tags as labels
for _, tag := range t.Tags {
if len(filteredECSTags) > 0 {
// check if the ECS tag is in filtered tags
if _, ok := filteredECSTags[*tag.Key]; !ok {
continue
}
}

tagKey := normalizeLabelName(*tag.Key)
ecsTags[tagKey] = *tag.Value
}

for _, i := range t.Containers {
// Let's go over the containers to see which ones are defined
var d ecstypes.ContainerDefinition
Expand Down Expand Up @@ -278,6 +326,11 @@ func (t *AugmentedTask) ExporterInformation() []*PrometheusTaskInfo {
host = ip
}

if host == "" {
// No hostname or IP found for this container. Skip.
continue
}

labels := labels{
TaskArn: *t.TaskArn,
TaskName: *t.TaskDefinition.Family,
Expand All @@ -288,6 +341,7 @@ func (t *AugmentedTask) ExporterInformation() []*PrometheusTaskInfo {
ContainerName: *i.Name,
ContainerArn: *i.ContainerArn,
DockerImage: *d.Image,
Tags: ecsTags,
}

exporterPath, ok = d.DockerLabels[*prometheusPathLabel]
Expand All @@ -297,7 +351,7 @@ func (t *AugmentedTask) ExporterInformation() []*PrometheusTaskInfo {

scheme, ok = d.DockerLabels[*prometheusSchemeLabel]
if ok {
labels.Scheme = scheme
labels.Scheme = scheme
}

ret = append(ret, &PrometheusTaskInfo{
Expand Down Expand Up @@ -414,9 +468,7 @@ func DescribeInstancesUnpaginated(svc *ec2.Client, instanceIds []string) ([]ec2t
}
result := []ec2types.Instance{}
for _, rsv := range finalOutput.Reservations {
for _, i := range rsv.Instances {
result = append(result, i)
}
result = append(result, rsv.Instances...)
}
return result, nil
}
Expand Down Expand Up @@ -531,6 +583,7 @@ func GetTasksOfClusters(svc *ecs.Client, clusterArns []*string) ([]ecstypes.Task
inDescribe := &ecs.DescribeTasksInput{
Cluster: clusterArn,
Tasks: output.TaskArns,
Include: ecstypes.TaskFieldTags.Values(),
}
descOutput, err2 := svc.DescribeTasks(context.Background(), inDescribe)
if err2 != nil {
Expand Down Expand Up @@ -568,9 +621,7 @@ func GetTasksOfClusters(svc *ecs.Client, clusterArns []*string) ([]ecstypes.Task
if result.err != nil {
return nil, result.err
}
for _, task := range result.out.Tasks {
tasks = append(tasks, task)
}
tasks = append(tasks, result.out.Tasks...)
}

return tasks, nil
Expand Down Expand Up @@ -601,24 +652,59 @@ func GetAugmentedTasks(svc *ecs.Client, svcec2 *ec2.Client, clusterArns []*strin
return tasks, nil
}

// getFilteredECSTags returns filtered ECS tags as a map,
// converts a comma separated ECS tags string into a map
func getFilteredECSTags(ecsTags string) map[string]interface{} {
if !filterECSTagsEnabled {
// filter not enabled
return nil
}

filteredECSTagsList := strings.Split(ecsTags, ",")
var filteredECSTagsMap = make(map[string]interface{})

for _, tag := range filteredECSTagsList {
filteredECSTagsMap[tag] = nil
}

return filteredECSTagsMap
}

func main() {
flag.Parse()

config, err := config.LoadDefaultConfig(context.Background())
// Configure AWS credentials based on the environment
var cfg aws.Config
var err error

if *useEksPodIdentity {
// Use EKS pod identity
cfg, err = config.LoadDefaultConfig(context.Background())
} else {
// Use default credential chain (environment variables, EC2 instance profile, etc.)
cfg, err = config.LoadDefaultConfig(context.Background())
}

if err != nil {
logError(err)
return
}

if *roleArn != "" {
// Assume role
stsSvc := sts.NewFromConfig(config)
config.Credentials = stscreds.NewAssumeRoleProvider(stsSvc, *roleArn)
// Assume role if specified
stsSvc := sts.NewFromConfig(cfg)
cfg.Credentials = stscreds.NewAssumeRoleProvider(stsSvc, *roleArn)
}

// Initialise AWS Service clients
svc := ecs.NewFromConfig(config)
svcec2 := ec2.NewFromConfig(config)
// Initialize AWS Service clients
svc := ecs.NewFromConfig(cfg)
svcec2 := ec2.NewFromConfig(cfg)

// Check if the --config.filter-ecs-tags is used in command line
filterECSTagsEnabled = isFlagPassed("config.filter-ecs-tags")

// Prepare filtered list of ECS tags
filteredECSTags = getFilteredECSTags(*filterECSTags)

work := func() {
var clusters *ecs.ListClustersOutput
Expand Down