feat(input): Adding max files option for filecount input plugin

plugins/inputs/filecount/README.md

@@ -49,13 +49,24 @@ plugin ordering. See [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ## duration. If mtime is negative, only count files that have been
   ## touched in this duration. Defaults to "0s".
   mtime = "0s"
+
+  ## Stop counting after this duration has passed.
+  ## Acceptable units: ns, us (µs), ms, s, m, h.
+  ## Defaults to "0s" (disabled).
+  timeout = "0s"
+
+  ## Stop counting after this many files have been processed.
+  ## Defaults to 0 (disabled).
+  maximum_files = 0
 ```
 
 ## Metrics
 
 - filecount
   - tags:
     - directory (the directory path)
+    - filecount_status (status of the count: "ok", "timeout",
+    or "maximum")
   - fields:
     - count (integer)
     - size_bytes (integer)
@@ -65,6 +76,6 @@ plugin ordering. See [CONFIGURATION.md][CONFIGURATION.md] for more details.
 ## Example Output
 
 ```text
-filecount,directory=/var/cache/apt count=7i,size_bytes=7438336i,oldest_file_timestamp=1507152973123456789i,newest_file_timestamp=1507152973123456789i 1530034445000000000
-filecount,directory=/tmp count=17i,size_bytes=28934786i,oldest_file_timestamp=1507152973123456789i,newest_file_timestamp=1507152973123456789i 1530034445000000000
+filecount,directory=/var/cache/apt,filecount_status=ok count=7i,size_bytes=7438336i,oldest_file_timestamp=1507152973123456789i,newest_file_timestamp=1507152973123456789i 1530034445000000000
+filecount,directory=/tmp,filecount_status=ok count=17i,size_bytes=28934786i,oldest_file_timestamp=1507152973123456789i,newest_file_timestamp=1507152973123456789i 1530034445000000000
 ```

plugins/inputs/filecount/filecount.go

@@ -29,6 +29,8 @@ type FileCount struct {
 	Size           config.Size     `toml:"size"`
 	MTime          config.Duration `toml:"mtime"`
 	Log            telegraf.Logger `toml:"-"`
+	Timeout        config.Duration `toml:"timeout"`
+	MaximumFiles   int64           `toml:"maximum_files"`
 
 	fs          fileSystem
 	fileFilters []fileFilterFunc
@@ -143,7 +145,21 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
 	oldestFileTimestamp := make(map[string]int64)
 	newestFileTimestamp := make(map[string]int64)
 
+	start := time.Now()
+	totalFiles := int64(0)
+	maxReached := false
+
 	walkFn := func(path string, _ *godirwalk.Dirent) error {
+		// Timeout check (priority)
+		if fc.Timeout > 0 && time.Since(start) > time.Duration(fc.Timeout) {
+			return filepath.SkipDir
+		}
+		// Maximum files check
+		if fc.MaximumFiles > 0 && totalFiles >= fc.MaximumFiles {
+			maxReached = true
+			return filepath.SkipDir
+		}
+
 		rel, err := filepath.Rel(basedir, path)
 		if err == nil && rel == "." {
 			return nil
@@ -161,6 +177,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
 			return nil
 		}
 		if match {
+			totalFiles++
 			parent := filepath.Dir(path)
 			childCount[parent]++
 			childSize[parent] += file.Size()
@@ -185,10 +202,17 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
 			}
 			gauge["oldest_file_timestamp"] = oldestFileTimestamp[path]
 			gauge["newest_file_timestamp"] = newestFileTimestamp[path]
-			acc.AddGauge("filecount", gauge,
-				map[string]string{
-					"directory": path,
-				})
+
+			tags := map[string]string{"directory": path}
+			if maxReached {
+				tags["filecount_status"] = "maximum"
+			} else if fc.Timeout > 0 && time.Since(start) > time.Duration(fc.Timeout) {
+				tags["filecount_status"] = "timeout"
+			} else {
+				tags["filecount_status"] = "ok"
+			}
+
+			acc.AddGauge("filecount", gauge, tags)
 		}
 		parent := filepath.Dir(path)
 		if fc.Recursive {
@@ -302,6 +326,8 @@ func newFileCount() *FileCount {
 		MTime:          config.Duration(0),
 		fileFilters:    nil,
 		fs:             osFS{},
+		Timeout:        config.Duration(0),
+		MaximumFiles:   0,
 	}
 }
 

plugins/inputs/filecount/filecount_test.go

@@ -35,7 +35,7 @@ func TestNoFiltersOnChildDir(t *testing.T) {
 	matches := []string{"subdir/quux", "subdir/quuz",
 		"subdir/nested2/qux", "subdir/nested2"}
 
-	tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
+	tags := map[string]string{"directory": getTestdataDir() + "/subdir", "filecount_status": "ok"}
 	acc := testutil.Accumulator{}
 	require.NoError(t, acc.GatherError(fc.Gather))
 	require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
@@ -50,7 +50,7 @@ func TestNoRecursiveButSuperMeta(t *testing.T) {
 	fc.Directories = []string{getTestdataDir() + "/**"}
 	matches := []string{"subdir/quux", "subdir/quuz", "subdir/nested2"}
 
-	tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
+	tags := map[string]string{"directory": getTestdataDir() + "/subdir", "filecount_status": "ok"}
 	acc := testutil.Accumulator{}
 	require.NoError(t, acc.GatherError(fc.Gather))
 
@@ -80,7 +80,7 @@ func TestDoubleAndSimpleStar(t *testing.T) {
 	fc.Directories = []string{getTestdataDir() + "/**/*"}
 	matches := []string{"qux"}
 
-	tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2"}
+	tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2", "filecount_status": "ok"}
 
 	acc := testutil.Accumulator{}
 	require.NoError(t, acc.GatherError(fc.Gather))
@@ -160,7 +160,8 @@ func TestDirectoryWithTrailingSlash(t *testing.T) {
 		metric.New(
 			"filecount",
 			map[string]string{
-				"directory": getTestdataDir(),
+				"directory":        getTestdataDir(),
+				"filecount_status": "ok",
 			},
 			map[string]interface{}{
 				"count":                 9,
@@ -176,6 +177,57 @@ func TestDirectoryWithTrailingSlash(t *testing.T) {
 	testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime())
 }
 
+func TestMaximumFiles(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.MaximumFiles = 3
+
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "maximum"}
+	acc := testutil.Accumulator{}
+	require.NoError(t, acc.GatherError(fc.Gather))
+	require.True(t, acc.HasPoint("filecount", tags, "count", int64(3)))
+}
+
+func TestMaximumFilesDisabled(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.MaximumFiles = 0
+
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "ok"}
+	acc := testutil.Accumulator{}
+	require.NoError(t, acc.GatherError(fc.Gather))
+	require.True(t, acc.HasPoint("filecount", tags, "count", int64(9)))
+}
+
+func TestTimeout(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.Timeout = config.Duration(5 * time.Second)
+
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "ok"}
+	acc := testutil.Accumulator{}
+	require.NoError(t, acc.GatherError(fc.Gather))
+	require.True(t, acc.HasPoint("filecount", tags, "count", int64(9)))
+	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(5096)))
+}
+
+func TestTimeoutDisabled(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.Timeout = config.Duration(0)
+
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "ok"}
+	acc := testutil.Accumulator{}
+	require.NoError(t, acc.GatherError(fc.Gather))
+	require.True(t, acc.HasPoint("filecount", tags, "count", int64(9)))
+}
+
+func TestMaximumFilesLargerThanTotal(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.MaximumFiles = 100
+
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "ok"}
+	acc := testutil.Accumulator{}
+	require.NoError(t, acc.GatherError(fc.Gather))
+	require.True(t, acc.HasPoint("filecount", tags, "count", int64(9)))
+}
+
 func getNoFilterFileCount() FileCount {
 	return FileCount{
 		Log:         testutil.Logger{},
@@ -242,7 +294,7 @@ func getFakeFileSystem(basePath string) fakeFileSystem {
 }
 
 func fileCountEquals(t *testing.T, fc FileCount, expectedCount, expectedSize int) {
-	tags := map[string]string{"directory": getTestdataDir()}
+	tags := map[string]string{"directory": getTestdataDir(), "filecount_status": "ok"}
 	acc := testutil.Accumulator{}
 	require.NoError(t, acc.GatherError(fc.Gather))
 	require.True(t, acc.HasPoint("filecount", tags, "count", int64(expectedCount)))

plugins/inputs/filecount/sample.conf

@@ -30,3 +30,12 @@
   ## duration. If mtime is negative, only count files that have been
   ## touched in this duration. Defaults to "0s".
   mtime = "0s"
+
+  ## Stop counting after this duration has passed.
+  ## Acceptable units: ns, us (µs), ms, s, m, h.
+  ## Defaults to "0s" (disabled).
+  timeout = "0s"
+
+  ## Stop counting after this many files have been processed.
+  ## Defaults to 0 (disabled).
+  maximum_files = 0