Added --include flag to buildah add and buildah copy

Complement to existing --exclude flag. When specified, only files matching
the given patterns are copied. Uses the same containerignore(5) pattern
format. Can be combined with --exclude, where exclude takes priority.

Signed-off-by: Joshua Arrevillaga <2004jarevillaga@gmail.com>

Author: Joshua Arrevillaga

add.go

@@ -126,6 +126,10 @@ type AddAndCopyOptions struct {
 	// FollowSymlink controls whether symlinks should be followed when copying content.
 	// When set to false, symlinks are not dereferenced.
 	FollowSymlink types.OptionalBool
+	// Includes is a list of patterns to include, the complement to Excludes.
+	// Only items matching one of these patterns are copied. Has the same
+	// pattern format as lines of a .containerignore file.
+	Includes []string
 }
 
 // getURL writes a tar archive containing the named content
@@ -608,6 +612,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 						UIDMap:             srcUIDMap,
 						GIDMap:             srcGIDMap,
 						Excludes:           options.Excludes,
+						Includes:           options.Includes,
 						ExpandArchives:     extract,
 						Chmod:              options.Chmod,
 						ChownDirs:          chownDirs,
@@ -774,6 +779,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					UIDMap:             srcUIDMap,
 					GIDMap:             srcGIDMap,
 					Excludes:           options.Excludes,
+					Includes:           options.Includes,
 					ExpandArchives:     extract,
 					Chmod:              options.Chmod,
 					ChownDirs:          chownDirs,

cmd/buildah/addcopy.go

@@ -46,6 +46,7 @@ type addCopyResults struct {
 	allowWildcard      bool
 	allowEmptyWildcard bool
 	noFollowSymlinks   bool
+	includes           []string
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -108,6 +109,7 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
 	flags.StringVar(&opts.timestamp, "timestamp", "", "set timestamps on new content to `seconds` after the epoch")
 	flags.BoolVar(&opts.allowWildcard, "allow-wildcard", true, "allow glob patterns in source paths")
 	flags.BoolVar(&opts.allowEmptyWildcard, "allow-empty-wildcard", false, "don't error when glob patterns match nothing")
+	flags.StringSliceVar(&opts.includes, "include", nil, "include pattern when copying files")
 }
 
 func addcopyInit() {
@@ -297,6 +299,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		Timestamp:             timestamp,
 		Link:                  iopts.link,
 		FollowSymlink:         followSymlink,
+		Includes:              iopts.includes,
 	}
 	if iopts.contextdir != "" {
 		var excludes []string

copier/copier.go

@@ -198,6 +198,15 @@ func (req *request) Excludes() []string {
 	}
 }
 
+func (req *request) Includes() []string {
+	switch req.Request {
+	case requestGet:
+		return req.GetOptions.Includes
+	default:
+		return nil
+	}
+}
+
 func (req *request) UIDMap() []idtools.IDMap {
 	switch req.Request {
 	case requestEval:
@@ -405,6 +414,7 @@ type GetOptions struct {
 	Timestamp          *time.Time        // timestamp to force on all contents
 	DisallowWildcard   bool              // reject glob patterns in source paths
 	AllowEmptyWildcard bool              // don't error when glob patterns match nothing
+	Includes           []string          // contents to include, using the OS-specific path separator.
 }
 
 // Get produces an archive containing items that match the specified glob
@@ -1024,11 +1034,19 @@ func copierHandler(bulkReader io.Reader, bulkWriter io.Writer, req request) (*re
 	// os.PathSeparator, implying that it expects OS-specific naming
 	// conventions.
 	excludes := req.Excludes()
-	pm, err := fileutils.NewPatternMatcher(excludes)
+	pmExcludes, err := fileutils.NewPatternMatcher(excludes)
 	if err != nil {
 		return nil, nil, fmt.Errorf("processing excludes list %v: %w", excludes, err)
 	}
 
+	var pmIncludes *fileutils.PatternMatcher
+	if includes := req.Includes(); len(includes) > 0 {
+		pmIncludes, err = fileutils.NewPatternMatcher(includes)
+		if err != nil {
+			return nil, nil, fmt.Errorf("processing includes list %v: %w", includes, err)
+		}
+	}
+
 	var idMappings *idtools.IDMappings
 	uidMap, gidMap := req.UIDMap(), req.GIDMap()
 	if len(uidMap) > 0 && len(gidMap) > 0 {
@@ -1042,10 +1060,10 @@ func copierHandler(bulkReader io.Reader, bulkWriter io.Writer, req request) (*re
 		resp := copierHandlerEval(req)
 		return resp, nil, nil
 	case requestStat:
-		resp := copierHandlerStat(req, pm, idMappings)
+		resp := copierHandlerStat(req, pmExcludes, idMappings)
 		return resp, nil, nil
 	case requestGet:
-		return copierHandlerGet(bulkWriter, req, pm, idMappings)
+		return copierHandlerGet(bulkWriter, req, pmExcludes, pmIncludes, idMappings)
 	case requestPut:
 		return copierHandlerPut(bulkReader, req, idMappings)
 	case requestMkdir:
@@ -1092,6 +1110,28 @@ func pathIsExcluded(root, path string, pm *fileutils.PatternMatcher) (string, bo
 	return rel, false, nil
 }
 
+func pathIsIncluded(root, path string, pm *fileutils.PatternMatcher) (bool, error) {
+	rel, err := convertToRelSubdirectory(root, path)
+	if err != nil {
+		return false, fmt.Errorf("copier: error computing path of %q relative to root %q: %w", path, root, err)
+	}
+	if pm == nil {
+		return true, nil
+	}
+	if rel == "." {
+		// special case
+		return true, nil
+	}
+	// Matches uses filepath.FromSlash() to convert candidates before
+	// checking if they match the patterns it's been given, implying that
+	// it expects Unix-style paths.
+	matches, err := pm.Matches(filepath.ToSlash(rel)) //nolint:staticcheck
+	if err != nil {
+		return false, fmt.Errorf("copier: error checking if %q is included: %w", rel, err)
+	}
+	return matches, nil
+}
+
 // resolvePath resolves symbolic links in paths, treating the specified
 // directory as the root.
 // Resolving the path this way, and using the result, is in no way secure
@@ -1173,7 +1213,7 @@ func containsWildcards(path string) bool {
 	return strings.ContainsAny(path, "*?[")
 }
 
-func copierHandlerStat(req request, pm *fileutils.PatternMatcher, idMappings *idtools.IDMappings) *response {
+func copierHandlerStat(req request, pmExcludes *fileutils.PatternMatcher, idMappings *idtools.IDMappings) *response {
 	errorResponse := func(fmtspec string, args ...any) *response {
 		return &response{Error: fmt.Sprintf(fmtspec, args...), Stat: statResponse{}}
 	}
@@ -1202,7 +1242,7 @@ func copierHandlerStat(req request, pm *fileutils.PatternMatcher, idMappings *id
 		s.Globbed = make([]string, 0, len(globMatched))
 		s.Results = make(map[string]*StatForItem)
 		for _, globbed := range globMatched {
-			rel, excluded, err := pathIsExcluded(req.Root, globbed, pm)
+			rel, excluded, err := pathIsExcluded(req.Root, globbed, pmExcludes)
 			if err != nil {
 				return errorResponse("copier: stat: %v", err)
 			}
@@ -1262,7 +1302,7 @@ func copierHandlerStat(req request, pm *fileutils.PatternMatcher, idMappings *id
 				// could be a relative link) and in the context
 				// of the chroot
 				result.ImmediateTarget = immediateTarget
-				resolvedTarget, err := resolvePath(req.Root, globbed, true, pm)
+				resolvedTarget, err := resolvePath(req.Root, globbed, true, pmExcludes)
 				if err != nil {
 					return errorResponse("copier: stat: error resolving %q: %v", globbed, err)
 				}
@@ -1355,8 +1395,8 @@ func checkLinks(item string, req request, info os.FileInfo) (string, os.FileInfo
 	return item, info, nil
 }
 
-func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMatcher, idMappings *idtools.IDMappings) (*response, func() error, error) {
-	statResponse := copierHandlerStat(req, pm, idMappings)
+func copierHandlerGet(bulkWriter io.Writer, req request, pmExcludes, pmIncludes *fileutils.PatternMatcher, idMappings *idtools.IDMappings) (*response, func() error, error) {
+	statResponse := copierHandlerStat(req, pmExcludes, idMappings)
 	errorResponse := func(fmtspec string, args ...any) (*response, func() error, error) {
 		return &response{Error: fmt.Sprintf(fmtspec, args...), Stat: statResponse.Stat, Get: getResponse{}}, nil, nil
 	}
@@ -1522,7 +1562,7 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 						// skip the "." entry
 						return nil
 					}
-					skippedPath, skip, err := pathIsExcluded(req.Root, path, pm)
+					skippedPath, skip, err := pathIsExcluded(req.Root, path, pmExcludes)
 					if err != nil {
 						return err
 					}
@@ -1533,14 +1573,14 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 							// all, we don't need to
 							// descend into this particular
 							// directory if it's a directory
-							if !pm.Exclusions() {
+							if !pmExcludes.Exclusions() {
 								return filepath.SkipDir
 							}
 							// if there are exclusion
 							// patterns for which this
 							// path is a prefix, we
 							// need to keep descending
-							for _, pattern := range pm.Patterns() {
+							for _, pattern := range pmExcludes.Patterns() {
 								if !pattern.Exclusion() {
 									continue
 								}
@@ -1563,6 +1603,16 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 						// also be in the excludes list
 						return nil
 					}
+					if pmIncludes != nil && !d.IsDir() {
+						included, err := pathIsIncluded(item, path, pmIncludes)
+						if err != nil {
+							return err
+						}
+
+						if !included {
+							return nil
+						}
+					}
 					// if it's a symlink, read its target
 					symlinkTarget := ""
 					if d.Type() == os.ModeSymlink {
@@ -1608,14 +1658,24 @@ func copierHandlerGet(bulkWriter io.Writer, req request, pm *fileutils.PatternMa
 				}
 				itemsCopied++
 			} else {
-				_, skip, err := pathIsExcluded(req.Root, item, pm)
+				_, skip, err := pathIsExcluded(req.Root, item, pmExcludes)
 				if err != nil {
 					return err
 				}
 				if skip {
 					continue
 				}
 
+				if pmIncludes != nil {
+					included, err := pathIsIncluded(req.Root, item, pmIncludes)
+					if err != nil {
+						return err
+					}
+					if !included {
+						continue
+					}
+				}
+
 				name := filepath.Base(queue[i].glob)
 				if req.GetOptions.Parents {
 					name, err = convertToRelSubdirectory(req.Directory, queue[i].glob)

docs/buildah-add.1.md

@@ -62,7 +62,7 @@ by symbolic links outside of the chroot will fail.
 
 **--exclude** *pattern*
 
-Exclude copying files matching the specified pattern. Option can be specified
+Exclude copying files matching the specified pattern. The option can be specified
 multiple times. Patterns are matched against each file's path relative to the
 context directory (or, with **--from**, relative to the source container or image root).
 See containerignore(5) for supported formats.
@@ -78,6 +78,12 @@ can be used.
 
 Path to an alternative .containerignore (.dockerignore) file. Requires \-\-contextdir be specified.
 
+**--include** *pattern*
+
+Only copy files matching the specific pattern, the complement of **--exclude**. The option
+can be specified multiple times. Patterns are matched against each file's path relative to
+the source directory being copied. See containerignore(5) for supported formats.
+
 **--link**
 
 Create an independent image layer for the added files instead of modifying the working

docs/buildah-copy.1.md

@@ -60,7 +60,7 @@ by symbolic links outside of the chroot will fail.
 
 **--exclude** *pattern*
 
-Exclude copying files matching the specified pattern. Option can be specified
+Exclude copying files matching the specified pattern. The option can be specified
 multiple times. Patterns are matched against each file's path relative to the
 context directory (or, with **--from**, relative to the source container or image root).
 See containerignore(5) for supported formats.
@@ -77,6 +77,12 @@ is preserved.
 
 Path to an alternative .containerignore (.dockerignore) file. Requires \-\-contextdir be specified.
 
+**--include** *pattern*
+
+Only copy files matching the specific pattern, the complement of **--exclude**. The option
+can be specified multiple times. Patterns are matched against each file's path relative to
+the source directory being copied. See containerignore(5) for supported formats.
+
 **--link**
 
 Create an independent image layer for the added files instead of modifying the working

tests/add.bats

@@ -668,3 +668,41 @@ EOF
   cmp $ubuntu/etc/passwd ${croot}/tmp/passwd
   cmp $ubuntu/etc/passwd ${croot}/tmp/passwd2
 }
+
+@test "add --include" {
+  mytest=${TEST_SCRATCH_DIR}/mytest
+  mkdir -p ${mytest}/subdir
+  touch ${mytest}/source.go
+  touch ${mytest}/readme.md
+  touch ${mytest}/subdir/nested.go
+  touch ${mytest}/subdir/nested.md
+
+expect="
+stuff
+stuff/source.go
+stuff/subdir
+stuff/subdir/nested.go"
+
+  run_buildah from $WITH_POLICY_JSON scratch
+  cid=$output
+  run_buildah add --include=**/*.go $cid ${mytest} /stuff
+
+  run_buildah_mount $cid
+  mnt=$output
+  run find $mnt -printf "%P\n"
+  filelist=$(LC_ALL=C sort <<<"$output")
+  run_buildah_umount $cid
+  expect_output --from="$filelist" "$expect" "add recursive include"
+
+  # else arm: a single named file that matches include copies cleanly
+  run_buildah from $WITH_POLICY_JSON scratch
+  cid=$output
+  run_buildah add --include=**/*.go $cid ${mytest}/source.go /stuff2/
+
+  run_buildah_mount $cid
+  mnt=$output
+  run find $mnt -printf "%P\n"
+  filelist=$(LC_ALL=C sort <<<"$output")
+  run_buildah_umount $cid
+  expect_output --from="$filelist" --substring "source.go" "add include single file"
+}

tests/copy.bats

@@ -807,3 +807,45 @@ parents/y/b.txt"
   run_buildah 125 copy --allow-empty-wildcard=true $cid ${TEST_SCRATCH_DIR}/no-such-file /dest4/
   expect_output --substring "no such file or directory"
 }
+
+@test "copy --include" {
+  mytest=${TEST_SCRATCH_DIR}/mytest
+  mkdir -p ${mytest}/subdir
+  touch ${mytest}/source.go
+  touch ${mytest}/readme.md
+  touch ${mytest}/subdir/nested.go
+  touch ${mytest}/subdir/nested.md
+
+  # recursive include: **/*.go keeps .go at all depths, drops .md
+expect="
+stuff
+stuff/source.go
+stuff/subdir
+stuff/subdir/nested.go"
+
+  run_buildah from $WITH_POLICY_JSON scratch
+  cid=$output
+  run_buildah copy --include=**/*.go $cid ${mytest} /stuff
+  run_buildah_mount $cid
+  mnt=$output
+  run find $mnt -printf "%P\n"
+  filelist=$(LC_ALL=C sort <<<"$output")
+  run_buildah_umount $cid
+  expect_output --from="$filelist" "$expect" "recursive include"
+
+  # include + exclude: exclude wins when both match
+expect="
+stuff
+stuff/source.go
+stuff/subdir"
+
+  run_buildah from $WITH_POLICY_JSON scratch
+  cid=$output
+  run_buildah copy --include=**/*.go --exclude=**/nested.go $cid ${mytest} /stuff
+  run_buildah_mount $cid
+  mnt=$output
+  run find $mnt -printf "%P\n"
+  filelist=$(LC_ALL=C sort <<<"$output")
+  run_buildah_umount $cid
+  expect_output --from="$filelist" "$expect" "include with exclude"
+}