Merge pull request #1477 from mritunjaysharma394/fetch

adds git checkout fetch,update,test and yams the melange apkbuild yamls

Author: imjasonh

pkg/convert/apkbuild/apkbuild.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"crypto/sha256"
 	"crypto/sha512"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -17,6 +19,7 @@ import (
 	rlhttp "chainguard.dev/melange/pkg/http"
 	"chainguard.dev/melange/pkg/manifest"
 	"github.com/chainguard-dev/clog"
+	"github.com/chainguard-dev/yam/pkg/yam/formatted"
 
 	apkotypes "chainguard.dev/apko/pkg/build/types"
 	"chainguard.dev/melange/pkg/config"
@@ -243,12 +246,135 @@ func (c Context) transitiveDependencyList(convertor ApkConvertor) []string {
 	return dependencies
 }
 
+// Helper function to check if a URL belongs to GitHub and extract the owner/repo
+func getGitHubIdentifierFromURL(packageURL string) (string, bool) {
+	u, err := url.Parse(packageURL)
+	if err != nil || u.Host != "github.com" {
+		// Not a GitHub URL
+		return "", false
+	}
+	// Extract the owner and repo from the URL path
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) < 2 {
+		// Invalid GitHub URL format
+		return "", false
+	}
+	owner, repo := parts[0], parts[1]
+	return path.Join(owner, repo), true
+}
+
+// Helper function to set up the update block based on the fetch source
+func (c *Context) setupUpdateBlock(packageURL string, packageVersion string, converter *ApkConvertor) {
+	// Check if the package was fetched from GitHub
+	if identifier, isGitHub := getGitHubIdentifierFromURL(packageURL); isGitHub {
+
+		// Enable GitHub monitoring
+		converter.GeneratedMelangeConfig.Update = config.Update{
+			Enabled: true,
+			GitHubMonitor: &config.GitHubMonitor{
+				Identifier: identifier, // Set the owner/repo identifier
+				// To add logic to improve this check
+				// StripPrefix:     "v",        // Strip "v" from tags like "v1.2.3"
+				// TagFilterPrefix: "v",        // Filter tags with a "v" prefix
+			},
+		}
+	} else {
+		// Fallback to release-monitoring.org if it's not a GitHub package
+		converter.GeneratedMelangeConfig.Update = config.Update{
+			Enabled: true,
+			ReleaseMonitor: &config.ReleaseMonitor{
+				Identifier: 12345, // Example ID, replace this with actual logic to get the ID
+			},
+		}
+	}
+}
+
+// Helper function to fetch the commit hash for a specific tag from a GitHub repository
+func getCommitForTagFromGitHub(repoURL, tag string) (string, error) {
+	// Parse the repository URL to extract the owner and repo name
+	u, err := url.Parse(repoURL)
+	if err != nil {
+		return "", fmt.Errorf("invalid repository URL: %w", err)
+	}
+
+	// Assume the URL is in the form of "https://github.com/owner/repo"
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("invalid GitHub repository URL format")
+	}
+	owner, repo := parts[0], parts[1]
+
+	// Build the API URL for fetching the tags in the repository
+	apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/tags/%s", owner, repo, tag)
+
+	// Send the request to the GitHub API
+	resp, err := http.Get(apiURL)
+	if err != nil {
+		return "", fmt.Errorf("error fetching tag information: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Parse the JSON response
+	var tagResponse struct {
+		Object struct {
+			Sha string `json:"sha"`
+		} `json:"object"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tagResponse); err != nil {
+		return "", fmt.Errorf("error parsing GitHub response: %w", err)
+	}
+
+	// Return the commit SHA associated with the tag
+	return tagResponse.Object.Sha, nil
+}
+
 // add pipeline fetch steps, validate checksums and generate mconvert expected sha
-func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) error {
+func (c *Context) buildFetchStep(ctx context.Context, converter ApkConvertor) error {
 	log := clog.FromContext(ctx)
 
 	apkBuild := converter.Apkbuild
 
+	// Check if the package URL is available
+	if apkBuild.Url != "" {
+		// Check if the URL belongs to GitHub
+		if _, isGitHub := getGitHubIdentifierFromURL(apkBuild.Url); isGitHub {
+			// GitHub URL, proceed with git-checkout pipeline
+			_, err := url.ParseRequestURI(apkBuild.Url)
+			if err != nil {
+				return fmt.Errorf("parsing URI %s: %w", apkBuild.Url, err)
+			}
+
+			// Fetch the commit hash for the package version tag
+			expectedCommit, err := getCommitForTagFromGitHub(apkBuild.Url, apkBuild.Pkgver) // Using the package version as the tag
+			if err != nil {
+				return fmt.Errorf("error fetching commit for tag: %w", err)
+			}
+
+			// Create a basic git-checkout pipeline
+			pipeline := config.Pipeline{
+				Uses: "melange/git-checkout",
+				With: map[string]string{
+					"repository":      apkBuild.Url,
+					"tag":             "${{package.version}}", // The version as the tag or branch reference
+					"expected-commit": expectedCommit,         // Use the dynamically fetched commit
+				},
+			}
+
+			// Add the pipeline to the generated configuration
+			converter.GeneratedMelangeConfig.Pipeline = append(converter.GeneratedMelangeConfig.Pipeline, pipeline)
+
+			// Set up the update block based on the package source (GitHub or release-monitoring)
+			c.setupUpdateBlock(apkBuild.Url, apkBuild.Pkgver, &converter)
+
+			log.Infof("Using git-checkout pipeline for package %s with repository %s and expected commit %s", converter.Pkgname, apkBuild.Url, expectedCommit)
+			return nil
+		} else {
+			log.Infof("Package URL is not from GitHub, falling back to tar.gz method")
+		}
+	}
+
+	// Fallback to fetching tar.gz if URL is missing or not GitHub
+	log.Infof("No valid GitHub URL found for package %s, using tar.gz method", converter.Pkgname)
 	if len(apkBuild.Source) == 0 {
 		log.Infof("skip adding pipeline for package %s, no source URL found", converter.Pkgname)
 		return nil
@@ -257,7 +383,7 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 		return fmt.Errorf("no package version")
 	}
 
-	// there can be multiple sources, let's add them all so, it's easier for users to remove from generated files if not needed
+	// Loop over sources and add fetch steps for tarball
 	for _, source := range apkBuild.Source {
 		location := source.Location
 
@@ -266,9 +392,14 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 			return fmt.Errorf("parsing URI %s: %w", location, err)
 		}
 
-		req, _ := http.NewRequestWithContext(ctx, "GET", location, nil)
-		resp, err := c.Client.Do(req)
+		// Create a request using standard http.NewRequestWithContext
+		req, err := http.NewRequestWithContext(ctx, "GET", location, nil)
+		if err != nil {
+			return fmt.Errorf("creating request for URI %s: %w", location, err)
+		}
 
+		// Use RLHTTPClient to send the request with rate limiting
+		resp, err := c.Client.Do(req)
 		if err != nil {
 			return fmt.Errorf("failed getting URI %s: %w", location, err)
 		}
@@ -287,7 +418,7 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 
 		var expectedSha string
 		if !failed {
-			// validate the source we are using matches the correct sha512 in the APKBIULD
+			// Validate the source matches the sha512 in the APKBUILD
 			validated := false
 			for _, shas := range apkBuild.Sha512sums {
 				if shas.Source == source.Filename {
@@ -300,7 +431,7 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 				}
 			}
 
-			// now generate the 256 sha we need for a mconvert config
+			// Now generate the 256 sha for the convert config
 			if !validated {
 				expectedSha = "SHA512 DOES NOT MATCH SOURCE - VALIDATE MANUALLY"
 				log.Infof("source %s expected sha512 do not match!", source.Filename)
@@ -314,6 +445,7 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 			expectedSha = "FIXME - SOURCE URL NOT VALID"
 		}
 
+		// Fallback to using the fetch pipeline with tarball location
 		pipeline := config.Pipeline{
 			Uses: "fetch",
 			With: map[string]string{
@@ -322,6 +454,7 @@ func (c Context) buildFetchStep(ctx context.Context, converter ApkConvertor) err
 			},
 		}
 		converter.GeneratedMelangeConfig.Pipeline = append(converter.GeneratedMelangeConfig.Pipeline, pipeline)
+
 	}
 
 	return nil
@@ -334,6 +467,22 @@ func (c ApkConvertor) mapconvert() {
 	c.GeneratedMelangeConfig.Package.Version = c.Apkbuild.Pkgver
 	c.GeneratedMelangeConfig.Package.Epoch = 0
 
+	// Add the version-check test block
+	testPipeline := config.Pipeline{
+		Name: "Verify " + c.Apkbuild.Pkgname + " installation, please improve the test as needed",
+		Runs: fmt.Sprintf("%s --version || exit 1", c.Apkbuild.Pkgname), // Basic version check
+	}
+
+	// Add the test block to the generated config
+	testBlock := &config.Test{
+		Pipeline: []config.Pipeline{
+			testPipeline,
+		},
+	}
+
+	// Add the test block to the configuration
+	c.GeneratedMelangeConfig.Test = testBlock
+
 	copyright := config.Copyright{
 		License: c.Apkbuild.License,
 	}
@@ -479,36 +628,38 @@ func contains(s []string, str string) bool {
 }
 
 func (c ApkConvertor) write(ctx context.Context, orderNumber, outdir string) error {
-	actual, err := yaml.Marshal(&c.GeneratedMelangeConfig)
-	if err != nil {
-		return fmt.Errorf("marshalling mconvert configuration: %w", err)
-	}
-
+	// Ensure output directory exists
 	if _, err := os.Stat(outdir); os.IsNotExist(err) {
 		err = os.MkdirAll(outdir, os.ModePerm)
 		if err != nil {
 			return fmt.Errorf("creating output directory %s: %w", outdir, err)
 		}
 	}
 
-	// write the mconvert config, prefix with our guessed order along with zero to help users easily rename / reorder generated files
-	mconvertFile := filepath.Join(outdir, orderNumber+"0-"+c.Apkbuild.Pkgname+".yaml")
-	f, err := os.Create(mconvertFile)
+	// Prepare the file path for the YAML output
+	manifestFile := filepath.Join(outdir, fmt.Sprintf("%s0-%s.yaml", orderNumber, c.Apkbuild.Pkgname))
+	f, err := os.Create(manifestFile)
 	if err != nil {
-		return fmt.Errorf("creating file %s: %w", mconvertFile, err)
+		return fmt.Errorf("creating file %s: %w", manifestFile, err)
 	}
 	defer f.Close()
 
-	_, err = f.WriteString(fmt.Sprintf("# Generated from %s\n", c.GeneratedMelangeConfig.GeneratedFromComment))
-	if err != nil {
-		return fmt.Errorf("creating writing to file %s: %w", mconvertFile, err)
+	// Write the initial comment to the YAML file
+	if _, err := f.WriteString(fmt.Sprintf("# Generated from %s\n", c.GeneratedMelangeConfig.GeneratedFromComment)); err != nil {
+		return fmt.Errorf("writing to file %s: %w", manifestFile, err)
 	}
 
-	_, err = f.WriteString(string(actual))
-	if err != nil {
-		return fmt.Errorf("creating writing to file %s: %w", mconvertFile, err)
+	// Marshal the configuration into a YAML node for formatting
+	var n yaml.Node
+	if err := n.Encode(c.GeneratedMelangeConfig); err != nil {
+		return fmt.Errorf("encoding YAML to node: %w", err)
+	}
+
+	// Use the formatted YAML encoder to write the YAML data
+	if err := formatted.NewEncoder(f).AutomaticConfig().Encode(&n); err != nil {
+		return fmt.Errorf("encoding formatted YAML to file %s: %w", manifestFile, err)
 	}
 
-	clog.FromContext(ctx).Infof("Generated melange config: %s", mconvertFile)
+	clog.FromContext(ctx).Infof("Generated melange config with update block: %s", manifestFile)
 	return nil
 }

pkg/convert/apkbuild/apkbuild_test.go

@@ -281,6 +281,14 @@ func Test_context_mapconvert(t *testing.T) {
 
 			assert.NoError(t, err)
 
+			// Ensure that the generated configuration contains the test block
+			assert.NotNil(t, config.Test, "Test block should be created in the generated config")
+			// assert.Nil(t, config.Test.Environment, "Expected environment to be nil or empty")
+
+			// Ensure that the test block contains the version-check pipeline
+			assert.NotEmpty(t, config.Test.Pipeline, "Test pipeline should not be empty")
+
+			// Check that the generated YAML matches the expected YAML
 			assert.YAMLEqf(t, string(expected), string(actual), "generated convert yaml not the same as expected")
 		})
 	}

pkg/convert/apkbuild/testdata/no_sub_packages.yaml

@@ -5,9 +5,20 @@ package:
   description: test package description
   copyright:
     - license: MIT
+
 environment: {}
+
 pipeline:
   - uses: autoconf/configure
+
   - uses: autoconf/make
+
   - uses: autoconf/make-install
+
   - uses: strip
+
+test:
+  environment: {}
+  pipeline:
+    - name: Verify test-pkg installation, please improve the test as needed
+      runs: test-pkg --version || exit 1

pkg/convert/apkbuild/testdata/scanner_error.yaml

@@ -1,33 +1,7 @@
 # Maintainer: Natanael Copa <ncopa@alpinelinux.org>
-  pkgname=libxext
-    pkgver=1.3.4
-    pkgrel=1
-    pkgdesc="X11 miscellaneous extensions library"
-    url="rate_limitted_http://xorg.freedesktop.org/"
-    arch="all"
-    license="MIT"
-    depends_dev="libxau-dev"
-    makedepends="$depends_dev libx11-dev xorgproto util-macros xmlto"
-    subpackages="$pkgname-dev $pkgname-doc"
-    options="!check"
-    source="https://www.x.org/releases/individual/lib/libXext-$pkgver.tar.bz2
-    "
-    
+|-
+    pkgname=libxext pkgver=1.3.4 pkgrel=1 pkgdesc="X11 miscellaneous extensions library" url="rate_limitted_http://xorg.freedesktop.org/" arch="all" license="MIT" depends_dev="libxau-dev" makedepends="$depends_dev libx11-dev xorgproto util-macros xmlto" subpackages="$pkgname-dev $pkgname-doc" options="!check" source="https://www.x.org/releases/individual/lib/libXext-$pkgver.tar.bz2 "
     builddir="$srcdir"/libXext-$pkgver
-    
-    build() {
-    ./configure \
-    --build=$CBUILD \
-    --host=$CHOST \
-    --prefix=/usr \
-    --sysconfdir=/etc \
-    --with-xmlto \
-    --without-fop
-    make
-}
-
-    package() {
-    make DESTDIR="$pkgdir" install
-}
-
-    sha512sums="09146397d95f80c04701be1cc0a9c580ab5a085842ac31d17dfb6d4c2e42b4253b89cba695e54444e520be359883a76ffd02f42484c9e2ba2c33a5a40c29df4a  libXext-1.3.4.tar.bz2"
+    build() { ./configure \ --build=$CBUILD \ --host=$CHOST \ --prefix=/usr \ --sysconfdir=/etc \ --with-xmlto \ --without-fop make }
+    package() { make DESTDIR="$pkgdir" install }
+    sha512sums="09146397d95f80c04701be1cc0a9c580ab5a085842ac31d17dfb6d4c2e42b4253b89cba695e54444e520be359883a76ffd02f42484c9e2ba2c33a5a40c29df4a  libXext-1.3.4.tar.bz2"