Merge pull request #10 from safedep/develop

Sync Develop with Main

Author: abhisek

services/pkg/common/db/adapters/mysql_adapter.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"time"
 
+	"github.com/safedep/gateway/services/pkg/common/utils"
 	"golang.org/x/net/context"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
@@ -29,20 +30,23 @@ func NewMySqlAdapter(config MySqlAdapterConfig) (SqlDataAdapter, error) {
 
 	log.Printf("Connecting to MySQL database with %s@%s:%d", config.Username, config.Host, config.Port)
 
-	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
-	retry := 5
-	t := 1
+	var db *gorm.DB
+	var err error
 
-	// Retry connection to avoid race with DB container init
-	for err != nil && t <= retry {
-		log.Printf("[%d/%d] Failed to connect to MySQL server: %v", t, retry, err)
+	retryN := 5
+	utils.InvokeWithRetry(utils.RetryConfig{
+		Count: retryN,
+		Sleep: 1 * time.Second,
+	}, func(n int) error {
 		db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
+		if err != nil {
+			log.Printf("[%d/%d] Failed to connect to MySQL server: %v",
+				n, retryN, err)
+		}
 
-		t += 1
-		time.Sleep(1 * time.Second)
-	}
+		return err
+	})
 
-	// Failed after retry
 	if err != nil {
 		return nil, err
 	}

services/pkg/common/utils/retry.go

@@ -0,0 +1,41 @@
+package utils
+
+import (
+	"errors"
+	"time"
+)
+
+var (
+	errInvalidRetryCount    = errors.New("invalid retry count")
+	errInvalidSleepDuration = errors.New("must have a valid sleep")
+)
+
+type RetriableFunc func(retryN int) error
+
+type RetryConfig struct {
+	Count int
+	Sleep time.Duration
+}
+
+func InvokeWithRetry(config RetryConfig, f RetriableFunc) error {
+	if config.Count <= 0 {
+		return errInvalidRetryCount
+	}
+
+	now := time.Now()
+	if now.Add(config.Sleep) == now {
+		return errInvalidSleepDuration
+	}
+
+	var err error
+	for i := 0; i < config.Count; i += 1 {
+		err = f(i + 1)
+		if err == nil {
+			break
+		}
+
+		time.Sleep(config.Sleep)
+	}
+
+	return err
+}

services/pkg/common/utils/retry_test.go

@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInvokeWithRetry(t *testing.T) {
+	cases := []struct {
+		name string
+		c    RetryConfig
+		f    RetriableFunc
+		err  error
+	}{
+		{
+			"Must fail with zero Count",
+			RetryConfig{
+				Count: 0,
+				Sleep: 0,
+			},
+			func(n int) error { return nil },
+			errInvalidRetryCount,
+		},
+		{
+			"Must fail with zero Sleep",
+			RetryConfig{
+				Count: 1,
+				Sleep: 0,
+			},
+			func(n int) error { return nil },
+			errInvalidSleepDuration,
+		},
+		{
+			"Retry is successful immediately",
+			RetryConfig{
+				Count: 1,
+				Sleep: time.Millisecond * 2,
+			},
+			func(n int) error {
+				return nil
+			},
+			nil,
+		},
+		{
+			"Retry is successful after 5 attempts",
+			RetryConfig{
+				Count: 5,
+				Sleep: time.Millisecond * 1,
+			},
+			func(n int) error {
+				if n < 5 {
+					return errors.New("< 5")
+				}
+
+				return nil
+			},
+			nil,
+		},
+		{
+			"Retry is never successful",
+			RetryConfig{
+				Count: 10,
+				Sleep: time.Millisecond * 1,
+			},
+			func(n int) error {
+				return errors.New("err")
+			},
+			errors.New("err"),
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			err := InvokeWithRetry(test.c, test.f)
+			if test.err == nil {
+				assert.Nil(t, err)
+			} else {
+				assert.Equal(t, test.err.Error(), err.Error())
+			}
+		})
+	}
+}

services/pkg/dcs/opensearch.go

@@ -121,6 +121,16 @@ func (s *opensearchIndexer) buildOpenSearchClient() (*opensearch.Client, error)
 }
 
 func (s *opensearchIndexer) initOpenSearchIndex(name string) error {
+	return utils.InvokeWithRetry(utils.RetryConfig{
+		Count: 30,
+		Sleep: time.Second * 1,
+	}, func(n int) error {
+		logger.Infof("Attempting to init opensearch index [retry=%d]", n)
+		return s.initOpenSearchIndexInternal(name)
+	})
+}
+
+func (s *opensearchIndexer) initOpenSearchIndexInternal(name string) error {
 	if s.client == nil {
 		return errors.New("client is nil")
 	}