Support network relevant settings for s3 output

Signed-off-by: Anson <anson.liu@live.com>

Author: smallc2009

apis/fluentbit/v1alpha2/plugins/output/s3_types.go

@@ -65,8 +65,40 @@ type S3 struct {
 	// Option to specify an AWS Profile for credentials.
 	Profile string `json:"Profile,omitempty"`
 	// Specify number of worker threads to use to output to S3
-	Workers      *int32 `json:"Workers,omitempty"`
-	*plugins.TLS `json:"tls,omitempty"`
+	Workers *int32 `json:"Workers,omitempty"`
+	// Set maximum time expressed in seconds to wait for a TCP connection to be established, this include the TLS handshake time.
+	ConnectTimeout *int32 `json:"ConnectTimeout,omitempty"`
+	// On connection timeout, specify if it should log an error. When disabled, the timeout is logged as a debug message.
+	ConnectTimeoutLogError *bool `json:"connectTimeoutLogError,omitempty"`
+	// Select the primary DNS connection type (TCP or UDP).
+	// +kubebuilder:validation:Enum:="TCP";"UDP"
+	DNSMode *string `json:"DNSMode,omitempty"`
+	// Prioritize IPv4 DNS results when trying to establish a connection.
+	DNSPreferIPv4 *bool `json:"DNSPreferIPv4,omitempty"`
+	// Prioritize IPV6 DNS results when trying to establish a connection.
+	DNSPreferIPv6 *bool `json:"DNSPreferIPv6,omitempty"`
+	// Set maximum time a connection can stay idle while assigned.
+	IoTimeout *int32 `json:"IoTimeout,omitempty"`
+	// Set maximum number of times a keepalive connection can be used before it is retired.
+	KeepaliveMaxRecycle *int32 `json:"keepaliveMaxRecycle,omitempty"`
+	// Set maximum number of TCP connections that can be established per worker.
+	MaxWorkerConnections *int32 `json:"maxWorkerConnections,omitempty"`
+	// Ignore the environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY when set.
+	ProxyEnvIgnore *bool `json:"proxyEnvIgnore,omitempty"`
+	// Specify network address to bind for data traffic.
+	SourceAddress *string `json:"sourceAddress,omitempty"`
+	// Enable or disable connection keepalive support. Accepts a boolean value: on / off.
+	// +kubebuilder:validation:Enum:="on";"off"
+	Keepalive *string `json:"keepalive,omitempty"`
+	// Interval between TCP keepalive probes when no response is received on a keepidle probe.
+	TCPKeepaliveInterval *int32 `json:"tcpKeepaliveInterval,omitempty"`
+	// Number of unacknowledged probes to consider a connection dead.
+	TCPKeepaliveProbes *int32 `json:"tcpKeepaliveProbes,omitempty"`
+	// Interval between the last data packet sent and the first TCP keepalive probe.
+	TCPKeepaliveTime *int32 `json:"tcpKeepaliveTime,omitempty"`
+	// Set maximum time expressed in seconds for an idle keepalive connection.
+	KeepaliveIdleTimeout *int32 `json:"keepaliveIdleTimeout,omitempty"`
+	*plugins.TLS         `json:"tls,omitempty"`
 }
 
 // Name implement Section() method
@@ -105,6 +137,21 @@ func (o *S3) Params(sl plugins.SecretLoader) (*params.KVs, error) {
 	plugins.InsertKVString(kvs, "external_id", o.ExternalId)
 	plugins.InsertKVString(kvs, "profile", o.Profile)
 	plugins.InsertKVField(kvs, "workers", o.Workers)
+	plugins.InsertKVField(kvs, "net.connect_timeout", o.ConnectTimeout)
+	plugins.InsertKVField(kvs, "net.connect_timeout_log_error", o.ConnectTimeoutLogError)
+	plugins.InsertKVField(kvs, "net.dns.mode", o.DNSMode)
+	plugins.InsertKVField(kvs, "net.dns.prefer_ipv4", o.DNSPreferIPv4)
+	plugins.InsertKVField(kvs, "net.dns.prefer_ipv6", o.DNSPreferIPv6)
+	plugins.InsertKVField(kvs, "net.io_timeout", o.IoTimeout)
+	plugins.InsertKVField(kvs, "net.keepalive_max_recycle", o.KeepaliveMaxRecycle)
+	plugins.InsertKVField(kvs, "net.max_worker_connections", o.MaxWorkerConnections)
+	plugins.InsertKVField(kvs, "net.proxy_env_ignore", o.ProxyEnvIgnore)
+	plugins.InsertKVField(kvs, "net.source_address", o.SourceAddress)
+	plugins.InsertKVField(kvs, "net.tcp_keepalive", o.Keepalive)
+	plugins.InsertKVField(kvs, "net.tcp_keepalive_interval", o.TCPKeepaliveInterval)
+	plugins.InsertKVField(kvs, "net.tcp_keepalive_probes", o.TCPKeepaliveProbes)
+	plugins.InsertKVField(kvs, "net.tcp_keepalive_time", o.TCPKeepaliveTime)
+	plugins.InsertKVField(kvs, "net.keepalive_idle_timeout", o.KeepaliveIdleTimeout)
 
 	if o.TLS != nil {
 		tls, err := o.TLS.Params(sl)

apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go

@@ -43,6 +43,20 @@ func TestOutput_S3_Params(t *testing.T) {
 		ExternalId:               "external_id",
 		Profile:                  "my-profile",
 		Workers:                  utils.ToPtr[int32](1),
+		ConnectTimeout:           utils.ToPtr[int32](10),
+		ConnectTimeoutLogError:   utils.ToPtr(true),
+		DNSMode:                  utils.ToPtr("TCP"),
+		DNSPreferIPv4:            utils.ToPtr(false),
+		DNSPreferIPv6:            utils.ToPtr(false),
+		IoTimeout:                utils.ToPtr[int32](0),
+		KeepaliveMaxRecycle:      utils.ToPtr[int32](2000),
+		MaxWorkerConnections:     utils.ToPtr[int32](0),
+		ProxyEnvIgnore:           utils.ToPtr(false),
+		SourceAddress:            utils.ToPtr("127.0.0.1"),
+		Keepalive:                utils.ToPtr("off"),
+		TCPKeepaliveInterval:     utils.ToPtr[int32](-1),
+		TCPKeepaliveProbes:       utils.ToPtr[int32](-1),
+		TCPKeepaliveTime:         utils.ToPtr[int32](-1),
 	}
 
 	expected := params.NewKVs()
@@ -74,6 +88,20 @@ func TestOutput_S3_Params(t *testing.T) {
 	expected.Insert("external_id", "external_id")
 	expected.Insert("profile", "my-profile")
 	expected.Insert("workers", "1")
+	expected.Insert("net.connect_timeout", "10")
+	expected.Insert("net.connect_timeout_log_error", "true")
+	expected.Insert("net.dns.mode", "TCP")
+	expected.Insert("net.dns.prefer_ipv4", "false")
+	expected.Insert("net.dns.prefer_ipv6", "false")
+	expected.Insert("net.io_timeout", "0")
+	expected.Insert("net.keepalive_max_recycle", "2000")
+	expected.Insert("net.max_worker_connections", "0")
+	expected.Insert("net.proxy_env_ignore", "false")
+	expected.Insert("net.source_address", "127.0.0.1")
+	expected.Insert("net.tcp_keepalive", "off")
+	expected.Insert("net.tcp_keepalive_interval", "-1")
+	expected.Insert("net.tcp_keepalive_probes", "-1")
+	expected.Insert("net.tcp_keepalive_time", "-1")
 
 	kvs, err := s3.Params(sl)
 	g.Expect(err).NotTo(HaveOccurred())

apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go

@@ -3396,10 +3396,30 @@ spec:
                   Compression:
                     description: Compression type for S3 objects.
                     type: string
+                  ConnectTimeout:
+                    description: Set maximum time expressed in seconds to wait for
+                      a TCP connection to be established, this include the TLS handshake
+                      time.
+                    format: int32
+                    type: integer
                   ContentType:
                     description: A standard MIME type for the S3 object; this will
                       be set as the Content-Type HTTP header.
                     type: string
+                  DNSMode:
+                    description: Select the primary DNS connection type (TCP or UDP).
+                    enum:
+                    - TCP
+                    - UDP
+                    type: string
+                  DNSPreferIPv4:
+                    description: Prioritize IPv4 DNS results when trying to establish
+                      a connection.
+                    type: boolean
+                  DNSPreferIPv6:
+                    description: Prioritize IPV6 DNS results when trying to establish
+                      a connection.
+                    type: boolean
                   Endpoint:
                     description: Custom endpoint for the S3 API.
                     type: string
@@ -3408,6 +3428,11 @@ spec:
                       with the role_arn parameter if your role requires an external
                       ID.
                     type: string
+                  IoTimeout:
+                    description: Set maximum time a connection can stay idle while
+                      assigned.
+                    format: int32
+                    type: integer
                   JsonDateFormat:
                     description: 'Specify the format of the date. Supported formats
                       are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z)
@@ -3498,6 +3523,54 @@ spec:
                       to S3
                     format: int32
                     type: integer
+                  connectTimeoutLogError:
+                    description: On connection timeout, specify if it should log an
+                      error. When disabled, the timeout is logged as a debug message.
+                    type: boolean
+                  keepalive:
+                    description: 'Enable or disable connection keepalive support.
+                      Accepts a boolean value: on / off.'
+                    enum:
+                    - "on"
+                    - "off"
+                    type: string
+                  keepaliveIdleTimeout:
+                    description: Set maximum time expressed in seconds for an idle
+                      keepalive connection.
+                    format: int32
+                    type: integer
+                  keepaliveMaxRecycle:
+                    description: Set maximum number of times a keepalive connection
+                      can be used before it is retired.
+                    format: int32
+                    type: integer
+                  maxWorkerConnections:
+                    description: Set maximum number of TCP connections that can be
+                      established per worker.
+                    format: int32
+                    type: integer
+                  proxyEnvIgnore:
+                    description: Ignore the environment variables HTTP_PROXY, HTTPS_PROXY
+                      and NO_PROXY when set.
+                    type: boolean
+                  sourceAddress:
+                    description: Specify network address to bind for data traffic.
+                    type: string
+                  tcpKeepaliveInterval:
+                    description: Interval between TCP keepalive probes when no response
+                      is received on a keepidle probe.
+                    format: int32
+                    type: integer
+                  tcpKeepaliveProbes:
+                    description: Number of unacknowledged probes to consider a connection
+                      dead.
+                    format: int32
+                    type: integer
+                  tcpKeepaliveTime:
+                    description: Interval between the last data packet sent and the
+                      first TCP keepalive probe.
+                    format: int32
+                    type: integer
                   tls:
                     description: Fluent Bit provides integrated support for Transport
                       Layer Security (TLS) and it predecessor Secure Sockets Layer

charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusteroutputs.yaml

@@ -3396,10 +3396,30 @@ spec:
                   Compression:
                     description: Compression type for S3 objects.
                     type: string
+                  ConnectTimeout:
+                    description: Set maximum time expressed in seconds to wait for
+                      a TCP connection to be established, this include the TLS handshake
+                      time.
+                    format: int32
+                    type: integer
                   ContentType:
                     description: A standard MIME type for the S3 object; this will
                       be set as the Content-Type HTTP header.
                     type: string
+                  DNSMode:
+                    description: Select the primary DNS connection type (TCP or UDP).
+                    enum:
+                    - TCP
+                    - UDP
+                    type: string
+                  DNSPreferIPv4:
+                    description: Prioritize IPv4 DNS results when trying to establish
+                      a connection.
+                    type: boolean
+                  DNSPreferIPv6:
+                    description: Prioritize IPV6 DNS results when trying to establish
+                      a connection.
+                    type: boolean
                   Endpoint:
                     description: Custom endpoint for the S3 API.
                     type: string
@@ -3408,6 +3428,11 @@ spec:
                       with the role_arn parameter if your role requires an external
                       ID.
                     type: string
+                  IoTimeout:
+                    description: Set maximum time a connection can stay idle while
+                      assigned.
+                    format: int32
+                    type: integer
                   JsonDateFormat:
                     description: 'Specify the format of the date. Supported formats
                       are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z)
@@ -3498,6 +3523,54 @@ spec:
                       to S3
                     format: int32
                     type: integer
+                  connectTimeoutLogError:
+                    description: On connection timeout, specify if it should log an
+                      error. When disabled, the timeout is logged as a debug message.
+                    type: boolean
+                  keepalive:
+                    description: 'Enable or disable connection keepalive support.
+                      Accepts a boolean value: on / off.'
+                    enum:
+                    - "on"
+                    - "off"
+                    type: string
+                  keepaliveIdleTimeout:
+                    description: Set maximum time expressed in seconds for an idle
+                      keepalive connection.
+                    format: int32
+                    type: integer
+                  keepaliveMaxRecycle:
+                    description: Set maximum number of times a keepalive connection
+                      can be used before it is retired.
+                    format: int32
+                    type: integer
+                  maxWorkerConnections:
+                    description: Set maximum number of TCP connections that can be
+                      established per worker.
+                    format: int32
+                    type: integer
+                  proxyEnvIgnore:
+                    description: Ignore the environment variables HTTP_PROXY, HTTPS_PROXY
+                      and NO_PROXY when set.
+                    type: boolean
+                  sourceAddress:
+                    description: Specify network address to bind for data traffic.
+                    type: string
+                  tcpKeepaliveInterval:
+                    description: Interval between TCP keepalive probes when no response
+                      is received on a keepidle probe.
+                    format: int32
+                    type: integer
+                  tcpKeepaliveProbes:
+                    description: Number of unacknowledged probes to consider a connection
+                      dead.
+                    format: int32
+                    type: integer
+                  tcpKeepaliveTime:
+                    description: Interval between the last data packet sent and the
+                      first TCP keepalive probe.
+                    format: int32
+                    type: integer
                   tls:
                     description: Fluent Bit provides integrated support for Transport
                       Layer Security (TLS) and it predecessor Secure Sockets Layer