Skip to content

Commit 785b4e1

Browse files
author
Morven Cao
authored
add pubsub support and gcp setup. (#409)
* add pubsub support and gcp setup. Signed-off-by: Morven Cao <lcao@redhat.com> * address comments. Signed-off-by: Morven Cao <lcao@redhat.com> --------- Signed-off-by: Morven Cao <lcao@redhat.com>
1 parent 985f709 commit 785b4e1

12 files changed

Lines changed: 1376 additions & 2 deletions

File tree

‎pkg/config/message_broker.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ func NewMessageBrokerConfig() *MessageBrokerConfig {
2727
}
2828

2929
func (c *MessageBrokerConfig) AddFlags(fs *pflag.FlagSet) {
30-
fs.BoolVar(&c.Disable, "disable-message-broker", c.Disable, "Disable MQTT message broker, default is false")
30+
fs.BoolVar(&c.Disable, "disable-message-broker", c.Disable, "Disable message broker, default is false")
3131
fs.MarkHidden("disable-message-broker") // This flag is hidden as it is not intended for regular use.
3232
fs.BoolVar(&c.EnableMock, "enable-message-broker-mock", c.EnableMock, "Enable message broker mock")
3333
fs.StringVar(&c.SourceID, "source-id", c.SourceID, "Source ID")
3434
fs.StringVar(&c.ClientID, "client-id", c.ClientID, "Client ID")
35-
fs.StringVar(&c.MessageBrokerType, "message-broker-type", c.MessageBrokerType, "Message broker type ('grpc' or 'mqtt'). Default is 'mqtt'.")
35+
fs.StringVar(&c.MessageBrokerType, "message-broker-type", c.MessageBrokerType, "Message broker type ('grpc', 'mqtt' or 'pubsub'). Default is 'mqtt'.")
3636
fs.StringVar(&c.MessageBrokerConfig, "message-broker-config-file", c.MessageBrokerConfig, "The config file path of message broker")
3737
}

‎templates/agent-template-gcp.yml‎

Lines changed: 324 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,324 @@
1+
---
2+
apiVersion: template.openshift.io/v1
3+
kind: Template
4+
metadata:
5+
name: maestro-agent
6+
annotations:
7+
openshift.io/display-name: maestro-agent
8+
description: agent to connect to maestro service.
9+
tags: maestro-agent
10+
iconClass: icon-shadowman
11+
template.openshift.io/provider-display-name: Red Hat, Inc.
12+
template.openshift.io/documentation-url: https://gitlab.cee.redhat.com/service/
13+
labels:
14+
template: maestro-agent
15+
parameters:
16+
17+
- name: PROJECT_ID
18+
description: Google project ID for PubSub and CloudSQL Service
19+
20+
- name: ENVIRONMENT
21+
displayName: Environment
22+
description: Which maestro environment to use for this deployment
23+
value: production
24+
25+
- name: AGENT_NAMESPACE
26+
description: namespace of maestro agent
27+
28+
- name: CONSUMER_NAME
29+
displayName: Treat CONSUMER_NAME as cluster name
30+
required: true
31+
32+
- name: IMAGE_REGISTRY
33+
displayName: Image Registry
34+
required: true
35+
36+
- name: IMAGE_REPOSITORY
37+
displayName: Image Repository
38+
required: true
39+
40+
- name: IMAGE_TAG
41+
displayName: Image tag
42+
value: latest
43+
44+
- name: KLOG_V
45+
displayName: KLOG V Level
46+
description: Log verbosity level
47+
value: "2"
48+
49+
objects:
50+
- apiVersion: apiextensions.k8s.io/v1
51+
kind: CustomResourceDefinition
52+
metadata:
53+
name: appliedmanifestworks.work.open-cluster-management.io
54+
spec:
55+
group: work.open-cluster-management.io
56+
names:
57+
kind: AppliedManifestWork
58+
listKind: AppliedManifestWorkList
59+
plural: appliedmanifestworks
60+
singular: appliedmanifestwork
61+
scope: Cluster
62+
preserveUnknownFields: false
63+
versions:
64+
- name: v1
65+
schema:
66+
openAPIV3Schema:
67+
description: AppliedManifestWork represents an applied manifestwork on managed cluster that is placed on a managed cluster. An AppliedManifestWork links to a manifestwork on a hub recording resources deployed in the managed cluster. When the agent is removed from managed cluster, cluster-admin on managed cluster can delete appliedmanifestwork to remove resources deployed by the agent. The name of the appliedmanifestwork must be in the format of {hash of hub's first kube-apiserver url}-{manifestwork name}
68+
type: object
69+
properties:
70+
apiVersion:
71+
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
72+
type: string
73+
kind:
74+
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
75+
type: string
76+
metadata:
77+
type: object
78+
spec:
79+
description: Spec represents the desired configuration of AppliedManifestWork.
80+
type: object
81+
properties:
82+
agentID:
83+
description: AgentID represents the ID of the work agent who is to handle this AppliedManifestWork.
84+
type: string
85+
hubHash:
86+
description: HubHash represents the hash of the first hub kube apiserver to identify which hub this AppliedManifestWork links to.
87+
type: string
88+
manifestWorkName:
89+
description: ManifestWorkName represents the name of the related manifestwork on the hub.
90+
type: string
91+
status:
92+
description: Status represents the current status of AppliedManifestWork.
93+
type: object
94+
properties:
95+
appliedResources:
96+
description: AppliedResources represents a list of resources defined within the manifestwork that are applied. Only resources with valid GroupVersionResource, namespace, and name are suitable. An item in this slice is deleted when there is no mapped manifest in manifestwork.Spec or by finalizer. The resource relating to the item will also be removed from managed cluster. The deleted resource may still be present until the finalizers for that resource are finished. However, the resource will not be undeleted, so it can be removed from this list and eventual consistency is preserved.
97+
type: array
98+
items:
99+
description: AppliedManifestResourceMeta represents the group, version, resource, name and namespace of a resource. Since these resources have been created, they must have valid group, version, resource, namespace, and name.
100+
type: object
101+
required:
102+
- name
103+
- resource
104+
- version
105+
properties:
106+
group:
107+
description: Group is the API Group of the Kubernetes resource, empty string indicates it is in core group.
108+
type: string
109+
name:
110+
description: Name is the name of the Kubernetes resource.
111+
type: string
112+
namespace:
113+
description: Name is the namespace of the Kubernetes resource, empty string indicates it is a cluster scoped resource.
114+
type: string
115+
resource:
116+
description: Resource is the resource name of the Kubernetes resource.
117+
type: string
118+
uid:
119+
description: UID is set on successful deletion of the Kubernetes resource by controller. The resource might be still visible on the managed cluster after this field is set. It is not directly settable by a client.
120+
type: string
121+
version:
122+
description: Version is the version of the Kubernetes resource.
123+
type: string
124+
evictionStartTime:
125+
description: 'EvictionStartTime represents the current appliedmanifestwork will be evicted after a grace period. An appliedmanifestwork will be evicted from the managed cluster in the following two scenarios: - the manifestwork of the current appliedmanifestwork is missing on the hub, or - the appliedmanifestwork hub hash does not match the current hub hash of the work agent.'
126+
type: string
127+
format: date-time
128+
served: true
129+
storage: true
130+
subresources:
131+
status: {}
132+
status:
133+
acceptedNames:
134+
kind: ""
135+
plural: ""
136+
conditions: []
137+
storedVersions: []
138+
139+
- apiVersion: rbac.authorization.k8s.io/v1
140+
kind: ClusterRole
141+
metadata:
142+
name: maestro-agent:agent
143+
rules:
144+
# Allow agent to managed appliedmanifestworks
145+
- apiGroups: ["work.open-cluster-management.io"]
146+
resources: ["appliedmanifestworks"]
147+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
148+
- apiGroups: ["work.open-cluster-management.io"]
149+
resources: ["appliedmanifestworks/status"]
150+
verbs: ["patch", "update"]
151+
- apiGroups: ["work.open-cluster-management.io"]
152+
resources: ["appliedmanifestworks/finalizers"]
153+
verbs: ["update"]
154+
# Allow agent to check executor permissions
155+
- apiGroups: ["authorization.k8s.io"]
156+
resources: ["subjectaccessreviews"]
157+
verbs: ["create"]
158+
- apiGroups: [""]
159+
resources: ["serviceaccounts"]
160+
verbs: ["impersonate"]
161+
162+
- apiVersion: rbac.authorization.k8s.io/v1
163+
kind: ClusterRoleBinding
164+
metadata:
165+
name: maestro-agent:agent
166+
roleRef:
167+
apiGroup: rbac.authorization.k8s.io
168+
kind: ClusterRole
169+
name: maestro-agent:agent
170+
subjects:
171+
- kind: ServiceAccount
172+
name: maestro-agent-sa
173+
namespace: ${AGENT_NAMESPACE}
174+
175+
- apiVersion: rbac.authorization.k8s.io/v1
176+
kind: ClusterRoleBinding
177+
metadata:
178+
name: maestro-agent:execution-admin
179+
roleRef:
180+
apiGroup: rbac.authorization.k8s.io
181+
kind: ClusterRole
182+
# We deploy a controller that could work with permission lower than cluster-admin, the tradeoff is
183+
# responsivity because list/watch cannot be maintained over too many namespaces.
184+
name: cluster-admin
185+
subjects:
186+
- kind: ServiceAccount
187+
name: maestro-agent-sa
188+
namespace: ${AGENT_NAMESPACE}
189+
190+
# - apiVersion: rbac.authorization.k8s.io/v1
191+
# kind: ClusterRoleBinding
192+
# metadata:
193+
# name: maestro-agent:execution
194+
# roleRef:
195+
# apiGroup: rbac.authorization.k8s.io
196+
# kind: ClusterRole
197+
# name: maestro-agent:execution
198+
# subjects:
199+
# - kind: ServiceAccount
200+
# name: maestro-agent-sa
201+
# namespace: ${AGENT_NAMESPACE}
202+
203+
- apiVersion: rbac.authorization.k8s.io/v1
204+
kind: Role
205+
metadata:
206+
name: maestro-agent:agent
207+
namespace: ${AGENT_NAMESPACE}
208+
rules:
209+
# leader election needs to operate configmaps
210+
- apiGroups: [""]
211+
resources: ["configmaps"]
212+
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
213+
- apiGroups: ["coordination.k8s.io"]
214+
resources: ["leases"]
215+
verbs: ["create", "get", "list", "update", "watch", "patch"]
216+
- apiGroups: ["", "events.k8s.io"]
217+
resources: ["events"]
218+
verbs: ["create", "patch", "update"]
219+
220+
- apiVersion: rbac.authorization.k8s.io/v1
221+
kind: RoleBinding
222+
metadata:
223+
name: maestro-agent:agent
224+
namespace: ${AGENT_NAMESPACE}
225+
roleRef:
226+
apiGroup: rbac.authorization.k8s.io
227+
kind: Role
228+
name: maestro-agent:agent
229+
subjects:
230+
- kind: ServiceAccount
231+
name: maestro-agent-sa
232+
namespace: ${AGENT_NAMESPACE}
233+
234+
- apiVersion: rbac.authorization.k8s.io/v1
235+
kind: RoleBinding
236+
metadata:
237+
name: maestro-agent:agent:extension-apiserver
238+
namespace: kube-system
239+
roleRef:
240+
apiGroup: rbac.authorization.k8s.io
241+
kind: Role
242+
name: maestro-agent:agent:extension-apiserver
243+
subjects:
244+
- kind: ServiceAccount
245+
name: maestro-agent-sa
246+
namespace: ${AGENT_NAMESPACE}
247+
248+
- apiVersion: rbac.authorization.k8s.io/v1
249+
kind: Role
250+
metadata:
251+
name: maestro-agent:agent:extension-apiserver
252+
namespace: kube-system
253+
rules:
254+
- apiGroups: [""]
255+
resources: ["configmaps"]
256+
resourceNames: ["extension-apiserver-authentication"]
257+
verbs: ["get", "list", "watch"]
258+
259+
- kind: Deployment
260+
apiVersion: apps/v1
261+
metadata:
262+
name: maestro-agent
263+
namespace: ${AGENT_NAMESPACE}
264+
labels:
265+
app: maestro-agent
266+
spec:
267+
replicas: 1
268+
selector:
269+
matchLabels:
270+
app: maestro-agent
271+
template:
272+
metadata:
273+
labels:
274+
app: maestro-agent
275+
spec:
276+
serviceAccountName: maestro-agent-sa
277+
containers:
278+
- name: maestro-agent
279+
image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG}
280+
imagePullPolicy: IfNotPresent
281+
env:
282+
- name: "MAESTRO_ENV"
283+
value: "${ENVIRONMENT}"
284+
command:
285+
- /usr/local/bin/maestro
286+
- agent
287+
- --consumer-name=${CONSUMER_NAME}
288+
- --workload-source-driver=pubsub
289+
- --workload-source-config=/secrets/pubsub/config.yaml
290+
- --cloudevents-client-id=${CONSUMER_NAME}
291+
- -v=${KLOG_V}
292+
volumeMounts:
293+
- name: pubsub
294+
mountPath: /secrets/pubsub
295+
volumes:
296+
- name: pubsub
297+
secret:
298+
secretName: maestro-agent-pubsub
299+
300+
- apiVersion: v1
301+
kind: Secret
302+
metadata:
303+
name: maestro-agent-pubsub
304+
namespace: ${AGENT_NAMESPACE}
305+
stringData:
306+
config.yaml: |
307+
projectID: ${PROJECT_ID}
308+
topics:
309+
agentEvents: projects/${PROJECT_ID}/topics/agentevents
310+
agentBroadcast: projects/${PROJECT_ID}/topics/agentbroadcast
311+
subscriptions:
312+
sourceEvents: projects/${PROJECT_ID}/subscriptions/sourceevents-${CONSUMER_NAME}
313+
sourceBroadcast: projects/${PROJECT_ID}/subscriptions/sourcebroadcast-${CONSUMER_NAME}
314+
315+
- kind: ServiceAccount
316+
apiVersion: v1
317+
metadata:
318+
name: maestro-agent-sa
319+
namespace: ${AGENT_NAMESPACE}
320+
labels:
321+
app: maestro-agent
322+
annotations:
323+
# bind KSA to GSA for workload identity
324+
iam.gke.io/gcp-service-account: maestro-agent-${CONSUMER_NAME}@${PROJECT_ID}.iam.gserviceaccount.com

0 commit comments

Comments
 (0)