Skip to content

Commit 83e3c74

Browse files
pcholakovpcholakov
committed
Install Kyverno cluster policies for Restate namespaces from TF
Moves the four kyverno cluster policies (linkerd-injection annotation mutate, linkerd authz generate, and the two RBAC ClusterRoles that grant kyverno permission to create policy.linkerd.io resources) into the sandbox TF as kubectl_manifest resources. They were previously declared as Nuon [[policy]] type=kubernetes_cluster entries in restate-byoc-gcp/policies.toml, but Nuon does not actually deploy them for this app -- the install config state has no policies field, and verifying on the canary cluster post-reprovision confirmed none were applied. Moving them into the sandbox makes them part of the substrate alongside kyverno itself, so a reprovision installs everything needed for new env namespaces to be linkerd-meshed correctly.
1 parent 127a444 commit 83e3c74

5 files changed

Lines changed: 389 additions & 0 deletions

‎kyverno_policies.tf‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# -----------------------------------------------------------------------------
2+
# Cluster-scoped Kyverno policies for Restate namespaces.
3+
#
4+
# These were previously declared as Nuon [[policy]] entries in nuon-byoc's
5+
# restate-byoc-gcp/policies.toml, but Nuon's `kubernetes_cluster` policy type
6+
# isn't actually deploying them for this app (the install config state has no
7+
# policies field), so they get installed here as part of the sandbox TF.
8+
#
9+
# Order:
10+
# 1. helm_release.linkerd_crds -- linkerd CRDs available for Kyverno
11+
# webhook validation
12+
# 2. helm_release.kyverno -- Kyverno controllers running
13+
# 3. ClusterRoles -- grant the kyverno background controller
14+
# permission to create policy.linkerd.io
15+
# resources (Server, HTTPRoute,
16+
# AuthorizationPolicy)
17+
# 4. ClusterPolicies -- the mutate/generate rules themselves
18+
# -----------------------------------------------------------------------------
19+
20+
resource "kubectl_manifest" "kyverno_rbac_linkerd_view" {
21+
yaml_body = file("${path.module}/policies/rbac-linkerd-view.yml")
22+
23+
depends_on = [helm_release.kyverno]
24+
}
25+
26+
resource "kubectl_manifest" "kyverno_rbac_linkerd_manage" {
27+
yaml_body = file("${path.module}/policies/rbac-linkerd-manage.yml")
28+
29+
depends_on = [helm_release.kyverno]
30+
}
31+
32+
resource "kubectl_manifest" "kyverno_add_linkerd_annotation" {
33+
yaml_body = file("${path.module}/policies/add-linkerd-annotation-to-restate-namespaces.yml")
34+
35+
depends_on = [
36+
helm_release.kyverno,
37+
helm_release.linkerd_crds,
38+
]
39+
}
40+
41+
resource "kubectl_manifest" "kyverno_linkerd_authz_for_restate" {
42+
yaml_body = file("${path.module}/policies/linkerd-authz-for-restate.yml")
43+
44+
depends_on = [
45+
helm_release.kyverno,
46+
helm_release.linkerd_crds,
47+
kubectl_manifest.kyverno_rbac_linkerd_manage,
48+
kubectl_manifest.kyverno_rbac_linkerd_view,
49+
]
50+
}

‎policies/add-linkerd-annotation-to-restate-namespaces.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: kyverno.io/v1
2+
kind: ClusterPolicy
3+
metadata:
4+
name: add-linkerd-annotation-to-restate-namespaces
5+
annotations:
6+
policies.kyverno.io/title: Add Linkerd Injection to Restate Namespaces
7+
policies.kyverno.io/category: Linkerd, Service Mesh
8+
policies.kyverno.io/subject: Namespace
9+
policies.kyverno.io/minversion: "1.6.0"
10+
kyverno.io/kyverno-version: "1.6.0"
11+
kyverno.io/kubernetes-version: "1.23"
12+
policies.kyverno.io/description: >-
13+
This policy automatically adds the 'linkerd.io/inject: enabled' annotation
14+
to namespaces that are labeled with 'app.kubernetes.io/name=restate'.
15+
This ensures that Linkerd sidecar injection is enabled for all pods
16+
created in Restate namespaces.
17+
spec:
18+
validationFailureAction: Enforce
19+
generateExisting: true
20+
rules:
21+
- name: add-linkerd-injection-annotation
22+
match:
23+
any:
24+
- resources:
25+
kinds:
26+
- Namespace
27+
selector:
28+
matchLabels:
29+
app.kubernetes.io/name: restate
30+
mutate:
31+
patchStrategicMerge:
32+
metadata:
33+
annotations:
34+
+(linkerd.io/inject): enabled
35+
+(config.linkerd.io/default-inbound-policy): deny

‎policies/linkerd-authz-for-restate.yml‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
apiVersion: kyverno.io/v1
2+
kind: ClusterPolicy
3+
metadata:
4+
name: linkerd-authz-for-restate
5+
annotations:
6+
policies.kyverno.io/title: Linkerd Authorization for Restate Namespaces
7+
policies.kyverno.io/category: Linkerd, Service Mesh, Security
8+
policies.kyverno.io/subject: Namespace
9+
policies.kyverno.io/minversion: "1.6.0"
10+
policies.kyverno.io/description: >-
11+
This policy generates Linkerd Server, HTTPRoute, and AuthorizationPolicy
12+
resources for Restate namespaces to control ingress traffic. It allows:
13+
- Restate pods to communicate with each other on port 5122
14+
- Cloud ingress to reach Restate on ports 8080, 9070, and 5122
15+
- vmagent to scrape /metrics on port 5122
16+
- Kubelet health probes on /health (port 9070) from any source
17+
# Keep the generated Linkerd kinds and apiVersions in sync with the active CRD
18+
# discovery barrier in the AWS sandbox repo before changing this policy.
19+
spec:
20+
validationFailureAction: Enforce
21+
generateExisting: true
22+
rules:
23+
# Servers
24+
- name: generate-server-node
25+
match: &restate-ns-match
26+
any:
27+
- resources:
28+
kinds:
29+
- Namespace
30+
selector:
31+
matchLabels:
32+
app.kubernetes.io/name: restate
33+
generate:
34+
apiVersion: policy.linkerd.io/v1beta3
35+
kind: Server
36+
name: restate-node
37+
namespace: "{{request.object.metadata.name}}"
38+
synchronize: true
39+
data:
40+
spec:
41+
podSelector:
42+
matchLabels:
43+
app.kubernetes.io/name: restate
44+
port: 5122
45+
46+
- name: generate-server-ingress
47+
match: *restate-ns-match
48+
generate:
49+
apiVersion: policy.linkerd.io/v1beta3
50+
kind: Server
51+
name: restate-ingress
52+
namespace: "{{request.object.metadata.name}}"
53+
synchronize: true
54+
data:
55+
spec:
56+
podSelector:
57+
matchLabels:
58+
app.kubernetes.io/name: restate
59+
port: 8080
60+
61+
- name: generate-server-admin
62+
match: *restate-ns-match
63+
generate:
64+
apiVersion: policy.linkerd.io/v1beta3
65+
kind: Server
66+
name: restate-admin
67+
namespace: "{{request.object.metadata.name}}"
68+
synchronize: true
69+
data:
70+
spec:
71+
podSelector:
72+
matchLabels:
73+
app.kubernetes.io/name: restate
74+
port: 9070
75+
76+
# HTTPRoutes for port 5122 (restate-node)
77+
- name: generate-metrics-route
78+
match: *restate-ns-match
79+
generate:
80+
apiVersion: policy.linkerd.io/v1beta3
81+
kind: HTTPRoute
82+
name: restate-node-metrics
83+
namespace: "{{request.object.metadata.name}}"
84+
synchronize: true
85+
data:
86+
spec:
87+
parentRefs:
88+
- name: restate-node
89+
kind: Server
90+
group: policy.linkerd.io
91+
rules:
92+
- matches:
93+
- path:
94+
type: PathPrefix
95+
value: /metrics
96+
97+
- name: generate-node-catchall-route
98+
match: *restate-ns-match
99+
generate:
100+
apiVersion: policy.linkerd.io/v1beta3
101+
kind: HTTPRoute
102+
name: restate-node-other
103+
namespace: "{{request.object.metadata.name}}"
104+
synchronize: true
105+
data:
106+
spec:
107+
parentRefs:
108+
- name: restate-node
109+
kind: Server
110+
group: policy.linkerd.io
111+
rules:
112+
- matches:
113+
- path:
114+
type: PathPrefix
115+
value: /
116+
117+
# HTTPRoutes for port 9070 (restate-admin)
118+
- name: generate-health-route
119+
match: *restate-ns-match
120+
generate:
121+
apiVersion: policy.linkerd.io/v1beta3
122+
kind: HTTPRoute
123+
name: restate-admin-health
124+
namespace: "{{request.object.metadata.name}}"
125+
synchronize: true
126+
data:
127+
spec:
128+
parentRefs:
129+
- name: restate-admin
130+
kind: Server
131+
group: policy.linkerd.io
132+
rules:
133+
- matches:
134+
- path:
135+
type: Exact
136+
value: /health
137+
138+
- name: generate-admin-catchall-route
139+
match: *restate-ns-match
140+
generate:
141+
apiVersion: policy.linkerd.io/v1beta3
142+
kind: HTTPRoute
143+
name: restate-admin-other
144+
namespace: "{{request.object.metadata.name}}"
145+
synchronize: true
146+
data:
147+
spec:
148+
parentRefs:
149+
- name: restate-admin
150+
kind: Server
151+
group: policy.linkerd.io
152+
rules:
153+
- matches:
154+
- path:
155+
type: PathPrefix
156+
value: /
157+
158+
# AuthorizationPolicies
159+
- name: generate-authz-restate-self
160+
match: *restate-ns-match
161+
generate:
162+
apiVersion: policy.linkerd.io/v1alpha1
163+
kind: AuthorizationPolicy
164+
name: allow-restate-self
165+
namespace: "{{request.object.metadata.name}}"
166+
synchronize: true
167+
data:
168+
spec:
169+
targetRef:
170+
group: policy.linkerd.io
171+
kind: HTTPRoute
172+
name: restate-node-other
173+
requiredAuthenticationRefs:
174+
- kind: ServiceAccount
175+
name: restate
176+
namespace: "{{request.object.metadata.name}}"
177+
178+
- name: generate-authz-cloud-ingress-node
179+
match: *restate-ns-match
180+
generate:
181+
apiVersion: policy.linkerd.io/v1alpha1
182+
kind: AuthorizationPolicy
183+
name: allow-cloud-ingress-node
184+
namespace: "{{request.object.metadata.name}}"
185+
synchronize: true
186+
data:
187+
spec:
188+
targetRef:
189+
group: policy.linkerd.io
190+
kind: HTTPRoute
191+
name: restate-node-other
192+
requiredAuthenticationRefs:
193+
- kind: ServiceAccount
194+
name: restate-cloud-ingress
195+
namespace: restate-cloud-ingress
196+
197+
- name: generate-authz-cloud-ingress-ingress
198+
match: *restate-ns-match
199+
generate:
200+
apiVersion: policy.linkerd.io/v1alpha1
201+
kind: AuthorizationPolicy
202+
name: allow-cloud-ingress-ingress
203+
namespace: "{{request.object.metadata.name}}"
204+
synchronize: true
205+
data:
206+
spec:
207+
targetRef:
208+
group: policy.linkerd.io
209+
kind: Server
210+
name: restate-ingress
211+
requiredAuthenticationRefs:
212+
- kind: ServiceAccount
213+
name: restate-cloud-ingress
214+
namespace: restate-cloud-ingress
215+
216+
- name: generate-authz-cloud-ingress-admin
217+
match: *restate-ns-match
218+
generate:
219+
apiVersion: policy.linkerd.io/v1alpha1
220+
kind: AuthorizationPolicy
221+
name: allow-cloud-ingress-admin
222+
namespace: "{{request.object.metadata.name}}"
223+
synchronize: true
224+
data:
225+
spec:
226+
targetRef:
227+
group: policy.linkerd.io
228+
kind: HTTPRoute
229+
name: restate-admin-other
230+
requiredAuthenticationRefs:
231+
- kind: ServiceAccount
232+
name: restate-cloud-ingress
233+
namespace: restate-cloud-ingress
234+
235+
- name: generate-authz-health-probes
236+
match: *restate-ns-match
237+
generate:
238+
apiVersion: policy.linkerd.io/v1alpha1
239+
kind: AuthorizationPolicy
240+
name: allow-health-probes
241+
namespace: "{{request.object.metadata.name}}"
242+
synchronize: true
243+
data:
244+
spec:
245+
targetRef:
246+
group: policy.linkerd.io
247+
kind: HTTPRoute
248+
name: restate-admin-health
249+
requiredAuthenticationRefs: []
250+
251+
- name: generate-authz-vmagent-metrics
252+
match: *restate-ns-match
253+
generate:
254+
apiVersion: policy.linkerd.io/v1alpha1
255+
kind: AuthorizationPolicy
256+
name: allow-vmagent-metrics
257+
namespace: "{{request.object.metadata.name}}"
258+
synchronize: true
259+
data:
260+
spec:
261+
targetRef:
262+
group: policy.linkerd.io
263+
kind: HTTPRoute
264+
name: restate-node-metrics
265+
requiredAuthenticationRefs:
266+
- kind: ServiceAccount
267+
name: vmagent
268+
namespace: observability

‎policies/rbac-linkerd-manage.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: kyverno:linkerd:manage
5+
labels:
6+
rbac.kyverno.io/aggregate-to-background-controller: "true"
7+
rules:
8+
- apiGroups:
9+
- policy.linkerd.io
10+
resources:
11+
- servers
12+
- httproutes
13+
- authorizationpolicies
14+
verbs:
15+
- create
16+
- delete
17+
- update

‎policies/rbac-linkerd-view.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: kyverno:linkerd:view
5+
labels:
6+
rbac.kyverno.io/aggregate-to-admission-controller: "true"
7+
rbac.kyverno.io/aggregate-to-reports-controller: "true"
8+
rbac.kyverno.io/aggregate-to-background-controller: "true"
9+
rules:
10+
- apiGroups:
11+
- policy.linkerd.io
12+
resources:
13+
- servers
14+
- httproutes
15+
- authorizationpolicies
16+
verbs:
17+
- get
18+
- list
19+
- watch

0 commit comments

Comments
 (0)