Skip to content

Commit aa5bc55

Browse files
ruanwenjunruanwenjun
committed
[KYUUBI #7495] Avoid overlong Kubernetes pod log names
1 parent 7067644 commit aa5bc55

5 files changed

Lines changed: 150 additions & 11 deletions

File tree

‎externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,9 +230,14 @@ object SparkSQLEngine extends Logging {
230230

231231
private val sparkSessionCreated = new AtomicBoolean(false)
232232

233-
// Kubernetes pod name max length - '-exec-' - Int.MAX_VALUE.length
234-
// 253 - 10 - 6
235-
val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH = 237
233+
final private val POD_NAME_MAX_LENGTH = 253
234+
final private val POD_UID_MAX_LENGTH = 36
235+
final private val POD_LOGS_DIRECTORY_SEPARATOR_LENGTH = 2
236+
final private val EXECUTOR_POD_NAME_RESERVED_LENGTH =
237+
"-exec-".length + Int.MaxValue.toString.length
238+
239+
val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH =
240+
executorPodNamePrefixMaxLength(KUBERNETES_NAMESPACE.defaultValStr)
236241

237242
SignalRegister.registerLogger(logger)
238243
setupConf()
@@ -292,7 +297,9 @@ object SparkSQLEngine extends Logging {
292297
// due to the long app name
293298
_sparkConf.setIfMissing(
294299
"spark.kubernetes.executor.podNamePrefix",
295-
generateExecutorPodNamePrefixForK8s(user))
300+
generateExecutorPodNamePrefixForK8s(
301+
user,
302+
_sparkConf.get("spark.kubernetes.namespace", KUBERNETES_NAMESPACE.defaultValStr)))
296303

297304
if (!isOnK8sClusterMode) {
298305
// set driver host to ip instead of kyuubi pod name
@@ -464,17 +471,27 @@ object SparkSQLEngine extends Logging {
464471
}
465472

466473
@VisibleForTesting
467-
def generateExecutorPodNamePrefixForK8s(userName: String): String = {
474+
def generateExecutorPodNamePrefixForK8s(
475+
userName: String,
476+
namespace: String = KUBERNETES_NAMESPACE.defaultValStr): String = {
468477
val resolvedUserName =
469478
userName.trim.toLowerCase(Locale.ROOT)
470479
.replaceAll("[^a-z0-9\\-]", "-")
471480
.replaceAll("-+", "-")
472481
.replaceAll("^-", "")
473482
val podNamePrefixWithUser = s"kyuubi-$resolvedUserName-${Instant.now().toEpochMilli}"
474-
if (podNamePrefixWithUser.length <= EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH) {
483+
if (podNamePrefixWithUser.length <= executorPodNamePrefixMaxLength(namespace)) {
475484
podNamePrefixWithUser
476485
} else {
477486
s"kyuubi-${UUID.randomUUID()}"
478487
}
479488
}
489+
490+
private def executorPodNamePrefixMaxLength(namespace: String): Int = {
491+
POD_NAME_MAX_LENGTH -
492+
namespace.length -
493+
POD_UID_MAX_LENGTH -
494+
POD_LOGS_DIRECTORY_SEPARATOR_LENGTH -
495+
EXECUTOR_POD_NAME_RESERVED_LENGTH
496+
}
480497
}

‎externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkSQLEngineSuite.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ import org.apache.kyuubi.KyuubiFunSuite
2121

2222
class SparkSQLEngineSuite extends KyuubiFunSuite {
2323

24+
private val namespace = "n" * 63
25+
private val podUid = "u" * 36
26+
2427
test("[KYUUBI #3385] generate executor pod name prefix with user or UUID") {
2528
val userName1 = "/kyuubi_user+-*"
2629
val executorPodNamePrefix1 = SparkSQLEngine.generateExecutorPodNamePrefixForK8s(userName1)
@@ -33,5 +36,16 @@ class SparkSQLEngineSuite extends KyuubiFunSuite {
3336
val executorPodNamePrefix2 = SparkSQLEngine.generateExecutorPodNamePrefixForK8s(userName2)
3437
assert(!executorPodNamePrefix2.contains(userName2))
3538
assert(executorPodNamePrefix2.length <= SparkSQLEngine.EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH)
39+
40+
val userName3 = "l" * 160
41+
val executorPodNamePrefix3 =
42+
SparkSQLEngine.generateExecutorPodNamePrefixForK8s(userName3, namespace)
43+
val executorPodName3 = s"$executorPodNamePrefix3-exec-${Int.MaxValue}"
44+
assert(!executorPodNamePrefix3.contains(userName3))
45+
assert(podLogsDirectoryNameLength(namespace, executorPodName3) <= 253)
46+
}
47+
48+
private def podLogsDirectoryNameLength(namespace: String, podName: String): Int = {
49+
s"${namespace}_${podName}_$podUid".length
3650
}
3751
}

‎kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,12 +261,16 @@ class SparkProcessBuilder(
261261

262262
def appendPodNameConf(conf: Map[String, String]): Map[String, String] = {
263263
val appName = conf.getOrElse(APP_KEY, "spark")
264+
val namespace = conf.get(KUBERNETES_NAMESPACE_KEY)
265+
.orElse(kubernetesNamespace())
266+
.getOrElse(KUBERNETES_NAMESPACE.defaultValStr)
264267
val map = mutable.Map.newBuilder[String, String]
265268
if (clusterManager().exists(cm => cm.toLowerCase(Locale.ROOT).startsWith("k8s"))) {
266269
if (!conf.contains(KUBERNETES_EXECUTOR_POD_NAME_PREFIX)) {
267270
val prefix = KubernetesUtils.generateExecutorPodNamePrefix(
268271
appName,
269272
engineRefId,
273+
namespace,
270274
forciblyRewriteExecPodNamePrefix)
271275
map += (KUBERNETES_EXECUTOR_POD_NAME_PREFIX -> prefix)
272276
}
@@ -275,6 +279,7 @@ class SparkProcessBuilder(
275279
val name = KubernetesUtils.generateDriverPodName(
276280
appName,
277281
engineRefId,
282+
namespace,
278283
forciblyRewriteDriverPodName)
279284
map += (KUBERNETES_DRIVER_POD_NAME -> name)
280285
}

‎kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,14 @@ import org.apache.kyuubi.config.KyuubiConf
3333
import org.apache.kyuubi.config.KyuubiConf._
3434

3535
object KubernetesUtils extends Logging {
36-
// Kubernetes pod name max length - '-exec-' - Int.MAX_VALUE.length
37-
// 253 - 10 - 6
38-
final val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH = 237
3936
final val DRIVER_POD_NAME_MAX_LENGTH = 253
37+
final private val POD_UID_MAX_LENGTH = 36
38+
final private val POD_LOGS_DIRECTORY_SEPARATOR_LENGTH = 2
39+
final private val EXECUTOR_POD_NAME_RESERVED_LENGTH =
40+
"-exec-".length + Int.MaxValue.toString.length
41+
42+
final val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH =
43+
maxExecutorPodNamePrefixLength(KUBERNETES_NAMESPACE.defaultValStr)
4044

4145
def buildKubernetesClient(conf: KyuubiConf): Option[KubernetesClient] = {
4246
val master = conf.get(KUBERNETES_MASTER)
@@ -138,6 +142,14 @@ object KubernetesUtils extends Logging {
138142
appName: String,
139143
engineRefId: String,
140144
forciblyRewrite: Boolean): String = {
145+
generateDriverPodName(appName, engineRefId, KUBERNETES_NAMESPACE.defaultValStr, forciblyRewrite)
146+
}
147+
148+
def generateDriverPodName(
149+
appName: String,
150+
engineRefId: String,
151+
namespace: String,
152+
forciblyRewrite: Boolean): String = {
141153
val resourceNamePrefix = if (appName.contains(engineRefId)) {
142154
getResourceNamePrefix(appName, None)
143155
} else {
@@ -148,7 +160,7 @@ object KubernetesUtils extends Logging {
148160
} else {
149161
s"kyuubi-$resourceNamePrefix-driver"
150162
}
151-
if (forciblyRewrite || resolvedResourceName.length > DRIVER_POD_NAME_MAX_LENGTH) {
163+
if (forciblyRewrite || resolvedResourceName.length > maxDriverPodNameLength(namespace)) {
152164
s"kyuubi-$engineRefId-driver"
153165
} else {
154166
resolvedResourceName
@@ -159,6 +171,18 @@ object KubernetesUtils extends Logging {
159171
appName: String,
160172
engineRefId: String,
161173
forciblyRewrite: Boolean): String = {
174+
generateExecutorPodNamePrefix(
175+
appName,
176+
engineRefId,
177+
KUBERNETES_NAMESPACE.defaultValStr,
178+
forciblyRewrite)
179+
}
180+
181+
def generateExecutorPodNamePrefix(
182+
appName: String,
183+
engineRefId: String,
184+
namespace: String,
185+
forciblyRewrite: Boolean): String = {
162186
val resourceNamePrefix = if (appName.contains(engineRefId)) {
163187
getResourceNamePrefix(appName, None)
164188
} else {
@@ -169,10 +193,22 @@ object KubernetesUtils extends Logging {
169193
} else {
170194
s"kyuubi-$resourceNamePrefix"
171195
}
172-
if (forciblyRewrite || resolvedResourceName.length > EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH) {
196+
if (forciblyRewrite || resolvedResourceName.length > maxExecutorPodNamePrefixLength(
197+
namespace)) {
173198
s"kyuubi-$engineRefId"
174199
} else {
175200
resolvedResourceName
176201
}
177202
}
203+
204+
private def maxDriverPodNameLength(namespace: String): Int = {
205+
DRIVER_POD_NAME_MAX_LENGTH -
206+
namespace.length -
207+
POD_UID_MAX_LENGTH -
208+
POD_LOGS_DIRECTORY_SEPARATOR_LENGTH
209+
}
210+
211+
private def maxExecutorPodNamePrefixLength(namespace: String): Int = {
212+
maxDriverPodNameLength(namespace) - EXECUTOR_POD_NAME_RESERVED_LENGTH
213+
}
178214
}

‎kyuubi-server/src/test/scala/org/apache/kyuubi/util/KubernetesPodNameSuite.scala‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.kyuubi.util
19+
20+
import org.apache.kyuubi.KyuubiFunSuite
21+
import org.apache.kyuubi.config.KyuubiConf
22+
import org.apache.kyuubi.engine.spark.SparkProcessBuilder
23+
import org.apache.kyuubi.engine.spark.SparkProcessBuilder._
24+
25+
class KubernetesPodNameSuite extends KyuubiFunSuite {
26+
27+
private val namespace = "n" * 63
28+
private val engineRefId = "kyuubi-test-engine"
29+
private val longAppName = "a" * 160
30+
private val podUid = "u" * 36
31+
32+
test("driver pod name should reserve kubelet pod logs directory budget") {
33+
val podName =
34+
KubernetesUtils.generateDriverPodName(longAppName, engineRefId, namespace, false)
35+
36+
assert(podName === s"kyuubi-$engineRefId-driver")
37+
assert(podLogsDirectoryNameLength(namespace, podName) <=
38+
KubernetesUtils.DRIVER_POD_NAME_MAX_LENGTH)
39+
}
40+
41+
test("executor pod name prefix should reserve kubelet pod logs directory budget") {
42+
val prefix =
43+
KubernetesUtils.generateExecutorPodNamePrefix(longAppName, engineRefId, namespace, false)
44+
val podName = s"$prefix-exec-${Int.MaxValue}"
45+
46+
assert(prefix === s"kyuubi-$engineRefId")
47+
assert(podLogsDirectoryNameLength(namespace, podName) <=
48+
KubernetesUtils.DRIVER_POD_NAME_MAX_LENGTH)
49+
}
50+
51+
test("SparkProcessBuilder should use spark kubernetes namespace for pod name budget") {
52+
val builder = new SparkProcessBuilder(
53+
"kyuubi",
54+
true,
55+
KyuubiConf().set(MASTER_KEY, "k8s://internal").set(DEPLOY_MODE_KEY, "cluster"),
56+
engineRefId)
57+
val conf = Map(APP_KEY -> longAppName, KUBERNETES_NAMESPACE_KEY -> namespace)
58+
val podNameConf = builder.appendPodNameConf(conf)
59+
60+
assert(podNameConf(KUBERNETES_DRIVER_POD_NAME) === s"kyuubi-$engineRefId-driver")
61+
assert(podNameConf(KUBERNETES_EXECUTOR_POD_NAME_PREFIX) === s"kyuubi-$engineRefId")
62+
}
63+
64+
private def podLogsDirectoryNameLength(namespace: String, podName: String): Int = {
65+
s"${namespace}_${podName}_$podUid".length
66+
}
67+
}

0 commit comments

Comments
 (0)