[KYUUBI #7415] Add a configuration to include additional Spark keys as path validation targets

docs/configuration/settings.md

@@ -537,6 +537,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 | kyuubi.session.local.dir.allow.list                                              || The local dir list that are allowed to access by the kyuubi session application.  End-users might set some parameters such as `spark.files` and it will  upload some local files when launching the kyuubi engine, if the local dir allow list is defined, kyuubi will check whether the path to upload is in the allow list. Note that, if it is empty, there is no limitation for that. And please use absolute paths. | set      | 1.6.0  |
 | kyuubi.session.name                                     | <undefined>       | A human readable name of the session and we use empty string by default. This name will be recorded in the event. Note that, we only apply this value from session conf.                                                                                                                                                                                                                                                 | string   | 1.4.0  |
 | kyuubi.session.proxy.user                               | <undefined>       | An alternative to hive.server2.proxy.user. The current behavior is consistent with hive.server2.proxy.user and now only takes effect in RESTFul API. When both parameters are set, kyuubi.session.proxy.user takes precedence.                                                                                                                                                                                           | string   | 1.9.0  |
+| kyuubi.session.spark.file.config.list                                            || A comma-separated list of additional Spark parameters for which Kyuubi checks whether the upload path is included in kyuubi.session.local.dir.allow.list.                                                                                                                                                                                                                                                                | set      | 1.12.0 |
 | kyuubi.session.timeout                                  | PT6H                    | (deprecated)session timeout, it will be closed when it's not accessed for this duration                                                                                                                                                                                                                                                                                                                                  | duration | 1.0.0  |
 | kyuubi.session.user.sign.enabled                        | false                   | Whether to verify the integrity of session user name on the engine side, e.g. Authz plugin in Spark.                                                                                                                                                                                                                                                                                                                     | boolean  | 1.7.0  |
 

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -1873,6 +1873,16 @@ object KyuubiConf {
       .toSet()
       .createWithDefault(Set.empty)
 
+  val SESSION_SPARK_FILE_CONFIG_LIST: ConfigEntry[Set[String]] =
+    buildConf("kyuubi.session.spark.file.config.list")
+      .doc(s"A comma-separated list of additional Spark parameters for which Kyuubi checks " +
+        s"whether the upload path is included in ${SESSION_LOCAL_DIR_ALLOW_LIST.key}.")
+      .version("1.12.0")
+      .serverOnly
+      .stringConf
+      .toSet()
+      .createWithDefault(Set.empty)
+
   val BATCH_APPLICATION_CHECK_INTERVAL: ConfigEntry[Long] =
     buildConf("kyuubi.batch.application.check.interval")
       .doc("The interval to check batch job application information.")

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala

@@ -176,7 +176,8 @@ object KyuubiApplicationManager {
       appConf: Map[String, String],
       kyuubiConf: KyuubiConf): Unit = {
     if (kyuubiConf.get(KyuubiConf.SESSION_LOCAL_DIR_ALLOW_LIST).nonEmpty) {
-      SparkProcessBuilder.PATH_CONFIGS.flatMap { key =>
+      (SparkProcessBuilder.PATH_CONFIGS.toSet ++
+        kyuubiConf.get(KyuubiConf.SESSION_SPARK_FILE_CONFIG_LIST)).flatMap { key =>
         appConf.get(key).map(_.split(",")).getOrElse(Array.empty)
       }.filter(_.nonEmpty).foreach { path =>
         checkApplicationAccessPath(path, kyuubiConf)

kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala

@@ -77,6 +77,42 @@ class KyuubiApplicationManagerSuite extends KyuubiFunSuite {
         appConf,
         localDirLimitConf)
     }
+
+    appConf = Map("spark.new.access.local" -> "/apache/kyuubi/jars/a.jar")
+    KyuubiApplicationManager.checkApplicationAccessPaths(
+      "SPARK",
+      appConf,
+      localDirLimitConf)
+    KyuubiApplicationManager.checkApplicationAccessPaths(
+      "SPARK",
+      appConf,
+      noLocalDirLimitConf)
+
+    appConf = Map("spark.new.access.local" -> "/apache/jars/a.jar")
+    KyuubiApplicationManager.checkApplicationAccessPaths(
+      "SPARK",
+      appConf,
+      localDirLimitConf)
+    KyuubiApplicationManager.checkApplicationAccessPaths(
+      "SPARK",
+      appConf,
+      noLocalDirLimitConf)
+
+    localDirLimitConf.set(KyuubiConf.SESSION_SPARK_FILE_CONFIG_LIST, Set("spark.new.access.local"))
+    appConf = Map("spark.new.access.local" -> "/apache/kyuubi/jars/a.jar")
+    KyuubiApplicationManager.checkApplicationAccessPaths(
+      "SPARK",
+      appConf,
+      localDirLimitConf)
+
+    appConf = Map("spark.new.access.local" -> "/apache/jars/a.jar")
+    intercept[KyuubiException] {
+      KyuubiApplicationManager.checkApplicationAccessPaths(
+        "SPARK",
+        appConf,
+        localDirLimitConf)
+    }
+    localDirLimitConf.unset(KyuubiConf.SESSION_SPARK_FILE_CONFIG_LIST)
   }
 
   test("Test kyuubi application Manager tag spark on kubernetes application") {