address review comments

Author: nartal1

integration_tests/src/main/python/spark_init_internal.py

@@ -51,6 +51,9 @@ def conf_for_env(env_name):
     _SPARK_JARS,
     _SPARK_JARS_PACKAGES
 }
+# Initialized in pytest_sessionstart; declared here so pytest_sessionfinish
+# does not raise NameError if session startup fails before assignment.
+_spark = None
 
 def findspark_init():
     import findspark
@@ -312,4 +315,6 @@ def pytest_sessionfinish(session, exitstatus):
             _spark.stop()
         except Exception as e:
             logging.warning(f"Exception while stopping SparkSession: {e}")
+        finally:
+            _spark = None