Change Paths to use pathlib.Path (#1431)

* Switch to using pathlib

* Sort linter list by code

* Add ignore for new rule violation

* Update ruff in pre-commit

* Fix s3 tests

* Fix newer test files with pathlib changes

* Fix agent unittests

* Fix path monkeypatching in docker container tests

* Fix additional os.path references

* Rename all os.path.pathsep to os.pathsep for brevity

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: TimPansino

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ default_install_hook_types:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.11.5
+    rev: v0.12.4
     hooks:
       # Run the linter.
       - id: ruff

newrelic/__init__.py

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os.path
+from pathlib import Path
 
-THIS_DIR = os.path.dirname(__file__)
+VERSION_FILE = Path(__file__).parent / "version.txt"
 
 try:
-    with open(os.path.join(THIS_DIR, "version.txt")) as f:
+    with VERSION_FILE.open() as f:
         version = f.read()
-except:
+except Exception:
     version = "0.0.0"
 
 version_info = list(map(int, version.split(".")))

newrelic/admin/debug_console.py

@@ -25,6 +25,7 @@
 )
 def debug_console(args):
     import sys
+    from pathlib import Path
 
     if len(args) == 0:
         usage("debug-console")
@@ -36,7 +37,7 @@ def debug_console(args):
     log_object = None
 
     if len(args) >= 2:
-        log_object = open(args[1], "w")
+        log_object = Path(args[1]).open("w")
 
     shell = ClientShell(config_file, log=log_object)
     shell.cmdloop()

newrelic/admin/generate_config.py

@@ -19,28 +19,25 @@
     "generate-config", "license_key [output_file]", """Generates a sample agent configuration file for <license_key>."""
 )
 def generate_config(args):
-    import os
     import sys
+    from pathlib import Path
 
     if len(args) == 0:
         usage("generate-config")
         sys.exit(1)
 
-    from newrelic import __file__ as package_root
+    import newrelic
 
-    package_root = os.path.dirname(package_root)
+    config_file = Path(newrelic.__file__).parent / "newrelic.ini"
 
-    config_file = os.path.join(package_root, "newrelic.ini")
-
-    with open(config_file) as f:
+    with config_file.open() as f:
         content = f.read()
 
     if len(args) >= 1:
         content = content.replace("*** REPLACE ME ***", args[0])
 
     if len(args) >= 2 and args[1] != "-":
-        output_file = open(args[1], "w")
-        output_file.write(content)
-        output_file.close()
+        with Path(args[1]).open("w") as output_file:
+            output_file.write(content)
     else:
         print(content)

newrelic/admin/license_key.py

@@ -26,6 +26,7 @@ def license_key(args):
     import logging
     import os
     import sys
+    from pathlib import Path
 
     if len(args) == 0:
         usage("license-key")
@@ -37,12 +38,12 @@ def license_key(args):
     if len(args) >= 2:
         log_file = args[1]
     else:
-        log_file = "/tmp/python-agent-test.log"
+        log_file = Path("/tmp/python-agent-test.log")
 
     log_level = logging.DEBUG
 
     try:
-        os.unlink(log_file)
+        log_file.unlink()
     except Exception:
         pass
 

newrelic/admin/local_config.py

@@ -25,6 +25,7 @@ def local_config(args):
     import logging
     import os
     import sys
+    from pathlib import Path
 
     if len(args) == 0:
         usage("local-config")
@@ -36,12 +37,12 @@ def local_config(args):
     if len(args) >= 2:
         log_file = args[1]
     else:
-        log_file = "/tmp/python-agent-test.log"
+        log_file = Path("/tmp/python-agent-test.log")
 
     log_level = logging.DEBUG
 
     try:
-        os.unlink(log_file)
+        log_file.unlink()
     except Exception:
         pass
 

newrelic/admin/network_config.py

@@ -25,6 +25,7 @@ def network_config(args):
     import logging
     import os
     import sys
+    from pathlib import Path
 
     if len(args) == 0:
         usage("network-config")
@@ -36,12 +37,12 @@ def network_config(args):
     if len(args) >= 2:
         log_file = args[1]
     else:
-        log_file = "/tmp/python-agent-test.log"
+        log_file = Path("/tmp/python-agent-test.log")
 
     log_level = logging.DEBUG
 
     try:
-        os.unlink(log_file)
+        log_file.unlink()
     except Exception:
         pass
 

newrelic/admin/run_program.py

@@ -31,6 +31,7 @@ def run_program(args):
     import os
     import sys
     import time
+    from pathlib import Path
 
     if len(args) == 0:
         usage("run-program")
@@ -46,10 +47,11 @@ def log_message(text, *args):
 
     log_message("New Relic Admin Script (%s)", __file__)
 
-    log_message("working_directory = %r", os.getcwd())
+    log_message("working_directory = %r", str(Path.cwd()))
     log_message("current_command = %r", sys.argv)
 
-    log_message("sys.prefix = %r", os.path.normpath(sys.prefix))
+    sys_prefix = str(Path(sys.prefix).resolve())
+    log_message("sys.prefix = %r", sys_prefix)
 
     try:
         log_message("sys.real_prefix = %r", sys.real_prefix)
@@ -67,44 +69,47 @@ def log_message(text, *args):
                 continue
             log_message("%s = %r", name, os.environ.get(name))
 
-    from newrelic import __file__ as root_directory
+    import newrelic
 
-    root_directory = os.path.dirname(root_directory)
-    boot_directory = os.path.join(root_directory, "bootstrap")
+    root_directory = Path(newrelic.__file__).parent
+    boot_directory = root_directory / "bootstrap"
+
+    root_directory = str(root_directory)
+    boot_directory = str(boot_directory)
 
     log_message("root_directory = %r", root_directory)
     log_message("boot_directory = %r", boot_directory)
 
     python_path = boot_directory
 
     if "PYTHONPATH" in os.environ:
-        path = os.environ["PYTHONPATH"].split(os.path.pathsep)
+        path = os.environ["PYTHONPATH"].split(os.pathsep)
         if boot_directory not in path:
-            python_path = f"{boot_directory}{os.path.pathsep}{os.environ['PYTHONPATH']}"
+            python_path = f"{boot_directory}{os.pathsep}{os.environ['PYTHONPATH']}"
 
     os.environ["PYTHONPATH"] = python_path
 
     os.environ["NEW_RELIC_ADMIN_COMMAND"] = repr(sys.argv)
 
-    os.environ["NEW_RELIC_PYTHON_PREFIX"] = os.path.realpath(os.path.normpath(sys.prefix))
+    os.environ["NEW_RELIC_PYTHON_PREFIX"] = sys_prefix
     os.environ["NEW_RELIC_PYTHON_VERSION"] = ".".join(map(str, sys.version_info[:2]))
 
     # If not an absolute or relative path, then we need to
     # see if program can be found in PATH. Note that can
     # be found in current working directory even though '.'
     # not in PATH.
 
-    program_exe_path = args[0]
+    program_exe_path = Path(args[0])
 
-    if not os.path.dirname(program_exe_path):
-        program_search_path = os.environ.get("PATH", "").split(os.path.pathsep)
+    if not program_exe_path.parent:
+        program_search_path = os.environ.get("PATH", "").split(os.pathsep)
         for path in program_search_path:
-            path = os.path.join(path, program_exe_path)
-            if os.path.exists(path) and os.access(path, os.X_OK):
+            path = Path(path) / program_exe_path
+            if path.exists() and os.access(path, os.X_OK):
                 program_exe_path = path
                 break
 
-    log_message("program_exe_path = %r", program_exe_path)
-    log_message("execl_arguments = %r", [program_exe_path] + args)
+    log_message("program_exe_path = %r", str(program_exe_path))
+    log_message("execl_arguments = %r", [program_exe_path, *args])
 
     os.execl(program_exe_path, *args)  # noqa: S606

newrelic/admin/run_python.py

@@ -31,6 +31,7 @@ def run_python(args):
     import os
     import sys
     import time
+    from pathlib import Path
 
     startup_debug = os.environ.get("NEW_RELIC_STARTUP_DEBUG", "off").lower() in ("on", "true", "1")
 
@@ -42,10 +43,11 @@ def log_message(text, *args):
 
     log_message("New Relic Admin Script (%s)", __file__)
 
-    log_message("working_directory = %r", os.getcwd())
+    log_message("working_directory = %r", str(Path.cwd()))
     log_message("current_command = %r", sys.argv)
 
-    log_message("sys.prefix = %r", os.path.normpath(sys.prefix))
+    sys_prefix = str(Path(sys.prefix).resolve())
+    log_message("sys.prefix = %r", sys_prefix)
 
     try:
         log_message("sys.real_prefix = %r", sys.real_prefix)
@@ -61,26 +63,29 @@ def log_message(text, *args):
         if name.startswith("NEW_RELIC_") or name.startswith("PYTHON"):
             log_message("%s = %r", name, os.environ.get(name))
 
-    from newrelic import __file__ as root_directory
+    import newrelic
 
-    root_directory = os.path.dirname(root_directory)
-    boot_directory = os.path.join(root_directory, "bootstrap")
+    root_directory = Path(newrelic.__file__).parent
+    boot_directory = root_directory / "bootstrap"
+
+    root_directory = str(root_directory)
+    boot_directory = str(boot_directory)
 
     log_message("root_directory = %r", root_directory)
     log_message("boot_directory = %r", boot_directory)
 
     python_path = boot_directory
 
     if "PYTHONPATH" in os.environ:
-        path = os.environ["PYTHONPATH"].split(os.path.pathsep)
+        path = os.environ["PYTHONPATH"].split(os.pathsep)
         if boot_directory not in path:
-            python_path = f"{boot_directory}{os.path.pathsep}{os.environ['PYTHONPATH']}"
+            python_path = f"{boot_directory}{os.pathsep}{os.environ['PYTHONPATH']}"
 
     os.environ["PYTHONPATH"] = python_path
 
     os.environ["NEW_RELIC_ADMIN_COMMAND"] = repr(sys.argv)
 
-    os.environ["NEW_RELIC_PYTHON_PREFIX"] = os.path.realpath(os.path.normpath(sys.prefix))
+    os.environ["NEW_RELIC_PYTHON_PREFIX"] = sys_prefix
     os.environ["NEW_RELIC_PYTHON_VERSION"] = ".".join(map(str, sys.version_info[:2]))
 
     # Heroku does not set #! line on installed Python scripts
@@ -91,17 +96,17 @@ def log_message(text, *args):
     # this script in preference to that used to execute this
     # script.
 
-    bin_directory = os.path.dirname(sys.argv[0])
+    bin_directory = Path(sys.argv[0]).parent
 
     if bin_directory:
-        python_exe = os.path.basename(sys.executable)
-        python_exe_path = os.path.join(bin_directory, python_exe)
-        if not os.path.exists(python_exe_path) or not os.access(python_exe_path, os.X_OK):
+        python_exe = Path(sys.executable).name
+        python_exe_path = bin_directory / python_exe
+        if not python_exe_path.exists() or not os.access(python_exe_path, os.X_OK):
             python_exe_path = sys.executable
     else:
         python_exe_path = sys.executable
 
-    log_message("python_exe_path = %r", python_exe_path)
-    log_message("execl_arguments = %r", [python_exe_path, python_exe_path] + args)
+    log_message("python_exe_path = %r", str(python_exe_path))
+    log_message("execl_arguments = %r", [python_exe_path, python_exe_path, *args])
 
     os.execl(python_exe_path, python_exe_path, *args)  # noqa: S606

newrelic/admin/server_config.py

@@ -28,6 +28,7 @@ def server_config(args):
     import os
     import sys
     import time
+    from pathlib import Path
 
     if len(args) == 0:
         usage("server-config")
@@ -39,12 +40,12 @@ def server_config(args):
     if len(args) >= 2:
         log_file = args[1]
     else:
-        log_file = "/tmp/python-agent-test.log"
+        log_file = Path("/tmp/python-agent-test.log")
 
     log_level = logging.DEBUG
 
     try:
-        os.unlink(log_file)
+        log_file.unlink()
     except Exception:
         pass