Merge pull request #142 from RRosio/fs_error

Capture inactive filesystem error

Author: RRosio

conftest.py

@@ -89,6 +89,24 @@ def malformed_config(tmp_path: Path):
         print(f"Patching jupyter_config_dir to: {config_dir}")
 
 
+@pytest.fixture(scope="function")
+def hdfs_config(tmp_path: Path):
+    config_dir = tmp_path / "config"
+    config_dir.mkdir(exist_ok=True)
+
+    yaml_content = """sources:
+  - name: "TestSourceHDF"
+    path: "hdfs://namenode.example.com:9000"
+    """
+    yaml_file = config_dir / "jupyter-fsspec.yaml"
+    yaml_file.write_text(yaml_content)
+
+    with patch(
+        "jupyter_fsspec.file_manager.jupyter_config_dir", return_value=str(config_dir)
+    ):
+        print(f"Patching jupyter_config_dir to: {config_dir}")
+
+
 @pytest.fixture(scope="function")
 def bad_info_config(tmp_path: Path):
     config_dir = tmp_path / "config"

jupyter_fsspec/file_manager.py

@@ -192,14 +192,6 @@ def initialize_filesystems(self):
                 )
                 continue
 
-            fs_class = fsspec.get_filesystem_class(fs_protocol)
-            if fs_class.async_impl:
-                fs = fsspec.filesystem(fs_protocol, asynchronous=True, *args, **kwargs)
-            else:
-                sync_fs = fsspec.filesystem(fs_protocol, *args, **kwargs)
-                fs = AsyncFileSystemWrapper(sync_fs)
-            logger.debug("fs_path: %s", fs_path)
-
             # Store the filesystem instance
             fs_info = {
                 "instance": None,
@@ -211,21 +203,46 @@ def initialize_filesystems(self):
                 "args": args,
                 "kwargs": kwargs,
             }
-            new_filesystems[key] = fs_info
-            fs_class = fsspec.get_filesystem_class(fs_protocol)
-            if fs_class.async_impl:
-                fs = FileSystemManager.construct_fs(fs_protocol, True, *args, **kwargs)
-                fs_info["instance"] = fs
-            else:
-                sync_fs = FileSystemManager.construct_fs(
-                    fs_protocol, False, *args, **kwargs
+            try:
+                fs_class = fsspec.get_filesystem_class(fs_protocol)
+
+                if fs_class.async_impl:
+                    fs = FileSystemManager.construct_fs(
+                        fs_protocol, True, *args, **kwargs
+                    )
+                    fs_info["instance"] = fs
+                else:
+                    sync_fs = FileSystemManager.construct_fs(
+                        fs_protocol, False, *args, **kwargs
+                    )
+                    fs = AsyncFileSystemWrapper(sync_fs)
+                    fs_info["instance"] = fs
+
+                logger.debug(
+                    f"Initialized filesystem '{fs_name}' with protocol '{fs_protocol}' at path '{fs_path}'"
+                )
+            except Exception:
+                fs_info["instance"] = None
+                logger.error(
+                    f"Failed to initialize filesystem '{fs_name}' at path '{fs_path}'."
                 )
-                fs = AsyncFileSystemWrapper(sync_fs)
-                fs_info["instance"] = fs
 
-            logger.debug(
-                f"Initialized filesystem '{fs_name}' with protocol '{fs_protocol}' at path '{fs_path}'"
-            )
+                import traceback
+
+                traceback.print_exc()
+
+                exc_type, exc_value, exc_tb = sys.exc_info()
+                error_info = {
+                    "type": exc_type.__name__,
+                    "message": str(exc_value),
+                    "short_traceback": traceback.format_exception_only(
+                        exc_type, exc_value
+                    )[-1].strip(),
+                    "traceback_list": traceback.format_tb(exc_tb),
+                }
+                fs_info["error"] = error_info
+
+            new_filesystems[key] = fs_info
 
         self.filesystems = new_filesystems
         self.name_to_prefix = name_to_prefix

jupyter_fsspec/handlers.py

@@ -96,6 +96,8 @@ async def get(self):
                 "canonical_path": fs_info["canonical_path"],
                 "kwargs": fs_info["kwargs"],
             }
+            if fs_info.get("error", None):
+                instance["error"] = fs_info["error"]
             file_systems.append(instance)
 
         self.set_status(200)

jupyter_fsspec/tests/test_api.py

@@ -424,6 +424,22 @@ async def xtest_action_same_fs_files(fs_manager_instance, jp_fetch):
     )
 
 
+@pytest.mark.no_setup_config_file_fs
+async def test_hdfs_config(hdfs_config, jp_fetch):
+    fetch_config = await jp_fetch("jupyter_fsspec", "config", method="GET")
+    assert fetch_config.code == 200
+    config_json = fetch_config.body.decode("utf-8")
+    config = json.loads(config_json)
+    content = config["content"]
+    assert len(content) == 1
+    item = content[0]
+    assert item["error"]
+    assert (
+        item["error"]["short_traceback"]
+        == "ModuleNotFoundError: No module named 'pyarrow'"
+    )
+
+
 async def test_upload_download(fs_manager_instance, jp_fetch):
     fs_manager = await fs_manager_instance
     remote_key = "TestSourceAWS"

src/FssFilesysItem.ts

@@ -36,6 +36,10 @@ class FssFilesysItem {
 
     const fsItem = document.createElement('div');
     fsItem.classList.add('jfss-fsitem-root');
+
+    if ('error' in fsInfo) {
+      fsItem.classList.add('jfss-fsitem-error');
+    }
     fsItem.addEventListener('mouseenter', this.handleFsysHover.bind(this));
     fsItem.addEventListener('mouseleave', this.handleFsysHover.bind(this));
     fsItem.dataset.fssname = fsInfo.name;
@@ -186,7 +190,11 @@ class FssFilesysItem {
       protocol: this.filesysProtocol,
       path: this.fsInfo.path
     });
-
+    if (this.fsInfo.error) {
+      this.logger.error('Inactive filesystem', {
+        ...this.fsInfo.error
+      });
+    }
     this.selected = true;
     this.filesysClicked.emit(this.fsInfo);
   }

style/base.css

@@ -159,6 +159,10 @@
   background-color: var(--jp-layout-color2);
 }
 
+.jfss-fsitem-error {
+  color: darkgrey;
+}
+
 .jfss-browseAreaLabel {
   width: 100%;
   margin: 0 0.5rem 0.5rem 0;

ui-tests/tests/filesystem_interaction.test.ts

@@ -60,6 +60,23 @@ const emptyConfig = {
   content: []
 };
 
+const hdfsConfig = {
+  status: 'success',
+  description: 'Retrieved available filesystems from configuration file.',
+  content: [
+    {
+      key: 'myhdfs',
+      name: 'myhdfs',
+      path: 'hdfs://namenode.example.com:9000',
+      prefix_path: 'myhdfs',
+      canonical_path: 'hdfs://namenode.example.com:9000',
+      args: [],
+      kwargs: {},
+      error: { name: 'ImportError' }
+    }
+  ]
+};
+
 const rootMyMemFs = {
   content: [
     {
@@ -151,6 +168,30 @@ test('test open jupyterFsspec with empty config', async ({ page }) => {
   await expect.soft(page.locator('.jfss-resultarea')).toBeVisible();
 });
 
+test('test open jupyterFsspec with hdfs config', async ({ page }) => {
+  await page.route('http://localhost:8888/jupyter_fsspec/config?**', route => {
+    route.fulfill({
+      status: 200,
+      contentType: 'application/json',
+      body: JSON.stringify(hdfsConfig)
+    });
+  });
+
+  await page.goto();
+  await page.getByText('FSSpec', { exact: true }).click();
+
+  // verify filesystem item was created
+  await expect.soft(page.locator('.jfss-fsitem-root')).toBeVisible();
+
+  // filesystem details should match as expected
+  await expect.soft(page.locator('.jfss-fsitem-error')).toBeVisible();
+  await expect.soft(page.locator('.jfss-fsitem-name')).toBeVisible();
+  await expect.soft(page.locator('.jfss-fsitem-name')).toHaveText('myhdfs');
+  await expect
+    .soft(page.locator('.jfss-fsitem-protocol'))
+    .toHaveText('Path: myhdfs');
+});
+
 test('test memory filesystem with mock config data', async ({ page }) => {
   await page.goto();
   await page.getByText('FSSpec', { exact: true }).click();