From cb99fa96130fd2fa2b377dbdc183dd71a1ea2cf0 Mon Sep 17 00:00:00 2001
From: Adam Binford <adamq43@gmail.com>
Date: Thu, 7 Nov 2024 16:05:48 +0000
Subject: [PATCH] Allow using default fs for fsspec and pass storage options to
 client

---
 python/hdfs_native/fsspec.py | 20 ++++++++++++--------
 python/tests/test_fsspec.py  |  5 +++++
 rust/src/hdfs/proxy.rs       | 18 ++++++++++++------
 3 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/python/hdfs_native/fsspec.py b/python/hdfs_native/fsspec.py
index 4970005..308020e 100644
--- a/python/hdfs_native/fsspec.py
+++ b/python/hdfs_native/fsspec.py
@@ -19,14 +19,16 @@
 class HdfsFileSystem(AbstractFileSystem):
     root_marker = "/"
 
-    def __init__(self, host: str, port: Optional[int] = None, *args, **storage_options):
+    def __init__(self, host: Optional[str] = None, port: Optional[int] = None, *args, **storage_options):
         super().__init__(host, port, *args, **storage_options)
         self.host = host
         self.port = port
-        url = f"{self.protocol}://{host}"
-        if port:
-            url += f":{port}"
-        self.client = Client(url)
+        url = f"{self.protocol}://"
+        if host:
+            url += host
+            if port:
+                url += f":{port}"
+        self.client = Client(url, storage_options)
 
     @property
     def fsid(self):
@@ -40,9 +42,11 @@ def _strip_protocol(cls, path: str) -> str:
     def unstrip_protocol(self, name: str) -> str:
         path = self._strip_protocol(name)
 
-        url = f"{self.protocol}://{self.host}"
-        if self.port:
-            url += f":{self.port}"
+        url = f"{self.protocol}://"
+        if self.host:
+            url += self.host
+            if self.port:
+                url += f":{self.port}"
 
         return f"{url}{path}"
 
diff --git a/python/tests/test_fsspec.py b/python/tests/test_fsspec.py
index d4d4e02..1be1c7d 100644
--- a/python/tests/test_fsspec.py
+++ b/python/tests/test_fsspec.py
@@ -6,6 +6,11 @@
 from hdfs_native.fsspec import HdfsFileSystem
 
 
+def test_config(minidfs: str):
+    url = urllib.parse.urlparse(minidfs)
+    fs: HdfsFileSystem = fsspec.filesystem(url.scheme, **{'fs.defaultFS': minidfs})
+    assert len(fs.ls("/")) == 0
+
 def test_dirs(fs: HdfsFileSystem):
     fs.mkdir("/testdir")
     assert fs.info("/testdir")["type"] == "directory"
diff --git a/rust/src/hdfs/proxy.rs b/rust/src/hdfs/proxy.rs
index 3875ab3..2b874b6 100644
--- a/rust/src/hdfs/proxy.rs
+++ b/rust/src/hdfs/proxy.rs
@@ -103,12 +103,18 @@ impl NameServiceProxy {
             todo!()
         };
 
-        Ok(NameServiceProxy {
-            proxy_connections,
-            current_active: AtomicUsize::new(0),
-            current_observers: Arc::new(Mutex::new(HashSet::new())),
-            msycned: AtomicBool::new(false),
-        })
+        if proxy_connections.is_empty() {
+            Err(HdfsError::InvalidArgument(
+                "No NameNode hosts found".to_string(),
+            ))
+        } else {
+            Ok(NameServiceProxy {
+                proxy_connections,
+                current_active: AtomicUsize::new(0),
+                current_observers: Arc::new(Mutex::new(HashSet::new())),
+                msycned: AtomicBool::new(false),
+            })
+        }
     }
 
     async fn msync_if_needed(&self, write: bool) -> Result<()> {