Allow using default fs for fsspec and pass storage options to client

python/hdfs_native/fsspec.py

@@ -19,14 +19,16 @@
 class HdfsFileSystem(AbstractFileSystem):
     root_marker = "/"
 
-    def __init__(self, host: str, port: Optional[int] = None, *args, **storage_options):
+    def __init__(self, host: Optional[str] = None, port: Optional[int] = None, *args, **storage_options):
         super().__init__(host, port, *args, **storage_options)
         self.host = host
         self.port = port
-        url = f"{self.protocol}://{host}"
-        if port:
-            url += f":{port}"
-        self.client = Client(url)
+        url = f"{self.protocol}://"
+        if host:
+            url += host
+            if port:
+                url += f":{port}"
+        self.client = Client(url, storage_options)
 
     @property
     def fsid(self):
@@ -40,9 +42,11 @@ def _strip_protocol(cls, path: str) -> str:
     def unstrip_protocol(self, name: str) -> str:
         path = self._strip_protocol(name)
 
-        url = f"{self.protocol}://{self.host}"
-        if self.port:
-            url += f":{self.port}"
+        url = f"{self.protocol}://"
+        if self.host:
+            url += self.host
+            if self.port:
+                url += f":{self.port}"
 
         return f"{url}{path}"
 

python/tests/test_fsspec.py

@@ -6,6 +6,11 @@
 from hdfs_native.fsspec import HdfsFileSystem
 
 
+def test_config(minidfs: str):
+    url = urllib.parse.urlparse(minidfs)
+    fs: HdfsFileSystem = fsspec.filesystem(url.scheme, **{'fs.defaultFS': minidfs})
+    assert len(fs.ls("/")) == 0
+
 def test_dirs(fs: HdfsFileSystem):
     fs.mkdir("/testdir")
     assert fs.info("/testdir")["type"] == "directory"

rust/src/hdfs/proxy.rs

@@ -103,12 +103,18 @@ impl NameServiceProxy {
             todo!()
         };
 
-        Ok(NameServiceProxy {
-            proxy_connections,
-            current_active: AtomicUsize::new(0),
-            current_observers: Arc::new(Mutex::new(HashSet::new())),
-            msycned: AtomicBool::new(false),
-        })
+        if proxy_connections.is_empty() {
+            Err(HdfsError::InvalidArgument(
+                "No NameNode hosts found".to_string(),
+            ))
+        } else {
+            Ok(NameServiceProxy {
+                proxy_connections,
+                current_active: AtomicUsize::new(0),
+                current_observers: Arc::new(Mutex::new(HashSet::new())),
+                msycned: AtomicBool::new(false),
+            })
+        }
     }
 
     async fn msync_if_needed(&self, write: bool) -> Result<()> {