From da2581379f00bd9c06acd4bf8fb4d56b445350a8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 26 Aug 2024 23:26:19 +0800 Subject: [PATCH 1/7] fix(core): Fix failed list related tests Signed-off-by: Xuanwo --- core/src/raw/oio/entry.rs | 7 ++++++- core/src/services/azdls/lister.rs | 9 ++++++++- core/src/services/azfile/lister.rs | 7 +++++++ core/src/services/b2/lister.rs | 10 +++++++--- core/src/services/cos/lister.rs | 7 +++---- core/src/services/gdrive/lister.rs | 7 +++++++ core/src/services/sqlite/backend.rs | 5 ++--- 7 files changed, 40 insertions(+), 12 deletions(-) diff --git a/core/src/raw/oio/entry.rs b/core/src/raw/oio/entry.rs index 43c4b96311a4..70320300b065 100644 --- a/core/src/raw/oio/entry.rs +++ b/core/src/raw/oio/entry.rs @@ -38,7 +38,12 @@ impl Entry { } /// Create a new entry with given value. - pub fn with(path: String, meta: Metadata) -> Entry { + pub fn with(mut path: String, meta: Metadata) -> Entry { + // Normalize path as `/` if it's empty. + if path.is_empty() { + path = "/".to_string(); + } + debug_assert!( meta.mode().is_dir() == path.ends_with('/'), "mode {:?} not match with path {}", diff --git a/core/src/services/azdls/lister.rs b/core/src/services/azdls/lister.rs index 27d51e266765..a14597961c41 100644 --- a/core/src/services/azdls/lister.rs +++ b/core/src/services/azdls/lister.rs @@ -56,6 +56,13 @@ impl oio::PageList for AzdlsLister { return Err(parse_error(resp).await?); } + // Return self at the first page. + if ctx.token.is_empty() && !ctx.done { + let path = build_rel_path(&self.core.root, &self.path); + let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + ctx.entries.push_back(e); + } + // Check whether this list is done. if let Some(value) = resp.headers().get("x-ms-continuation") { let value = value.to_str().map_err(|err| { @@ -90,7 +97,7 @@ impl oio::PageList for AzdlsLister { .with_last_modified(parse_datetime_from_rfc2822(&object.last_modified)?); let mut path = build_rel_path(&self.core.root, &object.name); - if mode == EntryMode::DIR { + if path.is_empty() { path += "/" }; diff --git a/core/src/services/azfile/lister.rs b/core/src/services/azfile/lister.rs index 913af07d838f..4378c4328df9 100644 --- a/core/src/services/azfile/lister.rs +++ b/core/src/services/azfile/lister.rs @@ -56,6 +56,13 @@ impl oio::PageList for AzfileLister { return Err(parse_error(resp).await?); } + // Return self at the first page. + if ctx.token.is_empty() && !ctx.done { + let path = build_rel_path(&self.core.root, &self.path); + let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + ctx.entries.push_back(e); + } + let bs = resp.into_body(); let results: EnumerationResults = diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 9b27b121c18b..e1a0c909e522 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -79,6 +79,13 @@ impl oio::PageList for B2Lister { return Err(parse_error(resp).await?); } + // Return self at the first page. + if ctx.token.is_empty() && !ctx.done { + let path = build_rel_path(&self.core.root, &self.path); + let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + ctx.entries.push_back(e); + } + let bs = resp.into_body(); let output: ListFileNamesResponse = @@ -96,9 +103,6 @@ impl oio::PageList for B2Lister { continue; } } - if file.file_name == build_abs_path(&self.core.root, &self.path) { - continue; - } let file_name = file.file_name.clone(); let metadata = parse_file_info(&file); diff --git a/core/src/services/cos/lister.rs b/core/src/services/cos/lister.rs index 7022aa9def0b..b5b66596faba 100644 --- a/core/src/services/cos/lister.rs +++ b/core/src/services/cos/lister.rs @@ -81,10 +81,9 @@ impl oio::PageList for CosLister { } for object in output.contents { - let path = build_rel_path(&self.core.root, &object.key); - - if path == self.path || path.is_empty() { - continue; + let mut path = build_rel_path(&self.core.root, &object.key); + if path.is_empty() { + path = "/".to_string(); } let meta = Metadata::new(EntryMode::from_path(&path)).with_content_length(object.size); diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs index e77a5022da7a..108df8b42ade 100644 --- a/core/src/services/gdrive/lister.rs +++ b/core/src/services/gdrive/lister.rs @@ -64,6 +64,13 @@ impl oio::PageList for GdriveLister { return Ok(()); } + // Return self at the first page. + if ctx.token.is_empty() && !ctx.done { + let path = build_rel_path(&self.core.root, &self.path); + let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + ctx.entries.push_back(e); + } + let decoded_response = serde_json::from_slice::(&bytes).map_err(new_json_deserialize_error)?; diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index a8f182a901e0..f353292ad7aa 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -251,11 +251,10 @@ impl kv::Adapter for Adapter { let pool = self.get_client().await?; let value = sqlx::query_scalar(&format!( - "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1 and `{}` <> $2", - self.key_field, self.table, self.key_field, self.key_field + "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1", + self.key_field, self.table, self.key_field )) .bind(format!("{path}%")) - .bind(path) .fetch_all(pool) .await .map_err(parse_sqlite_error)?; From a18440f97544007c2c5e74d20edc435b0fec25f6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 26 Aug 2024 23:42:50 +0800 Subject: [PATCH 2/7] Fix build Signed-off-by: Xuanwo --- core/src/services/azdls/lister.rs | 3 +-- core/src/services/azfile/lister.rs | 3 +-- core/src/services/b2/lister.rs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/core/src/services/azdls/lister.rs b/core/src/services/azdls/lister.rs index a14597961c41..6830f99ee264 100644 --- a/core/src/services/azdls/lister.rs +++ b/core/src/services/azdls/lister.rs @@ -58,8 +58,7 @@ impl oio::PageList for AzdlsLister { // Return self at the first page. if ctx.token.is_empty() && !ctx.done { - let path = build_rel_path(&self.core.root, &self.path); - let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); ctx.entries.push_back(e); } diff --git a/core/src/services/azfile/lister.rs b/core/src/services/azfile/lister.rs index 4378c4328df9..127072a561c0 100644 --- a/core/src/services/azfile/lister.rs +++ b/core/src/services/azfile/lister.rs @@ -58,8 +58,7 @@ impl oio::PageList for AzfileLister { // Return self at the first page. if ctx.token.is_empty() && !ctx.done { - let path = build_rel_path(&self.core.root, &self.path); - let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); ctx.entries.push_back(e); } diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index e1a0c909e522..8226b5974245 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -81,8 +81,7 @@ impl oio::PageList for B2Lister { // Return self at the first page. if ctx.token.is_empty() && !ctx.done { - let path = build_rel_path(&self.core.root, &self.path); - let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR)); + let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); ctx.entries.push_back(e); } From cd7f68ec0067ccbb32d50a84a0a785f244ec8834 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 26 Aug 2024 23:49:23 +0800 Subject: [PATCH 3/7] Fix azdls Signed-off-by: Xuanwo --- core/src/services/azdls/lister.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/azdls/lister.rs b/core/src/services/azdls/lister.rs index 6830f99ee264..7fd8c2cdc75c 100644 --- a/core/src/services/azdls/lister.rs +++ b/core/src/services/azdls/lister.rs @@ -96,7 +96,7 @@ impl oio::PageList for AzdlsLister { .with_last_modified(parse_datetime_from_rfc2822(&object.last_modified)?); let mut path = build_rel_path(&self.core.root, &object.name); - if path.is_empty() { + if mode.is_dir() { path += "/" }; From f3a8e75c5d5eea9d16d002741e56acf88902d658 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 26 Aug 2024 23:51:41 +0800 Subject: [PATCH 4/7] Fix b2 Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 8226b5974245..7086bc9fafac 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -80,7 +80,7 @@ impl oio::PageList for B2Lister { } // Return self at the first page. - if ctx.token.is_empty() && !ctx.done { + if ctx.token.is_empty() && !ctx.done && self.path.ends_with('/') { let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); ctx.entries.push_back(e); } From 0c12d406df13b6187e9dbe190bb761c303e9200f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Aug 2024 00:38:25 +0800 Subject: [PATCH 5/7] Fix b2 Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 7086bc9fafac..7b78edb76ebb 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -79,12 +79,6 @@ impl oio::PageList for B2Lister { return Err(parse_error(resp).await?); } - // Return self at the first page. - if ctx.token.is_empty() && !ctx.done && self.path.ends_with('/') { - let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); - ctx.entries.push_back(e); - } - let bs = resp.into_body(); let output: ListFileNamesResponse = From 38d67ed25ea4fcf830e153ae6f437b659ce98e57 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Aug 2024 00:58:27 +0800 Subject: [PATCH 6/7] Fix aliyun drive Signed-off-by: Xuanwo --- core/src/services/aliyun_drive/lister.rs | 39 +++++++++--------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/core/src/services/aliyun_drive/lister.rs b/core/src/services/aliyun_drive/lister.rs index a94db21bcd68..d5de627d111b 100644 --- a/core/src/services/aliyun_drive/lister.rs +++ b/core/src/services/aliyun_drive/lister.rs @@ -65,26 +65,19 @@ impl oio::PageList for AliyunDriveLister { }; let offset = if ctx.token.is_empty() { - if !parent.path.ends_with('/') { - // List "dir" should contains "dir/". - let path = if !parent.path.starts_with('/') { - format!("/{}", parent.path) - } else { - parent.path.clone() - }; - ctx.entries.push_back(Entry::new( - &format!("{}/", path), - Metadata::new(EntryMode::DIR).with_last_modified( - parent - .updated_at - .parse::>() - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "parse last modified time") - .set_source(e) - })?, - ), - )); - } + // Push self into the list result. + ctx.entries.push_back(Entry::new( + &parent.path, + Metadata::new(EntryMode::DIR).with_last_modified( + parent + .updated_at + .parse::>() + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "parse last modified time") + .set_source(e) + })?, + ), + )); None } else { Some(ctx.token.clone()) @@ -110,11 +103,7 @@ impl oio::PageList for AliyunDriveLister { let n = result.items.len(); for item in result.items { - let path = if parent.path.starts_with('/') { - build_abs_path(&parent.path, &item.name) - } else { - build_abs_path(&format!("/{}", &parent.path), &item.name) - }; + let path = build_abs_path(&parent.path, &item.name); let (path, md) = if item.path_type == "folder" { let path = format!("{}/", path); From d51df2e347ffe87d6a17bbb4d2cf7e08e0368899 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Aug 2024 01:22:52 +0800 Subject: [PATCH 7/7] Fix aliyun drive Signed-off-by: Xuanwo --- core/src/services/aliyun_drive/lister.rs | 56 ++++++++---------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/core/src/services/aliyun_drive/lister.rs b/core/src/services/aliyun_drive/lister.rs index d5de627d111b..4a28e1460023 100644 --- a/core/src/services/aliyun_drive/lister.rs +++ b/core/src/services/aliyun_drive/lister.rs @@ -100,53 +100,35 @@ impl oio::PageList for AliyunDriveLister { let result: AliyunDriveFileList = serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?; - let n = result.items.len(); - for item in result.items { - let path = build_abs_path(&parent.path, &item.name); - - let (path, md) = if item.path_type == "folder" { - let path = format!("{}/", path); - let meta = Metadata::new(EntryMode::DIR).with_last_modified( - item.updated_at - .parse::>() - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "parse last modified time") - .set_source(e) - })?, - ); - (path, meta) + let (path, mut md) = if item.path_type == "folder" { + let path = format!("{}{}/", &parent.path.trim_start_matches('/'), &item.name); + (path, Metadata::new(EntryMode::DIR)) } else { - let mut meta = Metadata::new(EntryMode::FILE).with_last_modified( - item.updated_at - .parse::>() - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "parse last modified time") - .set_source(e) - })?, - ); - if let Some(v) = item.size { - meta = meta.with_content_length(v); - } - if let Some(v) = item.content_type { - meta = meta.with_content_type(v); - } - (path, meta) + let path = format!("{}{}", &parent.path.trim_start_matches('/'), &item.name); + (path, Metadata::new(EntryMode::FILE)) }; + md = md.with_last_modified(item.updated_at.parse::>().map_err( + |e| Error::new(ErrorKind::Unexpected, "parse last modified time").set_source(e), + )?); + if let Some(v) = item.size { + md = md.with_content_length(v); + } + if let Some(v) = item.content_type { + md = md.with_content_type(v); + } + ctx.entries.push_back(Entry::new(&path, md)); } - if self.limit.is_some_and(|x| x < n) || result.next_marker.is_none() { + let next_marker = result.next_marker.unwrap_or_default(); + if next_marker.is_empty() { ctx.done = true; + } else { + ctx.token = next_marker; } - if let Some(marker) = result.next_marker { - if marker.is_empty() { - ctx.done = true; - } - ctx.token = marker; - } Ok(()) } }