feat(services-http): Implement write and delete for testing (#451)

* Implement write and delete for testing

Signed-off-by: Xuanwo <github@xuanwo.io>

* Add docs for http services

Signed-off-by: Xuanwo <github@xuanwo.io>

* Save work

Signed-off-by: Xuanwo <github@xuanwo.io>

* Implement http test

Signed-off-by: Xuanwo <github@xuanwo.io>

* Add CI for services test

Signed-off-by: Xuanwo <github@xuanwo.io>

* Allow use https as scheme

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix CI

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix CI

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix clippy

Signed-off-by: Xuanwo <github@xuanwo.io>

.github/workflows/service_test_http.yml

@@ -0,0 +1,46 @@
+name: Service Test HTTP
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  local_fs:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-11
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Start oay in background
+        shell: bash
+        working-directory: ./oay
+        run: |
+          cargo build
+          cargo run -- http &
+        env:
+          RUST_BACKTRACE: full
+          RUST_LOG: debug
+          OAY_BACKEND_FS_ROOT: /tmp/oay/
+
+      - name: Test
+        shell: bash
+        run: cargo test http --features compress,retry,services-http -- --nocapture
+        env:
+          RUST_BACKTRACE: full
+          RUST_LOG: debug
+          OPENDAL_HTTP_TEST: on
+          OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080

oay/src/services/http/mod.rs

@@ -33,8 +33,8 @@ use anyhow::anyhow;
 use futures::stream;
 use futures::AsyncWriteExt;
 use futures::StreamExt;
-use log::error;
 use log::warn;
+use log::{debug, error};
 use opendal::io_util::into_stream;
 use opendal::ops::BytesRange;
 use opendal::BytesReader;
@@ -75,22 +75,28 @@ impl Service {
 
         let meta = o.metadata().await?;
 
-        let r = if let Some(range) = req.headers().get(header::RANGE) {
+        let (size, r) = if let Some(range) = req.headers().get(header::RANGE) {
             let br = BytesRange::from_header_range(range.to_str().map_err(|e| {
                 Error::new(
                     ErrorKind::InvalidInput,
                     anyhow!("header range is invalid: {e}"),
                 )
             })?)?;
-            Box::new(o.range_reader(br.to_range(meta.content_length())).await?) as BytesReader
+
+            let range = br.to_range(meta.content_length());
+
+            (
+                range.size_hint().0 as u64,
+                Box::new(o.range_reader(range).await?) as BytesReader,
+            )
         } else {
-            Box::new(o.reader().await?) as BytesReader
+            (
+                meta.content_length(),
+                Box::new(o.reader().await?) as BytesReader,
+            )
         };
 
-        Ok(HttpResponse::Ok().body(SizedStream::new(
-            meta.content_length(),
-            into_stream(r, 8 * 1024),
-        )))
+        Ok(HttpResponse::Ok().body(SizedStream::new(size, into_stream(r, 8 * 1024))))
     }
 
     async fn put(&self, req: HttpRequest, body: web::Payload) -> Result<HttpResponse> {

src/operator.rs

@@ -129,23 +129,10 @@ impl Operator {
             Scheme::Fs => services::fs::Backend::from_iter(it).await?,
             #[cfg(feature = "services-hdfs")]
             Scheme::Hdfs => services::hdfs::Backend::from_iter(it).await?,
-            Scheme::S3 => services::s3::Backend::from_iter(it).await?,
-            Scheme::Memory => services::memory::Backend::build().finish().await?,
             #[cfg(feature = "services-http")]
-            Scheme::Http => {
-                use std::collections::HashMap;
-                use std::io::Error;
-
-                use crate::error::BackendError;
-
-                return Err(Error::new(
-                    ErrorKind::Unsupported,
-                    BackendError::new(
-                        it.collect::<HashMap<_, _>>(),
-                        anyhow::anyhow!("backend http doesn't support init from iter"),
-                    ),
-                ));
-            }
+            Scheme::Http => services::http::Backend::from_iter(it).await?,
+            Scheme::Memory => services::memory::Backend::build().finish().await?,
+            Scheme::S3 => services::s3::Backend::from_iter(it).await?,
         };
 
         Ok(Self { accessor })

src/ops.rs

@@ -430,8 +430,7 @@ impl BytesRange {
     /// Range: <unit>=<range-start>-<range-end>
     /// Range: <unit>=-<suffix-length>
     pub fn from_header_range(s: &str) -> Result<Self> {
-        let s = s.to_lowercase();
-        let s = s.strip_prefix("range: bytes=").ok_or_else(|| {
+        let s = s.strip_prefix("bytes=").ok_or_else(|| {
             Error::new(
                 ErrorKind::InvalidInput,
                 anyhow!("header range is invalid: {s}"),
@@ -486,8 +485,7 @@ impl BytesRange {
     /// Content-Range: <unit> <range-start>-<range-end>/*
     /// Content-Range: <unit> */<size>
     pub fn from_header_content_range(s: &str) -> Result<Self> {
-        let s = s.to_lowercase();
-        let s = s.strip_prefix("content-range: bytes ").ok_or_else(|| {
+        let s = s.strip_prefix("bytes ").ok_or_else(|| {
             Error::new(
                 ErrorKind::InvalidInput,
                 anyhow!("header content range is invalid: {s}"),
@@ -608,27 +606,19 @@ mod tests {
         let cases = vec![
             (
                 "range-start",
-                "Range: bytes=123-",
+                "bytes=123-",
                 BytesRange::new(Some(123), None),
             ),
-            (
-                "suffix",
-                "Range: bytes=-123",
-                BytesRange::new(None, Some(124)),
-            ),
+            ("suffix", "bytes=-123", BytesRange::new(None, Some(124))),
             (
                 "range",
-                "Range: bytes=123-124",
+                "bytes=123-124",
                 BytesRange::new(Some(123), Some(2)),
             ),
-            (
-                "one byte",
-                "Range: bytes=0-0",
-                BytesRange::new(Some(0), Some(1)),
-            ),
+            ("one byte", "bytes=0-0", BytesRange::new(Some(0), Some(1))),
             (
                 "lower case header",
-                "range: bytes=0-0",
+                "bytes=0-0",
                 BytesRange::new(Some(0), Some(1)),
             ),
         ];
@@ -647,17 +637,17 @@ mod tests {
         let cases = vec![
             (
                 "range start with unknown size",
-                "Content-Range: bytes 123-123/*",
+                "bytes 123-123/*",
                 BytesRange::new(Some(123), Some(1)),
             ),
             (
                 "range start with known size",
-                "Content-Range: bytes 123-123/1",
+                "bytes 123-123/1",
                 BytesRange::new(Some(123), Some(1)),
             ),
             (
                 "only have size",
-                "Content-Range: bytes */1024",
+                "bytes */1024",
                 BytesRange::new(Some(0), Some(1024)),
             ),
         ];

src/scheme.rs

@@ -31,7 +31,7 @@ pub enum Scheme {
     /// [hdfs][crate::services::hdfs]: Hadoop Distributed File System.
     #[cfg(feature = "services-hdfs")]
     Hdfs,
-    /// [http][crate::services::http]: HTTP read-only backend.
+    /// [http][crate::services::http]: HTTP backend.
     #[cfg(feature = "services-http")]
     Http,
     /// [memory][crate::services::memory]: In memory backend support.
@@ -71,6 +71,8 @@ impl FromStr for Scheme {
             "fs" => Ok(Scheme::Fs),
             #[cfg(feature = "services-hdfs")]
             "hdfs" => Ok(Scheme::Hdfs),
+            #[cfg(feature = "services-http")]
+            "http" | "https" => Ok(Scheme::Http),
             "memory" => Ok(Scheme::Memory),
             "s3" => Ok(Scheme::S3),
             v => Err(other(BackendError::new(