Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

refactor: Remove not used blocking http client #1895

Merged
merged 3 commits into from
Apr 10, 2023
Merged

refactor: Remove not used blocking http client #1895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fc01463
refactor: Remove not used blocking http client
Xuanwo Apr 10, 2023
9d27e1e
Polish naming
Xuanwo Apr 10, 2023
047e290
Fix API
Xuanwo Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ reqsign = "0.8.5"
reqwest = { version = "0.11.13", features = [
"multipart",
"stream",
"blocking",
], default-features = false }
rocksdb = { version = "0.20.1", default-features = false, optional = true }
serde = { version = "1", features = ["derive"] }
Expand Down
51 changes: 0 additions & 51 deletions core/src/raw/http_util/body.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,57 +33,6 @@ use crate::Error;
use crate::ErrorKind;
use crate::Result;

/// Body used in blocking HTTP requests.
#[derive(Default)]
pub enum Body {
/// An empty body.
#[default]
Empty,
/// Body with bytes.
Bytes(Bytes),
}

impl Body {
/// Consume the entire body.
pub fn consume(self) -> Result<()> {
Ok(())
}
}

impl From<Body> for reqwest::blocking::Body {
fn from(v: Body) -> Self {
match v {
Body::Empty => reqwest::blocking::Body::from(""),
Body::Bytes(bs) => reqwest::blocking::Body::from(bs),
}
}
}

/// IncomingBody carries the content returned by remote servers.
///
/// # Notes
///
/// Client SHOULD NEVER construct this body.
pub struct IncomingBody {
/// # TODO
///
/// hyper returns `impl Stream<Item = crate::Result<Bytes>>` but we can't
/// write the types in stable. So we will box here.
///
/// After [TAIT](https://rust-lang.github.io/rfcs/2515-type_alias_impl_trait.html)
/// has been stable, we can change `IncomingAsyncBody` into `IncomingAsyncBody<S>`.
#[allow(unused)]
inner: reqwest::blocking::Response,
#[allow(unused)]
size: Option<u64>,
}

impl IncomingBody {
pub fn new(resp: reqwest::blocking::Response, size: Option<u64>) -> Self {
IncomingBody { inner: resp, size }
}
}

/// Body used in async HTTP requests.
#[derive(Default)]
pub enum AsyncBody {
Expand Down
137 changes: 11 additions & 126 deletions core/src/raw/http_util/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
use std::fmt::Debug;
use std::fmt::Formatter;
use std::str::FromStr;
use std::thread;

use futures::TryStreamExt;
use http::Request;
Expand All @@ -27,19 +26,16 @@ use reqwest::redirect::Policy;
use reqwest::Url;

use super::body::IncomingAsyncBody;
use super::body::IncomingBody;
use super::parse_content_length;
use super::AsyncBody;
use super::Body;
use crate::Error;
use crate::ErrorKind;
use crate::Result;

/// HttpClient that used across opendal.
#[derive(Clone)]
pub struct HttpClient {
async_client: reqwest::Client,
blocking_client: reqwest::blocking::Client,
client: reqwest::Client,
}

/// We don't want users to know details about our clients.
Expand All @@ -52,25 +48,11 @@ impl Debug for HttpClient {
impl HttpClient {
/// Create a new http client in async context.
pub fn new() -> Result<Self> {
Self::build(
reqwest::ClientBuilder::new(),
reqwest::blocking::ClientBuilder::new(),
)
Self::build(reqwest::ClientBuilder::new())
}

/// Build a new http client in async context.
pub fn build(
async_builder: reqwest::ClientBuilder,
blocking_builder: reqwest::blocking::ClientBuilder,
) -> Result<Self> {
Ok(HttpClient {
async_client: Self::build_async_client(async_builder)?,
blocking_client: Self::build_blocking_client(blocking_builder)?,
})
}

/// Build a new blocking client with given builder.
fn build_async_client(mut builder: reqwest::ClientBuilder) -> Result<reqwest::Client> {
pub fn build(mut builder: reqwest::ClientBuilder) -> Result<Self> {
// Make sure we don't enable auto gzip decompress.
builder = builder.no_gzip();
// Make sure we don't enable auto brotli decompress.
Expand All @@ -83,122 +65,25 @@ impl HttpClient {
#[cfg(feature = "trust-dns")]
let builder = builder.trust_dns(true);

builder.build().map_err(|err| {
Error::new(ErrorKind::Unexpected, "async client build failed").set_source(err)
Ok(Self {
client: builder.build().map_err(|err| {
Error::new(ErrorKind::Unexpected, "async client build failed").set_source(err)
})?,
})
}

/// Build a new blocking client with given builder.
///
/// # Notes
///
/// `reqwest::blocking::ClientBuilder::build` will panic if called
/// inside async context. So we need to spawn a thread to build it.
///
/// And users SHOULD never call blocking clients in async context.
fn build_blocking_client(
mut builder: reqwest::blocking::ClientBuilder,
) -> Result<reqwest::blocking::Client> {
// Make sure we don't enable auto gzip decompress.
builder = builder.no_gzip();
// Make sure we don't enable auto brotli decompress.
builder = builder.no_brotli();
// Make sure we don't enable auto deflate decompress.
builder = builder.no_deflate();
// Redirect will be handled by ourselves.
builder = builder.redirect(Policy::none());

#[cfg(feature = "trust-dns")]
let builder = builder.trust_dns(true);

thread::spawn(|| {
builder.build().map_err(|err| {
Error::new(ErrorKind::Unexpected, "blocking client build failed").set_source(err)
})
})
.join()
.expect("the thread of building blocking client join failed")
}

/// Get the async client from http client.
pub async fn async_client(&self) -> reqwest::Client {
self.async_client.clone()
}

/// Get the blocking client from http client.
pub fn blocking_client(&self) -> reqwest::blocking::Client {
self.blocking_client.clone()
}

/// Send a request in blocking way.
pub fn send(&self, req: Request<Body>) -> Result<Response<IncomingBody>> {
let is_head = req.method() == http::Method::HEAD;
let (parts, body) = req.into_parts();

let mut req_builder = self
.blocking_client
.request(
parts.method,
Url::from_str(&parts.uri.to_string()).expect("input request url must be valid"),
)
.version(parts.version)
.headers(parts.headers);

req_builder = req_builder.body(body);

let resp = req_builder.send().map_err(|err| {
let is_temporary = !(
// Builder related error should not be retried.
err.is_builder() ||
// Error returned by RedirectPolicy.
//
// We don't set this by hand, just don't allow retry.
err.is_redirect() ||
// We never use `Response::error_for_status`, just don't allow retry.
//
// Status should be checked by our services.
err.is_status()
);

let mut oerr = Error::new(ErrorKind::Unexpected, "send blocking request")
.with_operation("http_util::Client::send")
.set_source(err);
if is_temporary {
oerr = oerr.set_temporary();
}

oerr
})?;

// Get content length from header so that we can check it.
// If the request method is HEAD, we will ignore this.
let content_length = if is_head {
None
} else {
parse_content_length(resp.headers()).expect("response content length must be valid")
};

let mut hr = Response::builder()
.version(resp.version())
.status(resp.status());
for (k, v) in resp.headers().iter() {
hr = hr.header(k, v);
}

let body = IncomingBody::new(resp, content_length);

let resp = hr.body(body).expect("response must build succeed");

Ok(resp)
pub fn client(&self) -> reqwest::Client {
self.client.clone()
}

/// Send a request in async way.
pub async fn send_async(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
let is_head = req.method() == http::Method::HEAD;
let (parts, body) = req.into_parts();

let mut req_builder = self
.async_client
.client
.request(
parts.method,
Url::from_str(&parts.uri.to_string()).expect("input request url must be valid"),
Expand Down
1 change: 0 additions & 1 deletion core/src/raw/http_util/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ pub use client::HttpClient;

mod body;
pub use body::AsyncBody;
pub use body::Body;
pub use body::IncomingAsyncBody;

mod header;
Expand Down
11 changes: 1 addition & 10 deletions core/src/raw/rps.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,16 +247,7 @@ mod tests {
},
};

let req: Request<AsyncBody> = pr.clone().into();
assert_eq!(Method::PATCH, req.method());
assert_eq!(
"https://opendal.apache.org/path/to/file",
req.uri().to_string()
);
assert_eq!("123", req.headers().get(CONTENT_LENGTH).unwrap());
assert_eq!("application/json", req.headers().get(CONTENT_TYPE).unwrap());

let req: Request<Body> = pr.into();
let req: Request<AsyncBody> = pr.into();
assert_eq!(Method::PATCH, req.method());
assert_eq!(
"https://opendal.apache.org/path/to/file",
Expand Down
14 changes: 7 additions & 7 deletions core/src/services/azblob/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ impl Accessor for AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

let resp = self.client.send_async(req).await?;
let resp = self.client.send(req).await?;

let status = resp.status();

Expand Down Expand Up @@ -706,7 +706,7 @@ impl AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

pub fn azblob_put_blob_request(
Expand Down Expand Up @@ -761,7 +761,7 @@ impl AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

async fn azblob_delete_blob(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
Expand All @@ -782,7 +782,7 @@ impl AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

async fn azblob_copy_blob(&self, from: &str, to: &str) -> Result<Response<IncomingAsyncBody>> {
Expand All @@ -809,7 +809,7 @@ impl AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

pub(crate) async fn azblob_list_blobs(
Expand Down Expand Up @@ -845,7 +845,7 @@ impl AzblobBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

async fn azblob_batch_delete(&self, paths: &[String]) -> Result<Response<IncomingAsyncBody>> {
Expand Down Expand Up @@ -881,7 +881,7 @@ impl AzblobBackend {
let mut req = batch_delete_req_builder.try_into_req()?;
self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/services/azblob/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ impl oio::Write for AzblobWriter {
.sign(&mut req)
.map_err(new_request_sign_error)?;

let resp = self.backend.client.send_async(req).await?;
let resp = self.backend.client.send(req).await?;

let status = resp.status();

Expand Down
10 changes: 5 additions & 5 deletions core/src/services/azdfs/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ impl Accessor for AzdfsBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

let resp = self.client.send_async(req).await?;
let resp = self.client.send(req).await?;

let status = resp.status();

Expand Down Expand Up @@ -462,7 +462,7 @@ impl AzdfsBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

/// resource should be one of `file` or `directory`
Expand Down Expand Up @@ -556,7 +556,7 @@ impl AzdfsBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

async fn azdfs_delete(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
Expand All @@ -579,7 +579,7 @@ impl AzdfsBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}

pub(crate) async fn azdfs_list(
Expand Down Expand Up @@ -613,7 +613,7 @@ impl AzdfsBackend {

self.signer.sign(&mut req).map_err(new_request_sign_error)?;

self.client.send_async(req).await
self.client.send(req).await
}
}

Expand Down
Loading