diff --git a/README.md b/README.md index 45ae423..d4a2b0e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Oxylabs Go SDK -This is a Go SDK for the [Oxylabs](https://oxylabs.io) [Scraper APIs](https://developers.oxylabs.io/scraper-apis/getting-started). +This is a Go SDK for the [Oxylabs](https://oxylabs.io) [Scraper APIs](https://developers.oxylabs.io/scraper-apis/serp-scraper-api#getting-started). This will help simplify integrating with Oxylabs's APIs, which can help you with retrieving search engine results (SERP), eCommerce data, real estate data, and more. @@ -86,7 +86,7 @@ There are three integration method for the Oxylabs SERP API, each exposed via di - Push-Pull (Async) - `serp.InitAsync(username, password)` - Proxy Endpoint - `proxy.Init(username, password)` -Learn more about integration methods [on the official documentation](https://developers.oxylabs.io/scraper-apis/getting-started/integration-methods) and how this SDK uses them [here](#integration-methods-1). +Learn more about integration methods [on the official documentation](https://developers.oxylabs.io/scraper-apis/serp-scraper-api/integration-methods) and how this SDK uses them [here](#integration-methods-1). ### Sources @@ -97,28 +97,26 @@ There are currently four search engines you can scrape with the Oxylabs SERP API | Search Engine | Sources | ------------- | -------------- | **Google** | `google`, `google_search`, `google_ads`, `google_hotels`, `google_travel_hotels`, `google_images`, `google_suggest`, `google_trends_explore` -| **Yandex** | `yandex`, `yandex_search` | **Bing** | `bing`, `bing_search` -| **Baidu** | `baidu`, `baidu_search` In the SDK you'll just need to call the relevant function name from the client. -For example if you wish to scrape Yandex with `yandex_search` as a source: +For example if you wish to scrape Google with `google_search` as a source: ```go -res, err := c.ScrapeYandexSearch("football") +res, err := c.ScrapeGoogleSearch("football") ``` ### Query Parameters -Each source has different accepted query parameters. For a detailed list of accepted parameters by each source you can head over to https://developers.oxylabs.io/scraper-apis/serp-scraper-api. +Each source has different accepted query parameters. For a detailed list of accepted parameters by each source you can head over to https://developers.oxylabs.io/scraper-apis/serp-scraper-api#request-parameter-values. By default, scrape functions will use default parameters. If you need to send specific query parameters, here is an example of how to do it: ```go -res, err := c.ScrapeYandexSearch( +res, err := c.ScrapeGoogleSearch( "football", - &serp.YandexSearchOpts{ + &serp.GoogleSearchOpts{ StartPage: 1, Pages: 3, Limit: 4, @@ -280,7 +278,7 @@ func main() { This method is also synchronous (like Realtime), but instead of using our service via a RESTful interface, you **can use our endpoint like a proxy**. Use Proxy Endpoint if you've used proxies before and would just like to get unblocked content from us. Since the parameters in this method are sent as as headers there are only a few parameters which this integration method accepts. You can find those parameters at -https://developers.oxylabs.io/scraper-apis/getting-started/integration-methods/proxy-endpoint#accepted-parameters. +https://developers.oxylabs.io/scraper-apis/serp-scraper-api/integration-methods/proxy-endpoint#accepted-parameters. The proxy endpoint integration is very open ended allowing many different use cases. To cater this, the user is provided a pre-configured `http.Client` and they can use it as they deem fit: diff --git a/oxylabs/types.go b/oxylabs/types.go index 0662772..d081667 100644 --- a/oxylabs/types.go +++ b/oxylabs/types.go @@ -71,12 +71,6 @@ const ( BingUrl Source = "bing" BingSearch Source = "bing_search" - YandexUrl Source = "yandex" - YandexSearch Source = "yandex_search" - - BaiduUrl Source = "baidu" - BaiduSearch Source = "baidu_search" - GoogleShoppingUrl Source = "google_shopping" GoogleShoppingSearch Source = "google_shopping_search" GoogleShoppingProduct Source = "google_shopping_product" diff --git a/serp/baidu.go b/serp/baidu.go deleted file mode 100644 index 2f0cb18..0000000 --- a/serp/baidu.go +++ /dev/null @@ -1,229 +0,0 @@ -package serp - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/oxylabs/oxylabs-sdk-go/internal" - "github.com/oxylabs/oxylabs-sdk-go/oxylabs" -) - -// Accepted parameters for baidu. -var BaiduSearchAcceptedDomainParameters = []oxylabs.Domain{ - oxylabs.DOMAIN_COM, - oxylabs.DOMAIN_CN, -} - -// checkParameterValidity checks validity of ScrapeBaiduSearch parameters. -func (opt *BaiduSearchOpts) checkParameterValidity() error { - if !internal.InList(opt.Domain, BaiduSearchAcceptedDomainParameters) { - return fmt.Errorf("invalid domain parameter: %s", opt.Domain) - } - - if !oxylabs.IsUserAgentValid(opt.UserAgent) { - return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent) - } - - if opt.Limit <= 0 || opt.Pages <= 0 || opt.StartPage <= 0 { - return fmt.Errorf("limit, pages and start_page parameters must be greater than 0") - } - - if opt.ParseInstructions != nil { - if err := oxylabs.ValidateParseInstructions(opt.ParseInstructions); err != nil { - return fmt.Errorf("invalid parse instructions: %w", err) - } - } - - return nil -} - -// checkParameterValidity checks validity of ScrapeBaiduUrl parameters. -func (opt *BaiduUrlOpts) checkParameterValidity() error { - if !oxylabs.IsUserAgentValid(opt.UserAgent) { - return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent) - } - - if opt.ParseInstructions != nil { - if err := oxylabs.ValidateParseInstructions(opt.ParseInstructions); err != nil { - return fmt.Errorf("invalid parse instructions: %w", err) - } - } - - return nil -} - -// BaiduSearchOpts contains all the query parameters available for baidu_search. -type BaiduSearchOpts struct { - Domain oxylabs.Domain - StartPage int - Pages int - Limit int - UserAgent oxylabs.UserAgent - CallbackUrl string - ParseInstructions *map[string]interface{} - PollInterval time.Duration -} - -// ScrapeBaiduSearch scrapes baidu via Oxylabs SERP API with baidu_search as source. -func (c *SerpClient) ScrapeBaiduSearch( - query string, - opts ...*BaiduSearchOpts, -) (*Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeBaiduSearchCtx(ctx, query, opts...) -} - -// ScrapeBaiduSearchCtx scrapes baidu via Oxylabs SERP API with baidu_search as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClient) ScrapeBaiduSearchCtx( - ctx context.Context, - query string, - opts ...*BaiduSearchOpts, -) (*Resp, error) { - // Prepare options. - opt := &BaiduSearchOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultDomain(&opt.Domain) - internal.SetDefaultStartPage(&opt.StartPage) - internal.SetDefaultLimit(&opt.Limit, internal.DefaultLimit_SERP) - internal.SetDefaultPages(&opt.Pages) - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err := opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.BaiduSearch, - "domain": opt.Domain, - "query": query, - "start_page": opt.StartPage, - "pages": opt.Pages, - "limit": opt.Limit, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Req. - httpResp, err := c.C.Req(ctx, jsonPayload, "POST") - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - return resp, nil -} - -// BaiduUrlOpts contains all the query parameters available for baidu. -type BaiduUrlOpts struct { - UserAgent oxylabs.UserAgent - CallbackUrl string - ParseInstructions *map[string]interface{} - PollInterval time.Duration -} - -// ScrapeBaiduUrl scrapes baidu via Oxylabs SERP API with baidu as source. -func (c *SerpClient) ScrapeBaiduUrl( - url string, - opts ...*BaiduUrlOpts, -) (*Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeBaiduUrlCtx(ctx, url, opts...) -} - -// ScrapeBaiduUrlCtx scrapes baidu via Oxylabs SERP API with baidu as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClient) ScrapeBaiduUrlCtx( - ctx context.Context, - url string, - opts ...*BaiduUrlOpts, -) (*Resp, error) { - // Check validity of URL. - err := internal.ValidateUrl(url, "baidu") - if err != nil { - return nil, err - } - - // Prepare options. - opt := &BaiduUrlOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err = opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.BaiduUrl, - "url": url, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Req. - httpResp, err := c.C.Req(ctx, jsonPayload, "POST") - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - return resp, nil -} diff --git a/serp/baidu_async.go b/serp/baidu_async.go deleted file mode 100644 index 6c62647..0000000 --- a/serp/baidu_async.go +++ /dev/null @@ -1,221 +0,0 @@ -package serp - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - - "github.com/oxylabs/oxylabs-sdk-go/internal" - "github.com/oxylabs/oxylabs-sdk-go/oxylabs" -) - -// ScrapeBaiduSearch scrapes baidu with async polling runtime via Oxylabs SERP API -// and baidu_search as source. -func (c *SerpClientAsync) ScrapeBaiduSearch( - query string, - opts ...*BaiduSearchOpts, -) (chan *Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeBaiduSearchCtx(ctx, query, opts...) -} - -// ScrapeBaiduSearchCtx scrapes baidu with async polling runtime via Oxylabs SERP API -// and baidu_search as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClientAsync) ScrapeBaiduSearchCtx( - ctx context.Context, - query string, - opts ...*BaiduSearchOpts, -) (chan *Resp, error) { - httpRespChan := make(chan *http.Response) - respChan := make(chan *Resp) - errChan := make(chan error) - - // Prepare options. - opt := &BaiduSearchOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultDomain(&opt.Domain) - internal.SetDefaultStartPage(&opt.StartPage) - internal.SetDefaultLimit(&opt.Limit, internal.DefaultLimit_SERP) - internal.SetDefaultPages(&opt.Pages) - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err := opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.BaiduSearch, - "domain": opt.Domain, - "query": query, - "start_page": opt.StartPage, - "pages": opt.Pages, - "limit": opt.Limit, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Get job ID. - jobID, err := c.C.GetJobID(jsonPayload) - if err != nil { - return nil, err - } - - // Poll job status. - go c.C.PollJobStatus( - ctx, - jobID, - opt.PollInterval, - httpRespChan, - errChan, - ) - - // Handle error. - err = <-errChan - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - httpResp := <-httpRespChan - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - // Retrieve internal resp and forward it to the - // resp channel. - go func() { - respChan <- resp - }() - - return respChan, nil -} - -// ScrapeBaiduUrl scrapes baidu with async polling runtime via Oxylabs SERP API -// and baidu as source. -func (c *SerpClientAsync) ScrapeBaiduUrl( - query string, - opts ...*BaiduUrlOpts, -) (chan *Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeBaiduUrlCtx(ctx, query, opts...) -} - -// ScrapeBaiduUrlCtx scrapes baidu with async polling runtime via Oxylabs SERP API -// and baidu as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClientAsync) ScrapeBaiduUrlCtx( - ctx context.Context, - url string, - opts ...*BaiduUrlOpts, -) (chan *Resp, error) { - httpRespChan := make(chan *http.Response) - respChan := make(chan *Resp) - errChan := make(chan error) - - // Check validity of URL. - err := internal.ValidateUrl(url, "baidu") - if err != nil { - return nil, err - } - - // Prepare options. - opt := &BaiduUrlOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err = opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.BaiduUrl, - "url": url, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Get job ID. - jobID, err := c.C.GetJobID(jsonPayload) - if err != nil { - return nil, err - } - - // Poll job status. - go c.C.PollJobStatus( - ctx, - jobID, - opt.PollInterval, - httpRespChan, - errChan, - ) - - // Handle error. - err = <-errChan - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - httpResp := <-httpRespChan - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - // Retrieve internal resp and forward it to the - // resp channel. - go func() { - respChan <- resp - }() - - return respChan, nil -} diff --git a/serp/yandex.go b/serp/yandex.go deleted file mode 100644 index 91c2add..0000000 --- a/serp/yandex.go +++ /dev/null @@ -1,259 +0,0 @@ -package serp - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/oxylabs/oxylabs-sdk-go/internal" - "github.com/oxylabs/oxylabs-sdk-go/oxylabs" -) - -// Accepted parameters for yandex. -var YandexSearchAcceptedDomainParameters = []oxylabs.Domain{ - oxylabs.DOMAIN_COM, - oxylabs.DOMAIN_RU, - oxylabs.DOMAIN_UA, - oxylabs.DOMAIN_BY, - oxylabs.DOMAIN_KZ, - oxylabs.DOMAIN_TR, -} -var YandexSearchAcceptedLocaleParameters = []oxylabs.Locale{ - oxylabs.LOCALE_EN, - oxylabs.LOCALE_RU, - oxylabs.LOCALE_BY, - oxylabs.LOCALE_DE, - oxylabs.LOCALE_FR, - oxylabs.LOCALE_ID, - oxylabs.LOCALE_KK, - oxylabs.LOCALE_TT, - oxylabs.LOCALE_TR, - oxylabs.LOCALE_UK, -} - -// checkParameterValidity checks validity of ScrapeYandexSearch parameters. -func (opt *YandexSearchOpts) checkParameterValidity() error { - if !internal.InList(opt.Domain, YandexSearchAcceptedDomainParameters) { - return fmt.Errorf("invalid domain parameter: %s", opt.Domain) - } - - if opt.Locale != "" && !internal.InList(opt.Locale, YandexSearchAcceptedLocaleParameters) { - return fmt.Errorf("invalid locale parameter: %s", opt.Locale) - } - - if !oxylabs.IsUserAgentValid(opt.UserAgent) { - return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent) - } - - if opt.Limit <= 0 || opt.Pages <= 0 || opt.StartPage <= 0 { - return fmt.Errorf("limit, pages and start_page parameters must be greater than 0") - } - - if opt.ParseInstructions != nil { - if err := oxylabs.ValidateParseInstructions(opt.ParseInstructions); err != nil { - return fmt.Errorf("invalid parse instructions: %w", err) - } - } - - return nil -} - -// checkParameterValidity checks validity of ScrapeYandexUrl parameters. -func (opt *YandexUrlOpts) checkParameterValidity() error { - if !oxylabs.IsUserAgentValid(opt.UserAgent) { - return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent) - } - - if opt.Render != "" && !oxylabs.IsRenderValid(opt.Render) { - return fmt.Errorf("invalid render parameter: %v", opt.Render) - } - - if opt.ParseInstructions != nil { - if err := oxylabs.ValidateParseInstructions(opt.ParseInstructions); err != nil { - return fmt.Errorf("invalid parse instructions: %w", err) - } - } - - return nil -} - -// YandexSearchOpts contains all the query parameters available for yandex_search. -type YandexSearchOpts struct { - Domain oxylabs.Domain - StartPage int - Pages int - Limit int - Locale oxylabs.Locale - GeoLocation string - UserAgent oxylabs.UserAgent - CallbackUrl string - ParseInstructions *map[string]interface{} - PollInterval time.Duration -} - -// ScrapeYandexSearch scrapes yandex via Oxylabs SERP API with yandex_search as source. -func (c *SerpClient) ScrapeYandexSearch( - query string, - opts ...*YandexSearchOpts, -) (*Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeYandexSearchCtx(ctx, query, opts...) -} - -// ScrapeYandexSearchCtx scrapes yandex via Oxylabs SERP API with yandex_search as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClient) ScrapeYandexSearchCtx( - ctx context.Context, - query string, - opts ...*YandexSearchOpts, -) (*Resp, error) { - // Prepare options. - opt := &YandexSearchOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultDomain(&opt.Domain) - internal.SetDefaultStartPage(&opt.StartPage) - internal.SetDefaultLimit(&opt.Limit, internal.DefaultLimit_SERP) - internal.SetDefaultPages(&opt.Pages) - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err := opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.YandexSearch, - "domain": opt.Domain, - "query": query, - "start_page": opt.StartPage, - "pages": opt.Pages, - "limit": opt.Limit, - "locale": opt.Locale, - "geo_location": opt.GeoLocation, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Req. - httpResp, err := c.C.Req(ctx, jsonPayload, "POST") - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - return resp, nil -} - -// YandexUrlOpts contains all the query parameters available for yandex. -type YandexUrlOpts struct { - UserAgent oxylabs.UserAgent - Render oxylabs.Render - CallbackUrl string - ParseInstructions *map[string]interface{} - PollInterval time.Duration -} - -// ScrapeYandexUrl scrapes a yandex url via Oxylabs SERP API with yandex as source. -func (c *SerpClient) ScrapeYandexUrl( - url string, - opts ...*YandexUrlOpts, -) (*Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeYandexUrlCtx(ctx, url, opts...) -} - -// ScapeYandexUrlCtx scrapes a yandex url via Oxylabs SERP API with yandex as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClient) ScrapeYandexUrlCtx( - ctx context.Context, - url string, - opts ...*YandexUrlOpts, -) (*Resp, error) { - // Check validity of URL. - err := internal.ValidateUrl(url, "yandex") - if err != nil { - return nil, err - } - - // Prepare options. - opt := &YandexUrlOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check validity of parameters. - err = opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare payload. - payload := map[string]interface{}{ - "source": oxylabs.YandexUrl, - "url": url, - "user_agent_type": opt.UserAgent, - "render": opt.Render, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Req. - httpResp, err := c.C.Req(ctx, jsonPayload, "POST") - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - return resp, nil -} diff --git a/serp/yandex_async.go b/serp/yandex_async.go deleted file mode 100644 index e2aa8d6..0000000 --- a/serp/yandex_async.go +++ /dev/null @@ -1,224 +0,0 @@ -package serp - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - - "github.com/oxylabs/oxylabs-sdk-go/internal" - "github.com/oxylabs/oxylabs-sdk-go/oxylabs" -) - -// ScrapeYandexSearch scrapes yandex with async polling runtime via Oxylabs SERP API -// and yandex_search as source. -func (c *SerpClientAsync) ScrapeYandexSearch( - query string, - opts ...*YandexSearchOpts, -) (chan *Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeYandexSearchCtx(ctx, query, opts...) -} - -// ScrapeYandexSearchCtx scrapes yandex with async polling runtime via Oxylabs SERP API -// and yandex_search as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClientAsync) ScrapeYandexSearchCtx( - ctx context.Context, - query string, - opts ...*YandexSearchOpts, -) (chan *Resp, error) { - httpRespChan := make(chan *http.Response) - respChan := make(chan *Resp) - errChan := make(chan error) - - // Prepare options. - opt := &YandexSearchOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultDomain(&opt.Domain) - internal.SetDefaultStartPage(&opt.StartPage) - internal.SetDefaultLimit(&opt.Limit, internal.DefaultLimit_SERP) - internal.SetDefaultPages(&opt.Pages) - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check the validity of the parameters. - err := opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare the payload. - payload := map[string]interface{}{ - "source": oxylabs.YandexSearch, - "domain": opt.Domain, - "query": query, - "start_page": opt.StartPage, - "pages": opt.Pages, - "limit": opt.Limit, - "locale": opt.Locale, - "geo_location": opt.GeoLocation, - "user_agent_type": opt.UserAgent, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - payload["parse"] = true - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Get the job ID. - jobID, err := c.C.GetJobID(jsonPayload) - if err != nil { - return nil, err - } - - // Poll job status. - go c.C.PollJobStatus( - ctx, - jobID, - opt.PollInterval, - httpRespChan, - errChan, - ) - - // Handle error. - err = <-errChan - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - httpResp := <-httpRespChan - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - // Retrieve internal resp and forward it to the - // resp channel. - go func() { - respChan <- resp - }() - - return respChan, nil -} - -// ScrapeYandexUrl scrapes yandex with async polling runtime via Oxylabs SERP API -// and yandex as source. -func (c *SerpClientAsync) ScrapeYandexUrl( - url string, - opts ...*YandexUrlOpts, -) (chan *Resp, error) { - ctx, cancel := context.WithTimeout(context.Background(), internal.DefaultTimeout) - defer cancel() - - return c.ScrapeYandexUrlCtx(ctx, url, opts...) -} - -// ScrapeYandexUrlCtx scrapes yandex with async polling runtime via Oxylabs SERP API -// and yandex as source. -// The provided context allows customization of the HTTP req, including setting timeouts. -func (c *SerpClientAsync) ScrapeYandexUrlCtx( - ctx context.Context, - url string, - opts ...*YandexUrlOpts, -) (chan *Resp, error) { - httpRespChan := make(chan *http.Response) - respChan := make(chan *Resp) - errChan := make(chan error) - - // Check the validity of the URL. - err := internal.ValidateUrl(url, "yandex") - if err != nil { - return nil, err - } - - // Prepare options. - opt := &YandexUrlOpts{} - if len(opts) > 0 && opts[len(opts)-1] != nil { - opt = opts[len(opts)-1] - } - - // Set defaults. - internal.SetDefaultUserAgent(&opt.UserAgent) - - // Check the validity of parameters. - err = opt.checkParameterValidity() - if err != nil { - return nil, err - } - - // Prepare the payload. - payload := map[string]interface{}{ - "source": oxylabs.YandexUrl, - "url": url, - "user_agent_type": opt.UserAgent, - "render": opt.Render, - "callback_url": opt.CallbackUrl, - } - - // Add custom parsing instructions to the payload if provided. - customParserFlag := false - if opt.ParseInstructions != nil { - payload["parse"] = true - payload["parsing_instructions"] = &opt.ParseInstructions - customParserFlag = true - } - - // Marshal. - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling payload: %v", err) - } - - // Get the job ID. - jobID, err := c.C.GetJobID(jsonPayload) - if err != nil { - return nil, err - } - - // Poll job status. - go c.C.PollJobStatus( - ctx, - jobID, - opt.PollInterval, - httpRespChan, - errChan, - ) - - // Handle error. - err = <-errChan - if err != nil { - return nil, err - } - - // Unmarshal the http Response and get the response. - httpResp := <-httpRespChan - resp, err := GetResp(httpResp, customParserFlag, customParserFlag) - if err != nil { - return nil, err - } - - // Retrieve internal resp and forward it to the - // resp channel. - go func() { - respChan <- resp - }() - - return respChan, nil -}