-
Notifications
You must be signed in to change notification settings - Fork 29
New issue
Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? # to your account
feat: add exponential and server-mediated backoff on retries #75
Conversation
Codecov Report
@@ Coverage Diff @@
## master #75 +/- ##
==========================================
+ Coverage 95.4% 96.83% +1.42%
==========================================
Files 6 6
Lines 305 348 +43
Branches 51 56 +5
==========================================
+ Hits 291 337 +46
+ Misses 14 11 -3
Continue to review full report at Codecov.
|
be3e1d9
to
a84a39b
Compare
ts/src/config.ts
Outdated
// fails. | ||
backoffMillis?: number; | ||
// On first error during profile creation, if the backoff is not specified | ||
// by the server response, then profiler will between 0 and |
ts/src/config.ts
Outdated
maxBackoffMillis?: number; | ||
|
||
// On each consecutive error in profile creation, the maximum backoff will | ||
// increase by this factor. The backoff will be random value selected |
ts/src/config.ts
Outdated
ts/src/config.ts
Outdated
|
||
// On each consecutive error in profile creation, the maximum backoff will | ||
// increase by this factor. The backoff will be random value selected | ||
// from a uniform distribution between 0 and the maximum backoff. |
ts/src/config.ts
Outdated
ts/src/profiler.ts
Outdated
* createProfile() should be retried when response indicates this request | ||
* should be retried or with exponential backoff (up to one hour) if the | ||
* response does not indicate when to retry this request. | ||
* collecting another profile, or retryer specifying exponential backoff. | ||
*/ | ||
async collectProfile(): Promise<number> { | ||
let prof: RequestProfile; | ||
try { | ||
prof = await this.createProfile(); | ||
} catch (err) { | ||
this.logger.error(`Failed to create profile: ${err}`); |
ts/src/profiler.ts
Outdated
ts/src/profiler.ts
Outdated
ts/src/profiler.ts
Outdated
ts/test/test-init-config.ts
Outdated
PTAL |
bf2dc9b
to
c5df7a6
Compare
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Some nits, mostly looks good. @ofrobots please take a look as well.
ts/src/config.ts
Outdated
// by the server response, then profiler will wait between 0 and | ||
// initialBackoffMillis before asking the server to create a profile again. | ||
// After a successful profile creation, the backoff envelope will be reset to | ||
// initialExpBackoffMillis. |
ts/src/config.ts
Outdated
// If the backoff is not specified by the server response, then profiler will | ||
// wait at most expBackoffMillisCap before asking server to create a profile | ||
// again. | ||
expBackoffMillisCap?: number; |
ts/src/config.ts
Outdated
// The backoff is capped here because setTimeout (which is used to control | ||
// when next profile is collected) will run immediately if the backoff is | ||
// to large. | ||
serverBackoffMillisCap?: number; |
ts/src/config.ts
Outdated
ts/src/profiler.ts
Outdated
@@ -120,6 +142,69 @@ async function profileBytes(p: perftools.profiles.IProfile): Promise<string> { | |||
return gzBuf.toString('base64'); | |||
} | |||
|
|||
/** | |||
* Error constructed from http server response which indicates backoff. |
ts/src/profiler.ts
Outdated
ts/src/config.ts
Outdated
// Server-specified backoffs will be capped at serverBackoffCapMillis. | ||
// The backoff is capped here because setTimeout (which is used to control | ||
// when next profile is collected) will run immediately if the backoff is | ||
// to large. |
ts/src/profiler.ts
Outdated
import {ProfilerConfig} from './config'; | ||
import {HeapProfiler} from './profilers/heap-profiler'; | ||
import {TimeProfiler} from './profilers/time-profiler'; | ||
|
||
export const common: Common = require('@google-cloud/common'); | ||
const parseDuration: (str: string) => number = require('parse-duration'); | ||
const msToStr: (ms: number) => string = require('pretty-ms'); |
ts/src/profiler.ts
Outdated
const item = response.body.error.details[i]; | ||
if (typeof item === 'object' && item.retryDelay && | ||
typeof item.retryDelay === 'string') { | ||
return parseDuration(item.retryDelay); |
ts/src/profiler.ts
Outdated
*/ | ||
function isBackoffResponseError(err: Error): err is BackoffResponseError { | ||
// tslint:disable-next-line: no-any | ||
return typeof (err as any).backoffMillis === 'number'; |
ts/src/profiler.ts
Outdated
@@ -97,7 +119,7 @@ function isRequestProfile(prof: any): prof is RequestProfile { | |||
} | |||
|
|||
/** | |||
* @return true if response has statusCode. | |||
* @return true iff response has statusCode. | |||
*/ | |||
// tslint:disable-next-line: no-any | |||
function hasHttpStatusCode(response: any): |
ts/src/profiler.ts
Outdated
try { | ||
await this.profileAndUpload(prof); | ||
} catch (err) { | ||
this.logger.error(`Failed to collect and upload profile: ${err}`); | ||
this.logger.warn(`Failed to collect and upload profile: ${err}`); |
PTAL |
e5495d2
to
e82d2d5
Compare
ts/src/config.ts
Outdated
initialBackoffMillis: 1000, | ||
backoffCapMillis: parseDuration('1h'), | ||
backoffMultiplier: 1.3, | ||
serverBackoffCapMillis: 2147483647 |
This comment was marked as spam.
This comment was marked as spam.
Sorry, something went wrong.
983ad4c
to
0ba909c
Compare
@ofrobots -- any additional comments? |
@ofrobots ping |
0ba909c
to
a31fc5b
Compare
package.json
Outdated
ts/src/profiler.ts
Outdated
* that backoff is greater than 0. Otherwise returns undefined. | ||
*/ | ||
// tslint:disable-next-line: no-any | ||
function getServerResponseBackoff(response: any): number|undefined { |
ts/src/profiler.ts
Outdated
*/ | ||
// tslint:disable-next-line: no-any | ||
function getServerResponseBackoff(response: any): number|undefined { | ||
if (response && response.body && response.body.error && |
ts/src/profiler.ts
Outdated
function getServerResponseBackoff(response: any): number|undefined { | ||
if (response && response.body && response.body.error && | ||
response.body.error.details && | ||
response.body.error.details instanceof Array) { |
ts/src/profiler.ts
Outdated
if (response && response.body && response.body.error && | ||
response.body.error.details && | ||
response.body.error.details instanceof Array) { | ||
for (let i = 0; i < response.body.error.details.length; i++) { |
ts/src/profiler.ts
Outdated
for (let i = 0; i < response.body.error.details.length; i++) { | ||
const item = response.body.error.details[i]; | ||
if (typeof item === 'object' && item.retryDelay && | ||
typeof item.retryDelay === 'string') { |
ts/src/profiler.ts
Outdated
@@ -97,7 +119,7 @@ function isRequestProfile(prof: any): prof is RequestProfile { | |||
} | |||
|
|||
/** | |||
* @return true if response has statusCode. | |||
* @return true iff response has statusCode. | |||
*/ | |||
// tslint:disable-next-line: no-any | |||
function hasHttpStatusCode(response: any): |
This comment was marked as spam.
This comment was marked as spam.
Sorry, something went wrong.
37a0a3b
to
5937b2d
Compare
PTAL |
ts/src/profiler.ts
Outdated
if (!hasHttpStatusCode(response)) { | ||
this.logger.debug( | ||
'Server response missing status information when attempting to upload profile.'); | ||
if (!(response instanceof http.ServerResponse)) { |
ts/src/profiler.ts
Outdated
function getServerResponseBackoff(response: any): number|undefined { | ||
if (response && response.body && response.body.error && | ||
response.body.error.details && | ||
response.body.error.details instanceof Array) { |
This comment was marked as spam.
This comment was marked as spam.
Sorry, something went wrong.
ts/src/profiler.ts
Outdated
for (let i = 0; i < response.body.error.details.length; i++) { | ||
const item = response.body.error.details[i]; | ||
if (typeof item === 'object' && item.retryDelay && | ||
typeof item.retryDelay === 'string') { |
This comment was marked as spam.
This comment was marked as spam.
Sorry, something went wrong.
I have stopped using instanceof. (Also, apologies for adding a comment instead of replying to comments; there were some comments I don't seem to be able to reply directly to). |
PTAL |
Fixes #43 and #44.
See #56 for previous discussion on this topic.