diff --git a/packages/gateway/src/durable-objects/summary-metrics.js b/packages/gateway/src/durable-objects/summary-metrics.js index 7ae0b679cb1..fe155d6ec1e 100644 --- a/packages/gateway/src/durable-objects/summary-metrics.js +++ b/packages/gateway/src/durable-objects/summary-metrics.js @@ -4,6 +4,8 @@ import { } from '../utils/histogram.js' /** + * @typedef {'CID'|'CID+PATH'} QueryType + * * @typedef {Object} SummaryMetrics * @property {number} totalWinnerResponseTime total response time of the requests * @property {number} totalWinnerSuccessfulRequests total number of successful requests @@ -11,12 +13,14 @@ import { * @property {number} totalCachedResponses total number of cached responses * @property {BigInt} totalContentLengthBytes total content length of responses * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses + * @property {Record} totalResponsesByQueryType * @property {Record} contentLengthHistogram * @property {Record} responseTimeHistogram * * @typedef {Object} FetchStats - * @property {number} responseTime number of milliseconds to get response - * @property {number} contentLength content length header content + * @property {string} [pathname] fetched pathname + * @property {number} [responseTime] number of milliseconds to get response + * @property {number} [contentLength] content length header content */ // Key to track total time for winner gateway to respond @@ -35,6 +39,8 @@ const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes' const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram' // Key to track response time histogram const RESPONSE_TIME_HISTOGRAM_ID = 'responseTimeHistogram' +// Key to track responses by query type +const TOTAL_RESPONSES_BY_QUERY_TYPE_ID = 'totalResponsesByQueryType' /** * Durable Object for keeping summary metrics of nft.storage Gateway @@ -57,19 +63,23 @@ export class SummaryMetrics0 { // Total cached requests this.totalCachedResponses = (await this.state.storage.get(TOTAL_CACHED_RESPONSES_ID)) || 0 - // Total content length responses + /** @type {BigInt} */ this.totalContentLengthBytes = (await this.state.storage.get(TOTAL_CONTENT_LENGTH_BYTES_ID)) || BigInt(0) - // Total cached content length responses + /** @type {BigInt} */ this.totalCachedContentLengthBytes = (await this.state.storage.get(TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID)) || BigInt(0) - // Content length histogram + /** @type {Record} */ + this.totalResponsesByQueryType = + (await this.state.storage.get(TOTAL_RESPONSES_BY_QUERY_TYPE_ID)) || + createResponsesByQueryTypeObject() + /** @type {Record} */ this.contentLengthHistogram = (await this.state.storage.get(CONTENT_LENGTH_HISTOGRAM_ID)) || createContentLengthHistogramObject() - // Response time histogram + /** @type {Record} */ this.responseTimeHistogram = (await this.state.storage.get(RESPONSE_TIME_HISTOGRAM_ID)) || createResponseTimeHistogramObject() @@ -94,6 +104,7 @@ export class SummaryMetrics0 { totalContentLengthBytes: this.totalContentLengthBytes.toString(), totalCachedContentLengthBytes: this.totalCachedContentLengthBytes.toString(), + totalResponsesByQueryType: this.totalResponsesByQueryType, contentLengthHistogram: this.contentLengthHistogram, responseTimeHistogram: this.responseTimeHistogram, }) @@ -109,13 +120,15 @@ export class SummaryMetrics0 { switch (url.pathname) { case '/metrics/winner': await this._updateWinnerMetrics(data) - return new Response() + break case '/metrics/cache': await this._updatedCacheMetrics(data) - return new Response() + break default: - return new Response('Not found', { status: 404 }) + throw new Error('Not found') } + + return new Response() } /** @@ -126,7 +139,7 @@ export class SummaryMetrics0 { this.totalCachedResponseTime += stats.responseTime this.totalCachedResponses += 1 this.totalCachedContentLengthBytes += BigInt(stats.contentLength) - this._updateContentLengthMetrics(stats) + this._updateContentMetrics(stats) this._updateResponseTimeHistogram(stats) // Save updated metrics await Promise.all([ @@ -164,7 +177,7 @@ export class SummaryMetrics0 { // Updated Metrics this.totalWinnerResponseTime += stats.responseTime this.totalWinnerSuccessfulRequests += 1 - this._updateContentLengthMetrics(stats) + this._updateContentMetrics(stats) this._updateResponseTimeHistogram(stats) // Save updated Metrics await Promise.all([ @@ -188,50 +201,65 @@ export class SummaryMetrics0 { RESPONSE_TIME_HISTOGRAM_ID, this.responseTimeHistogram ), + this.state.storage.put( + TOTAL_RESPONSES_BY_QUERY_TYPE_ID, + this.totalResponsesByQueryType + ), ]) } /** * @param {FetchStats} stats */ - _updateContentLengthMetrics(stats) { + _updateContentMetrics(stats) { + // Content Length this.totalContentLengthBytes += BigInt(stats.contentLength) - - // Update histogram - const tmpHistogram = { - ...this.contentLengthHistogram, - } - - // Get all the histogram buckets where the content size is smaller - const histogramCandidates = contentLengthHistogram.filter( - (h) => stats.contentLength < h + this.contentLengthHistogram = getUpdatedHistogram( + this.contentLengthHistogram, + contentLengthHistogram, + stats.contentLength ) - histogramCandidates.forEach((candidate) => { - tmpHistogram[candidate] += 1 - }) - this.contentLengthHistogram = tmpHistogram + // Query type + if (stats.pathname && stats.pathname !== '/') { + this.totalResponsesByQueryType['CID+PATH'] += 1 + } else { + this.totalResponsesByQueryType['CID'] += 1 + } } /** * @param {FetchStats} stats */ _updateResponseTimeHistogram(stats) { - const tmpHistogram = { - ...this.responseTimeHistogram, - } - - // Get all the histogram buckets where the response time is smaller - const histogramCandidates = responseTimeHistogram.filter( - (h) => stats.responseTime < h + this.responseTimeHistogram = getUpdatedHistogram( + this.responseTimeHistogram, + responseTimeHistogram, + stats.responseTime ) + } +} - histogramCandidates.forEach((candidate) => { - tmpHistogram[candidate] += 1 +function getUpdatedHistogram(histogramData, histogramBuckets, value) { + const updatedHistogram = { + ...histogramData, + } + // Update all the histogram buckets where the response time is smaller + histogramBuckets + .filter((h) => value < h) + .forEach((candidate) => { + updatedHistogram[candidate] += 1 }) - this.responseTimeHistogram = tmpHistogram - } + return updatedHistogram +} + +/** + * @return {Record} + */ +function createResponsesByQueryTypeObject() { + const e = queryType.map((t) => [t, 0]) + return Object.fromEntries(e) } function createContentLengthHistogramObject() { @@ -239,6 +267,9 @@ function createContentLengthHistogramObject() { return Object.fromEntries(h) } +// Either CID is stored in NFT.storage or not +export const queryType = ['CID', 'CID+PATH'] + // We will count occurences per bucket where content size is less or equal than bucket value export const contentLengthHistogram = [ 0.5, 1, 2, 5, 25, 50, 100, 500, 1000, 5000, 10000, 15000, 20000, 30000, 32000, diff --git a/packages/gateway/src/gateway.js b/packages/gateway/src/gateway.js index 8d24c86db01..8c437da93f8 100644 --- a/packages/gateway/src/gateway.js +++ b/packages/gateway/src/gateway.js @@ -38,6 +38,10 @@ import { */ export async function gatewayGet(request, env, ctx) { const startTs = Date.now() + const reqUrl = new URL(request.url) + const cid = getCidFromSubdomainUrl(reqUrl) + const pathname = reqUrl.pathname + const cache = caches.default let res = await cache.match(request.url) @@ -45,14 +49,14 @@ export async function gatewayGet(request, env, ctx) { // Update cache metrics in background const responseTime = Date.now() - startTs - ctx.waitUntil(updateSummaryCacheMetrics(request, env, res, responseTime)) + ctx.waitUntil( + updateSummaryCacheMetrics(request, env, res, responseTime, { + pathname, + }) + ) return res } - const reqUrl = new URL(request.url) - const cid = getCidFromSubdomainUrl(reqUrl) - const pathname = reqUrl.pathname - // Prepare IPFS gateway requests const shouldPreventRateLimit = await getGatewayRateLimitState(request, env) const gatewayReqs = env.ipfsGateways.map((gwUrl) => @@ -91,7 +95,9 @@ export async function gatewayGet(request, env, ctx) { ) await Promise.all([ - storeWinnerGwResponse(request, env, winnerGwResponse), + storeWinnerGwResponse(request, env, winnerGwResponse, { + pathname, + }), settleGatewayRequests(), // Cache request URL in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE contentLengthMb <= CF_CACHE_MAX_OBJECT_SIZE && @@ -120,7 +126,8 @@ export async function gatewayGet(request, env, ctx) { updateGatewayMetrics(request, env, r.value, false) ) ) - wasRateLimited && updateGatewayRedirectCounter(request, env) + // Update redirect counter + wasRateLimited && (await updateGatewayRedirectCounter(request, env)) })() ) @@ -147,6 +154,7 @@ export async function gatewayGet(request, env, ctx) { } } + // Throw server error throw err } } @@ -156,12 +164,19 @@ export async function gatewayGet(request, env, ctx) { * * @param {Request} request * @param {Env} env - * @param {GatewayResponse} winnerGwResponse + * @param {GatewayResponse} gwResponse + * @param {Object} [options] + * @param {string} [options.pathname] */ -async function storeWinnerGwResponse(request, env, winnerGwResponse) { +async function storeWinnerGwResponse( + request, + env, + gwResponse, + { pathname } = {} +) { await Promise.all([ - updateGatewayMetrics(request, env, winnerGwResponse, true), - updateSummaryWinnerMetrics(request, env, winnerGwResponse), + updateGatewayMetrics(request, env, gwResponse, true), + updateSummaryWinnerMetrics(request, env, { gwResponse, pathname }), ]) } @@ -245,9 +260,17 @@ function getHeaders(request) { * @param {import('./env').Env} env * @param {Response} response * @param {number} responseTime + * @param {Object} [options] + * @param {string} [options.pathname] */ -async function updateSummaryCacheMetrics(request, env, response, responseTime) { - // Get durable object for summary +async function updateSummaryCacheMetrics( + request, + env, + response, + responseTime, + { pathname } = {} +) { + // Get durable object for gateway const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID) const stub = env.summaryMetricsDurable.get(id) @@ -255,6 +278,7 @@ async function updateSummaryCacheMetrics(request, env, response, responseTime) { const contentLengthStats = { contentLength: Number(response.headers.get('content-length')), responseTime, + pathname, } await stub.fetch( @@ -294,17 +318,24 @@ async function getGatewayRateLimitState(request, env) { /** * @param {Request} request * @param {import('./env').Env} env - * @param {GatewayResponse} gwResponse + * @param {Object} options + * @param {GatewayResponse} [options.gwResponse] + * @param {string} [options.pathname] */ -async function updateSummaryWinnerMetrics(request, env, gwResponse) { +async function updateSummaryWinnerMetrics( + request, + env, + { gwResponse, pathname } +) { // Get durable object for gateway const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID) const stub = env.summaryMetricsDurable.get(id) /** @type {import('./durable-objects/summary-metrics').FetchStats} */ const fetchStats = { - responseTime: gwResponse.responseTime, - contentLength: Number(gwResponse.response.headers.get('content-length')), + contentLength: Number(gwResponse?.response.headers.get('content-length')), + responseTime: gwResponse?.responseTime, + pathname, } await stub.fetch(getDurableRequestUrl(request, 'metrics/winner', fetchStats)) diff --git a/packages/gateway/src/metrics.js b/packages/gateway/src/metrics.js index 5d4d180415b..9cc244f0447 100644 --- a/packages/gateway/src/metrics.js +++ b/packages/gateway/src/metrics.js @@ -267,6 +267,18 @@ export async function metricsGet(request, env, ctx) { `# HELP nftgateway_redirect_total Total redirects to gateway.`, `# TYPE nftgateway_redirect_total counter`, `nftgateway_redirect_total{env="${env.ENV}"} ${metricsCollected.gatewayRedirectCount}`, + `# HELP nftgateway_responses_by_query_type_total total of responses by query status. Either CID or CID+PATH.`, + `# TYPE nftgateway_responses_by_query_type_total counter`, + Object.keys(metricsCollected.summaryMetrics.totalResponsesByQueryType) + .map( + (type) => + `nftgateway_responses_by_query_type_total{env="${ + env.ENV + }",type="${type}"} ${ + metricsCollected.summaryMetrics.totalResponsesByQueryType[type] || 0 + }` + ) + .join('\n'), ].join('\n') res = new Response(metrics, { diff --git a/packages/gateway/test/metrics.spec.js b/packages/gateway/test/metrics.spec.js index 1bf08144081..5aad0e6b487 100644 --- a/packages/gateway/test/metrics.spec.js +++ b/packages/gateway/test/metrics.spec.js @@ -38,9 +38,11 @@ test('Gets Metrics content when empty state', async (t) => { ) t.is(metricsResponse.includes(`_responses_content_length_total{le=`), true) t.is( - metricsResponse.includes( - `_responses_content_length_bytes_total{env="test"} 0` - ), + metricsResponse.includes('nftgateway_winner_response_time_seconds_total'), + true + ) + t.is( + metricsResponse.includes('nftgateway_winner_response_time_seconds_total'), true ) gateways.forEach((gw) => { @@ -123,6 +125,38 @@ test('Gets Metrics content', async (t) => { }) }) +test('gets Metrics from query type', async (t) => { + const { mf } = t.context + + const p = await Promise.all([ + mf.dispatchFetch( + 'http://bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq.ipfs.localhost:8787' + ), + mf.dispatchFetch( + 'https://bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq.ipfs.localhost:8787/path' + ), + ]) + + // Wait for waitUntil + await Promise.all(p.map((p) => p.waitUntil())) + + const response = await mf.dispatchFetch('http://localhost:8787/metrics') + const metricsResponse = await response.text() + + t.is( + metricsResponse.includes( + `nftgateway_responses_by_query_type_total{env="test",type="CID"} 1` + ), + true + ) + t.is( + metricsResponse.includes( + `nftgateway_responses_by_query_type_total{env="test",type="CID+PATH"} 1` + ), + true + ) +}) + test('Gets Metrics from faster gateway', async (t) => { const { mf } = t.context