From cd34e35da4d97d7af27caa4ebfe5fb7f95311aaa Mon Sep 17 00:00:00 2001 From: Adam Weidman <65992621+adamfweidman@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:42:25 -0400 Subject: [PATCH] feat: Add screenshot storage support to broken link checker (#123) * get-cloud-region (#100) Add functionality to synthetics-sdk-api to extract cloud region during GCF execution * stoage proto api (#101) * expose resolveProjectId (#104) * update to capture_condition (#109) * chore(deps): bump ip from 1.1.8 to 1.1.9 (#105) * chore(deps): bump ip from 1.1.8 to 1.1.9 Bumps [ip](https://github.com/indutny/node-ip) from 1.1.8 to 1.1.9. - [Commits](https://github.com/indutny/node-ip/compare/v1.1.8...v1.1.9) --- updated-dependencies: - dependency-name: ip dependency-type: indirect ... Signed-off-by: dependabot[bot] * Empty-Commit --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam Weidman * add samples tags (#108) * blc-api-integration-def (#102) * resolveProjectId present (#106) * take-screenshots (#107) * rebase-capture-condition (#110) * refactor-integrations (#112) * sanitize strings (#113) * Take and populate screenshot (#114) * screenshots-prop * broken_links.spec working * fix naming * pass-args * response to comments * change default (#118) * update synthetics-sdk-api to point to new npm pkg --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 803 ++++++++++++++++-- .../synthetics-sdk-broken-links/package.json | 8 +- .../src/broken_links.ts | 87 +- .../src/handlers.ts | 15 +- .../synthetics-sdk-broken-links/src/index.ts | 4 +- .../src/link_utils.ts | 107 ++- .../src/navigation_func.ts | 30 +- .../src/options_func.ts | 89 +- .../src/storage_func.ts | 192 +++++ .../example_html_files/integration_server.js | 49 +- .../test/integration/integration.spec.ts | 341 +------- .../test/unit/broken_links.spec.ts | 364 ++++++-- .../test/unit/handlers.spec.ts | 60 ++ .../test/unit/link_utils.spec.ts | 157 +++- .../test/unit/navigation_func.spec.ts | 121 ++- .../test/unit/options_func.spec.ts | 61 +- .../test/unit/storage_func.spec.ts | 303 +++++++ 17 files changed, 2210 insertions(+), 581 deletions(-) create mode 100644 packages/synthetics-sdk-broken-links/src/storage_func.ts create mode 100644 packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts create mode 100644 packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts diff --git a/package-lock.json b/package-lock.json index 70c12324..108a717e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -665,6 +665,157 @@ "@opentelemetry/semantic-conventions": "^1.0.0" } }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage": { + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.7.0.tgz", + "integrity": "sha512-EMCEY+6JiIkx7Dt8NXVGGjy1vRdSGdHkoqZoqjJw7cEBkT7ZkX0c7puedfn1MamnzW5SX4xoa2jVq5u7OWBmkQ==", + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "compressible": "^2.0.12", + "duplexify": "^4.0.0", + "ent": "^2.2.0", + "fast-xml-parser": "^4.3.0", + "gaxios": "^6.0.2", + "google-auth-library": "^9.0.0", + "mime": "^3.0.0", + "mime-types": "^2.0.8", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@google-cloud/storage/node_modules/gaxios": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz", + "integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/google-auth-library": { + "version": "9.6.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.6.3.tgz", + "integrity": "sha512-4CacM29MLC2eT9Cey5GDVK4Q8t+MMp8+OEdOaqD9MG6b0dOyLORaaeJMPQ7EESVgm/+z5EKYyFLxgzBJlJgyHQ==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/storage/node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@google-cloud/storage/node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/@google-cloud/synthetics-sdk-api": { "resolved": "packages/synthetics-sdk-api", "link": true @@ -2889,6 +3040,14 @@ "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", "dev": true }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, "node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", @@ -2948,6 +3107,11 @@ "@types/node": "*" } }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "node_modules/@types/chai": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.5.tgz", @@ -3196,6 +3360,12 @@ "@types/pg": "*" } }, + "node_modules/@types/proxyquire": { + "version": "1.3.31", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.31.tgz", + "integrity": "sha512-uALowNG2TSM1HNPMMOR0AJwv4aPYPhqB0xlEhkeRTMuto5hjoSPZkvgu1nbPUkz3gEPAHv4sy4DmKsurZiEfRQ==", + "dev": true + }, "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -3206,6 +3376,30 @@ "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz", @@ -3271,6 +3465,11 @@ "@types/node": "*" } }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "node_modules/@types/triple-beam": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", @@ -3783,6 +3982,14 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "dependencies": { + "retry": "0.13.1" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -4379,6 +4586,17 @@ "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", "dev": true }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4795,6 +5013,17 @@ "detect-libc": "^1.0.3" } }, + "node_modules/duplexify": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", + "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -4849,6 +5078,11 @@ "node": ">=8.6" } }, + "node_modules/ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "node_modules/error": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/error/-/error-7.0.2.tgz", @@ -5483,6 +5717,27 @@ "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" }, + "node_modules/fast-xml-parser": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.4.tgz", + "integrity": "sha512-utnwm92SyozgA3hhH2I8qldf2lBqm6qHOICawRNRFu1qMe3+oqr+GcXjGqTmXTMGE5T4eC03kr/rlh5C1IRdZA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -5551,6 +5806,19 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/fill-keys": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", + "integrity": "sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA==", + "dev": true, + "dependencies": { + "is-object": "~1.0.1", + "merge-descriptors": "~1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -6705,6 +6973,15 @@ "node": ">=8" } }, + "node_modules/is-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", + "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-plain-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", @@ -7532,6 +7809,12 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, + "node_modules/module-not-found-error": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", + "integrity": "sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g==", + "dev": true + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -8228,6 +8511,17 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/proxyquire": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", + "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", + "dev": true, + "dependencies": { + "fill-keys": "^1.0.2", + "module-not-found-error": "^1.0.1", + "resolve": "^1.11.1" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -8620,6 +8914,27 @@ "node": ">=8" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -8891,9 +9206,9 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/sinon": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-16.0.0.tgz", - "integrity": "sha512-B8AaZZm9CT5pqe4l4uWJztfD/mOTa7dL8Qo0W4+s+t74xECOgSZDDQCBjNgIK3+n4kyxQrSTv2V5ul8K25qkiQ==", + "version": "16.1.3", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-16.1.3.tgz", + "integrity": "sha512-mjnWWeyxcAf9nC0bXcPmiDut+oE8HYridTNzBbF98AYVLmWwGRp2ISEpyhYflG1ifILT+eNn3BmKUJPxjXUPlA==", "dev": true, "dependencies": { "@sinonjs/commons": "^3.0.0", @@ -9087,6 +9402,19 @@ "node": ">= 0.8" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, "node_modules/streamx": { "version": "2.15.2", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.2.tgz", @@ -9178,6 +9506,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, "node_modules/superagent": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.0.9.tgz", @@ -9315,6 +9653,46 @@ "streamx": "^2.15.0" } }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/teeny-request/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -10554,7 +10932,8 @@ "version": "0.2.0", "license": "Apache-2.0", "dependencies": { - "@google-cloud/synthetics-sdk-api": "^0.5.1", + "@google-cloud/storage": "^7.7.0", + "@google-cloud/synthetics-sdk-api": "^0.6.0", "puppeteer": "21.3.6" }, "devDependencies": { @@ -10562,12 +10941,15 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", "chai-exclude": "^2.1.0", "express": "^4.18.2", - "sinon": "^15.2.0", + "node-mocks-http": "^1.13.0", + "proxyquire": "^2.1.3", + "sinon": "^16.1.1", "supertest": "^6.3.3", "synthetics-sdk-broken-links": "file:./" }, @@ -10576,9 +10958,9 @@ } }, "packages/synthetics-sdk-broken-links/node_modules/@google-cloud/synthetics-sdk-api": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-pzfCLlL2MvlxnGTqnSTKEE4icU+79Fuucm++Mnvifx9RMeq0gDwSRJYcx4nb1oKgfMS7TFoL+ejkIBn9uFDCOw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.6.0.tgz", + "integrity": "sha512-dRS3x7NAYUdET6SQ6+scu3eUGUtgH2PAY8Zx+0DDvB7Zo5ymVIP1b0hbAkLLDZzg9wNQFSDAh59A0lEuU3Zpyw==", "dependencies": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", "@opentelemetry/api": "1.6.0", @@ -10587,6 +10969,7 @@ "@opentelemetry/sdk-node": "0.43.0", "@opentelemetry/sdk-trace-base": "1.17.0", "@opentelemetry/sdk-trace-node": "1.17.0", + "axios": "1.6.7", "error-stack-parser": "2.1.4", "google-auth-library": "9.0.0", "ts-proto": "1.148.1", @@ -10849,15 +11232,6 @@ "node": ">= 14" } }, - "packages/synthetics-sdk-broken-links/node_modules/diff": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", - "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, "packages/synthetics-sdk-broken-links/node_modules/gaxios": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.1.1.tgz", @@ -10953,25 +11327,6 @@ "node": ">=12.0.0" } }, - "packages/synthetics-sdk-broken-links/node_modules/sinon": { - "version": "15.2.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", - "integrity": "sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==", - "deprecated": "16.1.1", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^10.3.0", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.4", - "supports-color": "^7.2.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/sinon" - } - }, "packages/synthetics-sdk-mocha": { "name": "@google-cloud/synthetics-sdk-mocha", "version": "0.1.1", @@ -11494,6 +11849,122 @@ "gcp-metadata": "^5.0.1" } }, + "@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "dependencies": { + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + } + } + }, + "@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==" + }, + "@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==" + }, + "@google-cloud/storage": { + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.7.0.tgz", + "integrity": "sha512-EMCEY+6JiIkx7Dt8NXVGGjy1vRdSGdHkoqZoqjJw7cEBkT7ZkX0c7puedfn1MamnzW5SX4xoa2jVq5u7OWBmkQ==", + "requires": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "compressible": "^2.0.12", + "duplexify": "^4.0.0", + "ent": "^2.2.0", + "fast-xml-parser": "^4.3.0", + "gaxios": "^6.0.2", + "google-auth-library": "^9.0.0", + "mime": "^3.0.0", + "mime-types": "^2.0.8", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "dependencies": { + "agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "requires": { + "debug": "^4.3.4" + } + }, + "gaxios": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz", + "integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + } + }, + "gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "requires": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "9.6.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.6.3.tgz", + "integrity": "sha512-4CacM29MLC2eT9Cey5GDVK4Q8t+MMp8+OEdOaqD9MG6b0dOyLORaaeJMPQ7EESVgm/+z5EKYyFLxgzBJlJgyHQ==", + "requires": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + } + }, + "gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "requires": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + } + }, + "https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==" + } + } + }, "@google-cloud/synthetics-sdk-api": { "version": "file:packages/synthetics-sdk-api", "requires": { @@ -11806,25 +12277,29 @@ "version": "file:packages/synthetics-sdk-broken-links", "requires": { "@google-cloud/functions-framework": "^3.1.3", - "@google-cloud/synthetics-sdk-api": "^0.5.1", + "@google-cloud/storage": "^7.7.0", + "@google-cloud/synthetics-sdk-api": "^0.6.0", "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", "chai-exclude": "^2.1.0", "express": "^4.18.2", + "node-mocks-http": "^1.13.0", + "proxyquire": "^2.1.3", "puppeteer": "21.3.6", - "sinon": "^15.2.0", + "sinon": "^16.1.1", "supertest": "^6.3.3", "synthetics-sdk-broken-links": "file:" }, "dependencies": { "@google-cloud/synthetics-sdk-api": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-pzfCLlL2MvlxnGTqnSTKEE4icU+79Fuucm++Mnvifx9RMeq0gDwSRJYcx4nb1oKgfMS7TFoL+ejkIBn9uFDCOw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.6.0.tgz", + "integrity": "sha512-dRS3x7NAYUdET6SQ6+scu3eUGUtgH2PAY8Zx+0DDvB7Zo5ymVIP1b0hbAkLLDZzg9wNQFSDAh59A0lEuU3Zpyw==", "requires": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", "@opentelemetry/api": "1.6.0", @@ -11833,6 +12308,7 @@ "@opentelemetry/sdk-node": "0.43.0", "@opentelemetry/sdk-trace-base": "1.17.0", "@opentelemetry/sdk-trace-node": "1.17.0", + "axios": "1.6.7", "error-stack-parser": "2.1.4", "google-auth-library": "9.0.0", "ts-proto": "1.148.1", @@ -12012,12 +12488,6 @@ "debug": "^4.3.4" } }, - "diff": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", - "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", - "dev": true - }, "gaxios": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.1.1.tgz", @@ -12093,20 +12563,6 @@ "@types/node": ">=13.7.0", "long": "^5.0.0" } - }, - "sinon": { - "version": "15.2.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", - "integrity": "sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^10.3.0", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.4", - "supports-color": "^7.2.0" - } } } }, @@ -13678,6 +14134,11 @@ "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", "dev": true }, + "@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" + }, "@tootallnate/quickjs-emscripten": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", @@ -13737,6 +14198,11 @@ "@types/node": "*" } }, + "@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "@types/chai": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.5.tgz", @@ -13981,6 +14447,12 @@ "@types/pg": "*" } }, + "@types/proxyquire": { + "version": "1.3.31", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.31.tgz", + "integrity": "sha512-uALowNG2TSM1HNPMMOR0AJwv4aPYPhqB0xlEhkeRTMuto5hjoSPZkvgu1nbPUkz3gEPAHv4sy4DmKsurZiEfRQ==", + "dev": true + }, "@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -13991,6 +14463,29 @@ "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" }, + "@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "requires": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + }, + "dependencies": { + "form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + } + } + }, "@types/send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz", @@ -14056,6 +14551,11 @@ "@types/node": "*" } }, + "@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "@types/triple-beam": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", @@ -14391,6 +14891,14 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, + "async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "requires": { + "retry": "0.13.1" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -14845,6 +15353,14 @@ "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", "dev": true }, + "compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "requires": { + "mime-db": ">= 1.43.0 < 2" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -15147,6 +15663,17 @@ "detect-libc": "^1.0.3" } }, + "duplexify": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", + "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -15194,6 +15721,11 @@ "ansi-colors": "^4.1.1" } }, + "ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "error": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/error/-/error-7.0.2.tgz", @@ -15663,6 +16195,14 @@ "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" }, + "fast-xml-parser": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.4.tgz", + "integrity": "sha512-utnwm92SyozgA3hhH2I8qldf2lBqm6qHOICawRNRFu1qMe3+oqr+GcXjGqTmXTMGE5T4eC03kr/rlh5C1IRdZA==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -15717,6 +16257,16 @@ "flat-cache": "^3.0.4" } }, + "fill-keys": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", + "integrity": "sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA==", + "dev": true, + "requires": { + "is-object": "~1.0.1", + "merge-descriptors": "~1.0.0" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -16546,6 +17096,12 @@ "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", "dev": true }, + "is-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", + "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==", + "dev": true + }, "is-plain-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", @@ -17172,6 +17728,12 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, + "module-not-found-error": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", + "integrity": "sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g==", + "dev": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -17674,6 +18236,17 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "proxyquire": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", + "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", + "dev": true, + "requires": { + "fill-keys": "^1.0.2", + "module-not-found-error": "^1.0.1", + "resolve": "^1.11.1" + } + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -17953,6 +18526,21 @@ "signal-exit": "^3.0.2" } }, + "retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" + }, + "retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "requires": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + } + }, "reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -18153,9 +18741,9 @@ } }, "sinon": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-16.0.0.tgz", - "integrity": "sha512-B8AaZZm9CT5pqe4l4uWJztfD/mOTa7dL8Qo0W4+s+t74xECOgSZDDQCBjNgIK3+n4kyxQrSTv2V5ul8K25qkiQ==", + "version": "16.1.3", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-16.1.3.tgz", + "integrity": "sha512-mjnWWeyxcAf9nC0bXcPmiDut+oE8HYridTNzBbF98AYVLmWwGRp2ISEpyhYflG1ifILT+eNn3BmKUJPxjXUPlA==", "dev": true, "requires": { "@sinonjs/commons": "^3.0.0", @@ -18311,6 +18899,19 @@ "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", "dev": true }, + "stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "requires": { + "stubs": "^3.0.0" + } + }, + "stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, "streamx": { "version": "2.15.2", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.2.tgz", @@ -18378,6 +18979,16 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, + "stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, "superagent": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.0.9.tgz", @@ -18431,25 +19042,29 @@ "version": "file:packages/synthetics-sdk-broken-links", "requires": { "@google-cloud/functions-framework": "^3.1.3", - "@google-cloud/synthetics-sdk-api": "^0.5.1", + "@google-cloud/storage": "^7.7.0", + "@google-cloud/synthetics-sdk-api": "^0.6.0", "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", "chai-exclude": "^2.1.0", "express": "^4.18.2", + "node-mocks-http": "^1.13.0", + "proxyquire": "^2.1.3", "puppeteer": "21.3.6", - "sinon": "^15.2.0", + "sinon": "^16.1.1", "supertest": "^6.3.3", "synthetics-sdk-broken-links": "file:" }, "dependencies": { "@google-cloud/synthetics-sdk-api": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-pzfCLlL2MvlxnGTqnSTKEE4icU+79Fuucm++Mnvifx9RMeq0gDwSRJYcx4nb1oKgfMS7TFoL+ejkIBn9uFDCOw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@google-cloud/synthetics-sdk-api/-/synthetics-sdk-api-0.6.0.tgz", + "integrity": "sha512-dRS3x7NAYUdET6SQ6+scu3eUGUtgH2PAY8Zx+0DDvB7Zo5ymVIP1b0hbAkLLDZzg9wNQFSDAh59A0lEuU3Zpyw==", "requires": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", "@opentelemetry/api": "1.6.0", @@ -18458,6 +19073,7 @@ "@opentelemetry/sdk-node": "0.43.0", "@opentelemetry/sdk-trace-base": "1.17.0", "@opentelemetry/sdk-trace-node": "1.17.0", + "axios": "1.6.7", "error-stack-parser": "2.1.4", "google-auth-library": "9.0.0", "ts-proto": "1.148.1", @@ -18637,12 +19253,6 @@ "debug": "^4.3.4" } }, - "diff": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", - "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", - "dev": true - }, "gaxios": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.1.1.tgz", @@ -18718,20 +19328,6 @@ "@types/node": ">=13.7.0", "long": "^5.0.0" } - }, - "sinon": { - "version": "15.2.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", - "integrity": "sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^10.3.0", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.4", - "supports-color": "^7.2.0" - } } } }, @@ -18823,6 +19419,35 @@ "streamx": "^2.15.0" } }, + "teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "requires": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "dependencies": { + "http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "requires": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + } + }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } + }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", diff --git a/packages/synthetics-sdk-broken-links/package.json b/packages/synthetics-sdk-broken-links/package.json index a9d91322..841b8a35 100644 --- a/packages/synthetics-sdk-broken-links/package.json +++ b/packages/synthetics-sdk-broken-links/package.json @@ -27,12 +27,15 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", "chai-exclude": "^2.1.0", "express": "^4.18.2", - "sinon": "^15.2.0", + "proxyquire": "^2.1.3", + "node-mocks-http": "^1.13.0", + "sinon": "^16.1.1", "supertest": "^6.3.3", "synthetics-sdk-broken-links": "file:./" }, @@ -40,7 +43,8 @@ "node": ">=18" }, "dependencies": { - "@google-cloud/synthetics-sdk-api": "^0.5.1", + "@google-cloud/storage": "^7.7.0", + "@google-cloud/synthetics-sdk-api": "^0.6.0", "puppeteer": "21.3.6" } } diff --git a/packages/synthetics-sdk-broken-links/src/broken_links.ts b/packages/synthetics-sdk-broken-links/src/broken_links.ts index 5e5f9865..c7a899ff 100644 --- a/packages/synthetics-sdk-broken-links/src/broken_links.ts +++ b/packages/synthetics-sdk-broken-links/src/broken_links.ts @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -import puppeteer, { Browser, Page } from 'puppeteer'; +// Internal Project Files import { + BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_SyntheticLinkResult, - instantiateMetadata, getRuntimeMetadata, + instantiateMetadata, SyntheticResult, } from '@google-cloud/synthetics-sdk-api'; import { @@ -32,10 +33,19 @@ import { checkLinks, closeBrowser, closePagePool, - retrieveLinksFromPage, openNewPage, + retrieveLinksFromPage, } from './navigation_func'; -import { setDefaultOptions, validateInputOptions } from './options_func'; +import { processOptions } from './options_func'; +import { + createStorageClientIfStorageSelected, + getOrCreateStorageBucket, + StorageParameters, +} from './storage_func'; + +// External Dependencies +import { Bucket } from '@google-cloud/storage'; +import puppeteer, { Browser, Page } from 'puppeteer'; export interface BrokenLinkCheckerOptions { origin_uri: string; @@ -48,6 +58,7 @@ export interface BrokenLinkCheckerOptions { wait_for_selector?: string; per_link_options?: { [key: string]: PerLinkOption }; total_synthetic_timeout_millis?: number; + screenshot_options?: ScreenshotOptions; } export interface PerLinkOption { @@ -70,6 +81,17 @@ export enum StatusClass { STATUS_CLASS_ANY = 'STATUS_CLASS_ANY', } +export interface ScreenshotOptions { + storage_location?: string; + capture_condition?: CaptureCondition; +} + +export enum CaptureCondition { + NONE = 'NONE', + FAILING = 'FAILING', + ALL = 'ALL', +} + let synthetics_sdk_broken_links_package; try { synthetics_sdk_broken_links_package = require('../package.json'); @@ -79,7 +101,11 @@ try { instantiateMetadata(synthetics_sdk_broken_links_package); export async function runBrokenLinks( - inputOptions: BrokenLinkCheckerOptions + inputOptions: BrokenLinkCheckerOptions, + args: { + executionId: string | undefined; + checkId: string | undefined; + } ): Promise { // init const startTime = new Date().toISOString(); @@ -96,6 +122,30 @@ export async function runBrokenLinks( const [timeLimitPromise, timeLimitTimeout, timeLimitresolver] = getTimeLimitPromise(startTime, adjusted_synthetic_timeout_millis); + const errors: BaseError[] = []; + + // Initialize Storage Client with Error Handling. Set to `null` if + // capture_condition is 'None' + const storageClient = createStorageClientIfStorageSelected( + errors, + options.screenshot_options!.capture_condition + ); + + // // Bucket Validation + const bucket: Bucket | null = await getOrCreateStorageBucket( + storageClient, + options.screenshot_options!.storage_location, + errors + ); + + const storageParams: StorageParameters = { + storageClient: storageClient, + bucket: bucket, + checkId: args.checkId || '_', + executionId: args.executionId || '_', + screenshotNumber: 1, + }; + const followed_links: BrokenLinksResultV1_SyntheticLinkResult[] = []; const checkLinksPromise = async () => { @@ -109,7 +159,8 @@ export async function runBrokenLinks( originPage, options, startTime, - adjusted_synthetic_timeout_millis + adjusted_synthetic_timeout_millis, + storageParams ) ); @@ -131,7 +182,8 @@ export async function runBrokenLinks( linksToFollow, options, startTime, - adjusted_synthetic_timeout_millis + adjusted_synthetic_timeout_millis, + storageParams )) ); return true; @@ -149,7 +201,9 @@ export async function runBrokenLinks( startTime, runtime_metadata, options, - followed_links + followed_links, + storageParams, + errors ); } catch (err) { const errorMessage = @@ -176,7 +230,8 @@ async function checkOriginLink( originPage: Page, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, startTime: string, - adjusted_synthetic_timeout_millis: number + adjusted_synthetic_timeout_millis: number, + storageParams: StorageParameters ): Promise { let originLinkResult: BrokenLinksResultV1_SyntheticLinkResult; @@ -193,6 +248,7 @@ async function checkOriginLink( originPage, { target_uri: options.origin_uri, anchor_text: '', html_element: '' }, options, + storageParams, true ); @@ -263,16 +319,3 @@ async function scrapeLinks( options.link_order ); } - -/** - * Validates input options and sets defaults in `options`. - * - * @param inputOptions - The input options for the broken link checker. - * @returns The processed broken link checker options. - */ -function processOptions( - inputOptions: BrokenLinkCheckerOptions -): BrokenLinksResultV1_BrokenLinkCheckerOptions { - const validOptions = validateInputOptions(inputOptions); - return setDefaultOptions(validOptions); -} diff --git a/packages/synthetics-sdk-broken-links/src/handlers.ts b/packages/synthetics-sdk-broken-links/src/handlers.ts index 2f55a619..b45caccd 100644 --- a/packages/synthetics-sdk-broken-links/src/handlers.ts +++ b/packages/synthetics-sdk-broken-links/src/handlers.ts @@ -12,9 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { runBrokenLinks, BrokenLinkCheckerOptions } from './broken_links'; +// Standard Libraries import { Request, Response } from 'express'; +// Internal Project Files +import { runBrokenLinks, BrokenLinkCheckerOptions } from './broken_links'; + +const syntheticExecutionIdHeader = 'Synthetic-Execution-Id'; +const checkIdHeader = 'Check-Id'; + /** * Middleware for easy invocation of SyntheticSDK broken links, and may be used to * register a GoogleCloudFunction http function, or express js compatible handler. @@ -26,5 +32,10 @@ import { Request, Response } from 'express'; export function runBrokenLinksHandler(options: BrokenLinkCheckerOptions) { // eslint-disable-next-line @typescript-eslint/no-explicit-any return async (req: Request, res: Response): Promise => - res.send(await runBrokenLinks(options)); + res.send( + await runBrokenLinks(options, { + executionId: req.get(syntheticExecutionIdHeader), + checkId: req.get(checkIdHeader), + }) + ); } diff --git a/packages/synthetics-sdk-broken-links/src/index.ts b/packages/synthetics-sdk-broken-links/src/index.ts index 775ecc9a..4d895a79 100644 --- a/packages/synthetics-sdk-broken-links/src/index.ts +++ b/packages/synthetics-sdk-broken-links/src/index.ts @@ -13,11 +13,11 @@ // limitations under the License. export { - runBrokenLinks, BrokenLinkCheckerOptions, + LinkOrder, PerLinkOption, + runBrokenLinks, StatusClass, - LinkOrder, } from './broken_links'; export * from './handlers'; export * from '@google-cloud/synthetics-sdk-api'; diff --git a/packages/synthetics-sdk-broken-links/src/link_utils.ts b/packages/synthetics-sdk-broken-links/src/link_utils.ts index fa46dedf..b90412c1 100644 --- a/packages/synthetics-sdk-broken-links/src/link_utils.ts +++ b/packages/synthetics-sdk-broken-links/src/link_utils.ts @@ -12,11 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { HTTPResponse } from 'puppeteer'; +// Standard Libraries +import * as path from 'path'; + +// Internal Project Files import { + BaseError, BrokenLinksResultV1, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, BrokenLinksResultV1_SyntheticLinkResult, GenericResultV1, getRuntimeMetadata, @@ -25,6 +30,10 @@ import { SyntheticResult, } from '@google-cloud/synthetics-sdk-api'; +// External Dependencies +import { HTTPResponse } from 'puppeteer'; +import { StorageParameters } from './storage_func'; + /** * Represents an intermediate link with its properties. */ @@ -153,6 +162,8 @@ function parseFollowedLinks( options: {} as BrokenLinksResultV1_BrokenLinkCheckerOptions, origin_link_result: {} as BrokenLinksResultV1_SyntheticLinkResult, followed_link_results: [], + execution_data_storage_path: '', + errors: [], }; for (const link of followed_links) { @@ -216,12 +227,21 @@ export function createSyntheticResult( start_time: string, runtime_metadata: { [key: string]: string }, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, - followed_links: BrokenLinksResultV1_SyntheticLinkResult[] + followed_links: BrokenLinksResultV1_SyntheticLinkResult[], + storageParams: StorageParameters, + errors: BaseError[] ): SyntheticResult { // Create BrokenLinksResultV1 by parsing followed links and setting options const broken_links_result: BrokenLinksResultV1 = parseFollowedLinks(followed_links); broken_links_result.options = options; + broken_links_result.errors = errors; + broken_links_result.execution_data_storage_path = storageParams.bucket + ? 'gs://' + + storageParams.bucket.name + + '/' + + getStoragePathToExecution(storageParams, options) + : ''; // Create SyntheticResult object const synthetic_result: SyntheticResult = { @@ -264,6 +284,89 @@ export function shuffleAndTruncate( return linksToFollow.slice(0, link_limit! - 1); } +/** + * Determines whether a screenshot should be taken based on screenshot options and link result. + * + * @param options - BrokenLinksResultV1_BrokenLinkCheckerOptions + * @param passed - boolean indicating whether the link navigation succeeded + * @returns true if a screenshot should be taken, false otherwise + */ +export function shouldTakeScreenshot( + options: BrokenLinksResultV1_BrokenLinkCheckerOptions, + passed: boolean +): boolean { + return ( + options.screenshot_options!.capture_condition === ApiCaptureCondition.ALL || + (options.screenshot_options!.capture_condition === + ApiCaptureCondition.FAILING && + !passed) + ); +} + +/** + + * Sanitizes an object name string for safe use, ensuring compliance with + * naming restrictions. + * + * @param {string} inputString - The original object name string. + * @returns {string} The sanitized object name. + * + * **Sanitization Rules:** + * * Removes control characters ([\u007F-\u009F]). + * * Removes disallowed characters (#, [, ], *, ?, ", <, >, |, /). + * * Replaces the forbidden prefix ".well-known/acme-challenge/" with an underscore. + * * Replaces standalone occurrences of "." or ".." with an underscore. + */ +export function sanitizeObjectName( + inputString: string | null | undefined +): string { + if (!inputString || inputString === '.' || inputString === '..') return '_'; + + // Regular expressions for: + /*eslint no-useless-escape: "off"*/ + const invalidCharactersRegex = /[\r\n\u007F-\u009F#\[\]*?:"<>|/]/g; // Control characters, special characters, path separator + const wellKnownPrefixRegex = /^\.well-known\/acme-challenge\//; + + // Core sanitization: + return inputString + .replace(wellKnownPrefixRegex, '_') // Replace forbidden prefix + .replace(invalidCharactersRegex, '_') // replace invalid characters + .trim() // Clean up any leading/trailing spaces + .replace(/\s+/g, '_'); // Replace one or more spaces with underscores +} + +export function getStoragePathToExecution( + storageParams: StorageParameters, + options: BrokenLinksResultV1_BrokenLinkCheckerOptions +) { + try { + const storageLocation = options.screenshot_options!.storage_location; + let writeDestination = ''; + + // extract folder name for a given storage location. If there is no '/' + // present then the storageLocation is just a folder + const firstSlashIndex = storageLocation.indexOf('/'); + if (firstSlashIndex !== -1) { + writeDestination = storageLocation.substring(firstSlashIndex + 1); + } + + // Ensure writeDestination ends with a slash for proper path joining + if (writeDestination && !writeDestination.endsWith('/')) { + writeDestination += '/'; + } + + writeDestination = path.join( + writeDestination, + storageParams.checkId, + storageParams.executionId + ); + + return writeDestination; + } catch (err) { + return ''; + } +} + export function getTimeLimitPromise( startTime: string, totalTimeoutMillis: number, diff --git a/packages/synthetics-sdk-broken-links/src/navigation_func.ts b/packages/synthetics-sdk-broken-links/src/navigation_func.ts index 030b705e..dcca8fd9 100644 --- a/packages/synthetics-sdk-broken-links/src/navigation_func.ts +++ b/packages/synthetics-sdk-broken-links/src/navigation_func.ts @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Browser, HTTPResponse, Page } from 'puppeteer'; +// Internal Project Files import { + BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_SyntheticLinkResult, ResponseStatusCode, ResponseStatusCode_StatusClass, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, } from '@google-cloud/synthetics-sdk-api'; import { checkStatusPassing, @@ -26,7 +28,12 @@ import { LinkIntermediate, NavigateResponse, getTimeLimitPromise, + shouldTakeScreenshot, } from './link_utils'; +import { StorageParameters, uploadScreenshotToGCS } from './storage_func'; + +// External Dependencies +import { Browser, HTTPResponse, Page } from 'puppeteer'; /** * Retrieves all links on the page using Puppeteer, handling relative and @@ -100,7 +107,8 @@ export async function checkLinks( links: LinkIntermediate[], options: BrokenLinksResultV1_BrokenLinkCheckerOptions, startTime: string, - total_timeout_millis: number + total_timeout_millis: number, + storageParams: StorageParameters ): Promise { let timeLimitReached = false; const followed_links: BrokenLinksResultV1_SyntheticLinkResult[] = []; @@ -116,7 +124,9 @@ export async function checkLinks( if (timeLimitReached) return false; try { - followed_links.push(await checkLink(page, link, options)); + followed_links.push( + await checkLink(page, link, options, storageParams) + ); /** In the case of a single page app, network requests can hang and cause * timeout issues in following links. To ensure this does not happen we * need to reset the page in between every link checked @@ -161,6 +171,7 @@ export async function checkLink( page: Page, link: LinkIntermediate, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, + storageParams: StorageParameters, isOrigin = false ): Promise { // Determine the expected status code for the link, using per-link setting if @@ -181,6 +192,18 @@ export async function checkLink( linkEndTime, } = await navigate(page, link, options, expectedStatusCode); + let screenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + if (shouldTakeScreenshot(options, passed)) { + screenshotOutput = await uploadScreenshotToGCS( + page, + storageParams, + options + ); + } + // Initialize variables for error information let errorType = ''; let errorMessage = ''; @@ -220,6 +243,7 @@ export async function checkLink( link_start_time: linkStartTime, link_end_time: linkEndTime, is_origin: isOrigin, + screenshot_output: screenshotOutput, }; } diff --git a/packages/synthetics-sdk-broken-links/src/options_func.ts b/packages/synthetics-sdk-broken-links/src/options_func.ts index 72b7d620..017eb2b2 100644 --- a/packages/synthetics-sdk-broken-links/src/options_func.ts +++ b/packages/synthetics-sdk-broken-links/src/options_func.ts @@ -12,18 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Internal Project Files +import { + BrokenLinkCheckerOptions, + CaptureCondition, + LinkOrder, + StatusClass, +} from './broken_links'; import { BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, BrokenLinksResultV1_BrokenLinkCheckerOptions_PerLinkOption, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, ResponseStatusCode, ResponseStatusCode_StatusClass, } from '@google-cloud/synthetics-sdk-api'; -import { - BrokenLinkCheckerOptions, - LinkOrder, - StatusClass, -} from './broken_links'; + +/** + * Validates input options and sets defaults in `options`. + * + * @param inputOptions - The input options for the broken link checker. + * @returns The processed broken link checker options. + */ +export function processOptions( + inputOptions: BrokenLinkCheckerOptions +): BrokenLinksResultV1_BrokenLinkCheckerOptions { + const validOptions = validateInputOptions(inputOptions); + return setDefaultOptions(validOptions); +} /** * Validates the input options for the Broken Link Checker. @@ -130,6 +147,26 @@ export function validateInputOptions( ); } + // Check storage_location + if ( + inputOptions.screenshot_options?.storage_location !== undefined && + typeof inputOptions.screenshot_options?.storage_location !== 'string' + ) { + throw new Error('Invalid storage_location value, must be a string'); + } + + // check storage_condition + if ( + inputOptions.screenshot_options?.capture_condition !== undefined && + !Object.values(CaptureCondition).includes( + inputOptions.screenshot_options?.capture_condition + ) + ) { + throw new Error( + 'Invalid capture_condition value, must be `ALL`, `FAILING`, OR `NONE`' + ); + } + // per_link_options for (const [key, value] of Object.entries( inputOptions.per_link_options || {} @@ -180,6 +217,10 @@ export function validateInputOptions( wait_for_selector: inputOptions.wait_for_selector, per_link_options: inputOptions.per_link_options, total_synthetic_timeout_millis: inputOptions.total_synthetic_timeout_millis, + screenshot_options: { + capture_condition: inputOptions.screenshot_options?.capture_condition, + storage_location: inputOptions.screenshot_options?.storage_location, + }, }; } @@ -192,7 +233,7 @@ export function validateInputOptions( export function setDefaultOptions( inputOptions: BrokenLinkCheckerOptions ): BrokenLinksResultV1_BrokenLinkCheckerOptions { - const defaulOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions = { + const defaultOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions = { origin_uri: '', link_limit: 10, query_selector_all: 'a', @@ -203,17 +244,25 @@ export function setDefaultOptions( wait_for_selector: '', per_link_options: {}, total_synthetic_timeout_millis: 60000, + screenshot_options: { + capture_condition: ApiCaptureCondition.NONE, + storage_location: '', + }, }; const outputOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions = {} as BrokenLinksResultV1_BrokenLinkCheckerOptions; - const optionsKeys = Object.keys(defaulOptions) as Array< + const optionsKeys = Object.keys(defaultOptions) as Array< keyof BrokenLinksResultV1_BrokenLinkCheckerOptions >; for (const optionName of optionsKeys) { // per_link_options and linkorder are handled below - if (optionName === 'per_link_options' || optionName === 'link_order') + if ( + optionName === 'per_link_options' || + optionName === 'link_order' || + optionName === 'screenshot_options' + ) continue; if ( @@ -222,13 +271,33 @@ export function setDefaultOptions( (inputOptions as any)[optionName] === undefined ) { // eslint-disable-next-line @typescript-eslint/no-explicit-any - (outputOptions as any)[optionName] = defaulOptions[optionName]; + (outputOptions as any)[optionName] = defaultOptions[optionName]; } else { // eslint-disable-next-line @typescript-eslint/no-explicit-any (outputOptions as any)[optionName] = inputOptions[optionName]; } } + // converting inputOptions.screenshot_options to + // BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions + outputOptions.screenshot_options = + {} as BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions; + if (inputOptions.screenshot_options?.capture_condition) { + outputOptions.screenshot_options!.capture_condition = + ApiCaptureCondition[inputOptions.screenshot_options.capture_condition]; + } else { + outputOptions.screenshot_options!.capture_condition = + defaultOptions.screenshot_options!.capture_condition; + } + + if (inputOptions.screenshot_options?.storage_location) { + outputOptions.screenshot_options.storage_location = + inputOptions.screenshot_options!.storage_location!; + } else { + outputOptions.screenshot_options.storage_location = + defaultOptions.screenshot_options!.storage_location!; + } + // converting inputOptions.link_order, type: LinkOrder to // outputOptions.link_order, type BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder if (inputOptions.link_order) { @@ -237,7 +306,7 @@ export function setDefaultOptions( inputOptions.link_order ]; } else { - outputOptions.link_order = defaulOptions.link_order; + outputOptions.link_order = defaultOptions.link_order; } // Convert `inputOptions.per_link_options`, type: {[key: string]: PerLinkOption} diff --git a/packages/synthetics-sdk-broken-links/src/storage_func.ts b/packages/synthetics-sdk-broken-links/src/storage_func.ts new file mode 100644 index 00000000..524919be --- /dev/null +++ b/packages/synthetics-sdk-broken-links/src/storage_func.ts @@ -0,0 +1,192 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Standard Libraries +import * as path from 'path'; + +// Internal Project Files +import { + BaseError, + BrokenLinksResultV1_BrokenLinkCheckerOptions, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, + getExecutionRegion, + resolveProjectId, +} from '@google-cloud/synthetics-sdk-api'; +import { getStoragePathToExecution, sanitizeObjectName } from './link_utils'; + +// External Dependencies +import { Storage, Bucket } from '@google-cloud/storage'; +import { Page } from 'puppeteer'; + +export interface StorageParameters { + storageClient: Storage | null; + bucket: Bucket | null; + checkId: string; + executionId: string; + screenshotNumber: number; +} + +/** + * Attempts to get an existing storage bucket if provided by the user OR + * create/use a dedicated synthetics bucket. + * Handles various errors gracefully, providing structured details in the `errors` array. + * + * @param storageClient - An initialized Storage client from the + * '@google-cloud/storage' SDK. + * @param storageLocation - The desired storage location (bucket or folder) + * provided by the user. Can be empty. + * @param errors - An array to accumulate potential errors of type `BaseError`. + * @returns A 'Bucket' object if successful, or null if errors occurred. + */ +export async function getOrCreateStorageBucket( + storageClient: Storage | null, + storageLocation: string, + errors: BaseError[] +): Promise { + let bucketName = ''; + + try { + if (!storageClient) return null; + + const projectId = sanitizeObjectName(await resolveProjectId()); + const region = sanitizeObjectName(await getExecutionRegion()); + + // if the user chose to use/create the default bucket but we were not able + // to resolve projectId or cloudRegion + if (!storageLocation && (!projectId || !region)) return null; + + bucketName = storageLocation + ? storageLocation.split('/')[0] + : `gcm-${projectId}-synthetics-${region}`; + + const bucket = storageClient.bucket(bucketName); + const [bucketExists] = await bucket.exists(); + + if (bucketExists) { + return bucket; // Bucket exists, return it + } else if (!storageLocation) { + // Create only if no location was provided + const [newBucket] = await bucket.create({ + location: region, + storageClass: 'STANDARD', + }); + return newBucket; + } else { + // User-provided invalid location + errors.push({ + error_type: 'InvalidStorageLocation', + error_message: `Invalid storage_location: Bucket ${bucketName} does not exist.`, + }); + } + } catch (err) { + const errorType = storageLocation + ? 'StorageValidationError' + : 'BucketCreationError'; + + // Using console.error rather than stderr.write since err type is unknown + console.error(errorType, err); + + errors.push({ + // General error handling + error_type: errorType, + error_message: `Failed to ${ + storageLocation ? 'validate' : 'create' + } bucket ${bucketName}. Please reference server logs for further information.`, + }); + } + + return null; // Return null if bucket retrieval or creation failed +} + +/** + * Initializes a Google Cloud Storage client, if storage is selected. Handles + * both expected and unexpected errors during initialization. + * + * @param errors - An array to accumulate potential errors of type `BaseError`. + * @returns A Storage client object if successful, or null if errors occurred. + */ +export function createStorageClientIfStorageSelected( + errors: BaseError[], + captureCondition: ApiCaptureCondition +): Storage | null { + if (captureCondition === ApiCaptureCondition.NONE) return null; + + try { + return new Storage(); + } catch (err) { + console.error('StorageClientInitializationError', err); + + errors.push({ + error_type: 'StorageClientInitializationError', + error_message: + 'Failed to initialize Storage client. Please reference server logs for further information.', + }); + return null; + } +} + +/** + * Uploads a screenshot to Google Cloud Storage. + * + * @param screenshot - Base64-encoded screenshot data. + * @param filename - Desired filename for the screenshot. + * @param storageParams - An object containing storageClient and bucket. + * @param options - Broken links checker options. + * @returns An ApiScreenshotOutput object indicating success or a screenshot_error. + */ +export async function uploadScreenshotToGCS( + page: Page, + storageParams: StorageParameters, + options: BrokenLinksResultV1_BrokenLinkCheckerOptions +): Promise { + const screenshot_output: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + try { + // Early exit if storage is not properly configured + if (!storageParams.storageClient || !storageParams.bucket) { + return screenshot_output; + } + + const screenshot: Buffer = await page.screenshot({ + fullPage: true, + encoding: 'binary', + }); + const filename = 'screenshot_' + storageParams.screenshotNumber + '.png'; + + const writeDestination = path.join( + getStoragePathToExecution(storageParams, options), + filename + ); + + // Upload to GCS + await storageParams.bucket.file(writeDestination).save(screenshot, { + contentType: 'image/png', + }); + + storageParams.screenshotNumber += 1; + screenshot_output.screenshot_file = filename; + } catch (err) { + console.error('ScreenshotFileUploadError', err); + + screenshot_output.screenshot_error = { + error_type: 'ScreenshotFileUploadError', + error_message: `Failed to take and/or upload screenshot for ${await page.url()}. Please reference server logs for further information.`, + }; + } + + return screenshot_output; +} diff --git a/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js b/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js index 8ae8bd4b..5e573891 100644 --- a/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js +++ b/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js @@ -12,56 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -const functions = require('@google-cloud/functions-framework'); -const SyntheticsSdkBrokenLinks = require('synthetics-sdk-broken-links'); +// Standard Libraries const path = require('path'); +// Internal Project Files +const SyntheticsSdkBrokenLinks = require('synthetics-sdk-broken-links'); + +// External Dependencies +const functions = require('@google-cloud/functions-framework'); + /* * This is the server template that is required to run a synthetic monitor in * Google Cloud Functions. */ -// Handles error when trying to visit page that does not exist -functions.http('BrokenLinksPageDoesNotExist', SyntheticsSdkBrokenLinks.runBrokenLinksHandler({ - origin_uri: `file:${path.join( - __dirname, - '../example_html_files/file_doesnt_exist.html' - )}` -})); - // Visits and checks empty page with no links functions.http('BrokenLinksEmptyPageOk', SyntheticsSdkBrokenLinks.runBrokenLinksHandler({ origin_uri: `file:${path.join( __dirname, '../example_html_files/200.html' - )}` -})); - -// Exits early when options cannot be parsed -functions.http('BrokenLinksInvalidOptionsNotOk', SyntheticsSdkBrokenLinks.runBrokenLinksHandler({ - origin_uri: `file:${path.join( - __dirname, - '../example_html_files/retrieve_links_test.html' - )}`, - link_order: 'incorrect' -})); - -// Completes full failing execution -functions.http('BrokenLinksFailingOk', SyntheticsSdkBrokenLinks.runBrokenLinksHandler({ - origin_uri: `file:${path.join( - __dirname, - '../example_html_files/retrieve_links_test.html' - )}`, - query_selector_all: 'a[src], img[href]', - get_attributes: ['href', 'src'] -})); - -// Completes full passing execution -functions.http('BrokenLinksPassingOk', SyntheticsSdkBrokenLinks.runBrokenLinksHandler({ - origin_uri: `file:${path.join( - __dirname, - '../example_html_files/retrieve_links_test.html' )}`, - query_selector_all: 'a[src]', - get_attributes: ['src'] + screenshot_options: { + capture_condition: 'NONE' + } })); diff --git a/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts b/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts index bdc873e2..93a24fff 100644 --- a/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts @@ -12,16 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Standard Libraries +import { expect } from 'chai'; +import supertest from 'supertest'; +const path = require('path'); + +// Internal Project Files import { - ResponseStatusCode_StatusClass, + BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput, + ResponseStatusCode_StatusClass, SyntheticResult, } from '@google-cloud/synthetics-sdk-api'; -import { expect } from 'chai'; -import supertest from 'supertest'; -const path = require('path'); - +// External Dependencies require('../../test/example_html_files/integration_server.js'); const { getTestServer } = require('@google-cloud/functions-framework/testing'); @@ -29,65 +36,17 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { const status_class_2xx = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, }; - - it('Handles error when trying to visit page that does not exist', async () => { - const server = getTestServer('BrokenLinksPageDoesNotExist'); - - // invoke SyntheticBrokenLinks with SuperTest - const response = await supertest(server) - .get('/') - .send() - .set('Content-Type', 'application/json') - .expect(200); - - const output: SyntheticResult = response.body as SyntheticResult; - const start_time = output.start_time; - const end_time = output.end_time; - const broken_links_result = output?.synthetic_broken_links_result_v1; - const origin_link = broken_links_result?.origin_link_result; - const followed_links = broken_links_result?.followed_link_results; - const runtime_metadata = output?.runtime_metadata; - - expect(start_time).to.be.a.string; - expect(end_time).to.be.a.string; - - expect(broken_links_result?.link_count).to.equal(1); - expect(broken_links_result?.passing_link_count).to.equal(0); - expect(broken_links_result?.failing_link_count).to.equal(1); - expect(broken_links_result?.unreachable_count).to.equal(1); - expect(broken_links_result?.status2xx_count).to.equal(0); - expect(broken_links_result?.status3xx_count).to.equal(0); - expect(broken_links_result?.status4xx_count).to.equal(0); - expect(broken_links_result?.status5xx_count).to.equal(0); - - const origin_uri = `file:${path.join( - __dirname, - '../example_html_files/file_doesnt_exist.html' - )}`; - - expect(origin_link) - .excluding(['link_start_time', 'link_end_time']) - .to.deep.equal({ - link_passed: false, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: origin_uri, - html_element: '', - anchor_text: '', - error_type: 'Error', - error_message: 'net::ERR_FILE_NOT_FOUND at ' + origin_uri, - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: true, - }); - - expect(followed_links).to.deep.equal([]); - - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-api']).to.not.be - .undefined; - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-broken-links']).to - .not.be.undefined; - }).timeout(10000); + const noneCaptureScreenshotOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions = + { + capture_condition: ApiCaptureCondition.NONE, + storage_location: '', + }; + + const defaultScreenshotOutput: BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput = + { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; it('Visits and checks empty page with no links', async () => { const server = getTestServer('BrokenLinksEmptyPageOk'); @@ -137,6 +96,7 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { wait_for_selector: '', per_link_options: {}, total_synthetic_timeout_millis: 60000, + screenshot_options: noneCaptureScreenshotOptions, }); expect(origin_link) @@ -154,6 +114,7 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, + screenshot_output: defaultScreenshotOutput, }); expect(followed_links).to.deep.equal([]); @@ -163,256 +124,4 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { expect(runtime_metadata?.['@google-cloud/synthetics-sdk-broken-links']).to .not.be.undefined; }).timeout(10000); - - it('Exits early with generic_result when options cannot be parsed', async () => { - const server = getTestServer('BrokenLinksInvalidOptionsNotOk'); - - // invoke SyntheticBrokenLinks with SuperTest - const response = await supertest(server) - .get('/') - .send() - .set('Content-Type', 'application/json') - .expect(200); - - const output: SyntheticResult = response.body as SyntheticResult; - const start_time = output.start_time; - const end_time = output.end_time; - const synthetic_generic_result = output?.synthetic_generic_result_v1; - const runtime_metadata = output?.runtime_metadata; - - expect(synthetic_generic_result?.ok).to.be.false; - expect(synthetic_generic_result?.generic_error?.error_type).to.equal( - 'Error' - ); - expect(synthetic_generic_result?.generic_error?.error_message).to.equal( - 'Invalid link_order value, must be `FIRST_N` or `RANDOM`' - ); - expect(start_time).to.be.a.string; - expect(end_time).to.be.a.string; - - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-api']).to.not.be - .undefined; - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-broken-links']).to - .not.be.undefined; - }).timeout(10000); - - it('Runs a failing Broken Links suite', async () => { - const server = getTestServer('BrokenLinksFailingOk'); - - // invoke SyntheticBrokenLinks with SuperTest - const response = await supertest(server) - .get('/') - .send() - .set('Content-Type', 'application/json') - .expect(200); - - const origin_uri = `file:${path.join( - __dirname, - '../example_html_files/retrieve_links_test.html' - )}`; - - const output: SyntheticResult = response.body as SyntheticResult; - const start_time = output.start_time; - const end_time = output.end_time; - const broken_links_result = output?.synthetic_broken_links_result_v1; - const options = broken_links_result?.options; - const origin_link = broken_links_result?.origin_link_result; - const followed_links = broken_links_result?.followed_link_results; - const runtime_metadata = output?.runtime_metadata; - - expect(start_time).to.be.a.string; - expect(end_time).to.be.a.string; - - expect(broken_links_result?.link_count).to.equal(3); - expect(broken_links_result?.passing_link_count).to.equal(2); - expect(broken_links_result?.failing_link_count).to.equal(1); - expect(broken_links_result?.unreachable_count).to.equal(1); - expect(broken_links_result?.status2xx_count).to.equal(2); - expect(broken_links_result?.status3xx_count).to.equal(0); - expect(broken_links_result?.status4xx_count).to.equal(0); - expect(broken_links_result?.status5xx_count).to.equal(0); - - expect(options).to.deep.equal({ - origin_uri: origin_uri, - link_limit: 10, - query_selector_all: 'a[src], img[href]', - get_attributes: ['href', 'src'], - link_order: - BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder.FIRST_N, - link_timeout_millis: 30000, - max_retries: 0, - wait_for_selector: '', - per_link_options: {}, - total_synthetic_timeout_millis: 60000, - }); - - expect(origin_link) - .excluding(['link_start_time', 'link_end_time']) - .to.deep.equal({ - link_passed: true, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: origin_uri, - html_element: '', - anchor_text: '', - status_code: 200, - error_type: '', - error_message: '', - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: true, - }); - - const sorted_followed_links = followed_links?.sort((a, b) => - a.target_uri.localeCompare(b.target_uri) - ); - - const doesnt_exist_path = `file://${path.join( - __dirname, - '../example_html_files/file_doesnt_exist.html' - )}` - .split(' ') - .join('%20'); - expect(sorted_followed_links) - .excluding(['target_uri', 'link_start_time', 'link_end_time']) - .to.deep.equal([ - { - link_passed: true, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: 'CHECKED_BELOW', - html_element: 'a', - anchor_text: 'External Link', - status_code: 200, - error_type: '', - error_message: '', - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: false, - }, - { - link_passed: false, - expected_status_code: { status_class: 200 }, - source_uri: origin_uri, - target_uri: 'CHECKED_BELOW', - html_element: 'img', - anchor_text: '', - error_type: 'Error', - error_message: 'net::ERR_FILE_NOT_FOUND at ' + doesnt_exist_path, - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: false, - }, - ]); - - const expectedTargetPaths = [ - 'example_html_files/200.html', - 'example_html_files/file_doesnt_exist.html', - ]; - followed_links?.forEach((link, index) => { - expect(link.target_uri.endsWith(expectedTargetPaths[index])); - }); - - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-api']).to.not.be - .undefined; - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-broken-links']).to - .not.be.undefined; - }).timeout(10000); - - it('Runs a passing Broken Links suite', async () => { - const server = getTestServer('BrokenLinksPassingOk'); - - // invoke SyntheticBrokenLinks with SuperTest - const response = await supertest(server) - .get('/') - .send() - .set('Content-Type', 'application/json') - .expect(200); - - const origin_uri = `file:${path.join( - __dirname, - '../example_html_files/retrieve_links_test.html' - )}`; - - const output: SyntheticResult = response.body as SyntheticResult; - const start_time = output.start_time; - const end_time = output.end_time; - const broken_links_result = output?.synthetic_broken_links_result_v1; - const options = broken_links_result?.options; - const origin_link = broken_links_result?.origin_link_result; - const followed_links = broken_links_result?.followed_link_results; - const runtime_metadata = output?.runtime_metadata; - - expect(start_time).to.be.a.string; - expect(end_time).to.be.a.string; - - expect(broken_links_result?.link_count).to.equal(2); - expect(broken_links_result?.passing_link_count).to.equal(2); - expect(broken_links_result?.failing_link_count).to.equal(0); - expect(broken_links_result?.unreachable_count).to.equal(0); - expect(broken_links_result?.status2xx_count).to.equal(2); - expect(broken_links_result?.status3xx_count).to.equal(0); - expect(broken_links_result?.status4xx_count).to.equal(0); - expect(broken_links_result?.status5xx_count).to.equal(0); - - expect(options).to.deep.equal({ - origin_uri: origin_uri, - link_limit: 10, - query_selector_all: 'a[src]', - get_attributes: ['src'], - link_order: - BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder.FIRST_N, - link_timeout_millis: 30000, - max_retries: 0, - wait_for_selector: '', - per_link_options: {}, - total_synthetic_timeout_millis: 60000, - }); - - expect(origin_link) - .excluding(['link_start_time', 'link_end_time']) - .to.deep.equal({ - link_passed: true, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: origin_uri, - html_element: '', - anchor_text: '', - status_code: 200, - error_type: '', - error_message: '', - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: true, - }); - - expect(followed_links) - .excluding(['target_uri', 'link_start_time', 'link_end_time']) - .to.deep.equal([ - { - link_passed: true, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: 'CHECKED_BELOW', - html_element: 'a', - anchor_text: 'External Link', - status_code: 200, - error_type: '', - error_message: '', - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: false, - }, - ]); - - const expectedTargetPaths = ['example_html_files/200.html']; - followed_links?.forEach((link, index) => { - expect(link.target_uri.endsWith(expectedTargetPaths[index])); - }); - - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-api']).to.not.be - .undefined; - expect(runtime_metadata?.['@google-cloud/synthetics-sdk-broken-links']).to - .not.be.undefined; - }).timeout(10000); }); diff --git a/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts index edb18d6c..0599d4fd 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts @@ -11,36 +11,103 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +// Standard Libraries import { expect, use } from 'chai'; import chaiExclude from 'chai-exclude'; use(chaiExclude); +const path = require('path'); +import sinon from 'sinon'; +// Internal Project Files import { - BrokenLinksResultV1_BrokenLinkCheckerOptions, + BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, BrokenLinksResultV1_SyntheticLinkResult, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, ResponseStatusCode, ResponseStatusCode_StatusClass, } from '@google-cloud/synthetics-sdk-api'; import { runBrokenLinks, BrokenLinkCheckerOptions, + CaptureCondition, } from '../../src/broken_links'; -const path = require('path'); + +// External Dependencies +const proxyquire = require('proxyquire'); +import { Page } from 'puppeteer'; +import { Bucket, Storage } from '@google-cloud/storage'; + +const TEST_BUCKET_NAME = 'gcm-test-project-id-synthetics-test-region'; describe('runBrokenLinks', async () => { const status_class_2xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, }; + const defaultScreenshotOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions = + { + capture_condition: ApiCaptureCondition.FAILING, + storage_location: '', + }; + const emptyScreenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + const successfulScreenshotOuput: ApiScreenshotOutput = { + screenshot_file: 'bucket/folder/file.png', + screenshot_error: {} as BaseError, + }; + const args = { checkId: 'test-check-id', executionId: 'test-execution-id' }; + + const mockedstorageFunc = proxyquire('../../src/storage_func', { + '@google-cloud/synthetics-sdk-api': { + getExecutionRegion: () => 'test-region', + resolveProjectId: () => 'test-project-id', + }, + }); - it('returns generic_result with appropriate error information if error thrown', async () => { + const mockedNavigationFunc = proxyquire('../../src/navigation_func', { + './storage_func': { + uploadScreenshotToGCS: () => successfulScreenshotOuput, + }, + }); + + let storageClientStub: sinon.SinonStubbedInstance; + let bucketStub: sinon.SinonStubbedInstance; + let pageStub: sinon.SinonStubbedInstance; + beforeEach(() => { + // Stub a storage bucket + bucketStub = sinon.createStubInstance(Bucket); + bucketStub.name = TEST_BUCKET_NAME; + bucketStub.create.resolves([bucketStub]); + // Simulate default_bucket not existing initially + bucketStub.exists.resolves([false]); // Simulate the bucket not existing initially + + // Stub the storage client + storageClientStub = sinon.createStubInstance(Storage); + storageClientStub.bucket.returns(bucketStub); + + // Stub a puppeteer page to return set Buffer when .screenshot() called + pageStub = sinon.createStubInstance(Page); + pageStub.screenshot.resolves(Buffer.from('screenshot-image-data', 'utf-8')); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('Exits early when options cannot be parsed', async () => { const inputOptions: BrokenLinkCheckerOptions = { origin_uri: 'uri-does-not-start-with-http', }; - const syntheticResult = await runBrokenLinks(inputOptions); + const result = await runBrokenLinks(inputOptions, args); + + const genericResult = result.synthetic_generic_result_v1; - const genericResult = syntheticResult.synthetic_generic_result_v1; expect(genericResult).to.be.exist; expect(genericResult?.ok).to.be.false; expect(genericResult?.generic_error?.error_type).to.equal('Error'); @@ -50,6 +117,16 @@ describe('runBrokenLinks', async () => { }).timeout(15000); it('returns broken_links_result with origin link failure when waitForSelector exceeds deadline', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -58,12 +135,12 @@ describe('runBrokenLinks', async () => { origin_uri: origin_uri, wait_for_selector: 'not_present', link_timeout_millis: 3001, + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const syntheticResult = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); - const broken_links_result = - syntheticResult?.synthetic_broken_links_result_v1; + const broken_links_result = result?.synthetic_broken_links_result_v1; const origin_link = broken_links_result?.origin_link_result; expect(broken_links_result?.followed_link_results).to.be.empty; @@ -75,6 +152,16 @@ describe('runBrokenLinks', async () => { }).timeout(15000); it('Global timeout occurs during checkOriginLink waiting for `wait_for_selector', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -86,8 +173,9 @@ describe('runBrokenLinks', async () => { wait_for_selector: 'none existent', link_timeout_millis: 35000, total_synthetic_timeout_millis: 31000, + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const result = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); const broken_links_result = result.synthetic_broken_links_result_v1; const expectedOriginLinkResult: BrokenLinksResultV1_SyntheticLinkResult = { @@ -104,6 +192,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, + screenshot_output: emptyScreenshotOutput, }; expect(broken_links_result?.origin_link_result) @@ -112,7 +201,66 @@ describe('runBrokenLinks', async () => { expect(broken_links_result?.followed_link_results.length).to.equal(0); }).timeout(40000); - it('successful execution with 1 failing link', async () => { + it('Handles error when trying to visit page that does not exist', async () => { + const origin_uri = `file:${path.join( + __dirname, + '../example_html_files/file_doesnt_exist.html' + )}`; + const inputOptions: BrokenLinkCheckerOptions = { + origin_uri: origin_uri, + screenshot_options: { capture_condition: CaptureCondition.NONE }, + }; + + const result = await runBrokenLinks(inputOptions, args); + + const broken_links_result = result?.synthetic_broken_links_result_v1; + const origin_link = broken_links_result?.origin_link_result; + const followed_links = broken_links_result?.followed_link_results; + + expect(result.start_time).to.be.a.string; + expect(result.end_time).to.be.a.string; + + expect(broken_links_result?.link_count).to.equal(1); + expect(broken_links_result?.passing_link_count).to.equal(0); + expect(broken_links_result?.failing_link_count).to.equal(1); + expect(broken_links_result?.unreachable_count).to.equal(1); + expect(broken_links_result?.status2xx_count).to.equal(0); + expect(broken_links_result?.status3xx_count).to.equal(0); + expect(broken_links_result?.status4xx_count).to.equal(0); + expect(broken_links_result?.status5xx_count).to.equal(0); + + expect(origin_link) + .excluding(['link_start_time', 'link_end_time']) + .to.deep.equal({ + link_passed: false, + expected_status_code: status_class_2xx, + source_uri: origin_uri, + target_uri: origin_uri, + html_element: '', + anchor_text: '', + status_code: undefined, + error_type: 'Error', + error_message: 'net::ERR_FILE_NOT_FOUND at ' + origin_uri, + link_start_time: 'NA', + link_end_time: 'NA', + is_origin: true, + screenshot_output: emptyScreenshotOutput, + }); + + expect(followed_links).to.deep.equal([]); + }).timeout(10000); + + it('Completes a full failing execution (1 failing link)', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -122,11 +270,31 @@ describe('runBrokenLinks', async () => { query_selector_all: 'a[src], img[href]', get_attributes: ['href', 'src'], wait_for_selector: '', + screenshot_options: { + capture_condition: CaptureCondition.FAILING, + }, }; - const result = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); + + const broken_links_result = result?.synthetic_broken_links_result_v1; + const options = broken_links_result?.options; + const origin_link = broken_links_result?.origin_link_result; + const followed_links = broken_links_result?.followed_link_results; + + expect(result.start_time).to.be.a.string; + expect(result.end_time).to.be.a.string; + + expect(broken_links_result?.link_count).to.equal(3); + expect(broken_links_result?.passing_link_count).to.equal(2); + expect(broken_links_result?.failing_link_count).to.equal(1); + expect(broken_links_result?.unreachable_count).to.equal(1); + expect(broken_links_result?.status2xx_count).to.equal(2); + expect(broken_links_result?.status3xx_count).to.equal(0); + expect(broken_links_result?.status4xx_count).to.equal(0); + expect(broken_links_result?.status5xx_count).to.equal(0); - const expectedOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions = { + expect(options).to.deep.equal({ origin_uri: origin_uri, link_limit: 10, query_selector_all: 'a[src], img[href]', @@ -138,31 +306,44 @@ describe('runBrokenLinks', async () => { wait_for_selector: '', per_link_options: {}, total_synthetic_timeout_millis: 60000, - }; + screenshot_options: defaultScreenshotOptions, + }); - const expectedOriginLinkResult: BrokenLinksResultV1_SyntheticLinkResult = { - link_passed: true, - expected_status_code: status_class_2xx, - source_uri: origin_uri, - target_uri: origin_uri, - html_element: '', - anchor_text: '', - status_code: 200, - error_type: '', - error_message: '', - link_start_time: 'NA', - link_end_time: 'NA', - is_origin: true, - }; + expect(origin_link) + .excluding(['link_start_time', 'link_end_time']) + .to.deep.equal({ + link_passed: true, + expected_status_code: status_class_2xx, + source_uri: origin_uri, + target_uri: origin_uri, + html_element: '', + anchor_text: '', + status_code: 200, + error_type: '', + error_message: '', + link_start_time: 'NA', + link_end_time: 'NA', + is_origin: true, + screenshot_output: emptyScreenshotOutput, + }); + + const sorted_followed_links = followed_links?.sort( + ( + a: BrokenLinksResultV1_SyntheticLinkResult, + b: BrokenLinksResultV1_SyntheticLinkResult + ) => a.target_uri.localeCompare(b.target_uri) + ); - const file_doesnt_exist_path = `file://${path.join( + const fileDoesntExistPath = `file://${path.join( __dirname, '../example_html_files/file_doesnt_exist.html' )}` .split(' ') .join('%20'); - const expectedFollowedLinksResults: BrokenLinksResultV1_SyntheticLinkResult[] = - [ + + expect(sorted_followed_links) + .excluding(['target_uri', 'link_start_time', 'link_end_time']) + .to.deep.equal([ { link_passed: true, expected_status_code: status_class_2xx, @@ -176,56 +357,127 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: emptyScreenshotOutput, }, { link_passed: false, - expected_status_code: status_class_2xx, + expected_status_code: { status_class: 200 }, source_uri: origin_uri, target_uri: 'CHECKED_BELOW', html_element: 'img', anchor_text: '', status_code: undefined, error_type: 'Error', - error_message: 'net::ERR_FILE_NOT_FOUND at ' + file_doesnt_exist_path, + error_message: 'net::ERR_FILE_NOT_FOUND at ' + fileDoesntExistPath, link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: successfulScreenshotOuput, }, - ]; + ]); - const broken_links_result = result.synthetic_broken_links_result_v1; - expect(broken_links_result?.options) - .excluding(['link_start_time', 'link_end_time']) - .to.deep.equal(expectedOptions); + // these are checked separately because path and puppeteer format space differently which cause the equality assertion to fail + const expectedTargeturis = [ + '/example_html_files/200.html', + '/example_html_files/file_doesnt_exist.html', + ]; + broken_links_result?.followed_link_results?.forEach( + (link: BrokenLinksResultV1_SyntheticLinkResult, index: number) => { + expect(link.target_uri.endsWith(expectedTargeturis[index])); + } + ); + }).timeout(150000); - expect(broken_links_result?.link_count).to.equal(3); + it('Completes a full passing execution', async () => { + const origin_uri = `file:${path.join( + __dirname, + '../example_html_files/retrieve_links_test.html' + )}`; + const inputOptions: BrokenLinkCheckerOptions = { + origin_uri: origin_uri, + query_selector_all: 'a[src]', + get_attributes: ['src'], + screenshot_options: { capture_condition: CaptureCondition.NONE }, + }; + + const result = await runBrokenLinks(inputOptions, args); + + const broken_links_result = result?.synthetic_broken_links_result_v1; + const options = broken_links_result?.options; + const origin_link = broken_links_result?.origin_link_result; + const followed_links = broken_links_result?.followed_link_results; + + expect(result.start_time).to.be.a.string; + expect(result.end_time).to.be.a.string; + + expect(broken_links_result?.link_count).to.equal(2); expect(broken_links_result?.passing_link_count).to.equal(2); - expect(broken_links_result?.failing_link_count).to.equal(1); - expect(broken_links_result?.unreachable_count).to.equal(1); + expect(broken_links_result?.failing_link_count).to.equal(0); + expect(broken_links_result?.unreachable_count).to.equal(0); expect(broken_links_result?.status2xx_count).to.equal(2); expect(broken_links_result?.status3xx_count).to.equal(0); expect(broken_links_result?.status4xx_count).to.equal(0); expect(broken_links_result?.status5xx_count).to.equal(0); - expect(broken_links_result?.origin_link_result) + expect(options).to.deep.equal({ + origin_uri: origin_uri, + link_limit: 10, + query_selector_all: 'a[src]', + get_attributes: ['src'], + link_order: + BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder.FIRST_N, + link_timeout_millis: 30000, + max_retries: 0, + wait_for_selector: '', + per_link_options: {}, + total_synthetic_timeout_millis: 60000, + screenshot_options: { + capture_condition: ApiCaptureCondition.NONE, + storage_location: '', + }, + }); + + expect(origin_link) .excluding(['link_start_time', 'link_end_time']) - .to.deep.equal(expectedOriginLinkResult); + .to.deep.equal({ + link_passed: true, + expected_status_code: status_class_2xx, + source_uri: origin_uri, + target_uri: origin_uri, + html_element: '', + anchor_text: '', + status_code: 200, + error_type: '', + error_message: '', + link_start_time: 'NA', + link_end_time: 'NA', + is_origin: true, + screenshot_output: emptyScreenshotOutput, + }); - const sorted_followed_links_result = - broken_links_result?.followed_link_results.sort((a, b) => - a.target_uri.localeCompare(b.target_uri) - ); - expect(sorted_followed_links_result) + expect(followed_links) .excluding(['target_uri', 'link_start_time', 'link_end_time']) - .to.deep.equal(expectedFollowedLinksResults); + .to.deep.equal([ + { + link_passed: true, + expected_status_code: status_class_2xx, + source_uri: origin_uri, + target_uri: 'CHECKED_BELOW', + html_element: 'a', + anchor_text: 'External Link', + status_code: 200, + error_type: '', + error_message: '', + link_start_time: 'NA', + link_end_time: 'NA', + is_origin: false, + screenshot_output: emptyScreenshotOutput, + }, + ]); - // these are checked separately because path and puppeteer format space differently which cause the equality assertion to fail - const expectedTargeturis = [ - '/example_html_files/200.html', - '/example_html_files/file_doesnt_exist.html', - ]; - broken_links_result?.followed_link_results?.forEach((link, index) => { - expect(link.target_uri.endsWith(expectedTargeturis[index])); + const expectedTargetPaths = ['example_html_files/200.html']; + followed_links?.forEach((link, index) => { + expect(link.target_uri.endsWith(expectedTargetPaths[index])); }); - }).timeout(150000); + }).timeout(10000); }); diff --git a/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts new file mode 100644 index 00000000..a228db4e --- /dev/null +++ b/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts @@ -0,0 +1,60 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Standard Libraries +import { expect } from 'chai'; +import sinon from 'sinon'; + +// External Dependency +import httpMocks from 'node-mocks-http'; +const proxyquire = require('proxyquire'); + +import { runBrokenLinksHandler } from '../../src/handlers'; +import { BrokenLinkCheckerOptions } from '../../src/broken_links'; + +describe('Broken Links Synthetic Handler', async () => { + it('has check id and execution id available', async () => { + // Stub the runBrokenLinks function using Sinon + const mockRunBrokenLinks = sinon.stub().callsFake(async (opts, args) => { + return Promise.resolve({ mocked_response: 'is unimportant' }); + }); + const mockedBrokenLinks = proxyquire('../../src/handlers', { + './broken_links': { runBrokenLinks: mockRunBrokenLinks }, + }); + + // Options for the runBrokenLinksHandler + const options: BrokenLinkCheckerOptions = { + origin_uri: 'https://example.com', + }; + + // Create mock request and response + const req = httpMocks.createRequest({ + headers: { + ['Synthetic-Execution-Id']: 'test-execution-id', + ['Check-Id']: 'test-check-id', + }, + }); + const res = httpMocks.createResponse(); + + // Call the middleware + await mockedBrokenLinks.runBrokenLinksHandler(options)(req, res); + + // Assertions with Sinon and Chai + sinon.assert.calledWith(mockRunBrokenLinks, options, { + executionId: 'test-execution-id', + checkId: 'test-check-id', + }); + expect(res.statusCode).to.equal(200); + }).timeout(5000); +}); diff --git a/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts index 9f52f2cb..e59d3f16 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts @@ -12,10 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Standard Libraries import { expect } from 'chai'; +import sinon from 'sinon'; + +// Internal Project Files import { + BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, BrokenLinksResultV1_SyntheticLinkResult, ResponseStatusCode, ResponseStatusCode_StatusClass, @@ -25,11 +31,19 @@ import { checkStatusPassing, createSyntheticResult, getGenericSyntheticResult, + getStoragePathToExecution, LinkIntermediate, + sanitizeObjectName, shuffleAndTruncate, + shouldTakeScreenshot, } from '../../src/link_utils'; import { setDefaultOptions } from '../../src/options_func'; +// External Dependencies +import { Bucket, Storage } from '@google-cloud/storage'; +import { StorageParameters } from '../../src/storage_func'; +import { TEST_BUCKET_NAME } from './storage_func.spec'; + describe('GCM Synthetics Broken Links Utilies', async () => { const status_value_200: ResponseStatusCode = { status_value: 200 }; const status_value_404: ResponseStatusCode = { status_value: 404 }; @@ -48,6 +62,19 @@ describe('GCM Synthetics Broken Links Utilies', async () => { const status_class_5xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_5XX, }; + const default_errors: BaseError[] = [ + { error_type: 'fake-error-type', error_message: 'fake-error-message' }, + ]; + const bucketStub: sinon.SinonStubbedInstance = + sinon.createStubInstance(Bucket); + bucketStub.name = TEST_BUCKET_NAME; + const storageParams = { + storageClient: {} as Storage, + bucket: bucketStub, + checkId: 'uptime123', + executionId: 'exec456', + screenshotNumber: 1, + }; it('checkStatusPassing returns correctly when passed a number as ResponseStatusCode', () => { // expecting success @@ -120,7 +147,9 @@ describe('GCM Synthetics Broken Links Utilies', async () => { start_time, runtime_metadata, options, - all_links + all_links, + storageParams, + default_errors ); // BrokenLinkResultV1 expectations (testing `parseFollowedLinks`) @@ -138,6 +167,9 @@ describe('GCM Synthetics Broken Links Utilies', async () => { options: options, origin_link_result: origin_link, followed_link_results: followed_links, + execution_data_storage_path: + 'gs://gcm-test-project-id-synthetics-test-region/uptime123/exec456', + errors: default_errors, }); expect( @@ -200,4 +232,127 @@ describe('GCM Synthetics Broken Links Utilies', async () => { expect(startTime).to.be.lessThan(endTime); expect(milliDifference).to.be.greaterThan(0); }); + + describe('sanitizeObjectName', () => { + it('should remove invalid characters', () => { + const input = 'test/@#$%^&*()/_+-=[]{};\':"|,.<>/?\r\n\t'; + const expectedOutput = "test_@_$%^&_()__+-=__{};'___,.______"; + expect(sanitizeObjectName(input)).to.equal(expectedOutput); + }); + + it('should replace the forbidden prefix', () => { + const input = '.well-known/acme-challenge/test'; + const expectedOutput = '_test'; + expect(sanitizeObjectName(input)).to.equal(expectedOutput); + }); + + it('should handle standalone "." and ".."', () => { + expect(sanitizeObjectName('.')).to.equal('_'); + expect(sanitizeObjectName('..')).to.equal('_'); + }); + + it('should handle null and undefined', () => { + expect(sanitizeObjectName(null)).to.equal('_'); + expect(sanitizeObjectName(undefined)).to.equal('_'); + }); + + it('should trim leading and trailing whitespace', () => { + const input = ' test name '; + const expectedOutput = 'test_name'; + expect(sanitizeObjectName(input)).to.equal(expectedOutput); + }); + }); + + describe('shouldTakeScreenshot', () => { + describe('screenshot_condition: ALL', () => { + const options = { + screenshot_options: { capture_condition: ApiCaptureCondition.ALL }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should return true when passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.true; + }); + + it('should return true when passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.true; + }); + }); + + describe('screenshot_condition: FAILING', () => { + const options = { + screenshot_options: { + capture_condition: ApiCaptureCondition.FAILING, + }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should return true if passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.true; + }); + + it('should return false if passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.false; + }); + }); + + describe('screenshot_condition: NONE', () => { + const options = { + screenshot_options: { capture_condition: ApiCaptureCondition.NONE }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should retrun true if passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.false; + }); + + it('should retrun true if passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.false; + }); + }); + }); + + describe('getStoragePathToExecution()', () => { + it('returns write_destination when given folder in storage location', () => { + const options = { + screenshot_options: { storage_location: 'bucket/folder1/folder2' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const writeDestination = getStoragePathToExecution( + storageParams, + options + ); + expect(writeDestination).to.equal('folder1/folder2/uptime123/exec456'); + }); + + it('should handle no folder and just bucket in storage_location', () => { + const options = { + screenshot_options: { storage_location: 'bucket' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const result = getStoragePathToExecution(storageParams, options); + expect(result).to.equal('uptime123/exec456'); + }); + + it('should handle error by returning empty string', () => { + const options = { + screenshot_options: { storage_location: 'bucket' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const storageParamsUndefiniedCheckId = { + storageClient: {} as Storage, + bucket: {} as Bucket, + executionId: 'exec456', + } as StorageParameters; + + const result = getStoragePathToExecution( + storageParamsUndefiniedCheckId, + options + ); + expect(result).to.equal(''); + }); + }); }); diff --git a/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts index c6256797..2f2a8e8d 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts @@ -12,51 +12,85 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Standard Libraries import { expect, use } from 'chai'; import chaiExclude from 'chai-exclude'; use(chaiExclude); - -import puppeteer, { Browser, HTTPResponse, Page } from 'puppeteer'; +const path = require('path'); import sinon from 'sinon'; + +// Internal Project Files import { + BaseError, BrokenLinksResultV1_SyntheticLinkResult, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, ResponseStatusCode, ResponseStatusCode_StatusClass, } from '@google-cloud/synthetics-sdk-api'; -import { LinkIntermediate } from '../../src/link_utils'; import { BrokenLinkCheckerOptions } from '../../src/broken_links'; -const path = require('path'); +import { LinkIntermediate } from '../../src/link_utils'; import { checkLink, navigate, retrieveLinksFromPage, } from '../../src/navigation_func'; import { setDefaultOptions } from '../../src/options_func'; +import * as storageFunc from '../../src/storage_func'; + +// External Dependencies +import { Bucket, Storage } from '@google-cloud/storage'; +const proxyquire = require('proxyquire'); + +// External Dependencies +import puppeteer, { Browser, HTTPResponse, Page } from 'puppeteer'; describe('GCM Synthetics Broken Links Navigation Functionality', async () => { - // constants + // Constants const link: LinkIntermediate = { target_uri: 'https://example.com', anchor_text: '', html_element: '', }; - const input_options: BrokenLinkCheckerOptions = { + const defaultOptions: BrokenLinkCheckerOptions = { origin_uri: 'http://origin.com', max_retries: 2, link_timeout_millis: 5000, }; - const options = setDefaultOptions(input_options); + const options = setDefaultOptions(defaultOptions); const response2xx: Partial = { status: () => 200 }; const response4xx: Partial = { status: () => 404 }; - const status_class_2xx: ResponseStatusCode = { + const statusClass2xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, }; - const status_class_4xx: ResponseStatusCode = { + const statusClass4xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_4XX, }; + const emptyScreenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + const successfulScreenshotOuput: ApiScreenshotOutput = { + screenshot_file: 'bucket/folder/file.png', + screenshot_error: {} as BaseError, + }; - // Puppeteer constants + const storageParams: storageFunc.StorageParameters = { + storageClient: sinon.createStubInstance(Storage), + bucket: sinon.createStubInstance(Bucket), + checkId: '', + executionId: '', + screenshotNumber: 1, + }; + + const navigStorageUploadSuccMocked = proxyquire('../../src/navigation_func', { + './storage_func': { + ...storageFunc, + uploadScreenshotToGCS: () => successfulScreenshotOuput, + }, + }); + + // Puppeteer Setup let browser: Browser; let page: Page; before(async () => { @@ -66,21 +100,35 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }); beforeEach(async () => { - // Create a new page for each test page = await browser.newPage(); page.setCacheEnabled(false); + + sinon + .stub(page, 'screenshot') + .resolves(Buffer.from('encoded-image-data', 'utf-8')); + }); + + afterEach(() => { + sinon.restore(); }); after(async () => { - // Close the browser after all tests browser && (await browser.close()); }); describe('navigate', async () => { - it('should pass after retries', async () => { - const pageStub = sinon.createStubInstance(Page); + let pageStub: sinon.SinonStubbedInstance; + + beforeEach(() => { + pageStub = sinon.createStubInstance(Page); pageStub.url.returns('fake-current-uri'); + }); + + afterEach(() => { + sinon.restore(); + }); + it('should pass after retries', async () => { // Configure the stub to simulate a failed navigation on the first call // and a successful one on the second pageStub.goto @@ -97,9 +145,6 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }); it('should fail after maximum retries', async () => { - const pageStub = sinon.createStubInstance(Page); - pageStub.url.returns('fake-current-uri'); - // Configure the stub to simulate a failed navigation on three // consecutive calls pageStub.goto @@ -120,11 +165,11 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { describe('checkLink', async () => { it('passes when navigating to real uri', async () => { - const synLinkResult = await checkLink(page, link, options); + const synLinkResult = await checkLink(page, link, options, storageParams); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: true, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: 'https://example.com', html_element: '', @@ -135,6 +180,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -156,11 +202,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { anchor_text: '', html_element: '', }; - const synLinkResult = await checkLink(page, json_link, options); + const synLinkResult = await checkLink( + page, + json_link, + options, + storageParams + ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: true, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: `file:${path.join( __dirname, @@ -174,6 +225,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -197,15 +249,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { html_element: 'img', }; - const synLinkResult = await checkLink( + const synLinkResult = await navigStorageUploadSuccMocked.checkLink( page, timeout_link, - options_with_timeout + options_with_timeout, + storageParams ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: false, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: target_uri, html_element: 'img', @@ -216,6 +269,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -224,14 +278,14 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }).timeout(5000); it('returns error when the actual response code does not match the expected', async () => { - // add expected 404 status to options of broken link checker - const optionsExp404 = Object.assign({}, options); const per_link_expected_404 = { expected_status_code: { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_4XX, }, link_timeout_millis: options.link_timeout_millis, }; + + const optionsExp404 = Object.assign({}, options); optionsExp404.per_link_options['https://expecting404.com'] = per_link_expected_404; @@ -251,15 +305,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { .onThirdCall() .resolves(response2xx as HTTPResponse); - const synLinkResult = await checkLink( + const synLinkResult = await navigStorageUploadSuccMocked.checkLink( pageStub, timeoutLink, - optionsExp404 + optionsExp404, + storageParams ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: false, - expected_status_code: status_class_4xx, + expected_status_code: statusClass4xx, source_uri: 'http://origin.com', target_uri: 'https://expecting404.com', html_element: 'a', @@ -271,6 +326,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -284,13 +340,11 @@ describe('retrieveLinksFromPage', async () => { // Puppeteer constants let browser: Browser; let page: Page; - let pageuriStub: sinon.SinonStub<[], string>; before(async () => { browser = await puppeteer.launch({ headless: 'new' }); }); beforeEach(async () => { - // Create a new page for each test page = await browser.newPage(); await page.goto( `file:${path.join( @@ -299,11 +353,10 @@ describe('retrieveLinksFromPage', async () => { )}` ); // Mock page.uri() to return a custom uri - pageuriStub = sinon.stub(page, 'url').returns('https://mocked.com'); + sinon.stub(page, 'url').returns('https://mocked.com'); }); after(async () => { - // Close the browser after all tests browser && (await browser.close()); }); @@ -357,7 +410,7 @@ describe('retrieveLinksFromPage', async () => { // note: does not return `mailto:...` link expect(results).to.deep.equal(expectations); - }); + }).timeout(5000); it('handles complicated query_selector_all', async () => { const query_selector_all = 'img[href], a[src]'; diff --git a/packages/synthetics-sdk-broken-links/test/unit/options_func.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/options_func.spec.ts index c431a8a1..0b8b0a87 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/options_func.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/options_func.spec.ts @@ -12,23 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Standard Libraries import { expect } from 'chai'; + +// Internal Project Files import { BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, ResponseStatusCode, ResponseStatusCode_StatusClass, } from '@google-cloud/synthetics-sdk-api'; import { BrokenLinkCheckerOptions, - StatusClass, LinkOrder, + StatusClass, } from '../../src/broken_links'; import { setDefaultOptions, validateInputOptions, } from '../../src/options_func'; -describe('GCM Synthetics Broken Links options_func suite testing', () => { +describe('GCM Synthetics Broken Links options_func suite testing', () => { const status_value_304: ResponseStatusCode = { status_value: 304 }; const status_class_2xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, @@ -92,6 +96,10 @@ describe('GCM Synthetics Broken Links options_func suite testing', () => { }, }; expect(options.per_link_options).to.deep.equal(link_options); + + expect(options.screenshot_options?.capture_condition).to.equal( + ApiCaptureCondition.NONE + ); }); describe('validateInputOptions', () => { @@ -368,6 +376,45 @@ describe('GCM Synthetics Broken Links options_func suite testing', () => { validateInputOptions(options); }).to.not.throw(); }); + it('throws error if storage_condition is not a valid StorageCondition value', () => { + const options = { + origin_uri: 'http://example.com', + screenshot_options: { capture_condition: 'invalid' }, + } as any as BrokenLinkCheckerOptions; + expect(() => { + validateInputOptions(options); + }).to.throw( + Error, + 'Invalid capture_condition value, must be `ALL`, `FAILING`, OR `NONE`' + ); + }); + it('storage_condition accepts string', () => { + const options = { + origin_uri: 'http://example.com', + screenshot_options: { capture_condition: 'FAILING' }, + } as any as BrokenLinkCheckerOptions; + expect(() => { + validateInputOptions(options); + }).to.not.throw(); + }); + it('throws error if storage_location is not a string', () => { + const options = { + origin_uri: 'http://example.com', + screenshot_options: { storage_location: 123 }, + } as any as BrokenLinkCheckerOptions; + expect(() => { + validateInputOptions(options); + }).to.throw(Error, 'Invalid storage_location value, must be a string'); + }); + it('storage_location can be an empty string', () => { + const options = { + origin_uri: 'http://example.com', + screenshot_options: { storage_location: '' }, + } as BrokenLinkCheckerOptions; + expect(() => { + validateInputOptions(options); + }).not.to.throw(); + }); it('validates input options when all values are valid', () => { const options = { origin_uri: 'http://example.com', @@ -384,6 +431,10 @@ describe('GCM Synthetics Broken Links options_func suite testing', () => { expected_status_code: StatusClass.STATUS_CLASS_2XX, }, }, + screenshot_options: { + storage_location: '', + capture_condition: 'FAILING', + }, } as BrokenLinkCheckerOptions; expect(() => { @@ -406,7 +457,11 @@ describe('GCM Synthetics Broken Links options_func suite testing', () => { max_retries: undefined, wait_for_selector: undefined, per_link_options: undefined, - total_synthetic_timeout_millis: undefined + total_synthetic_timeout_millis: undefined, + screenshot_options: { + storage_location: undefined, + capture_condition: undefined, + }, } as BrokenLinkCheckerOptions; expect(() => { diff --git a/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts new file mode 100644 index 00000000..9a29019a --- /dev/null +++ b/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts @@ -0,0 +1,303 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Standard Libraries +import { expect } from 'chai'; +import sinon from 'sinon'; + +// Internal Project Files +import { + BaseError, + BrokenLinksResultV1_BrokenLinkCheckerOptions, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, +} from '@google-cloud/synthetics-sdk-api'; +import { + createStorageClientIfStorageSelected, + StorageParameters, + uploadScreenshotToGCS, +} from '../../src/storage_func'; + +// External Dependencies +import { Bucket, File, Storage } from '@google-cloud/storage'; +const proxyquire = require('proxyquire'); +import { Page } from 'puppeteer'; + +// global test vars +export const TEST_BUCKET_NAME = 'gcm-test-project-id-synthetics-test-region'; + +describe('GCM Synthetics Broken Links storage_func suite testing', () => { + let storageClientStub: sinon.SinonStubbedInstance; + let bucketStub: sinon.SinonStubbedInstance; + + const storageFunc = proxyquire('../../src/storage_func', { + '@google-cloud/synthetics-sdk-api': { + getExecutionRegion: () => 'test-region', + resolveProjectId: () => 'test-project-id', + }, + }); + + const storage_condition_failing_links = ApiCaptureCondition.FAILING; + const storage_condition_none = ApiCaptureCondition.NONE; + + beforeEach(() => { + // Stub a storage bucket + bucketStub = sinon.createStubInstance(Bucket); + bucketStub.name = TEST_BUCKET_NAME; + bucketStub.create.resolves([bucketStub]); + + // Stub the storage client + storageClientStub = sinon.createStubInstance(Storage); + storageClientStub.bucket.returns(bucketStub); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('getOrCreateStorageBucket()', () => { + it('should create a bucket if no storage_location is provided', async () => { + bucketStub.exists.resolves([false]); // Simulate the bucket not existing initially + + const result = await storageFunc.getOrCreateStorageBucket( + storageClientStub, + '', + [] + ); + expect(result).to.equal(bucketStub); + expect(result.name).to.equal(TEST_BUCKET_NAME); + }); + + it('should return null if projectId or region cannot be resolved', async () => { + const failingProjectId = proxyquire('../../src/storage_func', { + '@google-cloud/synthetics-sdk-api': { + getExecutionRegion: () => 'test-region', + resolveProjectId: () => '', + }, + }); + + const result = await failingProjectId.getOrCreateStorageBucket( + storageClientStub, + '', + [] + ); + expect(result).to.be.null; + }); + + it('should return existing synthetics bucket if found when storage_location is not provided ', async () => { + bucketStub.exists.resolves([true]); // Simulate the bucket already exists + + const result = await storageFunc.getOrCreateStorageBucket( + storageClientStub, + TEST_BUCKET_NAME + '/fake-folder', + [] + ); + expect(result).to.equal(bucketStub); + sinon.assert.calledWithExactly( + storageClientStub.bucket, + TEST_BUCKET_NAME + ); + sinon.assert.notCalled(bucketStub.create); + }); + + it('should handle errors during bucket.exists()', async () => { + bucketStub.exists.throws(new Error('Simulated exists() error')); + + const errors: BaseError[] = []; + const result = await storageFunc.getOrCreateStorageBucket( + storageClientStub, + 'user-bucket', + errors + ); + + expect(result).to.be.null; + expect(errors.length).to.equal(1); + expect(errors[0].error_type).to.equal('StorageValidationError'); + }); + + it('should handle errors during bucket creation', async () => { + bucketStub.create.throws(new Error('Simulated creation error')); // Force an error + + const errors: BaseError[] = []; + const result = await storageFunc.getOrCreateStorageBucket( + storageClientStub, + '', + errors + ); + + expect(result).to.be.null; + expect(errors.length).to.equal(1); + expect(errors[0].error_type).to.equal('BucketCreationError'); + }); + }); + + describe('createStorageClient()', () => { + it('should return null if storage_condition is `None`', () => { + const result = createStorageClientIfStorageSelected( + [], + storage_condition_none + ); + expect(result).to.be.null; + }); + it('should successfully initialize a Storage client', () => { + const result = createStorageClientIfStorageSelected( + [], + storage_condition_failing_links + ); + expect(result).to.be.an.instanceOf(Storage); + }); + }); + + describe('uploadScreenshotToGCS', () => { + let storageClientStub: sinon.SinonStubbedInstance; + let bucketStub: sinon.SinonStubbedInstance; + let pageStub: sinon.SinonStubbedInstance; + + beforeEach(() => { + storageClientStub = sinon.createStubInstance(Storage); + bucketStub = sinon.createStubInstance(Bucket); + pageStub = sinon.createStubInstance(Page); + pageStub.url.resolves('https://fake-url'); + + storageClientStub.bucket.returns(bucketStub); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('Valid Storage Configuration', () => { + it('should upload the screenshots and return updated write_destination', async () => { + const storageParams = { + storageClient: storageClientStub, + bucket: bucketStub, + checkId: 'uptime123', + executionId: 'exec456', + screenshotNumber: 1, + }; + const options = { + screenshot_options: { storage_location: 'bucket/folder1/folder2' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const successPartialFileMock: Partial = { + save: sinon.stub().resolves(), + }; + bucketStub.file.returns(successPartialFileMock as File); + + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); + + expect(result.screenshot_file).to.equal('screenshot_1.png'); + expect(result.screenshot_error).to.deep.equal({}); + + const result2 = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); + + expect(result2.screenshot_file).to.equal('screenshot_2.png'); + expect(result2.screenshot_error).to.deep.equal({}); + }); + + it('should handle GCS upload errors', async () => { + const storageParams: StorageParameters = { + storageClient: storageClientStub, + bucket: bucketStub, + checkId: '', + executionId: '', + screenshotNumber: 1, + }; + const options = { + screenshot_options: {}, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const gcsError = new Error('Simulated GCS upload error'); + const failingPartialFileMock: Partial = { + save: sinon.stub().throws(gcsError), + }; + bucketStub.file.returns(failingPartialFileMock as File); + + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); + + expect(result.screenshot_file).to.equal(''); + expect(result.screenshot_error).to.deep.equal({ + error_type: 'ScreenshotFileUploadError', + error_message: + 'Failed to take and/or upload screenshot for https://fake-url. Please reference server logs for further information.', + }); + }); + }); + + describe('Invalid Storage Configuration', () => { + const emptyOptions = {} as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + beforeEach(() => { + pageStub.screenshot.resolves( + Buffer.from('encoded-image-data', 'utf-8') + ); + }); + + it('should return an empty result if storageClient is null', async () => { + // Missing storageClient + const storageParams = { + storageClient: null, + bucket: bucketStub, + checkId: '', + executionId: '', + screenshotNumber: 1, + }; + + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + emptyOptions + ); + + expect(result).to.deep.equal({ + screenshot_file: '', + screenshot_error: {}, + }); + }); + + it('should return an empty result if bucket is null', async () => { + // Missing bucket + const storageParams = { + storageClient: storageClientStub, + bucket: null, + checkId: '', + executionId: '', + screenshotNumber: 1, + } as StorageParameters; + + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + emptyOptions + ); + + expect(result).to.deep.equal({ + screenshot_file: '', + screenshot_error: {}, + }); + }); + }); + }); +});