From 1277ff2fe9b0c9e296d6022ccebdbbdacbe9913a Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Mon, 17 Jun 2024 07:28:06 +0000 Subject: [PATCH] provide robots.txt via config instead of static file --- cubedash/_pages.py | 10 ++++++++-- cubedash/static/robots.txt | 4 ---- 2 files changed, 8 insertions(+), 6 deletions(-) delete mode 100644 cubedash/static/robots.txt diff --git a/cubedash/_pages.py b/cubedash/_pages.py index 99644f99c..bb97760e3 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -8,7 +8,7 @@ from datacube.model import DatasetType, Range from datacube.scripts.dataset import build_dataset_info from dateutil import tz -from flask import abort, redirect, request, send_from_directory, url_for +from flask import abort, redirect, request, url_for from werkzeug.datastructures import MultiDict from werkzeug.exceptions import HTTPException @@ -51,6 +51,10 @@ _DEFAULT_ARRIVALS_DAYS: int = app.config.get("CUBEDASH_DEFAULT_ARRIVALS_DAY_COUNT", 14) +_ROBOTS_TXT_DEFAULT = ( + "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**" +) + # Add server timings to http headers. if app.config.get("CUBEDASH_SHOW_PERF_TIMES", False): _monitoring.init_app_monitoring() @@ -676,7 +680,9 @@ def about_page(): @app.route("/robots.txt") def robots_txt(): - return send_from_directory("static", "robots.txt") + return utils.as_json( + flask.current_app.config.get("ROBOTS_TXT", _ROBOTS_TXT_DEFAULT) + ) @app.route("/") diff --git a/cubedash/static/robots.txt b/cubedash/static/robots.txt deleted file mode 100644 index 7705abc24..000000000 --- a/cubedash/static/robots.txt +++ /dev/null @@ -1,4 +0,0 @@ -User-Agent: * -Allow: / -Disallow: /products/*/* -Disallow: /stac/**