Merge branch 'master' of https://github.com/datacommonsorg/website in…

…to fix

custom_dc/env.py

@@ -0,0 +1,24 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+from server.app_env import local
+
+
+class Config(_base.Config):
+  CUSTOM = True
+
+
+class LocalConfig(Config, local.Config):
+  LITE = True

deploy/overlays/climatetrace/custom_bigtable_info.yaml

@@ -1,4 +1,4 @@
 project: datcom-mixer-encode
 instance: dc-graph
 tables:
-  -
+  - climatetrace_2023_02_22_18_50_22

deploy/overlays/climatetrace/kustomization.yaml

@@ -24,8 +24,9 @@ resources:
 configMapGenerator:
   - behavior: merge
     literals:
-      - flaskEnv=custom
+      - flaskEnv=climate_trace
       - secretProject=datcom-mixer-encode
+      - enableModel=true
     name: website-configmap
   - behavior: create
     literals:

nl_server/__init__.py

@@ -29,7 +29,7 @@ def create_app():
   flask_env = os.environ.get('FLASK_ENV')
 
   model_config_path = '/datacommons/model/model.yaml'
-  if flask_env == 'local' or flask_env == 'test' or flask_env == 'integration-test':
+  if flask_env == 'local' or flask_env == 'test' or flask_env == 'integration_test':
     model_config_path = os.path.join(
         os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
         'deploy/base/model.yaml')

nl_server/loader.py

@@ -28,7 +28,7 @@ def load_model(app, embeddings_file):
   flask_env = os.environ.get('FLASK_ENV')
   # In local dev, cache the model in disk so each hot reload won't download
   # the model again.
-  if flask_env == 'local' or flask_env == 'integration-test':
+  if flask_env == 'local' or flask_env == 'integration_test':
     from diskcache import Cache
     cache = Cache(nl_cache_path)
     cache.expire()
@@ -46,7 +46,7 @@ def load_model(app, embeddings_file):
   nl_ner_places = NERPlaces()
   app.config["NL_NER_PLACES"] = nl_ner_places
 
-  if flask_env == 'local' or flask_env == 'integration-test':
+  if flask_env == 'local' or flask_env == 'integration_test':
     with Cache(cache.directory) as reference:
       reference.set(nl_embeddings_cache_key,
                     nl_embeddings,

run_server.sh

@@ -19,7 +19,6 @@ python3 -m venv .env
 source .env/bin/activate
 
 PORT=8080
-ENV=local
 ENABLE_MODEL=false
 PROTOC_VERSION=3.21.9
 
@@ -36,10 +35,10 @@ function help {
 while getopts ":e:p?m?d?l" OPTION; do
   case $OPTION in
     e)
-      ENV=$OPTARG
+      export FLASK_ENV=$OPTARG
       ;;
     p)
-      PORT=$OPTARG
+      export PORT=$OPTARG
       ;;
     m)
       export ENABLE_MODEL=true
@@ -62,20 +61,12 @@ if [[ "$(protoc --version)" != "libprotoc ${PROTOC_VERSION}" ]]; then
 fi
 
 export GOOGLE_CLOUD_PROJECT=datcom-website-dev
-if [[ $ENV == "lite" ]]; then
-  export FLASK_ENV=local-lite
-elif [[ $ENV == "custom" ]]; then
-  export FLASK_ENV=local-custom
-elif [[ $ENV == "iitm" ]]; then
-  export FLASK_ENV=local-iitm
-elif [[ $ENV == "feedingamerica" ]]; then
-  export FLASK_ENV=local-feedingamerica
-elif [[ $ENV == "stanford" ]]; then
-  export FLASK_ENV=local-stanford
-elif [[ ! -z ${ENV+x} ]]; then  # Use any specified env.
-  export FLASK_ENV=$ENV
-else
-  export FLASK_ENV=local
+
+if [[ $FLASK_ENV == "" ]]; then
+  export FLASK_ENV="local"
+fi
+if [[ $FLASK_ENV != "local" ]]; then
+  export ENV_PREFIX="Local"
 fi
 echo "Starting localhost with FLASK_ENV='$FLASK_ENV' on port='$PORT'"
 

run_test.sh

@@ -149,7 +149,7 @@ function run_webdriver_test {
 function run_integration_test {
   setup_python
   export ENABLE_MODEL=true
-  export FLASK_ENV=integration-test
+  export FLASK_ENV=integration_test
   export GOOGLE_CLOUD_PROJECT=datcom-website-dev
   export TEST_MODE=test
   python3 -m pytest -vv server/integration_tests/
@@ -158,7 +158,7 @@ function run_integration_test {
 function update_integration_test_golden {
   setup_python
   export ENABLE_MODEL=true
-  export FLASK_ENV=integration-test
+  export FLASK_ENV=integration_test
   export GOOGLE_CLOUD_PROJECT=datcom-website-dev
   export TEST_MODE=write
   python3 -m pytest -vv server/integration_tests/
@@ -205,7 +205,7 @@ while getopts tpwigotblcsaf OPTION; do
         ;;
     g)
         echo -e "### Updating integration test goldens"
-        update_integration_test_golden 
+        update_integration_test_golden
         ;;
     o)
         echo -e "### Production flag enabled"

server/__init__.py

@@ -213,7 +213,7 @@ def create_app():
   register_routes_common(app)
   if cfg.CUSTOM:
     register_routes_custom_dc(app)
-  if (cfg.ENV_NAME == 'STANFORD' or os.environ.get('ENABLE_MODEL') == 'true' or
+  if (cfg.ENV == 'stanford' or os.environ.get('ENABLE_MODEL') == 'true' or
       cfg.LOCAL and not cfg.LITE):
     register_routes_stanford_dc(app, cfg.LOCAL)
 
@@ -319,7 +319,7 @@ def before_request():
     g.locale_choices = i18n.locale_choices(requested_locale)
     g.locale = g.locale_choices[0]
     # Add commonly used config flags.
-    g.env_name = app.config.get('ENV_NAME', None)
+    g.env = app.config.get('ENV', None)
 
     scheme = request.headers.get('X-Forwarded-Proto')
     if scheme and scheme == 'http' and request.url.startswith('http://'):

server/app_env/_base.py

@@ -0,0 +1,58 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Module to hold Flask environment configuration base class.
+# All the flags used in the environmnet should be defined here with default
+# value.
+
+import os
+
+
+class Config:
+  ENV = ''
+  TEST = False
+  INTEGRATION = False
+  WEBDRIVER = False
+  LOCAL = False
+  LITE = False
+  VERSION = '{}-{}'.format(os.environ.get('WEBSITE_HASH'),
+                           os.environ.get('MIXER_HASH'))
+  API_ROOT = 'http://127.0.0.1:8081'  # Port for Kubernetes ESP.
+  NL_ROOT = 'http://127.0.0.1:6060'  # Port for Kubernetes ESP.
+  AI_CONFIG_PATH = '/datacommons/ai/ai.yaml'
+  SECRET_PROJECT = os.environ.get('SECRET_PROJECT') or ''
+  MAPS_API_KEY = os.environ.get('MAPS_API_KEY') or ''
+  GA_ACCOUNT = ''
+  SCHEME = 'https'
+  # Additional stat vars that need to be fetched for place page data.
+  # This is only needed for local development when cache is not up to date.
+  NEW_STAT_VARS = []
+  ENABLE_BLOCKLIST = False
+  # If the deployment is a custom instance.
+  CUSTOM = False
+  # Name of the site. The name is changed for custom instance.
+  NAME = 'Data Commons'
+  BASE_HTML_PATH = 'base.html'
+  # Whether to have account management page
+  ADMIN = False
+  # The GCP project of the mixer which Flask talks to. This only needs to
+  # be set for local development. Website deployed to GKE bundles the mixer
+  # as a custom service accessible via localhost.
+  API_PROJECT = ''
+  # Set this to False if the deployment has frequently updated data.
+  USE_MEMCACHE = True
+  # Whether to log the query (and make them avaiable in query history interface)
+  # Eanbling this to "True" requires adding "bigtable/user" acccess for the
+  # service account in datcom-store IAM settings
+  LOG_QUERY = False

server/app_env/autopush.py

@@ -0,0 +1,21 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+
+
+class Config(_base.Config):
+  GA_ACCOUNT = 'G-Y6ZXZ9JK3H'
+  GCS_BUCKET = 'datcom-website-autopush-resources'
+  LOG_QUERY = True

server/app_env/climate_trace.py

@@ -0,0 +1,26 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+from server.app_env import local
+
+
+class Config(_base.Config):
+  CUSTOM = True
+  NAME = "Climate Trace"
+  ENABLE_BLOCKLIST = True
+
+
+class LocalConfig(Config, local.Config):
+  pass

server/app_env/dev.py

@@ -0,0 +1,20 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+
+
+class Config(_base.Config):
+  GCS_BUCKET = 'datcom-website-autopush-resources'
+  LOG_QUERY = True

server/app_env/feeding_america.py

@@ -0,0 +1,33 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+from server.app_env import local
+
+
+class Config(_base.Config):
+  CUSTOM = True
+  NAME = "Feeding America"
+  BASE_HTML_PATH = 'custom_dc/feedingamerica/base.html'
+  GA_ACCOUNT = 'G-444S6716SQ'
+
+
+class LocalConfig(local.Config):
+  CUSTOM = True
+  NAME = "Feeding America"
+  BASE_HTML_PATH = 'custom_dc/feedingamerica/base.html'
+  # This needs to talk to local mixer that is setup as a custom mixer, which
+  # loads csv + tmcf files from GCS
+  API_ROOT = 'https://mixer.endpoints.datcom-mixer-statvar.cloud.goog'
+  API_PROJECT = 'datcom-mixer-statvar'

server/app_env/iitm.py

@@ -0,0 +1,29 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+from server.app_env import local
+
+
+class Config(_base.Config):
+  CUSTOM = True
+  NAME = "IITM"
+  BASE_HTML_PATH = 'custom_dc/iitm/base.html'
+  GA_ACCOUNT = 'G-32HPL4K4Y1'
+
+
+class LocalConfig(local.Config):
+  CUSTOM = True
+  NAME = "IITM"
+  BASE_HTML_PATH = 'custom_dc/iitm/base.html'

server/app_env/integration_test.py

@@ -0,0 +1,23 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from server.app_env import _base
+
+
+class Config(_base.Config):
+  INTEGRATION = True
+  API_ROOT = 'https://autopush.api.datacommons.org'
+  API_PROJECT = 'datcom-mixer-autopush'
+  SCHEME = 'http'
+  SECRET_PROJECT = 'datcom-website-dev'