Merge pull request #1 from friendtocephalopods/add-neptune

amundsen-gremlin: first main commit.

.github/CODEOWNERS

@@ -6,8 +6,8 @@
 
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence,
-# @amundsen-io/amundsen-committerswill be requested for
+#  @amundsen-io/amundsen-gremlin-committers will be requested for
 # review when someone opens a pull request.
-*       @amundsen-io/amundsen-committers
+*       @amundsen-io/amundsen-gremlin-committers
 
 *.py    @alran @cpu @dsimms @friendtocephalopods @kathawthorne @worldwise001

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Vscode project settings
+.vscode/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "amazon-neptune-tools"]
+	path = amazon-neptune-tools
+	url = https://github.com/awslabs/amazon-neptune-tools.git

.hooks/pre-commit

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+red='\033[0;31m'
+green='\033[0;32m'
+NC='\033[0m'
+
+# Sort imports
+echo -e "${green}[Isort]: Checking Sorting${NC}"
+isort -c
+if [ $? -ne 0 ]
+then
+  isort --apply 
+  echo -e "${red}Sorted imports; recommit${NC}"
+  exit 1
+fi
+
+# Check Neptune config
+
+echo -e "${green}[Config]: Checking for secrets${NC}"
+
+# Grep will return 0 if and only if it finds a changed amazonaws url. This suggests you may accidentally be publically
+# committing your config!
+git diff HEAD amundsen_gremlin/config.py | grep -q -c 'amazonaws.com'
+if [ $? -eq 0 ]
+then
+  echo -e "${red}Did you remember to remove your AWS config? If this is a false alarm, recommit with --no-verify${NC}"
+  exit 1
+fi

.hooks/pre-push

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+red='\033[0;31m'
+green='\033[0;32m'
+NC='\033[0m'
+
+set -e -o pipefail
+
+# Get only the files different on this branch
+BASE_SHA="$(git merge-base master HEAD)"
+FILES="$(git diff --name-only --diff-filter=AMC $BASE_SHA HEAD | grep "\.py$" | tr '\n' ' ')"
+
+echo -e "${green}[Python Checks][Info]: Checking Python Style, Types${NC}"
+
+if [ -n "$FILES" ]
+then
+
+  echo -e "${green}[Python Checks][Info]: ${FILES}${NC}"
+
+  # Run flake8
+  flake8 .
+
+  if [ $? -ne 0 ]
+  then
+    echo -e "${red}[Python Style][Error]: Fix the issues and commit again (or commit with --no-verify if you are sure)${NC}"
+    exit 1
+  fi
+
+  # Run mypy
+  mypy .
+
+  if [ $? -ne 0 ]
+  then
+    echo -e "${red}[Python Type Checks][Error]: Fix the issues and commit again (or commit with --no-verify if you are sure)${NC}"
+    exit 1
+  fi
+
+else
+  echo -e "${green}[Python Checks][Info]: No files to check${NC}"
+fi
+
+exit 0

README.md

@@ -1,2 +1,18 @@
 # amundsengremlin
 Amundsen Gremlin
+
+## Instructions to configure venv
+Virtual environments for python are convenient for avoiding dependency conflicts.
+The `venv` module built into python3 is recommended for ease of use, but any managed virtual environment will do.
+If you'd like to set up venv in this repo:
+```bash
+$ venv_path=[path_for_virtual_environment]
+$ python3 -m venv $venv_path
+$ source $venv_path/bin/activate
+$ pip install -r requirements.txt
+```
+
+If something goes wrong, you can always:
+```bash
+$ rm -rf $venv_path
+```

amundsen_gremlin/__init__.py

@@ -0,0 +1,2 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0

amundsen_gremlin/config.py

@@ -0,0 +1,50 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any, Mapping, Optional, Union
+
+
+class Config:
+    pass
+
+
+NEPTUNE_URLS_BY_USER: Mapping[str, Mapping[str, Any]] = {
+    "nobody": {
+        "neptune_endpoint": "nowhere.amazonaws.com",
+        "neptune_port": 8182,
+        "uri": "nowhere.amazonaws.com:8182/gremlin"
+    },
+}
+
+
+def neptune_url_for_development(*, user: Optional[str] = None) -> Optional[Mapping[str, Any]]:
+    # Hello!  If you get here and and your user is not above, ask one of them to borrow theirs. Or add your username
+    # to development_instance_users in terraform/deployments/development/main.tf and terraform apply
+    # TODO: add terraform files. One stopgap is to manually set up neptune instances for each dev user.
+    return NEPTUNE_URLS_BY_USER[os.getenv('USER', 'nobody')]
+
+
+class TestGremlinConfig(Config):
+    NEPTUNE_BULK_LOADER_S3_BUCKET_NAME = 'amundsen-gremlin-development-bulk-loader'
+    NEPTUNE_URL = 'something.amazonaws.com:8182/gremlin'
+    # TODO: populate a session here
+    NEPTUNE_SESSION = None
+
+
+class LocalGremlinConfig(Config):
+    LOG_LEVEL = 'DEBUG'
+    NEPTUNE_BULK_LOADER_S3_BUCKET_NAME = 'amundsen-gremlin-development-bulk-loader'
+    # The appropriate AWS region for your neptune setup
+    # ex: AWS_REGION_NAME = 'us-west-2'
+    AWS_REGION_NAME = None
+    # NB: Session should be shaped like:
+    # NEPTUNE_SESSION = boto3.session.Session(profile_name='youruserprofilehere',
+    #                                         region_name=AWS_REGION_NAME)
+    # Unfortunately this will always blow up without a legit profile name
+    NEPTUNE_SESSION = None
+    # NB: NEPTUNE_URL should be shaped like:
+    # NEPTUNE_URL = neptune_url_for_development()
+    # Unfortunately, this will blow up if your user is not present
+    NEPTUNE_URL = None
+    PROXY_HOST: Union[str, Mapping[str, Any], None] = NEPTUNE_URL