From 18a4301cdadaf005e3ad705e560256e8e3ffd9c6 Mon Sep 17 00:00:00 2001 From: Shreedhar Hardikar Date: Tue, 4 Apr 2017 15:50:11 -0700 Subject: [PATCH 1/2] Add --remote option This pushes the local repositories to a remote VM, since we cannot mount a volume from a local system remotely. --- common.bash | 25 +++++++++++++++++++++++++ setup_remote.bash | 30 ++++++++++++++++++++++++++++++ streamline-master/uber.bash | 4 ++++ 3 files changed, 59 insertions(+) create mode 100644 setup_remote.bash diff --git a/common.bash b/common.bash index af22e75..bae1a8d 100644 --- a/common.bash +++ b/common.bash @@ -22,6 +22,9 @@ parse_opts() { --enable-debug) build_mode=debug ;; + --remote) + remote=true + ;; esac done @@ -106,6 +109,28 @@ workspace() { dirname "$(dirname "${whereami}")" } +setup_remote() { + local repos + repos=(bug-free-fortnight gp-xerces gporca gpdb) + local ip + ip=$(docker-machine ip ${DOCKER_MACHINE_NAME}) + local user + user=$(docker-machine ssh ${DOCKER_MACHINE_NAME} whoami) + + docker-machine ssh ${DOCKER_MACHINE_NAME} bash -s $(workspace) ${repos[@]} < "$(dirname "$0")"/../setup_remote.bash + + for repo in ${repos[@]}; do + local repopath + repopath="$(workspace)/$repo" + + # -C - Path of the git repository locally + # -f - Replace the remote tree + # HEAD - current branch locally + # master - make sure to update the tree remotely which is by default at master + GIT_SSH_COMMAND="ssh -i ${DOCKER_CERT_PATH}/id_rsa" git -C "$repopath" push "$user@$ip":"$repopath" -f HEAD:master + done +} + absdir() { ( cd "$(dirname "$0")" diff --git a/setup_remote.bash b/setup_remote.bash new file mode 100644 index 0000000..f42c067 --- /dev/null +++ b/setup_remote.bash @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e -u -o pipefail + +main() { + local curpath + local user + curpath=$1 + shift + + user=$(whoami) + + git config --global receive.denyCurrentBranch updateInstead + + sudo mkdir -p "$curpath" + sudo chown -R "$user" "$curpath" + + for repo in $@; do + repopath="$curpath/$repo" + if [[ ! -e "$repopath" ]]; then + mkdir -p "$repopath" + git init "$repopath" + echo "Created directory on remote at $repopath." + else + echo "Directory already exists on remote at $repopath." + fi + done +} + +main "$@" diff --git a/streamline-master/uber.bash b/streamline-master/uber.bash index 6fbe2e5..59ed2fd 100755 --- a/streamline-master/uber.bash +++ b/streamline-master/uber.bash @@ -18,6 +18,10 @@ _main() { local build_mode parse_opts "$@" + if [[ "$remote" ]]; then + setup_remote + fi + local image_id image_id=$(build_image) From adbfa2cb0b7e7372bafb918ce894dc2c26328d62 Mon Sep 17 00:00:00 2001 From: Shreedhar Hardikar Date: Tue, 4 Apr 2017 16:29:04 -0700 Subject: [PATCH 2/2] Updated README --- README.md | 15 +++++++++++---- Remote_VM.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 Remote_VM.md diff --git a/README.md b/README.md index 745ae6f..bad8241 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ This repository is a simple on-ramp to help contributors run tests (`installcheck`) for GPDB ## Prerequisites +0. Prepare docker + 0. [MWare on Mac OSX](VMware_Fusion.md). + 0. [Remote VM on AWS](Remote_VM.md). 0. Hack on your code, commit them locally 0. Assuming all your code repositories are checked out in the same directory locally (e.g. `~/workspace`). Specifically, the following repositories @@ -27,6 +30,14 @@ This repository is a simple on-ramp to help contributors run tests (`installchec Set the `DEBUG` environment variable to reinstate debug output, e.g. `env DEBUG=1 streamline-master/uber.bash` +0. I want to run on AWS. Can I? + + Yes! Refer to [Remote VM on AWS](Remote_VM.md) on how to set it up. Add the + --remote flag when running uber.bash. + ``` + streamline-43/uber.bash --remote + ``` + 0. How do I set a GUC when running `installcheck`? Run with the `--interactive` flag first, e.g. @@ -36,10 +47,6 @@ This repository is a simple on-ramp to help contributors run tests (`installchec It will stop after starting the cluster, and you can follow the prompt to set any GUC before running `make installcheck` -0. Shit's *SLOW* - - If you are using Docker for Mac, [don't](VMware_Fusion.md). - 0. Shit don't work Please turn on debug output and attach the debug output when you ask for help diff --git a/Remote_VM.md b/Remote_VM.md new file mode 100644 index 0000000..db7db14 --- /dev/null +++ b/Remote_VM.md @@ -0,0 +1,54 @@ +# Using a remove VM on AWS + +0. (Optional) Uninstall Docker for Mac + 0. Click the whale in your tray, choose "Preferences..." + 0. Choose "Uninstall / Reset" tab + 0. Click "Uninstall" and follow the wizard + +0. Now try running `docker`, if you see the following error message: + + ``` + $ docker + bash: docker: command not found + ``` + + Then install docker, docker-machine, and docker-compose: + + ``` + brew install docker-compose + ``` + +0. Create your docker machine on AWS: + Follow the guide on https://docs.docker.com/machine/drivers/aws/#options for security options etc. + + ``` + docker-machine create --driver amazonec2 --amazonec2-instance-type c4.large --amazonec2-root-size 40 icg-$(whoami) + ``` + + The first command creates a EC2 VM of type c4.large and 40 GB root drive. + Remember to use the appropriate account credentials if they're not already in + `~/.aws/`. + Refer to the various instance types & their costs [here](https://aws.amazon.com/ec2/pricing/on-demand/). + + Remember to stop/delete the VM when you don't need it for long periods using the AWS console. + + +0. In your shell session where you want to run Docker, if you see the following error: + + ``` + Cannot connect to the Docker daemon. Is the docker daemon running on this host? + ``` + + You can fix it by doing the following: + + * (Fish) + + ``` + eval (docker-machine env icg-$(whoami)) + ``` + + * (Bash) + + ``` + eval "$(docker-machine icg-$(whoami))" + ```