From eb7c3a76837bee89783e572bcb39aeba625d0265 Mon Sep 17 00:00:00 2001 From: Johnny Graettinger Date: Tue, 24 Dec 2024 15:40:30 -0600 Subject: [PATCH] build infra WIP --- .circleci/config.yml | 70 ------- .github/workflows/ci-workflow.yaml | 156 +++------------ Dockerfile | 53 +++++ Makefile | 188 +++++++++++++----- consumer/protocol/protocol.pb.go | 2 +- docs/_static/cmd-gazctl-attach-uuids.txt | 15 +- docs/_static/cmd-gazctl-journals-append.txt | 36 ++-- docs/_static/cmd-gazctl-journals-apply.txt | 34 ++-- docs/_static/cmd-gazctl-journals-edit.txt | 32 +-- .../_static/cmd-gazctl-journals-fragments.txt | 36 ++-- docs/_static/cmd-gazctl-journals-list.txt | 41 ++-- docs/_static/cmd-gazctl-journals-prune.txt | 36 ++-- docs/_static/cmd-gazctl-journals-read.txt | 41 ++-- .../cmd-gazctl-journals-reset-head.txt | 28 +-- docs/_static/cmd-gazctl-journals-suspend.txt | 41 ++++ docs/_static/cmd-gazctl-print-config.txt | 9 +- docs/_static/cmd-gazctl-shards-apply.txt | 44 ++-- docs/_static/cmd-gazctl-shards-edit.txt | 42 ++-- docs/_static/cmd-gazctl-shards-list.txt | 50 +++-- docs/_static/cmd-gazctl-shards-prune.txt | 45 +++-- docs/_static/cmd-gazctl.txt | 9 +- docs/_static/cmd-gazette-print-config.txt | 56 ++++-- docs/_static/cmd-gazette-serve.txt | 56 ++++-- docs/reference-gazctl.rst | 4 + mk/ci-builder.Dockerfile | 69 ------- mk/ci-release.Dockerfile | 25 --- mk/cmd-reference.mk | 60 ------ mk/common-build.mk | 136 ------------- mk/common-config.mk | 41 ---- mk/microk8s.mk | 13 -- 30 files changed, 624 insertions(+), 844 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 Dockerfile create mode 100644 docs/_static/cmd-gazctl-journals-suspend.txt delete mode 100644 mk/ci-builder.Dockerfile delete mode 100644 mk/ci-release.Dockerfile delete mode 100644 mk/cmd-reference.mk delete mode 100644 mk/common-build.mk delete mode 100644 mk/common-config.mk delete mode 100644 mk/microk8s.mk diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f80db6ca..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: gazette/ci-builder:20200421 - environment: - # Instruct the go tool to place the module cache under the working - # directory (~/project) - GOPATH: /root/project/.build/go-path - # RocksDB version to build. Keep in sync with mk/config.mk - # Also be sure to update the save_cache step below. - ROCKSDB_VERSION: 6.22.1 - # `nproc` on CircleCI docker runners returns the number of host cores, - # not provisioned cores (eg, 36 cores when only 2 are available). - NPROC: 2 - # Same issue with GOMAXPROCS. - GOMAXPROCS: 2 - steps: - - checkout - - setup_remote_docker - - restore_cache: - # We can't use a variable here :( Must be hard-coded. - # https://discuss.circleci.com/t/cannot-use-circle-yml-environment-variables-in-cache-keys/10994 - key: rocksdb-v1-v6.22.1 - - run: - name: Build RocksDB - command: make $(pwd)/.build/rocksdb-v${ROCKSDB_VERSION}/librocksdb.so - - save_cache: - name: Cache RocksDB - key: rocksdb-v1-v6.22.1 - paths: - # We *also* can't use a variable here (seriously?!). - # https://discuss.circleci.com/t/can-i-use-variables-in-cache-paths/11393 - - .build/rocksdb-v6.22.1/ - - restore_cache: - keys: - - go-mod-v1-{{ checksum "go.sum" }} - - go-mod-v1- - - run: - name: Go Install - command: make go-install - - run: - name: Go Test - command: make go-test-ci - - save_cache: - name: Cache Go Modules - key: go-mod-v1-{{ checksum "go.sum" }} - paths: - - .build/go-path/pkg - release: - docker: - - image: gazette/ci-builder:20200421 - steps: - - run: - name: Publish Gazette - command: echo "release tag ${CIRCLE_TAG}" - -workflows: - version: 2 - build: - jobs: - - build - release: - jobs: - - release: - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml index cdfd870d..535c0d72 100644 --- a/.github/workflows/ci-workflow.yaml +++ b/.github/workflows/ci-workflow.yaml @@ -1,139 +1,47 @@ name: Gazette Continuous Integration -# We build on any push to a branch, or when a release is created. on: - pull_request: - paths-ignore: - - "docs/**" push: - branches: - - "master" - # Ignore pushes to tags, since those ought to be handled by the release created event. - tags-ignore: - - "*" - paths-ignore: - - "docs/**" - release: - # Without this additional restriction, GH actions will trigger multiple runs for a single - # release, because it fires off separate events creating vs publishing the release. - types: [created] - -env: - # This is only used as the cache key to prevent rebuilding rocksdb every time. Eventually - # we'll need to figure out a solution that doesn't duplicate this version everywhere. - # For now, ensure that it's changed both here and in mk/common-config.mk. - ROCKSDB_VERSION: "6.22.1" + branches: [master] + pull_request: + branches: [master] jobs: build: - name: "Build" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - name: "Checkout" - uses: actions/checkout@v2 - - # Sets outputs for version, docker tag, and whether we should push images and upload release - # artifacts. These outputs are used by later steps. - - name: "Release Info" - id: release_info - env: - # just having this in the env is enough to make it visible in the "raw" logs attached to the run - GITHUB_EVENT_JSON: "${{ toJson(github.event) }}" - run: | - is_release=${{ github.event_name == 'release' }} - if [[ "$is_release" == "true" ]]; then - push_images=true - tag_name=${{ github.event.release.tag_name }} - # The version regex doesn't need to be super sophisticated, since the goal is not to - # validate that this is a semantic version number. Rather the goal is just to see if - # matches the typical pattern we use for release tags (e.g. v0.86.1). If it does, then - # we'll remove the 'v' prefix and use the remainder as the docker tag - # If a release tag does not match that format, then we'll just use the tag value as is. - if echo "$tag_name" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then - version="${tag_name#v}" - else - version="${tag_name}" - fi - else - # This is not a release, so we'll use 'dev-' for the version number - # and just 'latest-dev' for the docker tag. - sha=${{ github.sha }} - version="dev-${sha:0:7}" - # If this is a master build, then we'll treat this as a release and just use the - # hard-coded tag as the docker image tag. - if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then - # We don't want to put the git sha in the docker tag because otherwise they'll - # accumulate forever and just clutter up the page on docker hub. So 'latest-dev' - # just always gets you the most recent master build, and if you want a specific master - # build, then you can use the '@sha256:...' syntax. - docker_tag="latest-dev" - push_images='true' - else - push_images='false' - fi - fi - echo ::set-output name=VERSION::${version} - echo ::set-output name=DOCKER_TAG::${docker_tag:-$version} - echo ::set-output name=PUSH_IMAGES::${push_images} - echo ::set-output name=IS_RELEASE::${is_release} - - # Try to load the ci-builder docker image from the cache. If this misses, then it will be - # built automatically by a make rule. A later step will then run 'docker save' to export the - # image as a tar archive so we can cache it. - - name: "CI Builder Docker Image Cache" - id: "ci_builder_cache" - uses: actions/cache@v2 + - uses: actions/checkout@v4 with: - key: ci-builder-${{ hashFiles('mk/ci-builder.Dockerfile') }} - path: ".build/ci-builder-image.tar" + fetch-depth: 0 # Full history. - # The 'c' in these Cache steps is just for changing the cache key so that - # we can manually invalidate the cache if we need. - - name: "RocksDB Cache" - uses: actions/cache@v2 - with: - key: rocksdb-c4-${{ env.ROCKSDB_VERSION }} - path: ".build-ci/rocksdb-v${{ env.ROCKSDB_VERSION }}" - - - name: "Go Module Cache" - uses: actions/cache@v2 - with: - key: go-mod-c4-${{ hashFiles('go.sum') }} - path: ".build-ci/go-path/pkg" - # If we don't have a cached directory that matches the hash exactly, - # then this will allow a non-matching directory to be pulled in. This is safe - # because go will use its own finer-grained cache invalidation logic. - restore-keys: "go-mod-c4-" + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Login to GitHub container registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | \ + docker login --username ${{ github.actor }} --password-stdin ghcr.io - - name: "Build Binaries" - run: "make as-ci target=release-linux-binaries VERSION=${{ steps.release_info.outputs.VERSION }}" + - uses: actions/setup-go@v4 + - run: sudo apt install -y protobuf-compiler librocksdb-dev libsqlite3-dev etcd-server + - run: echo "VERSION=$(git describe --dirty --tags)" >> "$GITHUB_ENV" - - name: "Test" - run: "make as-ci target=go-test-ci VERSION=${{ steps.release_info.outputs.VERSION }}" + - run: make go-test-ci + - run: make go-build go-build-arm64 - # We upload this to the artifacts that are attached to the action just to make it easy for - # someone to pull down a build from another branch. - - name: "Upload Binaries" - uses: actions/upload-artifact@v4 + - uses: docker/build-push-action@v5 with: - name: "gazette-x86_64-linux-gnu.zip" - path: ".build-ci/gazette-x86_64-linux-gnu.zip" - - - name: "Upload Release Binaries" - if: steps.release_info.outputs.IS_RELEASE == 'true' - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + context: .build + file: Dockerfile + target: broker + platforms: linux/amd64,linux/arm64 + tags: "ghcr.io/gazette/broker:${{ env.VERSION }}" + push: true + + - uses: docker/build-push-action@v5 with: - asset_name: gazette-x86_64-linux-gnu.zip - asset_path: ".build-ci/gazette-x86_64-linux-gnu.zip" - upload_url: "${{ github.event.release.upload_url }}" - asset_content_type: application/zip - - - name: "Build and Push Docker Images" - if: steps.release_info.outputs.PUSH_IMAGES == 'true' - run: | - docker login -u '${{ secrets.DOCKER_USERNAME }}' -p '${{ secrets.DOCKER_PASSWORD }}' ${{ secrets.DOCKER_REGISTRY }} - make as-ci target=ci-release-gazette-examples VERSION=${{ steps.release_info.outputs.VERSION }} - make as-ci target=ci-release-gazette-broker VERSION=${{ steps.release_info.outputs.VERSION }} - make push-to-registry REGISTRY=${{ secrets.DOCKER_REGISTRY }} RELEASE_TAG=${{ steps.release_info.outputs.DOCKER_TAG }} + context: .build + file: Dockerfile + target: examples + platforms: linux/amd64 + tags: "ghcr.io/gazette/examples:${{ env.VERSION }}" + push: true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..00aadf08 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +################################################################################ +# Gazette broker image. + +FROM ubuntu:24.04 AS broker + +ARG TARGETARCH + +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install --no-install-recommends -y \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY ${TARGETARCH}/gazette ${TARGETARCH}/gazctl /usr/local/bin + +# Run as non-privileged "gazette" user. +RUN useradd gazette --create-home --shell /usr/sbin/nologin +USER gazette +WORKDIR /home/gazette + +################################################################################ +# Gazette examples image. + +FROM ubuntu:24.04 AS examples + +ARG TARGETARCH + +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install --no-install-recommends -y \ + ca-certificates \ + curl \ + librocksdb8.9 \ + libsqlite3-0 \ + && rm -rf /var/lib/apt/lists/* + +# Copy binaries to the image. +COPY \ + ${TARGETARCH}/bike-share \ + ${TARGETARCH}/chunker \ + ${TARGETARCH}/counter \ + ${TARGETARCH}/gazctl \ + ${TARGETARCH}/integration.test \ + ${TARGETARCH}/summer \ + ${TARGETARCH}/wordcountctl \ + /usr/local/bin/ + +# Run as non-privileged "gazette" user. +RUN useradd gazette --create-home --shell /usr/sbin/nologin +USER gazette +WORKDIR /home/gazette + diff --git a/Makefile b/Makefile index 200a493d..09f15f3a 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,47 @@ -include mk/common-config.mk - -# Project-specific configuration: - -# Go binaries to package under the `gazette/broker` image. -ci-release-gazette-broker-targets = \ - ${WORKDIR}/go-path/bin/gazctl \ - ${WORKDIR}/go-path/bin/gazette - -# Go binaries to package under the `gazette/examples` image. -ci-release-gazette-examples-targets = \ - ${WORKDIR}/go-path/bin/bike-share \ - ${WORKDIR}/go-path/bin/chunker \ - ${WORKDIR}/go-path/bin/counter \ - ${WORKDIR}/go-path/bin/gazctl \ - ${WORKDIR}/go-path/bin/integration.test \ - ${WORKDIR}/go-path/bin/summer \ - ${WORKDIR}/go-path/bin/wordcountctl - -# Targets of protobufs which must be compiled. +# Git version & date which are injected into built binaries. +VERSION ?= "$(shell git describe --dirty --tags)" +DATE = $(shell date +%F-%T-%Z) +# Repository root (the directory of this Makefile). +ROOTDIR = $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) +# Location to place intermediate files and output artifacts. +# Note the go tool ignores directories with leading '.' or '_'. +WORKDIR = ${ROOTDIR}/.build + +############################################################################### +# Rules for generating protobuf sources: + +${WORKDIR}/protoc-gen-gogo: + go mod download github.com/golang/protobuf + go build -o $@ github.com/gogo/protobuf/protoc-gen-gogo + +${WORKDIR}/protoc-gen-grpc-gateway: + go mod download github.com/grpc-ecosystem/grpc-gateway + go build -o $@ github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway + +# PROTOC_INC_MODULES is an append-able list of Go modules +# which should be included with `protoc` invocations. +PROTOC_INC_MODULES += "github.com/golang/protobuf" +PROTOC_INC_MODULES += "github.com/gogo/protobuf" + +# module_path expands a $(module), like go.gazette.dev/core, to the local path +# of its repository as currently specified by go.mod. +module_path = $(shell go list -f '{{ .Dir }}' -m $(module)) + +# Run the protobuf compiler to generate message and gRPC service implementations. +# Invoke protoc with local and third-party include paths set. The `go list` tool +# is used to map submodules to corresponding go.mod versions and paths. +%.pb.go: %.proto ${WORKDIR}/protoc-gen-gogo + PATH=${WORKDIR}:$${PATH} ;\ + protoc -I . $(foreach module, $(PROTOC_INC_MODULES), -I$(module_path)) \ + --gogo_out=paths=source_relative,plugins=grpc:. \ + $*.proto + +%.pb.gw.go: %.proto ${WORKDIR}/protoc-gen-grpc-gateway + PATH=${WORKDIR}:$${PATH} ;\ + protoc -I . $(foreach module, $(PROTOC_INC_MODULES), -I$(module_path)) \ + --grpc-gateway_out=logtostderr=true,paths=source_relative,generate_unbound_methods=false,grpc_api_configuration=$*_gateway.yaml:. \ + $*.proto + protobuf-targets = \ ./broker/protocol/protocol.pb.go \ ./broker/protocol/protocol.pb.gw.go \ @@ -29,38 +53,96 @@ protobuf-targets = \ # consumer.proto depends on protocol.proto & recorded_op.proto. consumer/protocol/consumer.pb.go: broker/protocol/protocol.proto consumer/recoverylog/recorded_op.proto -# Rule for integration.test, depended on by ci-release-gazette-examples. -# It's an actual file, but we set it as PHONY so it's kept fresh -# with each build of that target. -${WORKDIR}/go-path/bin/integration.test: +############################################################################### +# Rules for building and testing: + +go-build: $(protobuf-targets) ${WORKDIR}/amd64/integration.test + MBP=go.gazette.dev/core/mainboilerplate ;\ + go build -o ${WORKDIR}/amd64/ -v --tags libsqlite3 \ + -ldflags "-X $${MBP}.Version=${VERSION} -X $${MBP}.BuildDate=${DATE}" ./... + +go-build-arm64: $(protobuf-targets) + MBP=go.gazette.dev/core/mainboilerplate ;\ + GOOS=linux ;\ + GOARCH=arm64 \ + go build -o ${WORKDIR}/arm64/ -v --tags nozstd ./cmd/gazette ./cmd/gazctl + +go-test-fast: ${protobuf-targets} + go test --tags libsqlite3 ./... + +go-test-ci: ${protobuf-targets} + GORACE="halt_on_error=1" go test -race -count=10 --tags libsqlite3 --failfast ./... + +${WORKDIR}/amd64/integration.test: go test -v -c -tags integration ./test/integration -o $@ -include mk/common-build.mk -include mk/microk8s.mk -include mk/cmd-reference.mk - -# Push the broker & example image to a specified private registry. -# Override the registry to use by passing a "REGISTRY=" flag to make. -REGISTRY=localhost:32000 -RELEASE_TAG=latest -push-to-registry: - docker tag gazette/broker:latest $(REGISTRY)/broker:$(RELEASE_TAG) - docker tag gazette/examples:latest $(REGISTRY)/examples:$(RELEASE_TAG) - docker push $(REGISTRY)/broker:$(RELEASE_TAG) - docker push $(REGISTRY)/examples:$(RELEASE_TAG) - -${WORKDIR}/gazette-x86_64-linux-gnu.zip: go-install - cd ${WORKDIR}/go-path/bin/ - zip gazette-x86_64-linux-gnu.zip gazette gazctl - -# Builds a zip file containing both the gazette and gazctl release binaries. This target may only be run -# on a linux host, since we don't currently support cross-compilation (we may want to in the -# future). -release-linux-binaries: go-install - @# sanity check, since our make builds don't support cross-compilation at the moment - @test "$(shell uname -io)" = "x86_64 GNU/Linux" || (echo "only x86_64 linux binaries are produced" && exit 1) - @rm -f ${WORKDIR}/gazette-x86_64-linux-gnu.zip - zip -j ${WORKDIR}/gazette-x86_64-linux-gnu.zip ${WORKDIR}/go-path/bin/gazette ${WORKDIR}/go-path/bin/gazctl - - -.PHONY: release-linux-binaries ${WORKDIR}/go-path/bin/integration.test +############################################################################### +# Rules for updating reference documentation: + +cmd-reference-targets = \ + docs/_static/cmd-gazette-serve.txt \ + docs/_static/cmd-gazette-print-config.txt \ + docs/_static/cmd-gazctl.txt \ + docs/_static/cmd-gazctl-attach-uuids.txt \ + docs/_static/cmd-gazctl-journals-append.txt \ + docs/_static/cmd-gazctl-journals-apply.txt \ + docs/_static/cmd-gazctl-journals-edit.txt \ + docs/_static/cmd-gazctl-journals-fragments.txt \ + docs/_static/cmd-gazctl-journals-list.txt \ + docs/_static/cmd-gazctl-journals-prune.txt \ + docs/_static/cmd-gazctl-journals-read.txt \ + docs/_static/cmd-gazctl-journals-reset-head.txt \ + docs/_static/cmd-gazctl-journals-suspend.txt \ + docs/_static/cmd-gazctl-print-config.txt \ + docs/_static/cmd-gazctl-shards-apply.txt \ + docs/_static/cmd-gazctl-shards-edit.txt \ + docs/_static/cmd-gazctl-shards-list.txt \ + docs/_static/cmd-gazctl-shards-prune.txt + +cmd-reference: ${cmd-reference-targets} + + +docs/_static/cmd-gazette-serve.txt: go-build + ${WORKDIR}/amd64/gazette serve --help > $@ || true +docs/_static/cmd-gazette-print-config.txt: go-build + ${WORKDIR}/amd64/gazette serve print config --help > $@ || true + +docs/_static/cmd-gazctl.txt: go-build + ${WORKDIR}/amd64/gazctl --help > $@ || true +docs/_static/cmd-gazctl-attach-uuids.txt: go-build + ${WORKDIR}/amd64/gazctl attach-uuids --help > $@ || true +docs/_static/cmd-gazctl-journals-append.txt: go-build + ${WORKDIR}/amd64/gazctl journals append --help > $@ || true +docs/_static/cmd-gazctl-journals-apply.txt: go-build + ${WORKDIR}/amd64/gazctl journals apply --help > $@ || true +docs/_static/cmd-gazctl-journals-edit.txt: go-build + ${WORKDIR}/amd64/gazctl journals edit --help > $@ || true +docs/_static/cmd-gazctl-journals-fragments.txt: go-build + ${WORKDIR}/amd64/gazctl journals fragments --help > $@ || true +docs/_static/cmd-gazctl-journals-list.txt: go-build + ${WORKDIR}/amd64/gazctl journals list --help > $@ || true +docs/_static/cmd-gazctl-journals-prune.txt: go-build + ${WORKDIR}/amd64/gazctl journals prune --help > $@ || true +docs/_static/cmd-gazctl-journals-read.txt: go-build + ${WORKDIR}/amd64/gazctl journals read --help > $@ || true +docs/_static/cmd-gazctl-journals-reset-head.txt: go-build + ${WORKDIR}/amd64/gazctl journals reset-head --help > $@ || true +docs/_static/cmd-gazctl-journals-suspend.txt: go-build + ${WORKDIR}/amd64/gazctl journals suspend --help > $@ || true +docs/_static/cmd-gazctl-print-config.txt: go-build + ${WORKDIR}/amd64/gazctl print-config --help > $@ || true +docs/_static/cmd-gazctl-shards-apply.txt: go-build + ${WORKDIR}/amd64/gazctl shards apply --help > $@ || true +docs/_static/cmd-gazctl-shards-edit.txt: go-build + ${WORKDIR}/amd64/gazctl shards edit --help > $@ || true +docs/_static/cmd-gazctl-shards-list.txt: go-build + ${WORKDIR}/amd64/gazctl shards list --help > $@ || true +docs/_static/cmd-gazctl-shards-prune.txt: go-build + ${WORKDIR}/amd64/gazctl shards prune --help > $@ || true + + +.PHONY: \ + cmd-reference \ + go-build \ + go-test-ci \ + go-test-fast \ No newline at end of file diff --git a/consumer/protocol/protocol.pb.go b/consumer/protocol/protocol.pb.go index 9b1a0e51..0ca62489 100644 --- a/consumer/protocol/protocol.pb.go +++ b/consumer/protocol/protocol.pb.go @@ -11,7 +11,6 @@ import ( proto "github.com/gogo/protobuf/proto" github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" golang_proto "github.com/golang/protobuf/proto" - _ "github.com/golang/protobuf/ptypes/duration" go_gazette_dev_core_broker_protocol "go.gazette.dev/core/broker/protocol" protocol "go.gazette.dev/core/broker/protocol" recoverylog "go.gazette.dev/core/consumer/recoverylog" @@ -19,6 +18,7 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + _ "google.golang.org/protobuf/types/known/durationpb" io "io" math "math" math_bits "math/bits" diff --git a/docs/_static/cmd-gazctl-attach-uuids.txt b/docs/_static/cmd-gazctl-attach-uuids.txt index 775a1dce..ebeeeadd 100644 --- a/docs/_static/cmd-gazctl-attach-uuids.txt +++ b/docs/_static/cmd-gazctl-attach-uuids.txt @@ -42,18 +42,11 @@ Optionally compose with "jq" to un-nest the JSON objects: > | jq -c '{uuid: .uuid} + .record' -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [attach-uuids command options] - --template= Go text/template for output (default: "{{.UUID}},{{.Line}}\n") - --max-length= Maximum allowed byte-length of an input line (default: 4194304) - --skip-header Omit the first line of each input file + --template= Go text/template for output (default: "{{.UUID}},{{.Line}}\n") + --max-length= Maximum allowed byte-length of an input line (default: 4194304) + --skip-header Omit the first line of each input file diff --git a/docs/_static/cmd-gazctl-journals-append.txt b/docs/_static/cmd-gazctl-journals-append.txt index 84eccdf2..8767335e 100644 --- a/docs/_static/cmd-gazctl-journals-append.txt +++ b/docs/_static/cmd-gazctl-journals-append.txt @@ -53,27 +53,31 @@ cat /dev/stdout > my-fifo & # Hold my-fifo open so gazctl doesn't read EOF. gazctl journals append -l my-label --framing 'lines' --mapping 'rendezvous' --input my-fifo -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [append command options] - -l, --selector= Label selector of journals to append to - -i, --input= Input file path. Use '-' for stdin (default: -) - -f, --framing=[none|lines|fixed] Framing of records in input, if any (default: none) - -m, --mapping=[random|modulo|rendezvous|direct] Mapping function of records to journals (default: random) - --base64 Partition keys under 'lines' framing are interpreted as base64 + -l, --selector= Label selector of journals to append to + -i, --input= Input file path. Use '-' for stdin (default: -) + -f, --framing=[none|lines|fixed] Framing of records in input, if any (default: none) + -m, --mapping=[random|modulo|rendezvous|direct] Mapping function of records to journals (default: random) + --base64 Partition keys under 'lines' framing are interpreted as base64 diff --git a/docs/_static/cmd-gazctl-journals-apply.txt b/docs/_static/cmd-gazctl-journals-apply.txt index d81de908..452f4f6d 100644 --- a/docs/_static/cmd-gazctl-journals-apply.txt +++ b/docs/_static/cmd-gazctl-journals-apply.txt @@ -35,26 +35,30 @@ gazctl supports a flag which will send changes in batches of at most a single Etcd transaction and it should therefore be used with caution. If possible, prefer to use label selectors to limit the number of changes. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [apply command options] - --specs= Input specifications path to apply. Use '-' for stdin (default: -) - --dry-run Perform a dry-run of the apply - --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction - (default: 0) + --specs= Input specifications path to apply. Use '-' for stdin (default: -) + --dry-run Perform a dry-run of the apply + --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction + (default: 0) diff --git a/docs/_static/cmd-gazctl-journals-edit.txt b/docs/_static/cmd-gazctl-journals-edit.txt index 130b1b94..31fd546b 100644 --- a/docs/_static/cmd-gazctl-journals-edit.txt +++ b/docs/_static/cmd-gazctl-journals-edit.txt @@ -29,25 +29,29 @@ gazctl supports a flag which will send changes in batches of at most a single Etcd transaction and it should therefore be used with caution. If possible, prefer to use label selectors to limit the number of changes. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [edit command options] - -l, --selector= Label Selector query to filter on - --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction - (default: 0) + -l, --selector= Label Selector query to filter on + --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction + (default: 0) diff --git a/docs/_static/cmd-gazctl-journals-fragments.txt b/docs/_static/cmd-gazctl-journals-fragments.txt index 846e4305..7ec98618 100644 --- a/docs/_static/cmd-gazctl-journals-fragments.txt +++ b/docs/_static/cmd-gazctl-journals-fragments.txt @@ -51,27 +51,31 @@ gazctl journals fragments -l name=my/journal --url-ttl 1m --from $(date -d "1 ho gazctl journals fragments -l my-label --format json --url-ttl 1h --from $(date -d 3AM '+%s') --to $(date -d 4:05AM '+%s') --format json -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [fragments command options] - -l, --selector= Label Selector query to filter on - -o, --format=[table|json|proto] Output format (default: table) - --from= Restrict to fragments created at or after this time, in unix seconds since epoch - --to= Restrict to fragments created before this time, in unix seconds since epoch - --url-ttl= Provide a signed GET URL with the given TTL + -l, --selector= Label Selector query to filter on + -o, --format=[table|json|proto] Output format (default: table) + --from= Restrict to fragments created at or after this time, in unix seconds since epoch + --to= Restrict to fragments created before this time, in unix seconds since epoch + --url-ttl= Provide a signed GET URL with the given TTL (default: 0s) diff --git a/docs/_static/cmd-gazctl-journals-list.txt b/docs/_static/cmd-gazctl-journals-list.txt index 5b23e646..e32fb3ef 100644 --- a/docs/_static/cmd-gazctl-journals-list.txt +++ b/docs/_static/cmd-gazctl-journals-list.txt @@ -24,29 +24,34 @@ typically, common configuration. This hierarchy is simply sugar for and is exactly equivalent to the original JournalSpecs. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [list command options] - -l, --selector= Label Selector query to filter on - -o, --format=[table|yaml|json|proto] Output format (default: table) - -L, --label-columns= Labels to present as columns, eg -L label-one -L label-two - -p, --primary Show primary column - -r, --replicas Show replicas column - --rf Show replication factor column - --stores Show fragment store column + -l, --selector= Label Selector query to filter on + -o, --format=[table|yaml|json|proto] Output format (default: table) + -L, --label-columns= Labels to present as columns, eg -L label-one -L label-two + -p, --primary Show primary column + -r, --replicas Show replicas column + --rf Show replication factor column + --stores Show fragment store column + --watch Use a long-lived watch diff --git a/docs/_static/cmd-gazctl-journals-prune.txt b/docs/_static/cmd-gazctl-journals-prune.txt index ff7f352e..94be0aed 100644 --- a/docs/_static/cmd-gazctl-journals-prune.txt +++ b/docs/_static/cmd-gazctl-journals-prune.txt @@ -3,32 +3,36 @@ Usage: Deletes fragments across all configured fragment stores of matching journals that are older than the configured retention. -There is a caveat when pruning journals. For a given journal, there could be multiple fragments covering the same offset. These fragments contain identical data at a given -offset, but the brokers are tracking only the largest fragment, i.e. the fragment that covers the largest span of offsets. As a result, the prune command will delete only this -tracked fragment, leaving the smaller fragments untouched. As a workaround, operators can wait for the fragment listing to refresh and prune the journals again. +There is a caveat when pruning journals. For a given journal, there could be multiple fragments covering the same offset. These fragments contain identical data at a given offset, but the brokers +are tracking only the largest fragment, i.e. the fragment that covers the largest span of offsets. As a result, the prune command will delete only this tracked fragment, leaving the smaller +fragments untouched. As a workaround, operators can wait for the fragment listing to refresh and prune the journals again. Use --selector to supply a LabelSelector to select journals to prune. See "journals list --help" for details and examples. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [prune command options] - -l, --selector= Label Selector query to filter on - --dry-run Perform a dry-run of the apply + -l, --selector= Label Selector query to filter on + --dry-run Perform a dry-run of the apply diff --git a/docs/_static/cmd-gazctl-journals-read.txt b/docs/_static/cmd-gazctl-journals-read.txt index 1affd294..c73ad467 100644 --- a/docs/_static/cmd-gazctl-journals-read.txt +++ b/docs/_static/cmd-gazctl-journals-read.txt @@ -58,29 +58,34 @@ echo "{}" > offsets.json # Must already exist. gazctl journals read -l my-label -o output --offsets offsets.json --offsets-out offsets.json --broker.cache.size=256 --zone=us-east-1 -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [read command options] - -l, --selector= Label selector of journals to read - -b, --block Do not exit on journal EOF; wait for new data until signaled - --tail Start reading from the journal write-head (rather than offset 0) - -o, --output= Output file path. Use '-' for stdout (default: -) - --offsets= Path from which initial journal offsets are read at startup - --offsets-out= Path to which final journal offsets are written at exit - --file-root= Filesystem path which roots file:// fragment store + -l, --selector= Label selector of journals to read + -b, --block Do not exit on journal EOF; wait for new data until signaled + --tail Start reading from the journal write-head (rather than offset 0) + -o, --output= Output file path. Use '-' for stdout (default: -) + --offsets= Path from which initial journal offsets are read at startup + --offsets-out= Path to which final journal offsets are written at exit + --file-root= Filesystem path which roots file:// fragment store + --from= Skip over fragments persisted before this time, in unix seconds since epoch diff --git a/docs/_static/cmd-gazctl-journals-reset-head.txt b/docs/_static/cmd-gazctl-journals-reset-head.txt index ca11d1d3..cb132249 100644 --- a/docs/_static/cmd-gazctl-journals-reset-head.txt +++ b/docs/_static/cmd-gazctl-journals-reset-head.txt @@ -37,23 +37,27 @@ reset-head is safe to run against journals which are already consistent and and are being actively appended to. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [journals command options] + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [reset-head command options] - -l, --selector= Label Selector query to filter on + -l, --selector= Label Selector query to filter on diff --git a/docs/_static/cmd-gazctl-journals-suspend.txt b/docs/_static/cmd-gazctl-journals-suspend.txt new file mode 100644 index 00000000..a07a606c --- /dev/null +++ b/docs/_static/cmd-gazctl-journals-suspend.txt @@ -0,0 +1,41 @@ +Usage: + gazctl [OPTIONS] journals [journals-OPTIONS] suspend [suspend-OPTIONS] + +Suspend idle journals to use fewer or zero broker replicas. + +Suspension updates the 'suspend' field of the JournalSpec with its suspended +status and its resumption offset. When applying other updates to JournalSpecs, +operators utilizing journal suspension must take care to not overwrite the +journal's suspension configuration. + +Typically this means reading the current JournalSpec and its ModRevision, +copying the current 'suspend' field alongside other changes being made, +and then applying the updated JournalSpec with ExpectModRevision. +Note that the 'journals edit' subcommand uses this workflow. + + +Help Options: + -h, --help Show this help message + +[journals command options] + + Interact with broker journals: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + + Broker: + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + +[suspend command options] + -l, --selector= Label Selector query to filter on + -f, --force Suspend journals even if they have local fragments + diff --git a/docs/_static/cmd-gazctl-print-config.txt b/docs/_static/cmd-gazctl-print-config.txt index 024a846f..4c798129 100644 --- a/docs/_static/cmd-gazctl-print-config.txt +++ b/docs/_static/cmd-gazctl-print-config.txt @@ -5,13 +5,6 @@ print-config parses the combined configuration from gazctl.ini, flags, and environment variables, and then writes the configuration to stdout in INI format. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message diff --git a/docs/_static/cmd-gazctl-shards-apply.txt b/docs/_static/cmd-gazctl-shards-apply.txt index 65398904..60592e44 100644 --- a/docs/_static/cmd-gazctl-shards-apply.txt +++ b/docs/_static/cmd-gazctl-shards-apply.txt @@ -25,31 +25,39 @@ gazctl supports a flag which will send changes in batches of at most a single Etcd transaction and it should therefore be used with caution. If possible, prefer to use label selectors to limit the number of changes. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [shards command options] + Interact with consumer shards: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Consumer: - --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] - --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] - --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] + --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] + --consumer.cert-file= Path to the client TLS certificate [$CONSUMER_CERT_FILE] + --consumer.cert-key-file= Path to the client TLS private key [$CONSUMER_CERT_KEY_FILE] + --consumer.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$CONSUMER_TRUSTED_CA_FILE] + --consumer.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$CONSUMER_AUTH_KEYS] + --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] + --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [apply command options] - --specs= Input specifications path to apply. Use '-' for stdin (default: -) - --dry-run Perform a dry-run of the apply - --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction - (default: 0) + --specs= Input specifications path to apply. Use '-' for stdin (default: -) + --dry-run Perform a dry-run of the apply + --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction + (default: 0) diff --git a/docs/_static/cmd-gazctl-shards-edit.txt b/docs/_static/cmd-gazctl-shards-edit.txt index 057c37bf..fee3d46f 100644 --- a/docs/_static/cmd-gazctl-shards-edit.txt +++ b/docs/_static/cmd-gazctl-shards-edit.txt @@ -29,30 +29,38 @@ gazctl supports a flag which will send changes in batches of at most a single Etcd transaction and it should therefore be used with caution. If possible, prefer to use label selectors to limit the number of changes. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [shards command options] + Interact with consumer shards: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Consumer: - --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] - --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] - --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] + --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] + --consumer.cert-file= Path to the client TLS certificate [$CONSUMER_CERT_FILE] + --consumer.cert-key-file= Path to the client TLS private key [$CONSUMER_CERT_KEY_FILE] + --consumer.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$CONSUMER_TRUSTED_CA_FILE] + --consumer.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$CONSUMER_AUTH_KEYS] + --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] + --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [edit command options] - -l, --selector= Label Selector query to filter on - --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction - (default: 0) + -l, --selector= Label Selector query to filter on + --max-txn-size= maximum number of specs to be processed within an apply transaction. If 0, the default, all changes are issued in a single transaction + (default: 0) diff --git a/docs/_static/cmd-gazctl-shards-list.txt b/docs/_static/cmd-gazctl-shards-list.txt index 452e306f..c791d9c1 100644 --- a/docs/_static/cmd-gazctl-shards-list.txt +++ b/docs/_static/cmd-gazctl-shards-list.txt @@ -19,34 +19,42 @@ It's recommended that --lag be used with a relatively focused --selector, as fetching consumption lag for a large number of shards may take a while. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message [shards command options] + Interact with consumer shards: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Consumer: - --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] - --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] - --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] + --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] + --consumer.cert-file= Path to the client TLS certificate [$CONSUMER_CERT_FILE] + --consumer.cert-key-file= Path to the client TLS private key [$CONSUMER_CERT_KEY_FILE] + --consumer.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$CONSUMER_TRUSTED_CA_FILE] + --consumer.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$CONSUMER_AUTH_KEYS] + --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] + --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [list command options] - -l, --selector= Label Selector query to filter on - -o, --format=[table|yaml|json|proto] Output format (default: table) - -L, --label-columns= Labels to present as columns, eg -L label-one -L label-two - -p, --primary Show primary column - -r, --replicas Show replicas column - --rf Show replication factor column - --lag Show the amount of unread data for each shard + -l, --selector= Label Selector query to filter on + -o, --format=[table|yaml|json|proto] Output format (default: table) + -L, --label-columns= Labels to present as columns, eg -L label-one -L label-two + -p, --primary Show primary column + -r, --replicas Show replicas column + --rf Show replication factor column + --lag Show the amount of unread data for each shard diff --git a/docs/_static/cmd-gazctl-shards-prune.txt b/docs/_static/cmd-gazctl-shards-prune.txt index 8a382105..c549b911 100644 --- a/docs/_static/cmd-gazctl-shards-prune.txt +++ b/docs/_static/cmd-gazctl-shards-prune.txt @@ -10,30 +10,47 @@ Prune log examines the provided hints to identify Fragments of the log which have no intersection with any live files of the DB, and can thus be safely deleted. +CAUTION: -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] +When pruning recovery logs which have been forked from other logs, +it's crucial that *all* shards which participate in the forked log +history are included in the prune operation. When pruning a log, +hints from all referencing shards are inspected to determine if a +fragment is overlapped, and a failure to include a shard which +references the log may cause data it depends on to be deleted. -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] Help Options: - -h, --help Show this help message + -h, --help Show this help message [shards command options] + Interact with consumer shards: + --zone= Availability zone within which this process is running (default: local) [$ZONE] + + Logging: + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + Consumer: - --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] - --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] - --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] + --consumer.address= Service address endpoint (default: http://localhost:8080) [$CONSUMER_ADDRESS] + --consumer.cert-file= Path to the client TLS certificate [$CONSUMER_CERT_FILE] + --consumer.cert-key-file= Path to the client TLS private key [$CONSUMER_CERT_KEY_FILE] + --consumer.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$CONSUMER_TRUSTED_CA_FILE] + --consumer.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$CONSUMER_AUTH_KEYS] + --consumer.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$CONSUMER_CACHE_SIZE] + --consumer.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$CONSUMER_CACHE_TTL] Broker: - --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] - --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] - --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] + --broker.address= Service address endpoint (default: http://localhost:8080) [$BROKER_ADDRESS] + --broker.cert-file= Path to the client TLS certificate [$BROKER_CERT_FILE] + --broker.cert-key-file= Path to the client TLS private key [$BROKER_CERT_KEY_FILE] + --broker.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$BROKER_TRUSTED_CA_FILE] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys. The first key is used to sign Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.cache.size= Size of client route cache. If <= zero, no cache is used (server always proxies) (default: 0) [$BROKER_CACHE_SIZE] + --broker.cache.ttl= Time-to-live of route cache entries. (default: 1m) [$BROKER_CACHE_TTL] [prune command options] - -l, --selector= Label Selector query to filter on - --dry-run Perform a dry-run of the apply + -l, --selector= Label Selector query to filter on + --dry-run Perform a dry-run of the apply diff --git a/docs/_static/cmd-gazctl.txt b/docs/_static/cmd-gazctl.txt index 1c355c2d..451b545e 100644 --- a/docs/_static/cmd-gazctl.txt +++ b/docs/_static/cmd-gazctl.txt @@ -9,15 +9,8 @@ or with '~/.config/gazette/gazctl.ini'. Use the 'print-config' sub-command to in the tool's current configuration. -Application Options: - --zone= Availability zone within which this process is running (default: local) [$ZONE] - -Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] - Help Options: - -h, --help Show this help message + -h, --help Show this help message Available commands: attach-uuids Generate and attach UUIDs to text input records diff --git a/docs/_static/cmd-gazette-print-config.txt b/docs/_static/cmd-gazette-print-config.txt index 3f3e8d6d..cba2715d 100644 --- a/docs/_static/cmd-gazette-print-config.txt +++ b/docs/_static/cmd-gazette-print-config.txt @@ -7,25 +7,51 @@ its responsible journals and will exit only when it can safely do so. Broker: - --broker.zone= Availability zone within which this process is running (default: local) [$BROKER_ZONE] - --broker.id= Unique ID of this process. Auto-generated if not set [$BROKER_ID] - --broker.host= Addressable, advertised hostname or IP of this process. Hostname is used if not set [$BROKER_HOST] - --broker.port= Service port for HTTP and gRPC requests. A random port is used if not set [$BROKER_PORT] - --broker.limit= Maximum number of Journals the broker will allocate (default: 1024) [$BROKER_LIMIT] - --broker.file-root= Local path which roots file:// fragment stores (optional) [$BROKER_FILE_ROOT] - --broker.max-append-rate= Max rate (in bytes-per-sec) that any one journal may be appended to. If zero, there is no max rate (default: 0) [$BROKER_MAX_APPEND_RATE] - --broker.min-append-rate= Min rate (in bytes-per-sec) at which a client may stream Append RPC content. RPCs unable to sustain this rate are aborted (default: 65536) - [$BROKER_MIN_APPEND_RATE] + --broker.zone= Availability zone within which this process is running (default: local) [$BROKER_ZONE] + --broker.id= Unique ID of this process. Auto-generated if not set [$BROKER_ID] + --broker.host= Addressable, advertised hostname or IP of this process. Hostname is used if not set [$BROKER_HOST] + --broker.port= Service port for HTTP and gRPC requests. A random port is used if not set. Port may also take the form 'unix:///path/to/socket' to use a Unix + Domain Socket [$BROKER_PORT] + --broker.server-cert-file= Path to the server TLS certificate. This option toggles whether TLS is used. If absent, all other TLS settings are ignored. + [$BROKER_SERVER_CERT_FILE] + --broker.server-cert-key-file= Path to the server TLS private key [$BROKER_SERVER_CERT_KEY_FILE] + --broker.server-ca-file= Path to the trusted CA for server verification of client certificates. When present, client certificates are required and verified against this + CA. When absent, client certificates are not required but are verified against the system CA pool if presented. [$BROKER_SERVER_CA_FILE] + --broker.peer-cert-file= Path to the client TLS certificate for peer-to-peer requests [$BROKER_PEER_CERT_FILE] + --broker.peer-cert-key-file= Path to the client TLS private key for peer-to-peer requests [$BROKER_PEER_CERT_KEY_FILE] + --broker.peer-ca-file= Path to the trusted CA for client verification of peer server certificates. When absent, the system CA pool is used instead. + [$BROKER_PEER_CA_FILE] + --broker.max-grpc-recv-size= Maximum size of gRPC messages accepted by this server, in bytes (default: 4194304) [$BROKER_MAX_GRPC_RECV_SIZE] + --broker.allow-origin= Origin to allow in CORS contexts [$BROKER_ALLOW_ORIGIN] + --broker.limit= Maximum number of Journals the broker will allocate (default: 1024) [$BROKER_LIMIT] + --broker.file-root= Local path which roots file:// fragment stores (optional) [$BROKER_FILE_ROOT] + --broker.max-append-rate= Max rate (in bytes-per-sec) that any one journal may be appended to. If zero, there is no max rate (default: 0) [$BROKER_MAX_APPEND_RATE] + --broker.max-replication= Maximum effective replication of any one journal, which upper-bounds its stated replication. (default: 9) [$BROKER_MAX_REPLICATION] + --broker.min-append-rate= Min rate (in bytes-per-sec) at which a client may stream Append RPC content. RPCs unable to sustain this rate are aborted (default: 65536) + [$BROKER_MIN_APPEND_RATE] + --broker.disable-stores Disable use of any configured journal fragment stores. The broker will neither list or persist remote fragments, and all data is discarded on + broker exit. [$BROKER_DISABLE_STORES] + --broker.watch-delay= Delay applied to the application of watched Etcd events. Larger values amortize the processing of fast-changing Etcd keys. (default: 30ms) + [$BROKER_WATCH_DELAY] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys used to sign (first key) and verify (all keys) Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.auto-suspend Automatically suspend journals which have persisted all fragments [$BROKER_AUTO_SUSPEND] Etcd: - --etcd.address= Etcd service address endpoint (default: http://localhost:2379) [$ETCD_ADDRESS] - --etcd.lease= Time-to-live of Etcd lease (default: 20s) [$ETCD_LEASE_TTL] - --etcd.prefix= Etcd base prefix for broker state and coordination (default: /gazette/cluster) [$ETCD_PREFIX] + --etcd.address= Etcd service address endpoint (default: http://localhost:2379) [$ETCD_ADDRESS] + --etcd.cert-file= Path to the client TLS certificate [$ETCD_CERT_FILE] + --etcd.cert-key-file= Path to the client TLS private key [$ETCD_CERT_KEY_FILE] + --etcd.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$ETCD_TRUSTED_CA_FILE] + --etcd.lease= Time-to-live of Etcd lease (default: 20s) [$ETCD_LEASE_TTL] + --etcd.prefix= Etcd base prefix for broker state and coordination (default: /gazette/cluster) [$ETCD_PREFIX] Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + +Debug: + --debug.port= Port for diagnostics [$DEBUG_PORT] + --debug.private When true, don't serve diagnostics over the main service port [$DEBUG_PRIVATE] Help Options: - -h, --help Show this help message + -h, --help Show this help message diff --git a/docs/_static/cmd-gazette-serve.txt b/docs/_static/cmd-gazette-serve.txt index 3f3e8d6d..cba2715d 100644 --- a/docs/_static/cmd-gazette-serve.txt +++ b/docs/_static/cmd-gazette-serve.txt @@ -7,25 +7,51 @@ its responsible journals and will exit only when it can safely do so. Broker: - --broker.zone= Availability zone within which this process is running (default: local) [$BROKER_ZONE] - --broker.id= Unique ID of this process. Auto-generated if not set [$BROKER_ID] - --broker.host= Addressable, advertised hostname or IP of this process. Hostname is used if not set [$BROKER_HOST] - --broker.port= Service port for HTTP and gRPC requests. A random port is used if not set [$BROKER_PORT] - --broker.limit= Maximum number of Journals the broker will allocate (default: 1024) [$BROKER_LIMIT] - --broker.file-root= Local path which roots file:// fragment stores (optional) [$BROKER_FILE_ROOT] - --broker.max-append-rate= Max rate (in bytes-per-sec) that any one journal may be appended to. If zero, there is no max rate (default: 0) [$BROKER_MAX_APPEND_RATE] - --broker.min-append-rate= Min rate (in bytes-per-sec) at which a client may stream Append RPC content. RPCs unable to sustain this rate are aborted (default: 65536) - [$BROKER_MIN_APPEND_RATE] + --broker.zone= Availability zone within which this process is running (default: local) [$BROKER_ZONE] + --broker.id= Unique ID of this process. Auto-generated if not set [$BROKER_ID] + --broker.host= Addressable, advertised hostname or IP of this process. Hostname is used if not set [$BROKER_HOST] + --broker.port= Service port for HTTP and gRPC requests. A random port is used if not set. Port may also take the form 'unix:///path/to/socket' to use a Unix + Domain Socket [$BROKER_PORT] + --broker.server-cert-file= Path to the server TLS certificate. This option toggles whether TLS is used. If absent, all other TLS settings are ignored. + [$BROKER_SERVER_CERT_FILE] + --broker.server-cert-key-file= Path to the server TLS private key [$BROKER_SERVER_CERT_KEY_FILE] + --broker.server-ca-file= Path to the trusted CA for server verification of client certificates. When present, client certificates are required and verified against this + CA. When absent, client certificates are not required but are verified against the system CA pool if presented. [$BROKER_SERVER_CA_FILE] + --broker.peer-cert-file= Path to the client TLS certificate for peer-to-peer requests [$BROKER_PEER_CERT_FILE] + --broker.peer-cert-key-file= Path to the client TLS private key for peer-to-peer requests [$BROKER_PEER_CERT_KEY_FILE] + --broker.peer-ca-file= Path to the trusted CA for client verification of peer server certificates. When absent, the system CA pool is used instead. + [$BROKER_PEER_CA_FILE] + --broker.max-grpc-recv-size= Maximum size of gRPC messages accepted by this server, in bytes (default: 4194304) [$BROKER_MAX_GRPC_RECV_SIZE] + --broker.allow-origin= Origin to allow in CORS contexts [$BROKER_ALLOW_ORIGIN] + --broker.limit= Maximum number of Journals the broker will allocate (default: 1024) [$BROKER_LIMIT] + --broker.file-root= Local path which roots file:// fragment stores (optional) [$BROKER_FILE_ROOT] + --broker.max-append-rate= Max rate (in bytes-per-sec) that any one journal may be appended to. If zero, there is no max rate (default: 0) [$BROKER_MAX_APPEND_RATE] + --broker.max-replication= Maximum effective replication of any one journal, which upper-bounds its stated replication. (default: 9) [$BROKER_MAX_REPLICATION] + --broker.min-append-rate= Min rate (in bytes-per-sec) at which a client may stream Append RPC content. RPCs unable to sustain this rate are aborted (default: 65536) + [$BROKER_MIN_APPEND_RATE] + --broker.disable-stores Disable use of any configured journal fragment stores. The broker will neither list or persist remote fragments, and all data is discarded on + broker exit. [$BROKER_DISABLE_STORES] + --broker.watch-delay= Delay applied to the application of watched Etcd events. Larger values amortize the processing of fast-changing Etcd keys. (default: 30ms) + [$BROKER_WATCH_DELAY] + --broker.auth-keys= Whitespace or comma separated, base64-encoded keys used to sign (first key) and verify (all keys) Authorization tokens. [$BROKER_AUTH_KEYS] + --broker.auto-suspend Automatically suspend journals which have persisted all fragments [$BROKER_AUTO_SUSPEND] Etcd: - --etcd.address= Etcd service address endpoint (default: http://localhost:2379) [$ETCD_ADDRESS] - --etcd.lease= Time-to-live of Etcd lease (default: 20s) [$ETCD_LEASE_TTL] - --etcd.prefix= Etcd base prefix for broker state and coordination (default: /gazette/cluster) [$ETCD_PREFIX] + --etcd.address= Etcd service address endpoint (default: http://localhost:2379) [$ETCD_ADDRESS] + --etcd.cert-file= Path to the client TLS certificate [$ETCD_CERT_FILE] + --etcd.cert-key-file= Path to the client TLS private key [$ETCD_CERT_KEY_FILE] + --etcd.trusted-ca-file= Path to the trusted CA for client verification of server certificates [$ETCD_TRUSTED_CA_FILE] + --etcd.lease= Time-to-live of Etcd lease (default: 20s) [$ETCD_LEASE_TTL] + --etcd.prefix= Etcd base prefix for broker state and coordination (default: /gazette/cluster) [$ETCD_PREFIX] Logging: - --log.level=[info|debug|warn] Logging level (default: info) [$LOG_LEVEL] - --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + --log.level=[trace|debug|info|warn|error|fatal] Logging level (default: warn) [$LOG_LEVEL] + --log.format=[json|text|color] Logging output format (default: text) [$LOG_FORMAT] + +Debug: + --debug.port= Port for diagnostics [$DEBUG_PORT] + --debug.private When true, don't serve diagnostics over the main service port [$DEBUG_PRIVATE] Help Options: - -h, --help Show this help message + -h, --help Show this help message diff --git a/docs/reference-gazctl.rst b/docs/reference-gazctl.rst index e4125f18..56a07222 100644 --- a/docs/reference-gazctl.rst +++ b/docs/reference-gazctl.rst @@ -41,6 +41,10 @@ gazctl journals reset-head --------------------------- .. literalinclude:: _static/cmd-gazctl-journals-reset-head.txt +gazctl journals suspend +--------------------------- +.. literalinclude:: _static/cmd-gazctl-journals-suspend.txt + gazctl print-config --------------------------- .. literalinclude:: _static/cmd-gazctl-print-config.txt diff --git a/mk/ci-builder.Dockerfile b/mk/ci-builder.Dockerfile deleted file mode 100644 index e341a266..00000000 --- a/mk/ci-builder.Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Note that this image will be cached in Github Actions, and the cache key is computed by hashing -# this file. This works only so long as there are no _other_ files that go into the final image. -# So if you add any ADD or COPY directives, be sure to update the cache key in the github actions -# workflow yaml -FROM ubuntu:22.04 - -RUN apt-get update -y \ - && apt-get upgrade -y \ - && apt-get install --no-install-recommends -y \ - build-essential \ - ca-certificates \ - curl \ - git \ - libbz2-dev \ - libjemalloc-dev \ - liblz4-dev \ - libprotobuf-dev \ - libsnappy-dev \ - libsqlite3-dev \ - libzstd-dev \ - protobuf-compiler \ - zlib1g-dev \ - zip \ - && rm -rf /var/lib/apt/lists/* - -ARG GOLANG_VERSION=1.22.4 -ARG GOLANG_SHA256=ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d - -ARG DOCKER_VERSION=19.03.8 -ARG DOCKER_SHA256=7f4115dc6a3c19c917f8b9664d7b51c904def1c984e082c4600097433323cf6f - -ARG ETCD_VERSION=v3.5.0 -ARG ETCD_SHA256=864baa0437f8368e0713d44b83afe21dce1fb4ee7dae4ca0f9dd5f0df22d01c4 - -ENV PATH=/usr/local/go/bin:$PATH - -RUN curl -L -o /tmp/golang.tgz \ - https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz \ - && echo "${GOLANG_SHA256} /tmp/golang.tgz" | sha256sum -c - \ - && tar --extract \ - --file /tmp/golang.tgz \ - --directory /usr/local \ - && rm /tmp/golang.tgz \ - && go version - -RUN curl -L -o /tmp/docker.tgz \ - https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \ - && echo "${DOCKER_SHA256} /tmp/docker.tgz" | sha256sum -c - \ - && tar --extract \ - --file /tmp/docker.tgz \ - --strip-components 1 \ - --directory /usr/local/bin/ \ - && rm /tmp/docker.tgz \ - && docker --version - -RUN curl -L -o /tmp/etcd.tgz \ - https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ - && echo "${ETCD_SHA256} /tmp/etcd.tgz" | sha256sum -c - \ - && tar --extract \ - --file /tmp/etcd.tgz \ - --directory /tmp/ \ - && mv /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcd /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin \ - && chown 1000:1000 /usr/local/bin/etcd /usr/local/bin/etcdctl \ - && rm -r /tmp/etcd-${ETCD_VERSION}-linux-amd64/ \ - && rm /tmp/etcd.tgz \ - && etcd --version - -WORKDIR /gazette - diff --git a/mk/ci-release.Dockerfile b/mk/ci-release.Dockerfile deleted file mode 100644 index 74bbab62..00000000 --- a/mk/ci-release.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM ubuntu:22.04 - -# Pick run-time library packages which match the development packages -# used by the ci-builder image. "curl" is included, to allow node-zone.sh -# mappings to directly query AWS/Azure/GCP metadata APIs. -RUN apt-get update -y \ - && apt-get upgrade -y \ - && apt-get install --no-install-recommends -y \ - ca-certificates \ - curl \ - libgflags2.2 \ - libjemalloc2 \ - libsnappy1v5 \ - libzstd1 \ - && rm -rf /var/lib/apt/lists/* - -# Copy binaries & librocks.so to the image. Configure Rocks for run-time linking. -COPY * /usr/local/bin/ -RUN mv /usr/local/bin/librocksdb.so* /usr/local/lib/ && ldconfig - -# Run as non-privileged "gazette" user. -RUN useradd gazette --create-home --shell /usr/sbin/nologin -USER gazette -WORKDIR /home/gazette - diff --git a/mk/cmd-reference.mk b/mk/cmd-reference.mk deleted file mode 100644 index 7e96b887..00000000 --- a/mk/cmd-reference.mk +++ /dev/null @@ -1,60 +0,0 @@ - -cmd-reference-targets = \ - docs/_static/cmd-gazette-serve.txt \ - docs/_static/cmd-gazette-print-config.txt \ - docs/_static/cmd-gazctl.txt \ - docs/_static/cmd-gazctl-attach-uuids.txt \ - docs/_static/cmd-gazctl-journals-append.txt \ - docs/_static/cmd-gazctl-journals-apply.txt \ - docs/_static/cmd-gazctl-journals-edit.txt \ - docs/_static/cmd-gazctl-journals-fragments.txt \ - docs/_static/cmd-gazctl-journals-list.txt \ - docs/_static/cmd-gazctl-journals-prune.txt \ - docs/_static/cmd-gazctl-journals-read.txt \ - docs/_static/cmd-gazctl-journals-reset-head.txt \ - docs/_static/cmd-gazctl-print-config.txt \ - docs/_static/cmd-gazctl-shards-apply.txt \ - docs/_static/cmd-gazctl-shards-edit.txt \ - docs/_static/cmd-gazctl-shards-list.txt \ - docs/_static/cmd-gazctl-shards-prune.txt - -cmd-reference: ${cmd-reference-targets} - - -docs/_static/cmd-gazette-serve.txt: go-install - gazette serve --help > $@ || true -docs/_static/cmd-gazette-print-config.txt: go-install - gazette serve print config --help > $@ || true - -docs/_static/cmd-gazctl.txt: go-install - gazctl --help > $@ || true -docs/_static/cmd-gazctl-attach-uuids.txt: go-install - gazctl attach-uuids --help > $@ || true -docs/_static/cmd-gazctl-journals-append.txt: go-install - gazctl journals append --help > $@ || true -docs/_static/cmd-gazctl-journals-apply.txt: go-install - gazctl journals apply --help > $@ || true -docs/_static/cmd-gazctl-journals-edit.txt: go-install - gazctl journals edit --help > $@ || true -docs/_static/cmd-gazctl-journals-fragments.txt: go-install - gazctl journals fragments --help > $@ || true -docs/_static/cmd-gazctl-journals-list.txt: go-install - gazctl journals list --help > $@ || true -docs/_static/cmd-gazctl-journals-prune.txt: go-install - gazctl journals prune --help > $@ || true -docs/_static/cmd-gazctl-journals-read.txt: go-install - gazctl journals read --help > $@ || true -docs/_static/cmd-gazctl-journals-reset-head.txt: go-install - gazctl journals reset-head --help > $@ || true -docs/_static/cmd-gazctl-print-config.txt: go-install - gazctl print-config --help > $@ || true -docs/_static/cmd-gazctl-shards-apply.txt: go-install - gazctl shards apply --help > $@ || true -docs/_static/cmd-gazctl-shards-edit.txt: go-install - gazctl shards edit --help > $@ || true -docs/_static/cmd-gazctl-shards-list.txt: go-install - gazctl shards list --help > $@ || true -docs/_static/cmd-gazctl-shards-prune.txt: go-install - gazctl shards prune --help > $@ || true - -.PHONY: cmd-reference diff --git a/mk/common-build.mk b/mk/common-build.mk deleted file mode 100644 index 4577fccf..00000000 --- a/mk/common-build.mk +++ /dev/null @@ -1,136 +0,0 @@ -# common-build.mk defines build rules which are common to both -# go.gazette.dev/core, and external consumer application projects -# which re-use the Gazette build infrastructure. - -# The ci-builder-image target builds a Docker image suitable for building -# gazette. It is the primary image used by gazette continuous integration builds. -ci-builder-image: ${WORKDIR}/ci-builder-image.tar - docker load -i ${WORKDIR}/ci-builder-image.tar - -# Builds the ci-builder docker image and also saves it as a tar file in the build directory -# This allows us to skip building the image without docker needing to check each individual -# layer, and also allows the tar file to be cached -${WORKDIR}/ci-builder-image.tar: - docker build -t gazette/ci-builder:latest - < ${COREDIR}/mk/ci-builder.Dockerfile - mkdir -p ${WORKDIR} - docker save -o ${WORKDIR}/ci-builder-image.tar gazette/ci-builder:latest - -host_os=$(shell uname -s) -AS_CI_RUN_ARGS= -ifeq ($(host_os),Linux) - AS_CI_RUN_ARGS = --user $(shell id -u):$(shell id -g) --group-add $(shell stat -c '%g' /var/run/docker.sock) -endif - -# The as-ci rule recursively calls `make` _within_ a instance of the ci-builder-image, -# and bind-mounting the gazette repository into the container. This rule allows for -# idempotent gazette builds which exactly match those produced by the CI builder. -# It uses a few tricks to keep builds fast: -# * The gazette repository checkout is bind-mounted into the container. -# * A ${WORKDIR}-ci directory is bind-mounted under the containers relative work -# directory. This keeps intermediate files of local vs "as-ci" builds separate -# so they don't clobber one another. -# * The GOPATH and GOCACHE variables are set to fall within the mounted work -# directory. Notably, this means repeat invocations can re-use the go modules -# cache, and the go build cache. -# * The Host's Docker socket is bind-mounted into the container, which has a docker -# client. This allows the ci-builder container to itself build Docker images. -# -# This will always run 'go mod download' before running the desired make target. This is -# done because users of gazette libraries are also depending on this makefile and so -# they need go to download the modules so that these files will be present before calling -# make. End-to-end, the process is to use 'go mod download' to download the makefile within -# the container, then the consumer's makefile will import this file, then make continues -# to build the target. -as-ci: ci-builder-image - mkdir -p ${WORKDIR} ${WORKDIR}-ci - # Strip root prefix from WORKDIR to build its equivalent within the container. - ROOT_CI=/gazette ;\ - WORK_CI=$${ROOT_CI}$(subst ${ROOTDIR},,${WORKDIR}) ;\ - docker run ${AS_CI_RUN_ARGS} \ - --rm \ - --tty \ - --mount src=${WORKDIR}-ci,target=$${WORK_CI},type=bind \ - --mount src=${ROOTDIR},target=$${ROOT_CI},type=bind \ - --env GOPATH=$${WORK_CI}/go-path \ - --env GOCACHE=$${WORK_CI}/go-build-cache \ - --mount src=/var/run/docker.sock,target=/var/run/docker.sock,type=bind \ - gazette/ci-builder /bin/sh -ec \ - "go mod download && \ - make ${target} VERSION=${VERSION} DATE=${DATE} REGISTRY=${REGISTRY} RELEASE_TAG=${RELEASE_TAG}" - -# Go build & test targets. -go-install: $(ROCKSDIR)/librocksdb.so $(protobuf-targets) - MBP=go.gazette.dev/core/mainboilerplate ;\ - go install -v --tags "${GO_BUILD_TAGS}" \ - -ldflags "-X $${MBP}.Version=${VERSION} -X $${MBP}.BuildDate=${DATE}" ./... -go-test-fast: ${ROCKSDIR}/librocksdb.so ${protobuf-targets} - go test -p ${NPROC} --tags "${GO_BUILD_TAGS}" ./... -go-test-ci: ${ROCKSDIR}/librocksdb.so ${protobuf-targets} - GORACE="halt_on_error=1" go test -p ${NPROC} -race -count=10 --tags "${GO_BUILD_TAGS}" --failfast ./... - -# The ci-release-% implicit rule builds a Docker image named by the rule -# stem, using binaries enumerated by a `-target` suffix. For example, -# an invocation with `ci-release-gazette-examples` has a stem `gazette-examples`, and will -# package binaries listed in `ci-release-gazette-examples-targets` into a docker -# image named `gazette/examples:latest`. -.SECONDEXPANSION: -ci-release-%: $(ROCKSDIR)/librocksdb.so go-install $$($$@-targets) - rm -rf ${WORKDIR}/ci-release - mkdir -p ${WORKDIR}/ci-release - ln ${$@-targets} ${ROCKSDIR}/librocksdb.so.${ROCKSDB_VERSION} \ - ${WORKDIR}/ci-release - docker build \ - -f ${COREDIR}/mk/ci-release.Dockerfile \ - -t $(subst -,/,$*):latest \ - ${WORKDIR}/ci-release/ - -# The librocksdb.so fetches and builds the version of RocksDB identified by -# the rule stem (eg, 5.17.2). We require a custom rule to build RocksDB as -# it's necessary to build with run-time type information (USE_RTTI=1), which -# is not enabled by default in third-party packages. -${WORKDIR}/rocksdb-v%/librocksdb.so: - # Fetch RocksDB source. - mkdir -p ${WORKDIR}/rocksdb-v$* - curl -L -o ${WORKDIR}/tmp.tgz https://github.com/facebook/rocksdb/archive/v$*.tar.gz - tar xzf ${WORKDIR}/tmp.tgz -C ${WORKDIR}/rocksdb-v$* --strip-components=1 - rm ${WORKDIR}/tmp.tgz - @# PORTABLE=1 prevents rocks from passing `-march=native`. This is important because it will cause gcc - @# to automatically use avx512 extensions if they're avaialable, which would cause it to break on CPUs - @# that don't support it. - PORTABLE=1 USE_SSE=1 DEBUG_LEVEL=0 USE_RTTI=1 \ - $(MAKE) -C $(dir $@) shared_lib -j${NPROC} - strip --strip-all $@ - - # Cleanup for less disk use / faster CI caching. - rm -rf $(dir $@)/shared-objects - find $(dir $@) -name "*.[oda]" -exec rm -f {} \; - -# Run the protobuf compiler to generate message and gRPC service implementations. -# Invoke protoc with local and third-party include paths set. The `go list` tool -# is used to map submodules to corresponding go.mod versions and paths. -%.pb.go: %.proto ${WORKDIR}/protoc-gen-gogo - PATH=${WORKDIR}:$${PATH} ;\ - protoc -I . $(foreach module, $(PROTOC_INC_MODULES), -I$(module_path)) \ - --gogo_out=paths=source_relative,plugins=grpc:. \ - $*.proto - -%.pb.gw.go: %.proto ${WORKDIR}/protoc-gen-grpc-gateway - PATH=${WORKDIR}:$${PATH} ;\ - protoc -I . $(foreach module, $(PROTOC_INC_MODULES), -I$(module_path)) \ - --grpc-gateway_out=logtostderr=true,paths=source_relative,generate_unbound_methods=false,grpc_api_configuration=$*_gateway.yaml:. \ - $*.proto - -# Rule to build protoc-gen-gogo. -${WORKDIR}/protoc-gen-gogo: - go mod download github.com/golang/protobuf - go build -o $@ github.com/gogo/protobuf/protoc-gen-gogo - -# Rule to build protoc-gen-grpc-gateway. -${WORKDIR}/protoc-gen-grpc-gateway: - go mod download github.com/grpc-ecosystem/grpc-gateway - go build -o $@ github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway - -# Rule for generic go-install-able Go binaries. -${WORKDIR}/go-path/bin/%: go-install - -.PHONY: ci-builder-image as-ci go-install go-test-fast go-test-ci diff --git a/mk/common-config.mk b/mk/common-config.mk deleted file mode 100644 index 9a51f213..00000000 --- a/mk/common-config.mk +++ /dev/null @@ -1,41 +0,0 @@ -# common-config.mk defines build configuration which is common to both -# go.gazette.dev/core, and external consumer application projects -# which re-use the Gazette build infrastructure. - -# Git version & date which are injected into built binaries. -VERSION = $(shell git describe --dirty --tags) -DATE = $(shell date +%F-%T-%Z) -# Number of available processors for parallel builds. -NPROC := $(if ${NPROC},${NPROC},$(shell nproc)) - -# Version of Rocks to build against. -# - This is tightly coupled github.com/tecbot/gorocksdb (update them together). -# - Also update .github/workflows/ci-workflow.yaml -ROCKSDB_VERSION = 6.22.1 - -# Repository root (the directory of the invoked Makefile). -ROOTDIR = $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) -# Location of the gazette/core repository (one level up from common-config.mk) -COREDIR = $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/..) -# Location to place intermediate files and output artifacts -# during the build process. Note the go tool ignores directories -# with leading '.' or '_'. -WORKDIR = ${ROOTDIR}/.build -# Location of RocksDB source under $WORKDIR. -ROCKSDIR = ${WORKDIR}/rocksdb-v${ROCKSDB_VERSION} - -# PROTOC_INC_MODULES is an append-able list of Go modules -# which should be included with `protoc` invocations. -PROTOC_INC_MODULES += "github.com/golang/protobuf" -PROTOC_INC_MODULES += "github.com/gogo/protobuf" - -# module_path expands a $(module), like go.gazette.dev/core, to the local path -# of its respository as currently specified by go.mod. -module_path = $(shell go list -f '{{ .Dir }}' -m $(module)) - -# Export appropriate CGO and run-time linker flags to build, link, -# and run against against our copy of Rocks. -export CGO_CFLAGS = -I${ROCKSDIR}/include -export CGO_CPPFLAGS = -I${ROCKSDIR}/include -export CGO_LDFLAGS = -L${ROCKSDIR} -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd -ldl -export LD_LIBRARY_PATH = ${ROCKSDIR} diff --git a/mk/microk8s.mk b/mk/microk8s.mk deleted file mode 100644 index 4d6a38d0..00000000 --- a/mk/microk8s.mk +++ /dev/null @@ -1,13 +0,0 @@ - -# Running microk8s.reset can take quite a while when many pods are running. -# This command nukes microk8s from orbit, by deleting its Etcd backing store -# and restarting, then spinning up Helm. In practice it's much faster for -# tearing down and re-initializing a local cluster. -microk8s-reset: - sudo microk8s.stop - sudo rm -r /var/snap/microk8s/common/var/run/etcd/member - sleep 1 - sudo microk8s.start || true - microk8s.enable dns storage registry - -.PHONY: microk8s-reset