Lighthouse v2.0.0

* Refactor configuration into `lighthouse.edn`
* Simplify cli and config detection
* Make graalvm-compatible
* Provide way to define many manifests in a single file
* Support visualization of dependencies with graphviz

.clj-kondo/.gitignore

@@ -0,0 +1 @@
+.cache

.clj-kondo/config.edn

@@ -0,0 +1,3 @@
+{:linters {:refer-all {:exclude [clojure.test]}}
+ :lint-as {lh.cli/defclifn clojure.core/defn
+           lh.test-support/with-tmp-files clojure.core/let}}

.github/workflows/commit.yml

@@ -0,0 +1,40 @@
+name: Commit
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+      - name: Setup Clojure
+        uses: DeLaGuardo/setup-clojure@3.5
+        with:
+          lein: latest
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: "m2-repo-${{ hashFiles('project.clj') }}"
+      - name: Run tests
+        run: lein test
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup clj-kondo
+        uses: DeLaGuardo/setup-clj-kondo@master
+        with:
+          version: '2022.01.15'
+      - name: Run clj-kondo
+        run: clj-kondo --lint src --lint test

.github/workflows/release.yml

@@ -0,0 +1,151 @@
+# This was heavily inspired by the build actions for clojure-lsp. Many thanks
+# to Eric Dallo for blazing the trail.
+
+name: Release
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  check:
+    name: Ensure consistency in release metadata
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      - name: Install Babashka
+        run: curl -s https://raw.githubusercontent.com/borkdude/babashka/master/install | sudo bash
+      - name: Check that version matches tag
+        run: bb check-version ${{ steps.get_version.outputs.VERSION }}
+
+  build-jar:
+    name: Build Uber Jar
+    runs-on: ubuntu-latest
+    needs: [check]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+      - name: Setup Clojure
+        uses: DeLaGuardo/setup-clojure@3.5
+        with:
+          lein: latest
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: "m2-repo-${{ hashFiles('project.clj') }}"
+      - name: Run uberjar
+        run: lein uberjar
+      - name: Upload jar
+        uses: actions/upload-artifact@v2
+        with:
+          path: target/lighthouse.jar
+          name: lighthouse.jar
+
+  macos-amd64:
+    name: Build MacOS native binary
+    runs-on: macos-latest
+    needs: [build-jar]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: lighthouse.jar
+      - name: Debugging
+        run: ls -R
+      - name: Install GraalVM
+        uses: DeLaGuardo/setup-graalvm@master
+        with:
+          graalvm: 21.3.1
+          java: java11
+      - name: Build native image
+        env:
+          THE_JAR: lighthouse.jar
+          THE_XMX: "-J-Xmx4500m"
+        run: ./script/compile
+      - name: Zip up binary
+        run: zip lighthouse-native-macos-amd64.zip lh
+      - name: Upload binary zip
+        uses: actions/upload-artifact@v2
+        with:
+          path: lighthouse-native-macos-amd64.zip
+          name: lighthouse-native-macos-amd64.zip
+
+  linux-amd64:
+    name: Build Linux native binary
+    runs-on: ubuntu-latest
+    needs: [build-jar]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: lighthouse.jar
+      - name: Debugging
+        run: ls -R
+      - name: Install GraalVM
+        uses: DeLaGuardo/setup-graalvm@master
+        with:
+          graalvm: 21.3.1
+          java: java11
+      - name: Build native image
+        env:
+          THE_JAR: lighthouse.jar
+          THE_XMX: "-J-Xmx4500m"
+          IS_STATIC: "true"
+        run: ./script/compile
+      - name: Zip up binary
+        run: zip lighthouse-native-linux-amd64.zip lh
+      - name: Upload binary zip
+        uses: actions/upload-artifact@v2
+        with:
+          path: lighthouse-native-linux-amd64.zip
+          name: lighthouse-native-linux-amd64.zip
+
+  upload-release:
+    name: Create release
+    runs-on: ubuntu-latest
+    needs: [build-jar, macos-amd64, linux-amd64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Display version
+        run: echo ${{ steps.get_version.outputs.VERSION }}
+
+      - name: Install Babashka
+        run: curl -s https://raw.githubusercontent.com/borkdude/babashka/master/install | sudo bash
+
+      - name: Get changelog entry
+        id: changelog
+        run: |
+          echo 'CHANGELOG_BODY<<EOF' >> $GITHUB_ENV
+          bb get-changelog-entry ${{ steps.get_version.outputs.VERSION }} >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v1
+        with:
+          body: ${{ env.CHANGELOG_BODY }}
+          prerelease: ${{ contains(steps.get_version.outputs.VERSION, 'pre') }}
+          files: |
+            lighthouse.jar/lighthouse.jar
+            lighthouse-native-macos-amd64.zip/lighthouse-native-macos-amd64.zip
+            lighthouse-native-linux-amd64.zip/lighthouse-native-linux-amd64.zip

.gitignore

@@ -0,0 +1,18 @@
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/lib/
+/classes/
+/target/
+/checkouts/
+.lein-deps-sum
+.lein-repl-history
+.lein-plugins/
+.lein-failures
+.nrepl-port
+.cpcache/
+lighthouse-*.log
+/lh*
+*.png
+*.jpeg

CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+## Unreleased
+
+## v2.0.0 - 2021-08-20
+
+Complete rewrite
+
+* Refactor configuration into `lighthouse.edn`
+* Simplify cli and config detection
+* Make graalvm-compatible
+* Provide way to define many manifests in a single file
+* Support visualization of dependencies with graphviz
+
+## v1.0.0 - 2020-12-11
+
+* Initial implementation

Dockerfile

@@ -0,0 +1,29 @@
+# this dockerfile and the associated compilation steps are
+# highly inspired by babashka's compilation process.
+#
+# See below for more details:
+#   https://github.com/babashka/babashka
+FROM clojure:openjdk-11-lein-2.9.6-bullseye AS BASE
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update
+RUN apt install --no-install-recommends -yy curl unzip build-essential zlib1g-dev sudo
+WORKDIR "/opt"
+RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.1.0/graalvm-ce-java11-linux-amd64-21.1.0.tar.gz
+RUN tar -xzf graalvm-ce-java11-linux-amd64-21.1.0.tar.gz
+
+ENV GRAALVM_HOME="/opt/graalvm-ce-java11-21.1.0"
+ENV JAVA_HOME="/opt/graalvm-ce-java11-21.1.0/bin"
+ENV PATH="$JAVA_HOME:$PATH"
+ENV IS_STATIC="true"
+ENV USE_MUSL="true"
+
+COPY . .
+RUN ./script/setup-musl
+RUN ./script/uberjar
+RUN ./script/compile
+
+FROM ubuntu:latest
+COPY --from=BASE /opt/lh /usr/local/bin/lh
+ENTRYPOINT ["lh"]
+CMD ["--help"]

LICENSE.md

@@ -0,0 +1,30 @@
+Copyright (c) 2022 Barracuda Networks, Inc.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of the copyright holder nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,28 @@
+# Lighthouse
+
+Generate Kubernetes manifests by combining EDN data instead of templating YAML.
+
+## Background
+
+There are [several issues](/doc/01-overview.md#rationale) with the existing
+solutions for generating Kubernetes manifests. Helm has idiosyncrasies and YAML
+is difficult to template.
+
+Lighthouse takes the approach of using EDN data structures in a directory tree
+with simple inheritance rules to support generating manifests in a flexible and
+reliable manner.
+
+* Read the [tutorial](doc/tutorial/01-getting-started.md) to get started.
+* Read the [manual](/doc/01-overview.md) for more information.
+
+# Installation
+
+## Static binary
+
+Download the latest binary from [Releases](https://github.com/barracudanetworks/lighthouse/releases), and put the `lh` binary in your path.
+
+# License
+
+Copyright 2022 Barracuda Networks, Inc.
+
+[3-clause BSD license](LICENSE.md)

bb.edn

@@ -0,0 +1,5 @@
+{:paths ["bin"]
+ :min-bb-version "0.4.0"
+ :tasks
+ {get-changelog-entry ci/get-changelog-entry
+  check-version ci/check-version}}

bin/ci.clj

@@ -0,0 +1,25 @@
+(ns ci
+  (:require [clojure.string :as string]))
+
+(defn get-changelog-entry
+  [release]
+  (->> (string/split-lines (slurp "CHANGELOG.md"))
+       (drop-while #(not (string/starts-with? % (str "## " release))))
+       (rest)
+       (take-while #(not (string/starts-with? % "##")))
+       (string/join "\n")
+       (string/trim)
+       (println)))
+
+(defn check-version
+  [check-version]
+  (let [resources-version (string/trim (slurp "resources/VERSION"))]
+    (if (= check-version resources-version)
+      (System/exit 0)
+      (do
+        (println (format (str "version does not match resources/VERSION!\n\n"
+                              "  checked version: '%s'\n"
+                              "resources/VERSION: '%s'")
+                         check-version
+                         resources-version))
+        (System/exit 1)))))

doc/01-overview.md

@@ -0,0 +1,31 @@
+# Overview
+
+This section will give a bird's-eye view of `lighthouse` and why it exists.
+
+## Intro
+
+`lighthouse` is a library that manages suites of EDN files and the connections between them. It provides:
+
+  1. simple inheritance rules
+  2. ways to provide metadata at the manifest and directory levels
+  3. access to fns via `sci`
+
+`lighthouse` takes these files, builds a tree out of them, merges down into a single map, and renders that map on disk.
+
+## Rationale
+
+During the development of our Kubernetes cluster, our team found issues with Helm/YAML manifests when trying to organize our deployments. Among other things:
+
+- Helm has some really annoying/bad idiosyncrasies (again, among others):
+  - You cannot provide an image tag that's completely digits. Helm will refuse to parse this as anything other than a number
+  - Helm will always pull down all required charts, even if the versions haven't changed.
+  - Helm's inheritance / value structure gets REALLY complicated / obtuse if you have nested charts.
+- Templates + YAML is a really, REALLY bad pairing.
+
+  When writing templates you need to worry about whitespace. In other languages (JSON, EDN, etc) this isn't a problem since whitespace is ignored. However, YAML applies syntactic significance to whitespaces. Because of this, when developing Helm templates, extra care has to be taken to ensure that the data is indented the proper amount in the final rendered YAML document. This leads to either `indents` being thrown around everywhere or specifically indented (and thus brittle) template functions.
+
+  All of this compounds and eventually the making sure the indentation levels in your template matches properly takes longer than actually implementing the template change in the first place.
+
+All-in-all, Helm and Go templates had just become a labor to deal with and manage, so we started looking for options. We decided we just wanted to deal with plain manifests. However, YAML (as mentioned above) isn't the best data language so we didn't want to use it if we didn't have to. Eventually, we settled on using EDN structures and a small library to convert EDN into YAML. That "small" library has transformed into this library.
+
+[The Basics :>](/doc/02-the-basics.md)