diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4fb048d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: +- package-ecosystem: maven + directory: "/" + schedule: + interval: daily + time: "12:00" + open-pull-requests-limit: 10 + ignore: + - dependency-name: org.projectlombok:lombok + versions: + - 1.18.18 + - dependency-name: org.mongodb:mongo-java-driver + versions: + - 3.12.7 + - dependency-name: org.apache.commons:commons-lang3 + versions: + - "3.11" diff --git a/.gitignore b/.gitignore index b3c173a..0e5a779 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,16 @@ -*.iws -*Db.properties -*Db.script -.settings -stacktrace.log -/*.zip -/plugin.xml -/*.log -/*DB.* -/cobertura.ser +Thumbs.db .DS_Store -/target/ -/out/ -/web-app/plugins -/web-app/WEB-INF/classes -.classpath -.project +.gradle +build/ +target/ +out/ +.idea *.iml *.ipr -.idea +*.iws +.project +.settings +.classpath +.factorypath +src/main/resources/application-local.yml +src/main/resources/application-prod.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9bcf999..0000000 --- a/.travis.yml +++ /dev/null @@ -1,3 +0,0 @@ -language: java -jdk: - - oraclejdk8 diff --git a/CHECKS b/CHECKS deleted file mode 100644 index 5fdfc8b..0000000 --- a/CHECKS +++ /dev/null @@ -1 +0,0 @@ -/admin/healthcheck \ No newline at end of file diff --git a/Procfile b/Procfile deleted file mode 100644 index 301fec2..0000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: java $JAVA_OPTS -Ddw.mongo.type=uri -Ddw.mongo.uri=$MONGOHQ_URL -Ddw.server.connector.type=http -Ddw.server.connector.port=$PORT -Ddw.blobManager.fileSystemBlogDataDirectory=/var/jsonblob/data -Ddw.blobManager.blobAccessTtl=$BLOB_TTL -jar target/jsonblob.jar server target/config/jsonblob.yml diff --git a/README.md b/README.md index a654d3b..fc53f05 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,20 @@ -JSON Blob -======== +## Micronaut 2.4.0 Documentation -JSON Blob was created to help parallelize client/server development. Mock JSON responses can be defined and stored using the online editor and then clients can use the JSON Blob API to retrieve and update the mock responses. +- [User Guide](https://docs.micronaut.io/2.4.0/guide/index.html) +- [API Reference](https://docs.micronaut.io/2.4.0/api/index.html) +- [Configuration Reference](https://docs.micronaut.io/2.4.0/guide/configurationreference.html) +- [Micronaut Guides](https://guides.micronaut.io/index.html) +--- -[![Build Status](https://travis-ci.org/tburch/jsonblob.svg?branch=master)](https://travis-ci.org/tburch/jsonblob) +## Feature assertj documentation + +- [https://assertj.github.io/doc/](https://assertj.github.io/doc/) + +## Feature http-client documentation + +- [Micronaut HTTP Client documentation](https://docs.micronaut.io/latest/guide/index.html#httpClient) + +## Feature management documentation + +- [Micronaut Management documentation](https://docs.micronaut.io/latest/guide/index.html#management) -##Building & Running JSON Blob -1. To run JSON Blob, you'll need the following things installed: - - Java (version 1.8+) - - Maven -1. Build the JSON Blob jar - from the command line run `mvn clean package`. -1. Start JSON Blob - from the command line run `java -Ddw.blobManager.fileSystemBlogDataDirectory= -jar target/jsonblob.jar server target/config/jsonblob.yml`. You'll need to replace `` with the path where you want to store blobs on the file system. diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..3ff45d1 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,93 @@ +plugins { + id("org.jetbrains.kotlin.jvm") version "1.5.21" + id("org.jetbrains.kotlin.kapt") version "1.5.21" + id("org.jetbrains.kotlin.plugin.allopen") version "1.5.21" + id("com.github.johnrengelman.shadow") version "6.1.0" + id("io.micronaut.application") version "2.0.2" +} + +version = "1.0.2" +group = "com.jsonblob" + +val kotlinVersion= project.properties["kotlinVersion"] +val testContainersVersion= project.properties["testContainersVersion"] +val jvmBrotliVersion= project.properties["jvmBrotliVersion"] + +repositories { + mavenCentral() + jcenter() +} + +micronaut { + runtime("netty") + testRuntime("junit5") + processing { + incremental(true) + annotations("jsonblob.*") + } +} + +dependencies { + implementation("io.micronaut:micronaut-validation") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion") + implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.1") + implementation("io.micronaut.kotlin:micronaut-kotlin-runtime") + implementation("io.micronaut:micronaut-runtime") + implementation("io.micronaut:micronaut-management") + implementation("io.micronaut.micrometer:micronaut-micrometer-core") + implementation("io.micronaut.micrometer:micronaut-micrometer-registry-new-relic") + implementation("io.micronaut.aws:micronaut-aws-sdk-v2") + implementation("software.amazon.awssdk:s3") + implementation("com.fasterxml.uuid:java-uuid-generator:3.1.4") + implementation("org.mongodb:mongo-java-driver:3.2.2") + implementation("com.google.guava:guava:30.1-jre") + implementation("io.github.microutils:kotlin-logging-jvm:2.0.6") + implementation("com.nixxcode.jvmbrotli:jvmbrotli:$jvmBrotliVersion") + implementation("io.micronaut.views:micronaut-views-handlebars") + implementation("commons-codec:commons-codec:1.15") + + runtimeOnly("ch.qos.logback:logback-classic:1.2.8") + runtimeOnly("com.nixxcode.jvmbrotli:jvmbrotli-darwin-x86-amd64:$jvmBrotliVersion") + runtimeOnly("com.nixxcode.jvmbrotli:jvmbrotli-linux-x86-amd64:$jvmBrotliVersion") + + testImplementation(platform("org.junit:junit-bom:5.7.1")) + testImplementation("org.junit.jupiter:junit-jupiter-params") + testImplementation("org.junit.jupiter:junit-jupiter-api") + testImplementation("io.micronaut.test:micronaut-test-junit5") + testImplementation("io.micronaut:micronaut-http-client") + testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin") + testImplementation("org.assertj:assertj-core") + testImplementation("org.skyscreamer:jsonassert:1.5.0") + testImplementation("org.testcontainers:testcontainers:$testContainersVersion") + testImplementation("org.testcontainers:junit-jupiter:$testContainersVersion") + testImplementation("org.testcontainers:localstack:$testContainersVersion") + testImplementation("com.amazonaws:aws-java-sdk-s3:1.11.1030") // for localstack + + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") + testRuntimeOnly("com.nixxcode.jvmbrotli:jvmbrotli-darwin-x86-amd64:$jvmBrotliVersion") +} + + +application { + mainClass.set("jsonblob.ApplicationKt") +} +java { + sourceCompatibility = JavaVersion.toVersion("1.8") +} + +tasks { + compileKotlin { + kotlinOptions { + jvmTarget = "1.8" + } + } + compileTestKotlin { + kotlinOptions { + jvmTarget = "1.8" + } + } + test { + useJUnitPlatform() + } +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..15f7f2c --- /dev/null +++ b/gradle.properties @@ -0,0 +1,4 @@ +micronautVersion=2.5.13 +kotlinVersion=1.4.21 +jvmBrotliVersion=0.2.0 +testContainersVersion=1.15.3 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..62d4c05 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..da9702f --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..fbd7c51 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100755 index 0000000..a9f778a --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/micronaut-cli.yml b/micronaut-cli.yml new file mode 100644 index 0000000..73c460e --- /dev/null +++ b/micronaut-cli.yml @@ -0,0 +1,6 @@ +applicationType: default +defaultPackage: jsonblob +testFramework: junit +sourceLanguage: kotlin +buildTool: gradle_kotlin +features: [app-name, assertj, gradle, http-client, junit, kotlin, kotlin-application, logback, management, micrometer, micrometer-new-relic, netty-server, readme, shade, yaml] diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 55ca5c9..0000000 --- a/pom.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - 4.0.0 - - com.lowtuna - jsonblob - 2.0.0-SNAPSHOT - - - 0.7.0 - - - - - sonatype-nexus-snapshots - Sonatype Nexus Snapshots - http://oss.sonatype.org/content/repositories/snapshots - - false - - - true - - - - - - - io.dropwizard - dropwizard-core - ${io.dropwizard.version} - - - io.dropwizard - dropwizard-assets - ${io.dropwizard.version} - - - com.lowtuna.dropwizard-extras - dropwizard-extras-view-handlebars - 1.0.0-SNAPSHOT - - - com.lowtuna.dropwizard-extras - dropwizard-extras-view-markdown - 1.0.0-SNAPSHOT - - - com.lowtuna.dropwizard-extras - dropwizard-extras-heroku - 1.0.0-SNAPSHOT - - - com.lowtuna.dropwizard-extras - dropwizard-extras-config - 1.0.0-SNAPSHOT - - - io.dropwizard - dropwizard-testing - ${io.dropwizard.version} - test - - - com.fasterxml.uuid - java-uuid-generator - 3.1.4 - - - org.projectlombok - lombok - 1.12.6 - - - org.mongodb - mongo-java-driver - 3.2.2 - - - org.apache.commons - commons-lang3 - 3.1 - - - commons-io - commons-io - 2.4 - - - junit - junit - 4.11 - test - - - com.github.jknack - handlebars-guava-cache - 1.1.2 - - - com.github.jknack - handlebars-jackson2 - 1.1.2 - - - com.fasterxml.jackson.core - jackson-databind - - - - - - io.dropwizard - dropwizard-testing - ${io.dropwizard.version} - test - - - - - jsonblob - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - - - - maven-resources-plugin - 2.6 - - - vagrant - package - - copy-resources - - - ${project.build.directory}/config/ - - - src/main/resources - true - - **/jsonblob.yml - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.6 - - true - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - package - - shade - - - - - - com.lowtuna.jsonblob.JsonBlobApplication - - - - - - - - - - diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..d47ac09 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name="jsonblob" diff --git a/src/main/java/com/lowtuna/jsonblob/JsonBlobApplication.java b/src/main/java/com/lowtuna/jsonblob/JsonBlobApplication.java deleted file mode 100644 index 4b36f74..0000000 --- a/src/main/java/com/lowtuna/jsonblob/JsonBlobApplication.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.lowtuna.jsonblob; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.MetricRegistry; -import com.github.jknack.handlebars.Handlebars; -import com.github.jknack.handlebars.Jackson2Helper; -import com.lowtuna.dropwizard.extras.heroku.RequestIdFilter; -import com.lowtuna.dropwizard.extras.view.handlebars.ConfiguredHandlebarsViewBundle; -import com.lowtuna.jsonblob.config.JsonBlobConfiguration; -import com.lowtuna.jsonblob.core.FileSystemJsonBlobManager; -import com.lowtuna.jsonblob.health.BlobDirectoryFreeSpaceHealthcheck; -import com.lowtuna.jsonblob.resource.ApiResource; -import com.lowtuna.jsonblob.resource.JsonBlobEditorResource; -import com.lowtuna.jsonblob.util.jersey.GitTipHeaderFilter; -import com.sun.jersey.spi.container.ContainerResponseFilter; -import io.dropwizard.Application; -import io.dropwizard.assets.AssetsBundle; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.joda.time.Duration; -import org.joda.time.format.PeriodFormatter; -import org.joda.time.format.PeriodFormatterBuilder; - -import javax.ws.rs.core.MultivaluedMap; -import java.util.concurrent.ScheduledExecutorService; - -@Slf4j -public class JsonBlobApplication extends Application { - - private final long startTIme = System.currentTimeMillis(); - private final PeriodFormatter uptimePeriodFormatter = new PeriodFormatterBuilder() - .appendDays() - .appendSuffix("d") - .appendHours() - .appendSuffix("h") - .appendMinutes() - .appendSuffix("m") - .appendSeconds() - .appendSuffix("s") - .toFormatter(); - - public static void main(String[] args) throws Exception { - if (args.length >= 2 && args[1].startsWith("~")) { - args[1] = System.getProperty("user.home") + args[1].substring(1); - } - new JsonBlobApplication().run(args); - } - - @Override - public String getName() { - return "jsonblob"; - } - - @Override - public void initialize(final Bootstrap bootstrap) { - bootstrap.addBundle(new AssetsBundle()); - - bootstrap.addBundle(new ConfiguredHandlebarsViewBundle() { - @Override - public Handlebars getInstance(JsonBlobConfiguration configuration) { - log.info("Using Handlebars configuration of {}", configuration.getHandlebarsConfig().getClass().getCanonicalName()); - Handlebars handlebars = configuration.getHandlebarsConfig().getInstance(bootstrap.getMetricRegistry()); - handlebars.registerHelper("json", new Jackson2Helper(bootstrap.getObjectMapper())); - return handlebars; - } - }); - } - - @Override - public void run(JsonBlobConfiguration configuration, Environment environment) throws ClassNotFoundException { - environment.metrics().register(MetricRegistry.name(getClass(), "uptime"), (Gauge) () -> uptimePeriodFormatter.print(new Duration(System.currentTimeMillis() - startTIme).toPeriod())); - - ScheduledExecutorService scheduledExecutorService = configuration.getBlobManagerConfig().getScheduledExecutorService().instance(environment); - ScheduledExecutorService cleanupScheduledExecutorService = configuration.getBlobManagerConfig().getCleanupScheduledExecutorService().instance(environment); - - FileSystemJsonBlobManager fileSystemBlobManager = new FileSystemJsonBlobManager(configuration.getBlobManagerConfig().getFileSystemBlogDataDirectory(), scheduledExecutorService, cleanupScheduledExecutorService, environment.getObjectMapper(), configuration.getBlobManagerConfig().getBlobAccessTtl(), configuration.getBlobManagerConfig().isDeleteEnabled(), environment.metrics()); - environment.lifecycle().manage(fileSystemBlobManager); - - environment.healthChecks().register("freeSpace", new BlobDirectoryFreeSpaceHealthcheck(configuration.getBlobManagerConfig().getFileSystemBlogDataDirectory(), 5242880)); - - environment.jersey().register(new ApiResource(fileSystemBlobManager, configuration.getGoogleAnalyticsConfig())); - environment.jersey().register(new JsonBlobEditorResource(fileSystemBlobManager, configuration.getGoogleAnalyticsConfig(), configuration.getBlobManagerConfig().getBlobAccessTtl())); - - environment.jersey().getResourceConfig().getContainerResponseFilters().add(new GitTipHeaderFilter()); - environment.jersey().getResourceConfig().getContainerRequestFilters().add(new RequestIdFilter("X-Request-ID")); - - // Support CORS - environment.jersey().getResourceConfig().getContainerResponseFilters().add((ContainerResponseFilter) (request, response) -> { - MultivaluedMap headers = response.getHttpHeaders(); - headers.add("Access-Control-Allow-Origin", "*"); - headers.add("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,HEAD,OPTIONS"); - headers.add("Access-Control-Expose-Headers", "X-Requested-With,X-jsonblob,X-Hello-Human,Location,Date,Content-Type,Accept,Origin"); - - String reqHead = request.getHeaderValue("Access-Control-Request-Headers"); - if (StringUtils.isNotEmpty(reqHead)) { - headers.add("Access-Control-Allow-Headers", reqHead); - } - - return response; - }); - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/config/BlobManagerConfig.java b/src/main/java/com/lowtuna/jsonblob/config/BlobManagerConfig.java deleted file mode 100644 index e3da812..0000000 --- a/src/main/java/com/lowtuna/jsonblob/config/BlobManagerConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.lowtuna.jsonblob.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.lowtuna.dropwizard.extras.config.executors.ScheduledExecutorServiceConfig; -import io.dropwizard.util.Duration; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.hibernate.validator.constraints.NotEmpty; - -import javax.validation.constraints.NotNull; -import java.io.File; - -@Data -@NoArgsConstructor -public class BlobManagerConfig { - - @JsonProperty - @NotEmpty - private String blobCollectionName = "blob"; - - @JsonProperty - @NotNull - private ScheduledExecutorServiceConfig scheduledExecutorService = new ScheduledExecutorServiceConfig("blobManagerScheduledExecutor-%d"); - - @JsonProperty - @NotNull - private ScheduledExecutorServiceConfig cleanupScheduledExecutorService = new ScheduledExecutorServiceConfig("blobManagerCleanupScheduledExecutor-%d"); - - @JsonProperty - @NotNull - private Duration blobAccessTtl = Duration.days(90); - - private boolean deleteEnabled = false; - - @NotNull - @JsonProperty("fileSystemBlogDataDirectory") - private File fileSystemBlogDataDirectory; - -} diff --git a/src/main/java/com/lowtuna/jsonblob/config/JsonBlobConfiguration.java b/src/main/java/com/lowtuna/jsonblob/config/JsonBlobConfiguration.java deleted file mode 100644 index 4f9b777..0000000 --- a/src/main/java/com/lowtuna/jsonblob/config/JsonBlobConfiguration.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.lowtuna.jsonblob.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import com.lowtuna.jsonblob.config.view.HandlebarsConfig; -import com.lowtuna.jsonblob.config.view.ProdHandlebarsConfig; -import io.dropwizard.Configuration; -import lombok.Getter; -import lombok.Setter; - -import javax.validation.Valid; -import javax.validation.constraints.NotNull; - -@Getter -@Setter -public class JsonBlobConfiguration extends Configuration { - - @Valid - @NotNull - @JsonProperty("blobManager") - private BlobManagerConfig blobManagerConfig = new BlobManagerConfig(); - - @Valid - @NotNull - @JsonProperty("ga") - private GoogleAnalyticsConfig googleAnalyticsConfig = new GoogleAnalyticsConfig(); - - @Valid - @JsonProperty("handlebars") - private HandlebarsConfig handlebarsConfig = new ProdHandlebarsConfig(); - -} diff --git a/src/main/java/com/lowtuna/jsonblob/config/view/DevHandlebarsConfig.java b/src/main/java/com/lowtuna/jsonblob/config/view/DevHandlebarsConfig.java deleted file mode 100644 index 898c075..0000000 --- a/src/main/java/com/lowtuna/jsonblob/config/view/DevHandlebarsConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.lowtuna.jsonblob.config.view; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.github.jknack.handlebars.Handlebars; -import com.github.jknack.handlebars.cache.NullTemplateCache; -import com.github.jknack.handlebars.io.FileTemplateLoader; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.hibernate.validator.constraints.NotEmpty; - -@Data -@NoArgsConstructor -public class DevHandlebarsConfig extends HandlebarsConfig { - - @JsonProperty - @NotEmpty - private String templateBaseDir; - - @Override - @JsonIgnore - public Handlebars createInstance() { - return new Handlebars().with(new FileTemplateLoader(templateBaseDir, StringUtils.EMPTY)); - } - - @Override - protected Handlebars setupTemplateCache(Handlebars handlebars, MetricRegistry metricRegistry) { - return handlebars.with(NullTemplateCache.INSTANCE); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/config/view/HandlebarsConfig.java b/src/main/java/com/lowtuna/jsonblob/config/view/HandlebarsConfig.java deleted file mode 100644 index 408ff2b..0000000 --- a/src/main/java/com/lowtuna/jsonblob/config/view/HandlebarsConfig.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.lowtuna.jsonblob.config.view; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.github.jknack.handlebars.Handlebars; -import com.github.jknack.handlebars.Template; -import com.github.jknack.handlebars.cache.GuavaTemplateCache; -import com.github.jknack.handlebars.helper.StringHelpers; -import com.github.jknack.handlebars.io.TemplateSource; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.lowtuna.jsonblob.util.mustache.Base64StringHelpers; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible = false, include = JsonTypeInfo.As.EXTERNAL_PROPERTY) -@JsonSubTypes({ - @JsonSubTypes.Type(value = DevHandlebarsConfig.class, name = "dev"), - @JsonSubTypes.Type(value = ProdHandlebarsConfig.class, name = "prod") -}) - -@JsonIgnoreProperties(ignoreUnknown = true, value = "type") -public abstract class HandlebarsConfig { - - @JsonIgnore - public Handlebars getInstance(MetricRegistry metricRegistry) { - Handlebars handlebars = createInstance(); - handlebars = setupTemplateCache(handlebars, metricRegistry); - StringHelpers.register(handlebars); - Base64StringHelpers.register(handlebars); - return handlebars; - } - - protected Handlebars setupTemplateCache(Handlebars handlebars, MetricRegistry metricRegistry) { - final Cache templateCache = CacheBuilder.newBuilder().recordStats().build(); - - metricRegistry.register(MetricRegistry.name(GuavaTemplateCache.class, "size"), new Gauge() { - @Override - public Long getValue() { - return templateCache.size(); - } - }); - metricRegistry.register(MetricRegistry.name(GuavaTemplateCache.class, "hits"), new Gauge() { - @Override - public Long getValue() { - return templateCache.stats().hitCount(); - } - }); - metricRegistry.register(MetricRegistry.name(GuavaTemplateCache.class, "misses"), new Gauge() { - @Override - public Long getValue() { - return templateCache.stats().missCount(); - } - }); - metricRegistry.register(MetricRegistry.name(GuavaTemplateCache.class, "eviction-count"), new Gauge() { - @Override - public Long getValue() { - return templateCache.stats().evictionCount(); - } - }); - metricRegistry.register(MetricRegistry.name(GuavaTemplateCache.class, "average-load-penalty"), new Gauge() { - @Override - public Double getValue() { - return templateCache.stats().averageLoadPenalty(); - } - }); - - return handlebars.with(new GuavaTemplateCache(templateCache)); - } - - @JsonIgnore - public abstract Handlebars createInstance(); - -} diff --git a/src/main/java/com/lowtuna/jsonblob/config/view/ProdHandlebarsConfig.java b/src/main/java/com/lowtuna/jsonblob/config/view/ProdHandlebarsConfig.java deleted file mode 100644 index 132d4b7..0000000 --- a/src/main/java/com/lowtuna/jsonblob/config/view/ProdHandlebarsConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.lowtuna.jsonblob.config.view; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.github.jknack.handlebars.Handlebars; -import com.github.jknack.handlebars.io.ClassPathTemplateLoader; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.hibernate.validator.constraints.NotEmpty; - -@Data -@NoArgsConstructor -public class ProdHandlebarsConfig extends HandlebarsConfig { - - @NotEmpty - @JsonProperty - private String classPathTemplatesBaseDir = "/views"; - - @Override - @JsonIgnore - public Handlebars createInstance() { - return new Handlebars().with(new ClassPathTemplateLoader(classPathTemplatesBaseDir, StringUtils.EMPTY)); - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/BlobCleanupProducer.java b/src/main/java/com/lowtuna/jsonblob/core/BlobCleanupProducer.java deleted file mode 100644 index 39ae2bf..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/BlobCleanupProducer.java +++ /dev/null @@ -1,134 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Optional; -import com.google.common.base.Stopwatch; -import io.dropwizard.util.Duration; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.DirectoryWalker; -import org.joda.time.DateTime; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDate; -import java.util.Collection; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by tburch on 8/18/17. - */ -@Slf4j -public class BlobCleanupProducer extends DirectoryWalker implements Runnable { - private final Path dataDirectoryPath; - private final Duration blobAccessTtl; - private final FileSystemJsonBlobManager fileSystemJsonBlobManager; - private final ObjectMapper om; - - public BlobCleanupProducer(Path dataDirectoryPath, Duration blobAccessTtl, FileSystemJsonBlobManager fileSystemJsonBlobManager, ObjectMapper om) { - super(null, 3); - this.dataDirectoryPath = dataDirectoryPath; - this.blobAccessTtl = blobAccessTtl; - this.fileSystemJsonBlobManager = fileSystemJsonBlobManager; - this.om = om; - } - - - @Override - protected boolean handleDirectory(File directory, int depth, Collection results) throws IOException { - if (isDataDir(directory.getAbsolutePath())) { - String[] dateParts = directory.getAbsolutePath().replace(dataDirectoryPath.toFile().getAbsolutePath(), "").split("/", 4); - LocalDate localDate = LocalDate.of(Integer.parseInt(dateParts[1]), Integer.parseInt(dateParts[2]), Integer.parseInt(dateParts[3])); - boolean process = localDate.isBefore(LocalDate.now().minusDays(blobAccessTtl.toDays())); - if (process) { - log.info("Processing {} blobs for un-accessed blobs", directory.getAbsolutePath()); - AtomicInteger fileCount = new AtomicInteger(0); - Files.newDirectoryStream(directory.toPath()) - .forEach(path -> { - fileCount.incrementAndGet(); - File file = path.toFile(); - if (file.getName().startsWith(FileSystemJsonBlobManager.BLOB_METADATA_FILE_NAME)) { - return; - } - - try { - log.debug("Processing {}", file.getAbsolutePath()); - String blobId = file.getName().split("\\.", 2)[0]; - File metadataFile = fileSystemJsonBlobManager.getMetaDataFile(file.getParentFile()); - - if (file.equals(metadataFile)) { - return; - } - - BlobMetadataContainer metadataContainer = metadataFile.exists() ? om.readValue(fileSystemJsonBlobManager.readFile(metadataFile), BlobMetadataContainer.class) : new BlobMetadataContainer(); - - Optional lastAccessed = fileSystemJsonBlobManager.resolveTimestamp(blobId); - if (metadataContainer.getLastAccessedByBlobId().containsKey(blobId)) { - lastAccessed = Optional.of(metadataContainer.getLastAccessedByBlobId().get(blobId)); - } - - if (!lastAccessed.isPresent()) { - log.warn("Couldn't get last accessed timestamp for blob {}", blobId); - return; - } - - log.debug("Blob {} was last accessed {}", blobId, lastAccessed.get()); - - if (lastAccessed.get().plusMillis((int) blobAccessTtl.toMilliseconds()).isBefore(DateTime.now())) { - if (file.delete()) { - log.info("Blob {} hasn't been accessed in {} (last accessed {}), so it's going to be deleted", blobId, blobAccessTtl, lastAccessed.get()); - } - } - } catch (JsonParseException e) { - log.warn("Couldn't parse JSON from BlobMetadataContainer", e); - } catch (JsonMappingException e) { - log.warn("Couldn't map JSON from BlobMetadataContainer", e); - } catch (IOException e) { - log.warn("Couldn't read json for BlobMetadataContainer file", e); - } - }); - - log.info("Processed {} blobs in {}", fileCount.get(), directory.getAbsolutePath()); - - if (fileCount.get() == 0) { - log.info("{} has no files, so it's being deleted", directory.getAbsolutePath()); - } else if (fileCount.get() == 1) { - File[] files = directory.listFiles(); - if (files != null && files.length > 0 && files[0].getName().startsWith(FileSystemJsonBlobManager.BLOB_METADATA_FILE_NAME)) { - if (files[0].delete() && directory.delete()) { - log.info("{} has only a metadata file, so it's being deleted", directory.getAbsolutePath()); - } - } - } - return false; - } - } else { - File[] files = directory.listFiles(); - if (files != null && files.length == 0) { - if (directory.delete()) log.info("{} has no files, so it's being deleted", directory.getAbsolutePath()); - return false; - } - } - return true; - } - - private boolean isDataDir(String path) { - return path.replace(dataDirectoryPath.toFile().getAbsolutePath(), "").split("/").length == 4; - } - - @Override - public void run() { - Stopwatch stopwatch = new Stopwatch().start(); - try { - walk(dataDirectoryPath.toFile(), null); - log.info("Completed cleaning up un-accessed blobs in {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - } catch (Exception e) { - e.printStackTrace(); - } - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/BlobMetadataContainer.java b/src/main/java/com/lowtuna/jsonblob/core/BlobMetadataContainer.java deleted file mode 100644 index 9e7b257..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/BlobMetadataContainer.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import com.google.common.collect.Maps; -import lombok.Data; -import org.joda.time.DateTime; - -import java.util.Map; - -/** - * Created by tburch on 2/9/17. - */ -@Data -public class BlobMetadataContainer { - private Map lastAccessedByBlobId = Maps.newHashMap(); -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/BlobNotFoundException.java b/src/main/java/com/lowtuna/jsonblob/core/BlobNotFoundException.java deleted file mode 100644 index 95cef87..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/BlobNotFoundException.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import lombok.Getter; - -@Getter -public class BlobNotFoundException extends Exception { - private final String id; - - public BlobNotFoundException(String id) { - this.id = id; - } - - public BlobNotFoundException(String message, String id) { - super(message); - this.id = id; - } - - public BlobNotFoundException(String message, Throwable cause, String id) { - super(message, cause); - this.id = id; - } - - public BlobNotFoundException(Throwable cause, String id) { - super(cause); - this.id = id; - } - - public BlobNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace, String id) { - super(message, cause, enableSuppression, writableStackTrace); - this.id = id; - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/FileSystemJsonBlobManager.java b/src/main/java/com/lowtuna/jsonblob/core/FileSystemJsonBlobManager.java deleted file mode 100644 index 4757baa..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/FileSystemJsonBlobManager.java +++ /dev/null @@ -1,321 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.uuid.Generators; -import com.google.common.base.Optional; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.Striped; -import com.mongodb.util.JSON; -import com.mongodb.util.JSONParseException; -import io.dropwizard.lifecycle.Managed; -import io.dropwizard.util.Duration; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.Charsets; -import org.bson.types.ObjectId; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import javax.annotation.concurrent.GuardedBy; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.Calendar; -import java.util.Map; -import java.util.TimeZone; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -/** - * Created by tburch on 11/15/16. - */ -@Slf4j -public class FileSystemJsonBlobManager implements JsonBlobManager, Runnable, Managed { - static final String BLOB_METADATA_FILE_NAME = "blobMetadata"; - - private static final DateTimeFormatter DIRECTORY_FORMAT = DateTimeFormat.forPattern("yyyy/MM/dd"); - - @GuardedBy("lastAccessedLock") - private ConcurrentMap lastAccessedUpdates = Maps.newConcurrentMap(); - private ReentrantReadWriteLock lastAccessedLock = new ReentrantReadWriteLock(); - private final Striped blobStripedLocks = Striped.lazyWeakReadWriteLock(5000); - - private final File blobDataDirectory; - private final ScheduledExecutorService scheduledExecutorService; - private final ScheduledExecutorService cleanupScheduledExecutorService; - private final ObjectMapper objectMapper; - private final Duration blobAccessTtl; - @Getter - private final boolean deleteEnabled; - private final MetricRegistry metricRegistry; - - public FileSystemJsonBlobManager(File blobDataDirectory, ScheduledExecutorService scheduledExecutorService, ScheduledExecutorService cleanupScheduledExecutorService, ObjectMapper objectMapper, Duration blobAccessTtl, boolean deleteEnabled, MetricRegistry metricRegistry) { - this.blobDataDirectory = blobDataDirectory; - this.scheduledExecutorService = scheduledExecutorService; - this.cleanupScheduledExecutorService = cleanupScheduledExecutorService; - this.objectMapper = objectMapper; - this.blobAccessTtl = blobAccessTtl; - this.deleteEnabled = deleteEnabled; - this.metricRegistry = metricRegistry; - - blobDataDirectory.mkdirs(); - } - - private File getBlobDirectory(DateTime createdTimestamp) { - return new File(blobDataDirectory, DIRECTORY_FORMAT.print(createdTimestamp)); - } - - private File getBlobFile(String blobId, DateTime createdTimestamp) { - File subDir = getBlobDirectory(createdTimestamp); - return new File(subDir, blobId + ".json.gz"); - } - - File getMetaDataFile(String blobId) { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - throw new IllegalStateException("Couldn't generate create timestamp from " + blobId); - } - - File blobDirectory = getBlobDirectory(createTimestamp.get()); - return getMetaDataFile(blobDirectory); - } - - File getMetaDataFile(File blobDirectory) { - return new File(blobDirectory, BLOB_METADATA_FILE_NAME + ".json.gz"); - } - - Optional resolveTimestamp(String blobId) { - try { - UUID uuid = UUID.fromString(blobId); - - Calendar uuidEpoch = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - uuidEpoch.clear(); - uuidEpoch.set(1582, 9, 15, 0, 0, 0); // 9 = October - long epochMillis = uuidEpoch.getTime().getTime(); - - long time = (uuid.timestamp() / 10000L) + epochMillis; - - return Optional.of(new DateTime(time)); - } catch (IllegalArgumentException e) { - try { - ObjectId objectId = new ObjectId(blobId); - return Optional.of(new DateTime(objectId.getTime())); - } catch (IllegalArgumentException e1) { - return Optional.absent(); - } - } - } - - @Override - public String createBlob(String blob) throws IllegalArgumentException { - if (!isValidJson(blob)) { - throw new IllegalArgumentException(); - } - - UUID uuid = Generators.timeBasedGenerator().generate(); - String blobId = createBlob(blob, uuid.toString()); - - return blobId; - } - - public String createBlob(String blob, String blobId) { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - throw new IllegalStateException("Couldn't generate create timestamp from " + blobId); - } - - File blobFile = getBlobFile(blobId, createTimestamp.get()); - blobFile.getParentFile().mkdirs(); - try { - writeFile(blobFile, blob); - return blobId; - } catch (IOException e) { - throw new IllegalStateException("Couldn't write blob", e); - } - } - - @Override - public String getBlob(String blobId) throws BlobNotFoundException { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - throw new BlobNotFoundException(blobId); - } - - File blobFile = getBlobFile(blobId, createTimestamp.get()); - - try { - String content = readFile(blobFile); - - updateLastAccessedTimestamp(blobId, createTimestamp.get()); - - return content; - } catch (FileNotFoundException e) { - throw new BlobNotFoundException(blobId); - } catch (IOException e) { - throw new RuntimeException("Couldn't read blob", e); - } - } - - @Override - public boolean updateBlob(String blobId, String blob) throws IllegalArgumentException, BlobNotFoundException { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - throw new BlobNotFoundException(blobId); - } - - File blobFile = getBlobFile(blobId, createTimestamp.get()); - if (!blobFile.exists()) { - throw new BlobNotFoundException(blobId); - } - - try { - writeFile(blobFile, blob); - - updateLastAccessedTimestamp(blobId, createTimestamp.get()); - - return true; - } catch (IOException e) { - throw new IllegalStateException("Couldn't write blob", e); - } - } - - @Override - public boolean deleteBlob(String blobId) throws BlobNotFoundException { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - throw new BlobNotFoundException(blobId); - } - - File blobFile = getBlobFile(blobId, createTimestamp.get()); - if (!blobFile.exists()) { - throw new BlobNotFoundException(blobId); - } - - boolean deleted = deleteFile(blobFile); - - updateLastAccessedTimestamp(blobId, createTimestamp.get()); - - return deleted; - } - - private void updateLastAccessedTimestamp(String blobId, DateTime createTimestamp) { - DateTime now = DateTime.now(); - if (now.toLocalDate().equals(createTimestamp.toLocalDate())) { - return; - } - - Lock lock = lastAccessedLock.writeLock(); - try { - lock.lock(); - lastAccessedUpdates.put(blobId, now); - } finally { - lock.unlock(); - } - } - - private boolean isValidJson(String json) { - try { - JSON.parse(json); - return true; - } catch (JSONParseException e) { - return false; - } - } - - void writeFile(File file, String content) throws IOException { - Lock lock = blobStripedLocks.get(file.getAbsolutePath()).writeLock(); - try { - lock.lock(); - try (Writer writer = new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(file)), Charsets.UTF_8)) { - writer.write(content); - } - } finally { - lock.unlock(); - } - } - - String readFile(File file) throws IOException { - Lock lock = blobStripedLocks.get(file.getAbsolutePath()).readLock(); - try { - lock.lock(); - StringBuilder sb = new StringBuilder(); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)))); - String line; - while ((line = bufferedReader.readLine()) != null) { - sb.append(line); - } - return sb.toString(); - } finally { - lock.unlock(); - } - } - - boolean deleteFile(File file) { - Lock lock = blobStripedLocks.get(file.getAbsolutePath()).writeLock(); - try { - lock.lock(); - return file.delete(); - } finally { - lock.unlock(); - } - } - - public boolean blobExists(String blobId) { - Optional createTimestamp = resolveTimestamp(blobId); - if (!createTimestamp.isPresent()) { - return false; - } - - File blobFile = getBlobFile(blobId, createTimestamp.get()); - return blobFile.exists(); - } - - @Override - public void run() { - Map lastAccessedUpdates = Maps.newHashMap(); - Lock lock = lastAccessedLock.writeLock(); - try { - lock.lock(); - lastAccessedUpdates.putAll(this.lastAccessedUpdates); - this.lastAccessedUpdates.clear(); - } finally { - lock.unlock(); - } - - if (lastAccessedUpdates.isEmpty()) { - return; - } - - log.debug("Updating last accessed time for {} blobs", lastAccessedUpdates.size()); - scheduledExecutorService.submit(new UpdateBlobLastAccessedJob(lastAccessedUpdates, this, objectMapper)); - } - - @Override - public void start() throws Exception { - log.info("Scheduling the updating of blob last accessed timestamps"); - scheduledExecutorService.scheduleWithFixedDelay(this, 1, 1, TimeUnit.MINUTES); - - BlobCleanupProducer dataDirectoryCleaner = new BlobCleanupProducer(blobDataDirectory.toPath(), blobAccessTtl, this, objectMapper); - cleanupScheduledExecutorService.scheduleWithFixedDelay(dataDirectoryCleaner, 0, 1, TimeUnit.DAYS); - } - - @Override - public void stop() throws Exception { - //nothing to do - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/JsonBlobManager.java b/src/main/java/com/lowtuna/jsonblob/core/JsonBlobManager.java deleted file mode 100644 index 441e034..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/JsonBlobManager.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.lowtuna.jsonblob.core; - -/** - * Created by tburch on 11/15/16. - */ -public interface JsonBlobManager { - String createBlob(String blob); - - String getBlob(String blobId) throws BlobNotFoundException; - - boolean updateBlob(String blobId, String blob) throws BlobNotFoundException; - - boolean deleteBlob(String blobId) throws BlobNotFoundException; -} diff --git a/src/main/java/com/lowtuna/jsonblob/core/UpdateBlobLastAccessedJob.java b/src/main/java/com/lowtuna/jsonblob/core/UpdateBlobLastAccessedJob.java deleted file mode 100644 index c811ae0..0000000 --- a/src/main/java/com/lowtuna/jsonblob/core/UpdateBlobLastAccessedJob.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.joda.time.DateTime; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Created by tburch on 2/9/17. - */ -@Slf4j -@RequiredArgsConstructor -public class UpdateBlobLastAccessedJob implements Runnable { - private final Map updates; - private final FileSystemJsonBlobManager fileSystemJsonBlobManager; - private final ObjectMapper om; - - @Override - public void run() { - log.info("Updating last access timestamp for {} blobs", updates.size()); - ListMultimap> updatesByMetadataFile = LinkedListMultimap.create(); - for (Map.Entry entry : updates.entrySet()) { - File metadataFile = fileSystemJsonBlobManager.getMetaDataFile(entry.getKey()); - updatesByMetadataFile.put(metadataFile, Pair.of(entry.getKey(), entry.getValue())); - } - - updatesByMetadataFile.keySet().stream().forEach(metadataFile -> { - try { - log.info(metadataFile.exists() ? "Reading metadata file at {}" : "No metadata file exists yet, so creating a new one", metadataFile.getAbsolutePath()); - BlobMetadataContainer metadataContainer = metadataFile.exists() ? om.readValue(fileSystemJsonBlobManager.readFile(metadataFile), BlobMetadataContainer.class) : new BlobMetadataContainer(); - updatesByMetadataFile.get(metadataFile).forEach(p -> metadataContainer.getLastAccessedByBlobId().put(p.getKey(), p.getValue())); - - log.info("Removing deleted blobs from last access map"); - Set blobs = Sets.newHashSet(Lists.transform(Arrays.asList(metadataFile.getParentFile().listFiles()), f -> f.getName().split(".", 1)[0])); - Set deletedBlobs = metadataContainer.getLastAccessedByBlobId().keySet().parallelStream().filter(blobId -> blobs.contains(blobId)).collect(Collectors.toSet()); - deletedBlobs.stream().forEach(blobId -> metadataContainer.getLastAccessedByBlobId().remove(blobId)); - - log.info("Writing metadata file at {}", metadataFile.getAbsolutePath()); - fileSystemJsonBlobManager.writeFile(metadataFile, om.writeValueAsString(metadataContainer)); - } catch (IOException e) { - log.warn("Couldn't read/write metadata file at {}", metadataFile.getAbsolutePath(), e); - } - }); - - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/health/BlobDirectoryFreeSpaceHealthcheck.java b/src/main/java/com/lowtuna/jsonblob/health/BlobDirectoryFreeSpaceHealthcheck.java deleted file mode 100644 index 61467fa..0000000 --- a/src/main/java/com/lowtuna/jsonblob/health/BlobDirectoryFreeSpaceHealthcheck.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.lowtuna.jsonblob.health; - -import com.codahale.metrics.health.HealthCheck; -import lombok.RequiredArgsConstructor; -import org.apache.commons.io.FileSystemUtils; - -import java.io.File; - -/** - * Created by tburch on 2/9/17. - */ -@RequiredArgsConstructor -public class BlobDirectoryFreeSpaceHealthcheck extends HealthCheck { - private final File blobDataDirectory; - private final long minFreeSpaceKb; - - @Override - protected Result check() throws Exception { - long freeSpaceKb = FileSystemUtils.freeSpaceKb(blobDataDirectory.getAbsolutePath()); - String message = freeSpaceKb + "Kb free for blob storage`"; - return freeSpaceKb > minFreeSpaceKb ? Result.healthy(message) : Result.unhealthy(message); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/resource/ApiResource.java b/src/main/java/com/lowtuna/jsonblob/resource/ApiResource.java deleted file mode 100644 index 4c6c705..0000000 --- a/src/main/java/com/lowtuna/jsonblob/resource/ApiResource.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.lowtuna.jsonblob.resource; - -import com.codahale.metrics.annotation.Timed; -import com.google.common.collect.Lists; -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import com.lowtuna.jsonblob.core.FileSystemJsonBlobManager; -import com.lowtuna.jsonblob.view.ApiView; -import com.sun.jersey.api.NotFoundException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriBuilder; -import java.util.Arrays; -import java.util.LinkedList; - -@Path("/api") -@Slf4j -public class ApiResource { - private final FileSystemJsonBlobManager fileSystemBlobManager; - private final GoogleAnalyticsConfig gaConfig; - - public ApiResource(FileSystemJsonBlobManager fileSystemBlobManager, GoogleAnalyticsConfig gaConfig) { - this.gaConfig = gaConfig; - this.fileSystemBlobManager = fileSystemBlobManager; - } - - @GET - @Timed - public ApiView getApiView() { - return new ApiView(gaConfig.getWebPropertyID(), "api", gaConfig.getCustomTrackingCodes()); - } - - @POST - @Path("jsonBlob") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - @Timed - public Response createJsonBlob(String json) { - try { - String blobId = fileSystemBlobManager.createBlob(json); - return Response.created(UriBuilder.fromResource(JsonBlobResource.class).build(blobId)).entity(json).header("X-jsonblob", blobId).build(); - } catch (IllegalArgumentException e) { - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } catch (IllegalStateException e) { - return Response.serverError().build(); - } - } - - @Path("{path: .*}") - @Timed - public JsonBlobResource getJsonBlobResource(@PathParam("path") String path, @HeaderParam("X-jsonblob") String jsonBlobId) { - LinkedList potentialIds = Lists.newLinkedList(Arrays.asList(path.split("/"))); - if (StringUtils.isNotEmpty(jsonBlobId)) { - potentialIds.addFirst(jsonBlobId); - } - - for (String candidate : potentialIds) { - if (fileSystemBlobManager.blobExists(candidate)) { - log.debug("Using FileSystemJsonBlobManager for loading blob"); - return new JsonBlobResource(candidate, fileSystemBlobManager); - } - } - - throw new NotFoundException(); - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobEditorResource.java b/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobEditorResource.java deleted file mode 100644 index 475b960..0000000 --- a/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobEditorResource.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.lowtuna.jsonblob.resource; - -import com.codahale.metrics.annotation.Timed; -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import com.lowtuna.jsonblob.core.BlobNotFoundException; -import com.lowtuna.jsonblob.core.FileSystemJsonBlobManager; -import com.lowtuna.jsonblob.view.AboutView; -import com.lowtuna.jsonblob.view.EditorView; -import com.sun.jersey.api.NotFoundException; -import io.dropwizard.util.Duration; -import lombok.RequiredArgsConstructor; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; - -@Path("/") -@RequiredArgsConstructor -public class JsonBlobEditorResource { - private final FileSystemJsonBlobManager fileSystemBlobManager; - private final GoogleAnalyticsConfig gaConfig; - private final Duration blobAccessTtl; - - @GET - @Timed - public EditorView defaultEditor() { - return new EditorView(gaConfig.getWebPropertyID(), "editor", gaConfig.getCustomTrackingCodes()); - } - - @GET - @Timed - @Path("new") - public EditorView emptyEditor() { - EditorView view = new EditorView(gaConfig.getWebPropertyID(), "editor", gaConfig.getCustomTrackingCodes()); - view.setJsonBlob("{}"); - return view; - } - - @GET - @Timed - @Path("about") - public AboutView about() { - AboutView view = new AboutView(gaConfig.getWebPropertyID(), "editor", gaConfig.getCustomTrackingCodes(), blobAccessTtl, fileSystemBlobManager.isDeleteEnabled()); - return view; - } - - @GET - @Timed - @Path("{blobId}") - public EditorView blobEditor(@PathParam("blobId") String blobId) { - try { - String json = fileSystemBlobManager.getBlob(blobId); - - EditorView view = new EditorView(gaConfig.getWebPropertyID(), "editor", gaConfig.getCustomTrackingCodes()); - view.setBlobId(blobId); - view.setJsonBlob(json); - return view; - } catch (BlobNotFoundException | IllegalArgumentException e) { - throw new NotFoundException(); - } - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobResource.java b/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobResource.java deleted file mode 100644 index f474357..0000000 --- a/src/main/java/com/lowtuna/jsonblob/resource/JsonBlobResource.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.lowtuna.jsonblob.resource; - -import com.codahale.metrics.annotation.Timed; -import com.lowtuna.jsonblob.core.BlobNotFoundException; -import com.lowtuna.jsonblob.core.JsonBlobManager; -import com.sun.jersey.api.NotFoundException; -import lombok.extern.slf4j.Slf4j; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -@Path("/{blobId}") -@Slf4j -public class JsonBlobResource { - private final String blobId; - private final JsonBlobManager jsonBlobManager; - - public JsonBlobResource(String blobId, JsonBlobManager jsonBlobManager) { - this.blobId = blobId; - this.jsonBlobManager = jsonBlobManager; - } - - @GET - @Timed - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response read() { - log.debug("Reading blob with id {} from {}", blobId, jsonBlobManager.getClass().getName()); - try { - String object = jsonBlobManager.getBlob(blobId); - return Response.ok(object).header("X-jsonblob", blobId).build(); - } catch (BlobNotFoundException e) { - throw new NotFoundException(); - } - } - - @PUT - @Timed - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response update(String json) { - log.debug("Updating blob with id {} from {}", blobId, jsonBlobManager.getClass().getName()); - try { - boolean updated = jsonBlobManager.updateBlob(blobId, json); - if (updated) { - return Response.ok(json).header("X-jsonblob", blobId).build(); - } else { - return Response.serverError().build(); - } - } catch (IllegalArgumentException e) { - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } catch (BlobNotFoundException e) { - throw new NotFoundException(); - } - } - - @DELETE - @Timed - public Response delete() { - log.debug("Deleting blob with id {} from {}", blobId, jsonBlobManager.getClass().getName()); - try { - jsonBlobManager.deleteBlob(blobId); - return Response.ok().build(); - } catch (BlobNotFoundException e) { - throw new NotFoundException(); - } - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/util/jersey/GitTipHeaderFilter.java b/src/main/java/com/lowtuna/jsonblob/util/jersey/GitTipHeaderFilter.java deleted file mode 100644 index dcefcb4..0000000 --- a/src/main/java/com/lowtuna/jsonblob/util/jersey/GitTipHeaderFilter.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.lowtuna.jsonblob.util.jersey; - -import com.sun.jersey.spi.container.ContainerRequest; -import com.sun.jersey.spi.container.ContainerResponse; -import com.sun.jersey.spi.container.ContainerResponseFilter; - -public class GitTipHeaderFilter implements ContainerResponseFilter { - - @Override - public ContainerResponse filter(ContainerRequest request, ContainerResponse response) { - response.getHttpHeaders().add("X-Hello-Human", "If you feel JSON Blob is useful, please consider supporting it! https://www.gittip.com/tburch/"); - return response; - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/util/mongo/JacksonMongoDbModule.java b/src/main/java/com/lowtuna/jsonblob/util/mongo/JacksonMongoDbModule.java deleted file mode 100644 index 082cf13..0000000 --- a/src/main/java/com/lowtuna/jsonblob/util/mongo/JacksonMongoDbModule.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.lowtuna.jsonblob.util.mongo; - -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.bson.types.ObjectId; - -public class JacksonMongoDbModule extends SimpleModule { - - public JacksonMongoDbModule() { - super("MongoModule"); - - addSerializer(new ObjectIdJacksonSerializer()); - addDeserializer(ObjectId.class, new ObjectIdJacksonDeserializer()); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonDeserializer.java b/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonDeserializer.java deleted file mode 100644 index 6320c46..0000000 --- a/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonDeserializer.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.lowtuna.jsonblob.util.mongo; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import org.bson.types.ObjectId; - -import java.io.IOException; - -public class ObjectIdJacksonDeserializer extends StdDeserializer { - - public ObjectIdJacksonDeserializer() { - super(ObjectId.class); - } - - @Override - public ObjectId deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { - return new ObjectId(jp.getValueAsString()); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonSerializer.java b/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonSerializer.java deleted file mode 100644 index 742c1c8..0000000 --- a/src/main/java/com/lowtuna/jsonblob/util/mongo/ObjectIdJacksonSerializer.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.lowtuna.jsonblob.util.mongo; - -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import org.bson.types.ObjectId; - -import java.io.IOException; - -public class ObjectIdJacksonSerializer extends StdSerializer { - - public ObjectIdJacksonSerializer() { - super(ObjectId.class); - } - - @Override - public void serialize(ObjectId value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonGenerationException { - jgen.writeString(value.toString()); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/util/mustache/Base64StringHelpers.java b/src/main/java/com/lowtuna/jsonblob/util/mustache/Base64StringHelpers.java deleted file mode 100644 index c8eb088..0000000 --- a/src/main/java/com/lowtuna/jsonblob/util/mustache/Base64StringHelpers.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.lowtuna.jsonblob.util.mustache; - -import com.github.jknack.handlebars.Handlebars; -import com.github.jknack.handlebars.Helper; -import com.github.jknack.handlebars.Options; -import org.apache.commons.codec.binary.Base64; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static org.apache.commons.lang3.Validate.notNull; - -public enum Base64StringHelpers implements Helper { - base64Encode { - @Override - protected CharSequence safeApply(final String value, final Options options) { - Boolean urlSafe = options.hash("urlSafe", Boolean.FALSE); - return urlSafe ? Base64.encodeBase64URLSafeString(value.getBytes(StandardCharsets.UTF_8)) : Base64.encodeBase64String(value.getBytes(StandardCharsets.UTF_8)); - } - - }, - - base64Decode { - @Override - protected CharSequence safeApply(final String value, final Options options) { - Boolean urlSafe = options.hash("urlSafe", Boolean.FALSE); - return new String(urlSafe ? Base64.encodeBase64URLSafe(value.getBytes(StandardCharsets.UTF_8)) : Base64.decodeBase64(value.getBytes(StandardCharsets.UTF_8))); - } - }; - - @Override - public CharSequence apply(final String context, final Options options) throws IOException { - if (options.isFalsy(context)) { - Object param = options.param(0, null); - return param == null ? null : param.toString(); - } - return safeApply(context, options); - } - - protected abstract CharSequence safeApply(final String context, final Options options); - - public void registerHelper(final Handlebars handlebars) { - notNull(handlebars, "The handlebars is required."); - handlebars.registerHelper(name(), this); - } - - public static void register(final Handlebars handlebars) { - notNull(handlebars, "A handlebars object is required."); - Base64StringHelpers[] helpers = values(); - for (Base64StringHelpers helper : helpers) { - helper.registerHelper(handlebars); - } - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/view/AboutView.java b/src/main/java/com/lowtuna/jsonblob/view/AboutView.java deleted file mode 100644 index 36db570..0000000 --- a/src/main/java/com/lowtuna/jsonblob/view/AboutView.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.lowtuna.jsonblob.view; - -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import io.dropwizard.util.Duration; -import lombok.Data; - -import java.util.Set; - -@Data -public class AboutView extends JsonBlobView { - private final Duration blobAccessTtl; - private final boolean deletionEnabled; - - public AboutView(String gaWebPropertyID, String pageName, Set customTrackingCodes, Duration blobAccessTtl, boolean deletionEnabled) { - super("/about.hbs", gaWebPropertyID, pageName, customTrackingCodes); - this.blobAccessTtl = blobAccessTtl; - this.deletionEnabled = deletionEnabled; - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/view/ApiView.java b/src/main/java/com/lowtuna/jsonblob/view/ApiView.java deleted file mode 100644 index 4440f9e..0000000 --- a/src/main/java/com/lowtuna/jsonblob/view/ApiView.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.lowtuna.jsonblob.view; - -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Set; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ApiView extends JsonBlobView { - - public ApiView(String gaWebPropertyID, String pageName, Set customTrackingCodes) { - super("/api.hbs", gaWebPropertyID, pageName, customTrackingCodes); - } - -} diff --git a/src/main/java/com/lowtuna/jsonblob/view/EditorView.java b/src/main/java/com/lowtuna/jsonblob/view/EditorView.java deleted file mode 100644 index 64c7d15..0000000 --- a/src/main/java/com/lowtuna/jsonblob/view/EditorView.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.lowtuna.jsonblob.view; - -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import lombok.Data; - -import java.util.Set; - -@Data -public class EditorView extends JsonBlobView { - private String blobId; - private String jsonBlob; - - public EditorView(String gaWebPropertyID, String pageName, Set customTrackingCodes) { - super("/editor.hbs", gaWebPropertyID, pageName, customTrackingCodes); - } -} diff --git a/src/main/java/com/lowtuna/jsonblob/view/JsonBlobView.java b/src/main/java/com/lowtuna/jsonblob/view/JsonBlobView.java deleted file mode 100644 index 55bd282..0000000 --- a/src/main/java/com/lowtuna/jsonblob/view/JsonBlobView.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.lowtuna.jsonblob.view; - -import com.lowtuna.dropwizard.extras.config.GoogleAnalyticsConfig; -import io.dropwizard.views.View; -import lombok.Data; - -import java.util.Date; -import java.util.Set; - -@Data -public abstract class JsonBlobView extends View { - private final String gaWebPropertyID; - private final String pageName; - private final Set gaCustomTrackingCodes; - private final Date now = new Date(); - - protected JsonBlobView(String view, String gaWebPropertyID, String pageName, Set customTrackingCodes) { - super(view); - this.gaWebPropertyID = gaWebPropertyID; - this.pageName = pageName; - this.gaCustomTrackingCodes = customTrackingCodes; - } - -} diff --git a/src/main/kotlin/jsonblob/Application.kt b/src/main/kotlin/jsonblob/Application.kt new file mode 100644 index 0000000..87950a1 --- /dev/null +++ b/src/main/kotlin/jsonblob/Application.kt @@ -0,0 +1,11 @@ +package jsonblob + +import io.micronaut.runtime.Micronaut.build + +fun main(args: Array) { + build() + .args(*args) + .packages("jsonblob") + .start() +} + diff --git a/src/main/kotlin/jsonblob/api/http/AboutController.kt b/src/main/kotlin/jsonblob/api/http/AboutController.kt new file mode 100644 index 0000000..588b4b4 --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/AboutController.kt @@ -0,0 +1,23 @@ +package jsonblob.api.http + +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Get +import io.micronaut.views.View +import jsonblob.api.http.view.AboutView +import jsonblob.config.GoogleAnalyticsConfig +import jsonblob.config.JsonBlobConfig + +@Controller("/about") +class AboutController( + private val jsonBlobConfig: JsonBlobConfig, + private val googleAnalyticsConfig: GoogleAnalyticsConfig +) { + @Get + @View("about") + fun get() = AboutView( + googleAnalyticsConfig.webPropertyId, + "about", + googleAnalyticsConfig.customTrackingCodes, + jsonBlobConfig.deleteAfter.toDays() + ) +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/ApiController.kt b/src/main/kotlin/jsonblob/api/http/ApiController.kt new file mode 100644 index 0000000..d0586aa --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/ApiController.kt @@ -0,0 +1,210 @@ +package jsonblob.api.http + +import io.micronaut.http.HttpRequest +import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus +import io.micronaut.http.MediaType +import io.micronaut.http.annotation.Body +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Delete +import io.micronaut.http.annotation.Get +import io.micronaut.http.annotation.Header +import io.micronaut.http.annotation.PathVariable +import io.micronaut.http.annotation.Post +import io.micronaut.http.annotation.Produces +import io.micronaut.http.annotation.Put +import io.micronaut.http.exceptions.HttpStatusException +import io.micronaut.http.server.util.HttpHostResolver +import io.micronaut.views.View +import io.micronaut.web.router.RouteBuilder +import jsonblob.api.http.view.ApiView +import jsonblob.config.GoogleAnalyticsConfig +import jsonblob.config.JsonBlobConfig +import jsonblob.core.id.IdHandler +import jsonblob.core.store.JsonBlobStore +import jsonblob.model.JsonBlob +import jsonblob.util.JsonCleaner +import mu.KotlinLogging +import java.net.URI + + +private val log = KotlinLogging.logger {} + +@Controller("/${ApiController.apiPath}") +class ApiController( + private val jsonBlobConfig: JsonBlobConfig, + private val idResolvers: List>, + private val idGenerator: IdHandler<*>, + private val jsonBlobStore: JsonBlobStore, + private val httpHostResolver: HttpHostResolver, + private val uriNamingStrategy: RouteBuilder.UriNamingStrategy, + private val googleAnalyticsConfig: GoogleAnalyticsConfig +) { + companion object { + const val apiPath = "api" + const val jsonBlobPath = "jsonBlob" + const val jsonBlobHeader = "X-jsonblob" + } + + @Get + @View("api") + fun apiView() = ApiView( + googleAnalyticsConfig.webPropertyId, + "api", + googleAnalyticsConfig.customTrackingCodes + ) + + @Post("/$jsonBlobPath") + @Produces(MediaType.APPLICATION_JSON) + fun createBlob(@Body json: String, httpRequest: HttpRequest): HttpResponse { + if (JsonCleaner.validJson(json)) { + val jsonBlob = JsonBlob( + id = idGenerator.generate(), + json = json + ) + val blob = jsonBlobStore.write(jsonBlob) + val host = httpHostResolver.resolve(httpRequest) + uriNamingStrategy.resolveUri("/api/jsonBlob/${blob.id}") + return HttpResponse.created(blob.json, URI.create(host)) + } else { + throw HttpStatusException(HttpStatus.BAD_REQUEST, "Invalid JSON") + } + } + + @Get("/$jsonBlobPath/{blobId}") + @Produces(MediaType.APPLICATION_JSON) + fun getBlob(@PathVariable blobId: String) = readJsonBlob(blobId)?.json + ?: throw HttpStatusException(HttpStatus.NOT_FOUND, "Blob with id $blobId does not exist") + + @Get("/{path:.*}") + @Produces(MediaType.APPLICATION_JSON) + fun getCustomPath(@PathVariable path: String, @Header(jsonBlobHeader) jsonbBlobHeader: String?): String { + if (jsonbBlobHeader != null) { + val headerBlob = readJsonBlob(jsonbBlobHeader) + if (headerBlob != null) { + return headerBlob.json + } + } + return readFirstBlobFromPath(path)?.json ?: throw HttpStatusException( + HttpStatus.NOT_FOUND, + "Blob does not exist" + ) + } + + @Put("/$jsonBlobPath/{blobId}") + @Produces(MediaType.APPLICATION_JSON) + fun updateBlob(@PathVariable blobId: String, @Body json: String) = + update(blobId, json)?.json ?: throw HttpStatusException( + HttpStatus.NOT_FOUND, + "Blob with id $blobId does not exist" + ) + + @Put("/{path:.*}") + @Produces(MediaType.APPLICATION_JSON) + fun updateCustomPathOrHeader( + @PathVariable path: String, + @Header(jsonBlobHeader) jsonbBlobHeader: String?, + @Body json: String + ): String { + if (jsonbBlobHeader != null) { + val headerBlob = update(jsonbBlobHeader, json) + if (headerBlob != null) { + return headerBlob.json + } + } + return updateFirstBlobFromPath(path, json)?.json ?: throw HttpStatusException( + HttpStatus.NOT_FOUND, + "Blob does not exist" + ) + } + + @Delete("/$jsonBlobPath/{blobId}") + fun deleteBlob(@PathVariable blobId: String): HttpStatus { + if (!jsonBlobConfig.deleteEnabled) { + throw HttpStatusException(HttpStatus.METHOD_NOT_ALLOWED, "Delete is not enabled") + } + return if (delete(blobId)) { + HttpStatus.OK + } else { + throw HttpStatusException(HttpStatus.NOT_FOUND, "Blob with id $blobId does not exist") + } + } + + @Delete("/{path:.*}") + fun deleteCustomPathOrHeader( + @PathVariable path: String, + @Header(jsonBlobHeader) jsonbBlobHeader: String? + ): HttpStatus { + if (!jsonBlobConfig.deleteEnabled) { + throw HttpStatusException(HttpStatus.METHOD_NOT_ALLOWED, "Delete is not enabled") + } + if (jsonbBlobHeader != null) { + if (delete(jsonbBlobHeader)) { + return HttpStatus.OK + } + } + return if (deleteFirstBlobFromPath(path)) { + HttpStatus.OK + } else { + throw HttpStatusException(HttpStatus.NOT_FOUND, "Blob does not exist") + } + } + + private fun deleteFirstBlobFromPath(path: String): Boolean { + val ids = blobIdsFromPath(path) + if (ids.isNotEmpty()) { + return delete(ids.first()) + } + return false + } + + private fun delete(blobId: String) = jsonBlobStore.exists(blobId) && jsonBlobStore.remove(blobId) + + private fun updateFirstBlobFromPath(path: String, json: String): JsonBlob? { + val ids = blobIdsFromPath(path) + return if (ids.isNotEmpty()) { + update(ids.first(), json) + } else { + null + } + } + + private fun update(blobId: String, json: String): JsonBlob? { + if (JsonCleaner.validJson(json)) { + val resolver = idResolvers.firstOrNull { it.handles(blobId) } + return if (resolver != null) { + val created = resolver.resolveTimestamp(blobId) + val jsonBlob = JsonBlob( + id = blobId, + json = json, + created = created + ) + jsonBlobStore.write(jsonBlob) + } else { + null + } + } else { + throw HttpStatusException(HttpStatus.BAD_REQUEST, "Invalid JSON") + } + } + + private fun readFirstBlobFromPath(path: String): JsonBlob? { + val ids = blobIdsFromPath(path) + return if (ids.isNotEmpty()) { + readJsonBlob(ids.first()) + } else { + null + } + } + + private fun readJsonBlob(blobId: String) = jsonBlobStore.read(blobId) + + private fun blobIdsFromPath(path: String) = path + .split("/") + .mapNotNull { pathPart -> + if (idResolvers.any { it.handles(pathPart) }) { + pathPart + } else { + null + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/EditorController.kt b/src/main/kotlin/jsonblob/api/http/EditorController.kt new file mode 100644 index 0000000..136e7c5 --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/EditorController.kt @@ -0,0 +1,69 @@ +package jsonblob.api.http + +import io.micronaut.http.HttpStatus +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Get +import io.micronaut.http.annotation.PathVariable +import io.micronaut.http.exceptions.HttpStatusException +import io.micronaut.views.View +import jsonblob.api.http.view.EditorView +import jsonblob.config.AdsenseConfig +import jsonblob.config.GoogleAnalyticsConfig +import jsonblob.core.store.JsonBlobStore + +@Controller("/") +class EditorController( + private val jsonBlobStore: JsonBlobStore, + private val googleAnalyticsConfig: GoogleAnalyticsConfig, + private val adsenseConfig: AdsenseConfig +) { + companion object { + private const val pageName = "editor" + } + + private fun readJsonBlob(blobId: String) = jsonBlobStore.read(blobId) + + @Get("ads.txt") + fun ads() : String { + if (adsenseConfig.adsConfig.value.isNotBlank()) { + return "google.com, ${adsenseConfig.publisherId}, ${adsenseConfig.adsConfig.type}, ${adsenseConfig.adsConfig.value}" + } + throw HttpStatusException(HttpStatus.NOT_FOUND, "ads.txt doesn't exist") + } + + @Get + @View("editor") + fun get() = EditorView( + googleAnalyticsConfig.webPropertyId, + pageName, + googleAnalyticsConfig.customTrackingCodes + ) + + @Get("{blobId}") + @View("editor") + fun getBlob(@PathVariable blobId: String) : EditorView { + if (blobId == "new") { + return EditorView( + googleAnalyticsConfig.webPropertyId, + pageName, + googleAnalyticsConfig.customTrackingCodes, + "{}" + ) + } + + val blob = readJsonBlob(blobId) + + if (blob == null) { + throw HttpStatusException(HttpStatus.NOT_FOUND, "Blob with id $blobId does not exist") + } else { + return EditorView( + googleAnalyticsConfig.webPropertyId, + pageName, + googleAnalyticsConfig.customTrackingCodes, + blob.json, + blob.id + ) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/view/AboutView.kt b/src/main/kotlin/jsonblob/api/http/view/AboutView.kt new file mode 100644 index 0000000..9ef39a7 --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/view/AboutView.kt @@ -0,0 +1,12 @@ +package jsonblob.api.http.view + +import io.micronaut.core.annotation.Introspected +import jsonblob.config.GoogleAnalyticsConfig + +@Introspected +class AboutView( + gaWebPropertyID: String, + pageName: String, + gaCustomTrackingCodes: Set, + val deleteAfterDays: Long +) : JsonBlobView(gaWebPropertyID, pageName, gaCustomTrackingCodes) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/view/ApiView.kt b/src/main/kotlin/jsonblob/api/http/view/ApiView.kt new file mode 100644 index 0000000..edf45cd --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/view/ApiView.kt @@ -0,0 +1,11 @@ +package jsonblob.api.http.view + +import io.micronaut.core.annotation.Introspected +import jsonblob.config.GoogleAnalyticsConfig + +@Introspected +class ApiView( + gaWebPropertyID: String, + pageName: String, + gaCustomTrackingCodes: Set +) : JsonBlobView(gaWebPropertyID, pageName, gaCustomTrackingCodes) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/view/EditorView.kt b/src/main/kotlin/jsonblob/api/http/view/EditorView.kt new file mode 100644 index 0000000..fca0740 --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/view/EditorView.kt @@ -0,0 +1,13 @@ +package jsonblob.api.http.view + +import io.micronaut.core.annotation.Introspected +import jsonblob.config.GoogleAnalyticsConfig + +@Introspected +class EditorView( + gaWebPropertyID: String, + pageName: String, + gaCustomTrackingCodes: Set, + val jsonBlob: String? = null, + val blobId: String? = null +) : JsonBlobView(gaWebPropertyID, pageName, gaCustomTrackingCodes) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/api/http/view/JsonBlobView.kt b/src/main/kotlin/jsonblob/api/http/view/JsonBlobView.kt new file mode 100644 index 0000000..f61cb07 --- /dev/null +++ b/src/main/kotlin/jsonblob/api/http/view/JsonBlobView.kt @@ -0,0 +1,11 @@ +package jsonblob.api.http.view + +import jsonblob.config.GoogleAnalyticsConfig +import java.time.Instant + +open class JsonBlobView( + val gaWebPropertyID: String, + val pageName: String, + val gaCustomTrackingCodes: Set, + val now: Instant = Instant.now() +) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/AdsenseConfig.kt b/src/main/kotlin/jsonblob/config/AdsenseConfig.kt new file mode 100644 index 0000000..41688e9 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/AdsenseConfig.kt @@ -0,0 +1,17 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties + +@ConfigurationProperties("adsense") +class AdsenseConfig { + var publisherId = "" + + var adsConfig = AdsConfig() + + @ConfigurationProperties("ads-config") + class AdsConfig { + var type : String = "DIRECT" + + var value: String = "" + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/BrotliBlobCompressorConfig.kt b/src/main/kotlin/jsonblob/config/BrotliBlobCompressorConfig.kt new file mode 100644 index 0000000..9d21358 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/BrotliBlobCompressorConfig.kt @@ -0,0 +1,39 @@ +package jsonblob.config + +import com.nixxcode.jvmbrotli.common.BrotliLoader +import io.micronaut.context.annotation.ConfigurationProperties +import io.micronaut.context.annotation.Factory +import io.micronaut.context.annotation.Primary +import io.micronaut.context.exceptions.DisabledBeanException +import io.micronaut.core.annotation.Order +import io.micronaut.core.order.Ordered +import jsonblob.core.compression.compressor.BrotliBlobCompressor +import mu.KotlinLogging +import javax.inject.Singleton +import javax.validation.constraints.Max +import javax.validation.constraints.Min + + +private val log = KotlinLogging.logger {} + +@ConfigurationProperties("brotli") +class BrotliBlobCompressorConfig { + @get:Min(-1) + @get:Max(11) + var quality = -1 +} + +@Factory +class BrotliBlobCompressorFactory { + @Primary + @Singleton + @Order(Ordered.HIGHEST_PRECEDENCE) + fun brotliBlobCompressor(brotliBlobCompressorConfig: BrotliBlobCompressorConfig): BrotliBlobCompressor { + if (BrotliLoader.isBrotliAvailable()) { + return BrotliBlobCompressor(brotliBlobCompressorConfig) + } else { + log.warn { "Brotli is not available" } + throw DisabledBeanException("Brotli is not available") + } + } +} diff --git a/src/main/kotlin/jsonblob/config/FileSystemJsonBlobStoreConfig.kt b/src/main/kotlin/jsonblob/config/FileSystemJsonBlobStoreConfig.kt new file mode 100644 index 0000000..aeb4f9d --- /dev/null +++ b/src/main/kotlin/jsonblob/config/FileSystemJsonBlobStoreConfig.kt @@ -0,0 +1,20 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties +import io.micronaut.context.annotation.Requires +import javax.validation.constraints.NotBlank + +@ConfigurationProperties(FileSystemJsonBlobStoreConfig.PREFIX) +@Requires(property = FileSystemJsonBlobStoreConfig.PREFIX + ".base-path") +class FileSystemJsonBlobStoreConfig { + companion object { + const val PREFIX = "file-system-blob-store" + } + + var deleteConcurrency = 250 + + var stripes = 512 + + @get:NotBlank + var basePath = "" +} diff --git a/src/main/kotlin/jsonblob/config/GoogleAnalyticsConfig.kt b/src/main/kotlin/jsonblob/config/GoogleAnalyticsConfig.kt new file mode 100644 index 0000000..d32a117 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/GoogleAnalyticsConfig.kt @@ -0,0 +1,18 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties +import javax.validation.constraints.NotBlank + +@ConfigurationProperties("ga") +class GoogleAnalyticsConfig { + var webPropertyId = "" + + var customTrackingCodes = emptySet() + + class CustomTrackingCode { + @get:NotBlank + var key : String = "" + + var value: String = "" + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/HandlerbarsWithHelpersFactory.kt b/src/main/kotlin/jsonblob/config/HandlerbarsWithHelpersFactory.kt new file mode 100644 index 0000000..3bbc11a --- /dev/null +++ b/src/main/kotlin/jsonblob/config/HandlerbarsWithHelpersFactory.kt @@ -0,0 +1,28 @@ +package jsonblob.config + +import com.github.jknack.handlebars.Handlebars +import com.github.jknack.handlebars.cache.ConcurrentMapTemplateCache +import io.micronaut.context.annotation.Factory +import io.micronaut.context.annotation.Replaces +import io.micronaut.views.handlebars.HandlebarsFactory +import jsonblob.handlebars.helper.NamedHelper +import mu.KotlinLogging +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Factory +class HandlebarsWithHelpersFactory { + @Singleton + @Replaces(factory = HandlebarsFactory::class) + fun handlebarsWithHandlers(helpers: Collection>) : Handlebars { + val handlebars = Handlebars().with(ConcurrentMapTemplateCache()) + log.info { "Adding Helpers to Handlebars" } + helpers.forEach { + handlebars.registerHelper(it.getName(), it) + } + log.info { "Competed adding Helpers to Handlebars" } + return handlebars + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/JsonBlobConfig.kt b/src/main/kotlin/jsonblob/config/JsonBlobConfig.kt new file mode 100644 index 0000000..22d4a62 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/JsonBlobConfig.kt @@ -0,0 +1,13 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties +import java.time.Duration + +@ConfigurationProperties("json-blob") +class JsonBlobConfig { + var deleteAfter: Duration = Duration.ofDays(180) + + var deleteEnabled = true + + var pruneEnabled = true +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/S3ClientBuilderListener.kt b/src/main/kotlin/jsonblob/config/S3ClientBuilderListener.kt new file mode 100644 index 0000000..61e3d59 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/S3ClientBuilderListener.kt @@ -0,0 +1,35 @@ +package jsonblob.config + +import io.micronaut.context.annotation.Property +import io.micronaut.context.annotation.Requires +import io.micronaut.context.event.BeanCreatedEvent +import io.micronaut.context.event.BeanCreatedEventListener +import mu.KotlinLogging +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration +import software.amazon.awssdk.core.retry.RetryMode +import software.amazon.awssdk.services.s3.S3ClientBuilder +import java.net.URI +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Singleton +@Requires(beans = [S3ClientBuilder::class]) +class S3ClientBuilderListener( + @Property(name = endpointProp) private val endpoint: String? +) : BeanCreatedEventListener { + companion object { + const val endpointProp = "aws.s3.endpoint" + } + + override fun onCreated(event: BeanCreatedEvent): S3ClientBuilder { + return event.bean + .overrideConfiguration(ClientOverrideConfiguration.builder().retryPolicy(RetryMode.LEGACY).build()) + .apply { + endpoint?.let { + endpointOverride(URI.create(it)) + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/config/S3JsonBlobStoreConfig.kt b/src/main/kotlin/jsonblob/config/S3JsonBlobStoreConfig.kt new file mode 100644 index 0000000..33abdba --- /dev/null +++ b/src/main/kotlin/jsonblob/config/S3JsonBlobStoreConfig.kt @@ -0,0 +1,23 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties +import io.micronaut.context.annotation.Requires +import javax.validation.constraints.NotBlank + +@ConfigurationProperties(S3JsonBlobStoreConfig.PREFIX) +@Requires(property = S3JsonBlobStoreConfig.PREFIX + ".bucket") +class S3JsonBlobStoreConfig { + companion object { + const val PREFIX = "s3-blob-store" + } + + @get:NotBlank + var bucket = "" + + @get:NotBlank + var basePath = "json-blobs" + + var setupLifecycle = false + + var copyToResetLastModified = false +} diff --git a/src/main/kotlin/jsonblob/config/SnowflakeConfig.kt b/src/main/kotlin/jsonblob/config/SnowflakeConfig.kt new file mode 100644 index 0000000..2234865 --- /dev/null +++ b/src/main/kotlin/jsonblob/config/SnowflakeConfig.kt @@ -0,0 +1,29 @@ +package jsonblob.config + +import io.micronaut.context.annotation.ConfigurationProperties +import io.micronaut.context.annotation.Factory +import jsonblob.core.Snowflake +import javax.inject.Singleton +import javax.validation.constraints.Max +import javax.validation.constraints.Min + +@ConfigurationProperties("snowflake") +class SnowflakeConfig { + @get:Min(1) + @get:Max(Snowflake.maxNodeId) + var nodeId: Long = 1 + + var autoConfigureNodeId = true +} + +@Factory +class SnowFlakeFactory { + @Singleton + fun snowflake(config: SnowflakeConfig): Snowflake { + return if (config.autoConfigureNodeId) { + Snowflake() + } else { + Snowflake(config) + } + } +} diff --git a/src/main/kotlin/jsonblob/core/Snowflake.kt b/src/main/kotlin/jsonblob/core/Snowflake.kt new file mode 100644 index 0000000..0455c27 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/Snowflake.kt @@ -0,0 +1,125 @@ +package jsonblob.core + +import jsonblob.config.SnowflakeConfig +import mu.KotlinLogging +import java.net.NetworkInterface +import java.security.SecureRandom +import java.time.Instant + + +private val log = KotlinLogging.logger {} + +/** + * Distributed Sequence Generator. + * Inspired by Twitter snowflake: https://github.com/twitter/snowflake/tree/snowflake-2010 + * + * This class should be used as a Singleton. + * Make sure that you create and reuse a Single instance of Snowflake per node in your distributed system cluster. + */ +class Snowflake { + constructor() { + nodeId = createNodeId().also { log.info { "Using nodeId=$it" } } + customEpoch = DEFAULT_CUSTOM_EPOCH + } + + constructor(config: SnowflakeConfig) { + nodeId = config.nodeId + customEpoch = DEFAULT_CUSTOM_EPOCH + } + + constructor(nodeId: Long, customEpoch: Long = DEFAULT_CUSTOM_EPOCH) { + require(!(nodeId < 0 || nodeId > maxNodeId)) { String.format("NodeId must be between %d and %d", 0, maxNodeId) } + this.nodeId = nodeId + this.customEpoch = customEpoch + } + + private val nodeId: Long + private val customEpoch: Long + + @Volatile + private var lastTimestamp = -1L + + @Volatile + private var sequence = 0L + + @Synchronized + fun nextId(): Long { + var currentTimestamp = timestamp() + check(currentTimestamp >= lastTimestamp) { "Invalid System Clock!" } + if (currentTimestamp == lastTimestamp) { + sequence = sequence + 1 and maxSequence + if (sequence == 0L) { + // Sequence Exhausted, wait till next millisecond. + currentTimestamp = waitNextMillis(currentTimestamp) + } + } else { + // reset sequence to start with zero for the next millisecond + sequence = 0 + } + lastTimestamp = currentTimestamp + return (currentTimestamp shl NODE_ID_BITS + SEQUENCE_BITS or (nodeId shl SEQUENCE_BITS) + or sequence) + } + + // Get current timestamp in milliseconds, adjust for the custom epoch. + private fun timestamp(): Long { + return Instant.now().toEpochMilli() - customEpoch + } + + // Block and wait till next millisecond + private fun waitNextMillis(currentTimestamp: Long): Long { + var ct = currentTimestamp + while (currentTimestamp == lastTimestamp) { + ct = timestamp() + } + return ct + } + + private fun createNodeId(): Long { + var nodeId: Long = try { + val sb = StringBuilder() + val networkInterfaces = NetworkInterface.getNetworkInterfaces() + while (networkInterfaces.hasMoreElements()) { + val networkInterface = networkInterfaces.nextElement() + val mac = networkInterface.hardwareAddress + if (mac != null) { + for (macPort in mac) { + sb.append(String.format("%02X", macPort)) + } + } + } + sb.toString().hashCode().toLong() + } catch (ex: Exception) { + SecureRandom().nextInt().toLong() + } + nodeId = nodeId and maxNodeId + return nodeId + } + + fun parse(id: Long): LongArray { + val maskNodeId = (1L shl NODE_ID_BITS) - 1 shl SEQUENCE_BITS + val maskSequence = (1L shl SEQUENCE_BITS) - 1 + val timestamp = (id shr NODE_ID_BITS + SEQUENCE_BITS) + customEpoch + val nodeId = id and maskNodeId shr SEQUENCE_BITS + val sequence = id and maskSequence + return longArrayOf(timestamp, nodeId, sequence) + } + + override fun toString(): String { + return ("Snowflake Settings [EPOCH_BITS=" + EPOCH_BITS + ", NODE_ID_BITS=" + NODE_ID_BITS + + ", SEQUENCE_BITS=" + SEQUENCE_BITS + ", CUSTOM_EPOCH=" + customEpoch + + ", NodeId=" + nodeId + "]") + } + + companion object { + private const val UNUSED_BITS = 1 // Sign bit, Unused (always set to 0) + private const val EPOCH_BITS = 41 + private const val NODE_ID_BITS = 10 + private const val SEQUENCE_BITS = 12 + const val maxNodeId = (1L shl NODE_ID_BITS) - 1 + private const val maxSequence = (1L shl SEQUENCE_BITS) - 1 + + // Custom Epoch (January 1, 2015 Midnight UTC = 2015-01-01T00:00:00Z) + private const val DEFAULT_CUSTOM_EPOCH = 1420070400000L + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/compression/CompressorPicker.kt b/src/main/kotlin/jsonblob/core/compression/CompressorPicker.kt new file mode 100644 index 0000000..1e4ed3b --- /dev/null +++ b/src/main/kotlin/jsonblob/core/compression/CompressorPicker.kt @@ -0,0 +1,36 @@ +package jsonblob.core.compression + +import jsonblob.core.compression.compressor.BlobCompressor +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import java.io.ByteArrayOutputStream +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Singleton +class BlobCompressorPicker( + private val compressors: List +) { + fun bestCompressor(json: String) : BlobCompressor { + return runBlocking { + val compressorsToBytes = compressors.map { + async { + val bytes = ByteArrayOutputStream().apply { + it.getOutputStream(this).use { + it.write(json.toByteArray()) + } + } + log.debug { "${it::class.simpleName} compressed JSON (${json.toByteArray().size} bytes) to ${bytes.size()} bytes" } + Pair(it, bytes) + } + }.awaitAll() + val sorted = compressorsToBytes.sortedBy { (_, value) -> value.size()}.toMap() + val best = sorted.entries.first() + best.key + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/compression/compressor/BlobCompressor.kt b/src/main/kotlin/jsonblob/core/compression/compressor/BlobCompressor.kt new file mode 100644 index 0000000..19c6b6e --- /dev/null +++ b/src/main/kotlin/jsonblob/core/compression/compressor/BlobCompressor.kt @@ -0,0 +1,11 @@ +package jsonblob.core.compression.compressor + +import java.io.InputStream +import java.io.OutputStream + +interface BlobCompressor { + fun handles(fileExtension: String) : Boolean + fun getFileExtension() : String + fun getInputStream(inputStream: InputStream) : InputStream + fun getOutputStream(outputStream: OutputStream) : OutputStream +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/compression/compressor/BrotliBlobCompressor.kt b/src/main/kotlin/jsonblob/core/compression/compressor/BrotliBlobCompressor.kt new file mode 100644 index 0000000..70d642a --- /dev/null +++ b/src/main/kotlin/jsonblob/core/compression/compressor/BrotliBlobCompressor.kt @@ -0,0 +1,32 @@ +package jsonblob.core.compression.compressor + +import com.nixxcode.jvmbrotli.dec.BrotliInputStream +import com.nixxcode.jvmbrotli.enc.BrotliOutputStream +import com.nixxcode.jvmbrotli.enc.Encoder +import jsonblob.config.BrotliBlobCompressorConfig +import mu.KotlinLogging +import java.io.InputStream +import java.io.OutputStream + + +private val log = KotlinLogging.logger {} + +class BrotliBlobCompressor(config: BrotliBlobCompressorConfig) : NoCompressionJsonBlobCompressor() { + companion object { + const val fileExtension = "br" + } + + init { + log.info { "Using quality of ${config.quality}" } + } + + private val params = Encoder.Parameters().setQuality(config.quality) + + override fun handles(fileExtension: String) = fileExtension.endsWith(Companion.fileExtension) + + override fun getFileExtension() = listOf(super.getFileExtension(), fileExtension).joinToString(separator = ".") + + override fun getInputStream(inputStream: InputStream) = BrotliInputStream(inputStream) + + override fun getOutputStream(outputStream: OutputStream) = BrotliOutputStream(outputStream, params) +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/compression/compressor/GZIPBlobCompressor.kt b/src/main/kotlin/jsonblob/core/compression/compressor/GZIPBlobCompressor.kt new file mode 100644 index 0000000..6c1be56 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/compression/compressor/GZIPBlobCompressor.kt @@ -0,0 +1,24 @@ +package jsonblob.core.compression.compressor + +import io.micronaut.context.annotation.Secondary +import java.io.InputStream +import java.io.OutputStream +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream +import javax.inject.Singleton + +@Singleton +@Secondary +class GZIPBlobCompressor : NoCompressionJsonBlobCompressor() { + companion object { + const val fileExtension = "gz" + } + + override fun handles(fileExtension: String) = fileExtension.endsWith(Companion.fileExtension) + + override fun getFileExtension() = listOf(super.getFileExtension(), fileExtension).joinToString(separator = ".") + + override fun getInputStream(inputStream: InputStream) = GZIPInputStream(inputStream) + + override fun getOutputStream(outputStream: OutputStream) = GZIPOutputStream(outputStream) +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/compression/compressor/NoCompressionBlobCompressor.kt b/src/main/kotlin/jsonblob/core/compression/compressor/NoCompressionBlobCompressor.kt new file mode 100644 index 0000000..903f715 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/compression/compressor/NoCompressionBlobCompressor.kt @@ -0,0 +1,23 @@ +package jsonblob.core.compression.compressor + +import io.micronaut.core.annotation.Order +import io.micronaut.core.order.Ordered +import java.io.InputStream +import java.io.OutputStream +import javax.inject.Singleton + +@Order(Ordered.LOWEST_PRECEDENCE) +@Singleton +open class NoCompressionJsonBlobCompressor : BlobCompressor { + companion object { + const val fileExtension = "json" + } + + override fun getFileExtension() = fileExtension + + override fun handles(fileExtension: String) = fileExtension == getFileExtension() + + override fun getInputStream(inputStream: InputStream) = inputStream + + override fun getOutputStream(outputStream: OutputStream) = outputStream +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/id/IdHandler.kt b/src/main/kotlin/jsonblob/core/id/IdHandler.kt new file mode 100644 index 0000000..23024d7 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/id/IdHandler.kt @@ -0,0 +1,15 @@ +package jsonblob.core.id + +import java.time.Instant + +abstract class IdHandler { + abstract fun idFrom(t: T) : String + abstract fun to(id: String) : T + abstract fun resolveTimestamp(t: T) : Instant + abstract fun generate(): String + open fun handles(id: String) = kotlin.runCatching { + to(id) + true + }.getOrDefault(false) + open fun resolveTimestamp(id: String) = resolveTimestamp(to(id)) +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/id/ObjectIdJsonBlobHandler.kt b/src/main/kotlin/jsonblob/core/id/ObjectIdJsonBlobHandler.kt new file mode 100644 index 0000000..a92327e --- /dev/null +++ b/src/main/kotlin/jsonblob/core/id/ObjectIdJsonBlobHandler.kt @@ -0,0 +1,19 @@ +package jsonblob.core.id + +import io.micronaut.core.annotation.Order +import org.bson.types.ObjectId +import java.time.Instant +import javax.inject.Singleton + + +@Singleton +@Order(1) +class ObjectIdJsonBlobHandler : IdHandler() { + override fun resolveTimestamp(t: ObjectId): Instant = Instant.ofEpochSecond(t.timestamp.toLong()) + + override fun idFrom(t: ObjectId): String = t.toHexString() + + override fun to(id: String) = ObjectId(id) + + override fun generate(): String = ObjectId().toHexString() +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/id/SnowflakeJsonBlobHandler.kt b/src/main/kotlin/jsonblob/core/id/SnowflakeJsonBlobHandler.kt new file mode 100644 index 0000000..a0cf864 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/id/SnowflakeJsonBlobHandler.kt @@ -0,0 +1,25 @@ +package jsonblob.core.id + +import io.micronaut.context.annotation.Primary +import io.micronaut.core.annotation.Order +import jsonblob.core.Snowflake +import java.time.Instant +import javax.inject.Singleton + +@Singleton +@Order(3) +@Primary +class SnowflakeJsonBlobHandler( + private val snowflake: Snowflake +) : IdHandler() { + override fun resolveTimestamp(t: Long): Instant { + val snowflake = snowflake.parse(t) + return Instant.ofEpochMilli(snowflake.first()) + } + + override fun generate() = snowflake.nextId().toString() + + override fun idFrom(t: Long): String = t.toString() + + override fun to(id: String) = id.toLong() +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/id/Type1UUIDJsonBlobHandler.kt b/src/main/kotlin/jsonblob/core/id/Type1UUIDJsonBlobHandler.kt new file mode 100644 index 0000000..5e0e4cf --- /dev/null +++ b/src/main/kotlin/jsonblob/core/id/Type1UUIDJsonBlobHandler.kt @@ -0,0 +1,29 @@ +package jsonblob.core.id + +import com.fasterxml.uuid.Generators +import io.micronaut.core.annotation.Order +import java.time.Instant +import java.util.* +import javax.inject.Singleton + +@Singleton +@Order(2) +class Type1UUIDJsonBlobHandler : IdHandler() { + override fun resolveTimestamp(t: UUID): Instant { + val epochMillis = Calendar.getInstance(TimeZone.getTimeZone("UTC")) + .apply { + this.clear() + this.set(1582, 9, 15, 0, 0, 0) // 9 = October + }.time.time + + val time: Long = t.timestamp() / 10000L + epochMillis + + return Instant.ofEpochMilli(time) + } + + override fun idFrom(t: UUID): String = t.toString() + + override fun to(id: String): UUID = UUID.fromString(id) + + override fun generate() = Generators.timeBasedGenerator().generate().toString() +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/store/JsonBlobBase.kt b/src/main/kotlin/jsonblob/core/store/JsonBlobBase.kt new file mode 100644 index 0000000..adc8a03 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/store/JsonBlobBase.kt @@ -0,0 +1,20 @@ +package jsonblob.core.store + +import jsonblob.core.id.IdHandler +import java.time.ZoneId +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.temporal.Temporal + +abstract class JsonBlobBase( + private val basePath: String, + private val idResolvers: List> +) { + companion object { + internal val directoryFormat = DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneId.from(ZoneOffset.UTC)) + } + + protected fun resolveTimestamp(id: String) = idResolvers.findLast { it.handles(id) }?.resolveTimestamp(id = id) ?: throw IllegalStateException("No Resolver for '$id'") + + protected fun getPrefix(timestamp: Temporal) = "$basePath/${directoryFormat.format(timestamp)}" +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/store/JsonBlobStore.kt b/src/main/kotlin/jsonblob/core/store/JsonBlobStore.kt new file mode 100644 index 0000000..2f1573f --- /dev/null +++ b/src/main/kotlin/jsonblob/core/store/JsonBlobStore.kt @@ -0,0 +1,61 @@ +package jsonblob.core.store; + +import jsonblob.core.compression.BlobCompressorPicker +import jsonblob.core.compression.compressor.BlobCompressor +import jsonblob.core.id.IdHandler +import jsonblob.model.JsonBlob +import mu.KotlinLogging +import java.io.ByteArrayOutputStream +import java.io.InputStream + + +private val log = KotlinLogging.logger {} + +abstract class JsonBlobStore( + basePath: String, + idResolvers: List>, + private val compressorPicker: BlobCompressorPicker, +) : JsonBlobBase(basePath, idResolvers) { + abstract fun retrieve(id: String): InputStream? + abstract fun store(id: String, json: String, compressor: BlobCompressor): Boolean + abstract fun remove(id: String): Boolean + abstract fun path(id: String): String? + + fun exists(id: String) = path(id) != null + + fun read(id: String): JsonBlob? { + return kotlin.runCatching { + val created = resolveTimestamp(id) + JsonBlob( + id = id, + json = json(retrieve(id) ?: throw IllegalStateException("Json blob $id doesn't exist")), + created = created + ) + }.onFailure { + log.debug { "Couldn't read JsonBlob with id=$id " } + }.getOrNull() + } + + fun write(jsonBlob: JsonBlob): JsonBlob { + return kotlin.runCatching { + val compressor = compressorPicker.bestCompressor(jsonBlob.json) + if (!store(jsonBlob.id, jsonBlob.json, compressor)) { + throw IllegalStateException("Couldn't store blob with id=${jsonBlob.id}") + } + JsonBlob( + id = jsonBlob.id, + json = jsonBlob.json, + created = resolveTimestamp(jsonBlob.id) + ) + }.onFailure { + log.warn(it) { "Couldn't write JsonBlob with id=${jsonBlob.id} " } + }.getOrElse { + throw IllegalStateException("Could not store json blob ${jsonBlob.id}") + } + } + + private fun json(inputStream: InputStream) = ByteArrayOutputStream() + .apply { + inputStream.copyTo(this) + }.toString() +} diff --git a/src/main/kotlin/jsonblob/core/store/file/FileSystemBlobPruner.kt b/src/main/kotlin/jsonblob/core/store/file/FileSystemBlobPruner.kt new file mode 100644 index 0000000..8df3f25 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/store/file/FileSystemBlobPruner.kt @@ -0,0 +1,98 @@ +package jsonblob.core.store.file + +import com.google.common.base.Stopwatch +import io.micronaut.context.annotation.Requires +import io.micronaut.scheduling.annotation.Scheduled +import jsonblob.config.FileSystemJsonBlobStoreConfig +import jsonblob.config.JsonBlobConfig +import jsonblob.core.id.IdHandler +import jsonblob.core.store.JsonBlobBase +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.count +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flatMapMerge +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import java.io.File +import java.nio.file.Files +import java.nio.file.attribute.BasicFileAttributes +import java.time.Instant +import java.util.concurrent.TimeUnit +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Singleton +@Requires(beans = [FileSystemJsonBlobStoreConfig::class]) +class FileSystemBlobPruner( + private val idResolvers: List>, + private val config: FileSystemJsonBlobStoreConfig, + private val jsonBlobConfig: JsonBlobConfig, +): JsonBlobBase(config.basePath, idResolvers) { + + @Scheduled(fixedDelay = "6h", initialDelay = "1m") + fun removeUnAccessedFiles() { + if (jsonBlobConfig.pruneEnabled) { + Stopwatch.createStarted().apply { + removeUnAccessedFilesSince() + log.info { "Removing unaccessed files took ${this.elapsed(TimeUnit.SECONDS)} seconds" } + } + } + } + + @OptIn(FlowPreview::class) + internal fun removeUnAccessedFilesSince() { + val deleteBefore = Instant.now().minus(jsonBlobConfig.deleteAfter) + log.info { "Removing blobs not accessed since $deleteBefore" } + val baseDir = File(config.basePath) + val blobFiles = baseDir + .walkBottomUp() + .filterNot { it == baseDir } + .asSequence() + .asFlow() + + val count = runBlocking { + flow { + emitAll(blobFiles) + }.flatMapMerge(config.deleteConcurrency) { file -> + flow { + try { + val attrs = Files.readAttributes(file.toPath(), BasicFileAttributes::class.java) + if (attrs.isDirectory && file.listFiles().isEmpty()) { // empty directory + emit(file.delete()).apply { + log.info { "Deleted empty directory $file" } + } + } else if (!attrs.isDirectory) { + if (idResolvers.any { it.handles(file.blobId()) }) { // a json blob + val lastAccessed = attrs.lastAccessTime().toInstant() + when { + lastAccessed == Instant.EPOCH -> log.warn { "Last Access Time was the Epoch, which typically means lastAccessTime cannot be determined" } + lastAccessed.isBefore(deleteBefore) -> emit(file.delete()).apply { + log.info { "Deleted blob ${file.blobId()} at $file" } + } + } + } else { // some other file... likely old metadata about blobs from previous versions of json blob + emit(file.delete()).apply { + log.info { "Deleted file at $file" } + } + } + } + } catch (e: Exception) { + log.warn(e) { "Caught exception while checking $file to see if it needed to be pruned" } + } + } + }.count { + it + } + } + log.info { "Completed removing $count files not accessed since $deleteBefore" } + } + +} + +internal fun File.blobId(): String = runCatching { + name.substring(0, name.indexOf(".")) +}.getOrElse { throw IllegalStateException("$name doesn't look like a JSON Blob Id") } \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/store/file/FileSystemJsonBlobStore.kt b/src/main/kotlin/jsonblob/core/store/file/FileSystemJsonBlobStore.kt new file mode 100644 index 0000000..5e80f6d --- /dev/null +++ b/src/main/kotlin/jsonblob/core/store/file/FileSystemJsonBlobStore.kt @@ -0,0 +1,147 @@ +package jsonblob.core.store.file + +import com.google.common.util.concurrent.Striped +import io.micronaut.context.annotation.Primary +import io.micronaut.context.annotation.Requires +import jsonblob.config.FileSystemJsonBlobStoreConfig +import jsonblob.core.compression.BlobCompressorPicker +import jsonblob.core.compression.compressor.BlobCompressor +import jsonblob.core.id.IdHandler +import jsonblob.core.store.JsonBlobStore +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import java.io.DataOutputStream +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.InputStream +import java.time.Instant +import java.time.temporal.Temporal +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Singleton +@Primary +@Requires(beans = [FileSystemJsonBlobStoreConfig::class]) +open class FileSystemJsonBlobStore( + private val config: FileSystemJsonBlobStoreConfig, + idResolvers: List>, + compressorPicker: BlobCompressorPicker, + private val blobCompressors: List, +) : JsonBlobStore(config.basePath, idResolvers, compressorPicker) { + + init { + log.info { "Creating ${config.basePath} if it doesn't already exist" } + File(config.basePath).mkdirs() + } + + private val blobStripedLocks = Striped.lazyWeakReadWriteLock(config.stripes) + + override fun retrieve(id: String): InputStream? { + return kotlin.runCatching { + val created = resolveTimestamp(id) + val file = getBlobFile(created, id) ?: throw NoSuchElementException("No blob with id $id") + + val compressor = blobCompressors.find { it.handles(file.extension) } + ?: throw IllegalStateException("No Blob compressor for .${file.extension}") + + log.debug { "Reading blob from ${file.absolutePath} using ${compressor::class.simpleName}" } + + val lock = blobStripedLocks[file.absolutePath].readLock() + try { + lock.lock() + + compressor.getInputStream(FileInputStream(file)) + } finally { + lock.unlock() + } + }.onFailure { + log.debug { "Couldn't retrieve JsonBlob with id=$id " } + }.getOrNull() + } + + override fun store(id: String, json: String, compressor: BlobCompressor): Boolean { + val created = resolveTimestamp(id) + val file = File(getDataDirectory(created).apply { mkdirs() }, "$id.${compressor.getFileExtension()}") + + mutableListOf(*blobFiles(created, id).toTypedArray()).apply { + remove(file) + }.forEach { + it.delete() + } + + log.debug { "Storing blob to ${file.absolutePath} using ${compressor::class.simpleName}" } + + val lock = blobStripedLocks[file.absolutePath].writeLock() + + try { + lock.lock() + + DataOutputStream(compressor.getOutputStream(FileOutputStream(file))).use { + it.write(json.toByteArray()) + } + + return true.apply { + log.debug { "Stored blob to ${file.absolutePath}" } + } + } finally { + lock.unlock() + } + } + + override fun remove(id: String): Boolean { + val created = resolveTimestamp(id) + getBlobFile(created, id)?.let { + val lock = blobStripedLocks[it.absolutePath].writeLock() + try { + lock.lock() + return kotlin.runCatching { + it.delete() + }.getOrElse { false } + } finally { + lock.unlock() + } + } + return false + } + + override fun path(id: String): String? { + val created = resolveTimestamp(id) + val file = getBlobFile(created, id) ?: return null + return if (blobCompressors.any { it.handles(file.extension) }) { + file.absolutePath + } else { + null + } + } + + private fun getDataDirectory(timestamp: Temporal) = File(getPrefix(timestamp)) + + private fun getBlobFile( + created: Instant, + id: String + ): File? { + return kotlin.runCatching { + runBlocking { + blobFiles(created, id).maxByOrNull { + it.lastModified() + }!! + } + }.getOrNull() + } + + private fun blobFiles(created: Instant, id: String) = blobCompressors + .map { + File(getDataDirectory(created), "$id.${it.getFileExtension()}") + }.filter { + it.exists() + } + +} + +data class FileSystemJsonBlobAccessedEvent( + val id: String, + val timestamp: Instant = Instant.now() +) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/core/store/s3/S3JsonBlobStore.kt b/src/main/kotlin/jsonblob/core/store/s3/S3JsonBlobStore.kt new file mode 100644 index 0000000..9ff9fc5 --- /dev/null +++ b/src/main/kotlin/jsonblob/core/store/s3/S3JsonBlobStore.kt @@ -0,0 +1,246 @@ +package jsonblob.core.store.s3 + +import io.micronaut.context.annotation.Requires +import io.micronaut.context.event.ApplicationEventPublisher +import io.micronaut.runtime.event.annotation.EventListener +import io.micronaut.scheduling.annotation.Async +import jsonblob.config.JsonBlobConfig +import jsonblob.config.S3JsonBlobStoreConfig +import jsonblob.core.compression.BlobCompressorPicker +import jsonblob.core.compression.compressor.BlobCompressor +import jsonblob.core.id.IdHandler +import jsonblob.core.store.JsonBlobStore +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.core.sync.ResponseTransformer +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration +import software.amazon.awssdk.services.s3.model.CopyObjectRequest +import software.amazon.awssdk.services.s3.model.ExpirationStatus +import software.amazon.awssdk.services.s3.model.GetObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectResponse +import software.amazon.awssdk.services.s3.model.LifecycleExpiration +import software.amazon.awssdk.services.s3.model.LifecycleRule +import software.amazon.awssdk.services.s3.model.LifecycleRuleAndOperator +import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter +import software.amazon.awssdk.services.s3.model.PutBucketLifecycleConfigurationRequest +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import software.amazon.awssdk.services.s3.model.Tag +import software.amazon.awssdk.services.s3.model.Tagging +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.io.InputStream +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import javax.inject.Singleton + + +private val log = KotlinLogging.logger {} + +@Singleton +@Requires(beans = [S3JsonBlobStoreConfig::class]) +open class S3JsonBlobStore( + idResolvers: List>, + compressorPicker: BlobCompressorPicker, + private val s3: S3Client, + private val config: JsonBlobConfig, + private val s3JsonBlobStoreConfig: S3JsonBlobStoreConfig, + private val blobCompressors: List, + private val eventPublisher: ApplicationEventPublisher, +) : JsonBlobStore(s3JsonBlobStoreConfig.basePath, idResolvers, compressorPicker) { + private val lastAccessedFormat = DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZoneId.of("UTC")) + private val lastAccessedTag = "last-accessed" + private val expirationTag = "expire-after" + private val typeTag = "type" + private val jsonBlobTypeValue = "json-blob" + + init { + if (s3JsonBlobStoreConfig.setupLifecycle) { + val bucketPolicyId = "$lastAccessedTag-lifecycle-policy-${config.deleteAfter}" + log.info { "Setting up Lifecycle Policy for expiring objects with prefix ${s3JsonBlobStoreConfig.basePath} after ${config.deleteAfter} with id $bucketPolicyId" } + s3.putBucketLifecycleConfiguration( + PutBucketLifecycleConfigurationRequest.builder() + .bucket("json-blobs-dev") + .lifecycleConfiguration( + BucketLifecycleConfiguration.builder() + .rules( + LifecycleRule.builder() + .expiration( + LifecycleExpiration.builder() + .days(config.deleteAfter.toDays().toInt()) + .build() + ) + .id(bucketPolicyId) + .filter( + LifecycleRuleFilter.builder() + .and( + LifecycleRuleAndOperator.builder() + .prefix(s3JsonBlobStoreConfig.basePath) + .tags( + jsonBlobTag(), + expirationTag() + ) + .build() + ).build() + ) + .status(ExpirationStatus.ENABLED) + .build() + ) + .build() + ) + .build() + ) + log.info { "Completed setting up Lifecycle Policy for expiring objects with id $bucketPolicyId" } + } + } + + override fun retrieve(id: String): InputStream? { + return kotlin.runCatching { + val latestObjectKey = lastModifiedS3Blob(id) + val compression = latestObjectKey.substringAfterLast(".") + val compressor = blobCompressors.find { it.handles(compression) } + ?: throw IllegalStateException("No Blob compressor for .$compression") + + log.debug { "Reading blob from $latestObjectKey using ${compressor::class.simpleName}" } + + return compressor.getInputStream( + s3.getObject( + GetObjectRequest.builder() + .bucket(s3JsonBlobStoreConfig.bucket) + .key(latestObjectKey) + .build(), + ResponseTransformer.toBytes() + ).also { + log.debug { "Read blob from $latestObjectKey" } + if (s3JsonBlobStoreConfig.copyToResetLastModified) { + eventPublisher.publishEventAsync(S3JsonBlobAccessedEvent(latestObjectKey)) + } + }.asInputStream() + ) + }.onFailure { + log.warn { "Couldn't retrieve blobId=$id from S3" } + }.getOrNull() + } + + override fun store(id: String, json: String, compressor: BlobCompressor): Boolean { + return kotlin.runCatching { + val key = getKey(id, compressor) + + log.debug { "Storing blob to $key using ${compressor::class.simpleName}" } + + val byteArrayOutputStream = ByteArrayOutputStream().apply { + DataOutputStream(compressor.getOutputStream(this)).use { + it.write(json.toByteArray()) + } + } + + s3.putObject( + PutObjectRequest.builder() + .bucket(s3JsonBlobStoreConfig.bucket) + .key(key) + .tagging(objectTags()) + .build(), + RequestBody.fromBytes(byteArrayOutputStream.toByteArray()) + ) + return true.apply { + log.debug { "Stored blob to $key" } + } + }.onFailure { + log.warn(it) { "Couldn't store blobId=$id from S3" } + }.getOrElse { false } + } + + override fun remove(id: String): Boolean { + return kotlin.runCatching { + s3Blobs(id) + .map { it.first } + .forEach { blob -> + s3.deleteObject { + it.bucket( + s3JsonBlobStoreConfig.bucket + ).key(blob) + } + } + return true + }.getOrElse { false } + } + + override fun path(id: String) = kotlin.runCatching { + lastModifiedS3Blob(id) + }.getOrElse { null } + + @EventListener + @Async + open fun onAccessedEvent(event: S3JsonBlobAccessedEvent) { + val now = lastAccessedFormat.format(Instant.now()) + val tags = s3.getObjectTagging { + it + .bucket(s3JsonBlobStoreConfig.bucket) + .key(event.s3ObjectKey) + }.tagSet() + if (tags.isEmpty() || tags.filter { it.key() == lastAccessedTag }.filterNot { it.value() == now }.any()) { + kotlin.runCatching { + log.info { "Copying ${s3JsonBlobStoreConfig.bucket}/${event.s3ObjectKey} to reset the timestamp on the object" } + s3.copyObject( + CopyObjectRequest.builder() + .copySource("${s3JsonBlobStoreConfig.bucket}/${event.s3ObjectKey}") + .destinationBucket(s3JsonBlobStoreConfig.bucket) + .destinationKey(event.s3ObjectKey) + .tagging(objectTags()) + .build() + ) + log.info { "Copied ${event.s3ObjectKey}" } + }.onFailure { + log.warn { "Couldn't copy ${s3JsonBlobStoreConfig.bucket}/${event.s3ObjectKey}" } + } + } + } + + private fun lastModifiedS3Blob(id: String) = s3Blobs(id).maxByOrNull { it.second.lastModified() }!!.first + + private fun objectTags() = + Tagging.builder() + .tagSet(listOf( + lastAccessedTag(), + expirationTag(), + jsonBlobTag() + )).build() + + private fun lastAccessedTag() = tagOf(lastAccessedTag, lastAccessedFormat.format(Instant.now())) + + private fun jsonBlobTag() = tagOf(typeTag, jsonBlobTypeValue) + + private fun expirationTag() = tagOf(expirationTag, config.deleteAfter.toString()) + + private fun tagOf(key: String, value: String) = Tag.builder().key(key).value(value).build() + + private fun getKey(id: String, compressor: BlobCompressor) = + "${getPrefix(resolveTimestamp(id))}/$id.${compressor.getFileExtension()}" + + private fun s3Blobs(id: String) : List> { + return runBlocking { + blobCompressors.associateBy { + getKey(id, it) + }.map { + async { + kotlin.runCatching { + it.key to s3.headObject( + HeadObjectRequest.builder() + .bucket(s3JsonBlobStoreConfig.bucket) + .key(getKey(id, it.value)) + .build() + ) + }.getOrNull() + } + }.awaitAll().filterNotNull() + } + } + +} + +data class S3JsonBlobAccessedEvent(val s3ObjectKey: String) \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/handlebars/helper/Base64Helpers.kt b/src/main/kotlin/jsonblob/handlebars/helper/Base64Helpers.kt new file mode 100644 index 0000000..6640e40 --- /dev/null +++ b/src/main/kotlin/jsonblob/handlebars/helper/Base64Helpers.kt @@ -0,0 +1,35 @@ +package jsonblob.handlebars.helper + +import com.github.jknack.handlebars.Options +import org.apache.commons.codec.binary.Base64 +import java.nio.charset.StandardCharsets +import javax.inject.Singleton + +@Singleton +class Base64EncodeHelper : NamedHelper { + override fun getName() = "base64Encode" + + override fun apply(context: String, options: Options): String { + val urlSafe: Boolean = options.hash("urlSafe", java.lang.Boolean.FALSE) + return if (urlSafe) { + Base64.encodeBase64URLSafeString(context.toByteArray(StandardCharsets.UTF_8)) + } else { + Base64.encodeBase64String(context.toByteArray(StandardCharsets.UTF_8)) + } + } +} + +@Singleton +class Base64DecodeHelper: NamedHelper { + override fun getName() = "base64Decode" + + override fun apply(context: String, options: Options): Any { + val urlSafe: Boolean = options.hash("urlSafe", java.lang.Boolean.FALSE) + return if (urlSafe) { + Base64.encodeBase64URLSafe(context.toByteArray(StandardCharsets.UTF_8)) + } else { + Base64.decodeBase64(context.toByteArray(StandardCharsets.UTF_8)) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/handlebars/helper/InstantFormatter.kt b/src/main/kotlin/jsonblob/handlebars/helper/InstantFormatter.kt new file mode 100644 index 0000000..7401445 --- /dev/null +++ b/src/main/kotlin/jsonblob/handlebars/helper/InstantFormatter.kt @@ -0,0 +1,17 @@ +package jsonblob.handlebars.helper + +import com.github.jknack.handlebars.Options +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import javax.inject.Singleton + +@Singleton +class InstantFormatter : NamedHelper { + override fun apply(context: Instant, options: Options): String { + val format = options.params.first() as String + return DateTimeFormatter.ofPattern(format).withZone(ZoneId.of("UTC")).format(context) + } + + override fun getName() = "dateFormat" +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/handlebars/helper/JsonHelper.kt b/src/main/kotlin/jsonblob/handlebars/helper/JsonHelper.kt new file mode 100644 index 0000000..39cc6f7 --- /dev/null +++ b/src/main/kotlin/jsonblob/handlebars/helper/JsonHelper.kt @@ -0,0 +1,12 @@ +package jsonblob.handlebars.helper + +import com.github.jknack.handlebars.Options +import jsonblob.util.JsonCleaner +import javax.inject.Singleton + +@Singleton +class JsonHelper: NamedHelper { + override fun getName() = "json" + + override fun apply(context: String, options: Options) = JsonCleaner.removeWhiteSpace(context) +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/handlebars/helper/NamedHelper.kt b/src/main/kotlin/jsonblob/handlebars/helper/NamedHelper.kt new file mode 100644 index 0000000..deada56 --- /dev/null +++ b/src/main/kotlin/jsonblob/handlebars/helper/NamedHelper.kt @@ -0,0 +1,7 @@ +package jsonblob.handlebars.helper + +import com.github.jknack.handlebars.Helper + +interface NamedHelper : Helper { + fun getName(): String +} \ No newline at end of file diff --git a/src/main/kotlin/jsonblob/model/JsonBlob.kt b/src/main/kotlin/jsonblob/model/JsonBlob.kt new file mode 100644 index 0000000..b84a472 --- /dev/null +++ b/src/main/kotlin/jsonblob/model/JsonBlob.kt @@ -0,0 +1,9 @@ +package jsonblob.model + +import java.time.Instant + +data class JsonBlob( + val id: String, + val json: String, + val created: Instant = Instant.now() +) diff --git a/src/main/kotlin/jsonblob/util/JsonCleaner.kt b/src/main/kotlin/jsonblob/util/JsonCleaner.kt new file mode 100644 index 0000000..da052b2 --- /dev/null +++ b/src/main/kotlin/jsonblob/util/JsonCleaner.kt @@ -0,0 +1,86 @@ +package jsonblob.util + +import com.fasterxml.jackson.core.JsonEncoding +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonParser.NumberType.BIG_DECIMAL +import com.fasterxml.jackson.core.JsonParser.NumberType.BIG_INTEGER +import com.fasterxml.jackson.core.JsonParser.NumberType.DOUBLE +import com.fasterxml.jackson.core.JsonParser.NumberType.FLOAT +import com.fasterxml.jackson.core.JsonParser.NumberType.INT +import com.fasterxml.jackson.core.JsonParser.NumberType.LONG +import com.fasterxml.jackson.core.JsonToken +import com.fasterxml.jackson.core.JsonToken.END_ARRAY +import com.fasterxml.jackson.core.JsonToken.END_OBJECT +import com.fasterxml.jackson.core.JsonToken.FIELD_NAME +import com.fasterxml.jackson.core.JsonToken.NOT_AVAILABLE +import com.fasterxml.jackson.core.JsonToken.START_ARRAY +import com.fasterxml.jackson.core.JsonToken.START_OBJECT +import com.fasterxml.jackson.core.JsonToken.VALUE_EMBEDDED_OBJECT +import com.fasterxml.jackson.core.JsonToken.VALUE_FALSE +import com.fasterxml.jackson.core.JsonToken.VALUE_NULL +import com.fasterxml.jackson.core.JsonToken.VALUE_NUMBER_FLOAT +import com.fasterxml.jackson.core.JsonToken.VALUE_NUMBER_INT +import com.fasterxml.jackson.core.JsonToken.VALUE_STRING +import com.fasterxml.jackson.core.JsonToken.VALUE_TRUE +import java.io.ByteArrayOutputStream + +class JsonCleaner { + companion object { + fun validJson(json: String) = kotlin + .runCatching { + removeWhiteSpace(json) + true + }.getOrElse { false } + + fun removeWhiteSpace(json: String): String { + val outputStream = ByteArrayOutputStream() + val jsonFactory = JsonFactory() + jsonFactory.createParser(json).use { jp -> + jsonFactory.createGenerator(outputStream, JsonEncoding.UTF8).use { jg -> + var next: JsonToken? + do { + next = jp.nextToken().also { + when (it) { + NOT_AVAILABLE -> { + // no-op for non-blocking + } + START_OBJECT -> jg.writeStartObject() + START_ARRAY -> jg.writeStartArray() + END_OBJECT -> jg.writeEndObject() + END_ARRAY -> jg.writeEndArray() + FIELD_NAME -> jg.writeFieldName(jp.text) + VALUE_EMBEDDED_OBJECT -> jg.writeEmbeddedObject(jp.embeddedObject) + VALUE_STRING -> jg.writeString(jp.valueAsString) + VALUE_NUMBER_INT -> writeNumber(jp, jg) + VALUE_NUMBER_FLOAT -> writeNumber(jp, jg) + VALUE_TRUE -> jg.writeBoolean(true) + VALUE_FALSE -> jg.writeBoolean(false) + VALUE_NULL -> jg.writeNull() + else -> { + } + } + } + } while (next != null) + + } + return String(outputStream.toByteArray()) + } + } + + private fun writeNumber(jp: JsonParser, jg: JsonGenerator) { + when (jp.numberType) { + INT -> jg.writeNumber(jp.intValue) + LONG -> jg.writeNumber(jp.longValue) + BIG_INTEGER -> jg.writeNumber(jp.bigIntegerValue) + FLOAT -> jg.writeNumber(jp.floatValue) + DOUBLE -> jg.writeNumber(jp.doubleValue) + BIG_DECIMAL -> jg.writeNumber(jp.decimalValue) + else -> { + } + } + + } + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..32a21eb --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,27 @@ +micronaut: + application: + name: jsonblob + metrics: + enabled: true + export: + newrelic: + enabled: false + router: + static-resources: + json-blob: + paths: + - classpath:static-resources + server: + cors: + enabled: true + configurations: + web: + exposedHeaders: + - X-Requested-With + - X-jsonblob + - X-Hello-Human + - Location + - Date + - Content-Type + - Accept + - Origin \ No newline at end of file diff --git a/src/main/resources/assets/favicon.ico b/src/main/resources/assets/favicon.ico deleted file mode 100644 index e535075..0000000 Binary files a/src/main/resources/assets/favicon.ico and /dev/null differ diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt deleted file mode 100644 index 59869a8..0000000 --- a/src/main/resources/banner.txt +++ /dev/null @@ -1,6 +0,0 @@ - _ __ __ __ - (_)________ ____ / /_ / /___ / /_ - / / ___/ __ \/ __ \/ __ \/ / __ \/ __ \ - / (__ ) /_/ / / / / /_/ / / /_/ / /_/ / - __/ /____/\____/_/ /_/_.___/_/\____/_.___/ -/___/ \ No newline at end of file diff --git a/src/main/resources/jsonblob.yml b/src/main/resources/jsonblob.yml deleted file mode 100644 index 4e33a58..0000000 --- a/src/main/resources/jsonblob.yml +++ /dev/null @@ -1,23 +0,0 @@ -ga: - webPropertyID: UA-36870706-1 - customTrackingCodes: - - - key: _setDomainName - value: jsonblob.com - - - key: _trackPageview - -server: - type: simple - applicationContextPath: / - requestLog: - appenders: - - type: "console" -# Logging settings. -logging: - level: INFO - loggers: - "com.lowtuna": DEBUG - appenders: - - type: "console" - logFormat: "%-5p [%d{ISO8601}] [%thread] [%X{requestId}] %c{5}: %m%n%xEx" \ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..4346bfe --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,31 @@ + + + + true + + + %cyan(%date{"yyyy-MM-dd'T'HH:mm:ss,SSSXXX", UTC}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n + + + + + /var/log/jsonblob/log.txt + true + + %cyan(%date{"yyyy-MM-dd'T'HH:mm:ss,SSSXXX", UTC}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n + + + /var/log/jsonblob/logs/logFile.%d{yyyy-MM-dd}.%i.txt + + 50MB + + 30 + + + + + + + + diff --git a/src/main/resources/assets/css/api.css b/src/main/resources/static-resources/assets/css/api.css similarity index 100% rename from src/main/resources/assets/css/api.css rename to src/main/resources/static-resources/assets/css/api.css diff --git a/src/main/resources/assets/css/base.css b/src/main/resources/static-resources/assets/css/base.css similarity index 100% rename from src/main/resources/assets/css/base.css rename to src/main/resources/static-resources/assets/css/base.css diff --git a/src/main/resources/assets/css/bootstrap/bootstrap-theme.min.css b/src/main/resources/static-resources/assets/css/bootstrap/bootstrap-theme.min.css similarity index 100% rename from src/main/resources/assets/css/bootstrap/bootstrap-theme.min.css rename to src/main/resources/static-resources/assets/css/bootstrap/bootstrap-theme.min.css diff --git a/src/main/resources/assets/css/bootstrap/bootstrap.min.css b/src/main/resources/static-resources/assets/css/bootstrap/bootstrap.min.css similarity index 100% rename from src/main/resources/assets/css/bootstrap/bootstrap.min.css rename to src/main/resources/static-resources/assets/css/bootstrap/bootstrap.min.css diff --git a/src/main/resources/assets/css/joyride/joyride-2.1.css b/src/main/resources/static-resources/assets/css/joyride/joyride-2.1.css similarity index 100% rename from src/main/resources/assets/css/joyride/joyride-2.1.css rename to src/main/resources/static-resources/assets/css/joyride/joyride-2.1.css diff --git a/src/main/resources/assets/css/jsoneditor/img/jsoneditor-icons.png b/src/main/resources/static-resources/assets/css/jsoneditor/img/jsoneditor-icons.png similarity index 100% rename from src/main/resources/assets/css/jsoneditor/img/jsoneditor-icons.png rename to src/main/resources/static-resources/assets/css/jsoneditor/img/jsoneditor-icons.png diff --git a/src/main/resources/assets/css/jsoneditor/jsoneditor-min.css b/src/main/resources/static-resources/assets/css/jsoneditor/jsoneditor-min.css similarity index 100% rename from src/main/resources/assets/css/jsoneditor/jsoneditor-min.css rename to src/main/resources/static-resources/assets/css/jsoneditor/jsoneditor-min.css diff --git a/src/main/resources/assets/css/jsoneditor/jsoneditor.css b/src/main/resources/static-resources/assets/css/jsoneditor/jsoneditor.css similarity index 100% rename from src/main/resources/assets/css/jsoneditor/jsoneditor.css rename to src/main/resources/static-resources/assets/css/jsoneditor/jsoneditor.css diff --git a/src/main/resources/static-resources/assets/favicon-16x16.png b/src/main/resources/static-resources/assets/favicon-16x16.png new file mode 100644 index 0000000..9bc0875 Binary files /dev/null and b/src/main/resources/static-resources/assets/favicon-16x16.png differ diff --git a/src/main/resources/static-resources/assets/favicon-32x32.png b/src/main/resources/static-resources/assets/favicon-32x32.png new file mode 100644 index 0000000..7ff54d0 Binary files /dev/null and b/src/main/resources/static-resources/assets/favicon-32x32.png differ diff --git a/src/main/resources/static-resources/assets/favicon.ico b/src/main/resources/static-resources/assets/favicon.ico new file mode 100644 index 0000000..fe57e1f Binary files /dev/null and b/src/main/resources/static-resources/assets/favicon.ico differ diff --git a/src/main/resources/assets/fonts/glyphicons-halflings-regular.eot b/src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.eot similarity index 100% rename from src/main/resources/assets/fonts/glyphicons-halflings-regular.eot rename to src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.eot diff --git a/src/main/resources/assets/fonts/glyphicons-halflings-regular.svg b/src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.svg similarity index 100% rename from src/main/resources/assets/fonts/glyphicons-halflings-regular.svg rename to src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.svg diff --git a/src/main/resources/assets/fonts/glyphicons-halflings-regular.ttf b/src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.ttf similarity index 100% rename from src/main/resources/assets/fonts/glyphicons-halflings-regular.ttf rename to src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.ttf diff --git a/src/main/resources/assets/fonts/glyphicons-halflings-regular.woff b/src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.woff similarity index 100% rename from src/main/resources/assets/fonts/glyphicons-halflings-regular.woff rename to src/main/resources/static-resources/assets/fonts/glyphicons-halflings-regular.woff diff --git a/src/main/resources/assets/index.html b/src/main/resources/static-resources/assets/index.html similarity index 100% rename from src/main/resources/assets/index.html rename to src/main/resources/static-resources/assets/index.html diff --git a/src/main/resources/assets/js/ace/ace-min.js b/src/main/resources/static-resources/assets/js/ace/ace-min.js similarity index 100% rename from src/main/resources/assets/js/ace/ace-min.js rename to src/main/resources/static-resources/assets/js/ace/ace-min.js diff --git a/src/main/resources/assets/js/ace/ace.js b/src/main/resources/static-resources/assets/js/ace/ace.js similarity index 100% rename from src/main/resources/assets/js/ace/ace.js rename to src/main/resources/static-resources/assets/js/ace/ace.js diff --git a/src/main/resources/assets/js/ace/ext-searchbox.js b/src/main/resources/static-resources/assets/js/ace/ext-searchbox.js similarity index 100% rename from src/main/resources/assets/js/ace/ext-searchbox.js rename to src/main/resources/static-resources/assets/js/ace/ext-searchbox.js diff --git a/src/main/resources/assets/js/ace/keybinding-vim.js b/src/main/resources/static-resources/assets/js/ace/keybinding-vim.js similarity index 100% rename from src/main/resources/assets/js/ace/keybinding-vim.js rename to src/main/resources/static-resources/assets/js/ace/keybinding-vim.js diff --git a/src/main/resources/assets/js/ace/mode-json.js b/src/main/resources/static-resources/assets/js/ace/mode-json.js similarity index 100% rename from src/main/resources/assets/js/ace/mode-json.js rename to src/main/resources/static-resources/assets/js/ace/mode-json.js diff --git a/src/main/resources/assets/js/ace/theme-jsoneditor.js b/src/main/resources/static-resources/assets/js/ace/theme-jsoneditor.js similarity index 100% rename from src/main/resources/assets/js/ace/theme-jsoneditor.js rename to src/main/resources/static-resources/assets/js/ace/theme-jsoneditor.js diff --git a/src/main/resources/assets/js/ace/theme-textmate.js b/src/main/resources/static-resources/assets/js/ace/theme-textmate.js similarity index 100% rename from src/main/resources/assets/js/ace/theme-textmate.js rename to src/main/resources/static-resources/assets/js/ace/theme-textmate.js diff --git a/src/main/resources/assets/js/ace/worker-json.js b/src/main/resources/static-resources/assets/js/ace/worker-json.js similarity index 100% rename from src/main/resources/assets/js/ace/worker-json.js rename to src/main/resources/static-resources/assets/js/ace/worker-json.js diff --git a/src/main/resources/assets/js/base64.js b/src/main/resources/static-resources/assets/js/base64.js similarity index 100% rename from src/main/resources/assets/js/base64.js rename to src/main/resources/static-resources/assets/js/base64.js diff --git a/src/main/resources/assets/js/bootstrap/bootstrap.min.js b/src/main/resources/static-resources/assets/js/bootstrap/bootstrap.min.js similarity index 100% rename from src/main/resources/assets/js/bootstrap/bootstrap.min.js rename to src/main/resources/static-resources/assets/js/bootstrap/bootstrap.min.js diff --git a/src/main/resources/assets/js/joyride/jquery.joyride-2.1.js b/src/main/resources/static-resources/assets/js/joyride/jquery.joyride-2.1.js similarity index 100% rename from src/main/resources/assets/js/joyride/jquery.joyride-2.1.js rename to src/main/resources/static-resources/assets/js/joyride/jquery.joyride-2.1.js diff --git a/src/main/resources/assets/js/jquery/1.11.0.min.js b/src/main/resources/static-resources/assets/js/jquery/1.11.0.min.js similarity index 100% rename from src/main/resources/assets/js/jquery/1.11.0.min.js rename to src/main/resources/static-resources/assets/js/jquery/1.11.0.min.js diff --git a/src/main/resources/assets/js/jquery/jquery.cookie.js b/src/main/resources/static-resources/assets/js/jquery/jquery.cookie.js similarity index 100% rename from src/main/resources/assets/js/jquery/jquery.cookie.js rename to src/main/resources/static-resources/assets/js/jquery/jquery.cookie.js diff --git a/src/main/resources/assets/js/json2.js b/src/main/resources/static-resources/assets/js/json2.js similarity index 100% rename from src/main/resources/assets/js/json2.js rename to src/main/resources/static-resources/assets/js/json2.js diff --git a/src/main/resources/assets/js/jsoneditor/jsoneditor-min.js b/src/main/resources/static-resources/assets/js/jsoneditor/jsoneditor-min.js similarity index 100% rename from src/main/resources/assets/js/jsoneditor/jsoneditor-min.js rename to src/main/resources/static-resources/assets/js/jsoneditor/jsoneditor-min.js diff --git a/src/main/resources/assets/js/jsoneditor/jsoneditor.js b/src/main/resources/static-resources/assets/js/jsoneditor/jsoneditor.js similarity index 100% rename from src/main/resources/assets/js/jsoneditor/jsoneditor.js rename to src/main/resources/static-resources/assets/js/jsoneditor/jsoneditor.js diff --git a/src/main/resources/assets/js/jwerty/jwerty.js b/src/main/resources/static-resources/assets/js/jwerty/jwerty.js similarity index 100% rename from src/main/resources/assets/js/jwerty/jwerty.js rename to src/main/resources/static-resources/assets/js/jwerty/jwerty.js diff --git a/src/main/resources/views/about.hbs b/src/main/resources/views/about.hbs index fa88f22..bdb0c39 100644 --- a/src/main/resources/views/about.hbs +++ b/src/main/resources/views/about.hbs @@ -8,19 +8,17 @@

JSON Blob was created to help parallelize client/server development. Mock JSON responses can be defined using the online editor and then clients can use the JSON Blob API to retrieve and update the mock responses.

- {{#if deletionEnabled}}

- Blobs that are not accessed in {{blobAccessTtl.quantity}} {{blobAccessTtl.unit}} will be removed. + Blobs that are not accessed in {{deleteAfterDays}} days will be removed.

- {{/if}}

- {{> copyright.hbs}} + {{> views/copyright}}
{{/partial}} -{{> base.hbs}} \ No newline at end of file +{{> views/base}} \ No newline at end of file diff --git a/src/main/resources/views/api.hbs b/src/main/resources/views/api.hbs index 223c8a6..2d10735 100644 --- a/src/main/resources/views/api.hbs +++ b/src/main/resources/views/api.hbs @@ -82,7 +82,7 @@ HTTP/1.1 200 OK

Custom URLs

Paths with blob Ids

-

GET, PUT, and DELETE requests can be customized to support any url path scheme. The only requirement is that the first path part is /api/. For GET, PUT, and DELETE requests, the blobId must be present somewhere as a URL path part following /api/ or the blobId is set in the X-jsonblob header and then you are free to use any URL path as long as the first path part is /api/.

+

GET, PUT, and DELETE requests can be customized to support any url path scheme. The only requirement is that the first path part is /api/. For GET, PUT, and DELETE requests, the blobId must be present somewhere as a URL path part following /api/ or the blobId is set in the X-jsonblob header and then you are free to use any URL path as long as the first path part is /api/. The first matching blobId will be returned.

As an example, if we were trying to mimic a RESTful url structure for getting the names of the employees in a company with a particular role, we may use want to use a url like /api/company/<companyId>/employees/engineers"> where 5226571730043f8b22dadc20 is the <companyId> that represents the data the client is expecting.

Example request and response

@@ -107,7 +107,7 @@ Transfer-Encoding: chunked
 {"people":["bill","steve","bob"]}
- {{> copyright.hbs}} + {{> views/copyright}}
@@ -184,4 +184,4 @@ Transfer-Encoding: chunked
{{/partial}} -{{> base.hbs}} \ No newline at end of file +{{> views/base}} \ No newline at end of file diff --git a/src/main/resources/views/base.hbs b/src/main/resources/views/base.hbs index 0840dd6..2ae200f 100644 --- a/src/main/resources/views/base.hbs +++ b/src/main/resources/views/base.hbs @@ -31,7 +31,7 @@ - {{> ga.hbs}} + {{> views/ga}} {{#block "head" }} @@ -84,4 +84,4 @@ {{/block}} - + \ No newline at end of file diff --git a/src/main/resources/views/copyright.hbs b/src/main/resources/views/copyright.hbs index 1ef42f3..d4113f4 100644 --- a/src/main/resources/views/copyright.hbs +++ b/src/main/resources/views/copyright.hbs @@ -1 +1 @@ -

© {{dateFormat now "yyyy"}} Tristan Burch

\ No newline at end of file +

© {{dateFormat now "yyyy"}} Tristan Burch

\ No newline at end of file diff --git a/src/main/resources/views/editor.hbs b/src/main/resources/views/editor.hbs index f3c65b2..07a44a7 100644 --- a/src/main/resources/views/editor.hbs +++ b/src/main/resources/views/editor.hbs @@ -18,7 +18,7 @@
- {{> copyright.hbs}} + {{> views/copyright}}
@@ -289,13 +289,11 @@ {{/partial}} -{{> base.hbs}} +{{> views/base}} \ No newline at end of file diff --git a/src/main/resources/views/ga.hbs b/src/main/resources/views/ga.hbs index 9758225..f752d54 100644 --- a/src/main/resources/views/ga.hbs +++ b/src/main/resources/views/ga.hbs @@ -1,10 +1,9 @@ + + \ No newline at end of file diff --git a/src/test/java/com/lowtuna/jsonblob/core/TryBlobCleanupJob.java b/src/test/java/com/lowtuna/jsonblob/core/TryBlobCleanupJob.java deleted file mode 100644 index c316956..0000000 --- a/src/test/java/com/lowtuna/jsonblob/core/TryBlobCleanupJob.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.lowtuna.jsonblob.core; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.joda.JodaModule; -import io.dropwizard.util.Duration; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.bson.types.ObjectId; -import org.joda.time.DateTime; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.UUID; -import java.util.concurrent.Executors; - -/** - * Created by tburch on 8/16/17. - */ -@Slf4j -public class TryBlobCleanupJob { - private final Duration blobTtl = Duration.days(1); - - private File tempDir; - private FileSystemJsonBlobManager blobManager; - - @Before - public void initBlobManage() { - File temp = FileUtils.getTempDirectory(); - File dir = new File(temp, UUID.randomUUID().toString()); - dir.deleteOnExit(); - this.tempDir = dir; - - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new JodaModule()); - this.blobManager = new FileSystemJsonBlobManager(tempDir, Executors.newScheduledThreadPool(10), Executors.newScheduledThreadPool(10), objectMapper, blobTtl, true, new MetricRegistry()); - } - - @Test - public void testCleanupWithAccessed() throws Exception { - DateTime now = DateTime.now(); - - String oldBlobId = (new ObjectId(now.minusDays((int) (blobTtl.toMinutes() * 2)).toDate())).toString(); - String newBlobId = new ObjectId(now.toDate()).toString(); - log.info("newBlobId={}, oldBlobId={}", newBlobId, oldBlobId); - - Assert.assertEquals(0, countFiles()); - blobManager.createBlob("{\"foo\":|\"bar\"}", oldBlobId); - Assert.assertEquals(1, countFiles()); - blobManager.createBlob("{\"foo\":|\"bar\"}", newBlobId); - Assert.assertEquals(2, countFiles()); - - blobManager.getBlob(oldBlobId); - - blobManager.run(); - - log.info("Starting blob manager"); - blobManager.start(); - - Thread.sleep(2000); - - Assert.assertEquals(2, countFiles()); - } - - @Test - public void testCleanup() throws Exception { - DateTime now = DateTime.now(); - - String oldBlobId = (new ObjectId(now.minusDays((int) (blobTtl.toMinutes() * 2)).toDate())).toString(); - String newBlobId = new ObjectId(now.toDate()).toString(); - log.info("newBlobId={}, oldBlobId={}", newBlobId, oldBlobId); - - Assert.assertEquals(0, countFiles()); - blobManager.createBlob("{\"foo\":|\"bar\"}", oldBlobId); - Assert.assertEquals(1, countFiles()); - blobManager.createBlob("{\"foo\":|\"bar\"}", newBlobId); - Assert.assertEquals(2, countFiles()); - - blobManager.run(); - - log.info("Starting blob manager"); - blobManager.start(); - - Thread.sleep(2000); - - Assert.assertEquals(1, countFiles()); - } - - private long countFiles() throws IOException { - return Files.find(tempDir.toPath(), 999, (p, bfa) -> bfa.isRegularFile()).count(); - } - -} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/JsonblobsTest.kt b/src/test/kotlin/jsonblob/JsonblobsTest.kt new file mode 100644 index 0000000..e8b9a47 --- /dev/null +++ b/src/test/kotlin/jsonblob/JsonblobsTest.kt @@ -0,0 +1,19 @@ +package jsonblob +import io.micronaut.runtime.EmbeddedApplication +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import javax.inject.Inject + +@MicronautTest +class JsonblobTest { + + @Inject + lateinit var application: EmbeddedApplication<*> + + @Test + fun testItWorks() { + Assertions.assertTrue(application.isRunning) + } + +} diff --git a/src/test/kotlin/jsonblob/api/http/ApiTest.kt b/src/test/kotlin/jsonblob/api/http/ApiTest.kt new file mode 100644 index 0000000..e5a0739 --- /dev/null +++ b/src/test/kotlin/jsonblob/api/http/ApiTest.kt @@ -0,0 +1,219 @@ +package jsonblob.api.http + +import io.micronaut.http.HttpRequest +import io.micronaut.http.HttpRequest.DELETE +import io.micronaut.http.HttpRequest.POST +import io.micronaut.http.HttpRequest.PUT +import io.micronaut.http.HttpResponse +import io.micronaut.http.MediaType +import io.micronaut.http.client.HttpClient +import io.micronaut.http.client.annotation.Client +import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.core.compression.compressor.GZIPBlobCompressor +import jsonblob.core.id.Type1UUIDJsonBlobHandler +import jsonblob.core.store.JsonBlobStore +import mu.KotlinLogging +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.skyscreamer.jsonassert.JSONAssert.assertEquals +import org.testcontainers.shaded.com.google.common.io.Files +import java.util.UUID +import javax.inject.Inject + + +private val log = KotlinLogging.logger {} + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class ApiTest: TestPropertyProvider { + private val tempDir = Files.createTempDir().apply { deleteOnExit() } + + private val bucket = "fubar" + + private val json = """ + { + "name" : "bob", + "age": 38 + } + """.trimIndent() + + private val updateJson = json.replace("38", "39") + + @Inject + lateinit var type1UUIDJsonBlobHandler: Type1UUIDJsonBlobHandler + + @Inject + lateinit var blobStore : JsonBlobStore + + @Inject + lateinit var gzipBlobCompressor: GZIPBlobCompressor + + @Inject + @field:Client("/") + lateinit var client: HttpClient + + @Test + fun `new blobs created via API POST are created`() { + val resp = client + .toBlocking() + .exchange(POST("/api/jsonBlob", json).contentType(MediaType.APPLICATION_JSON_TYPE), String::class.java) + assertThat(resp.code()).isEqualTo(201) + val locationHeader = resp.header("Location").also { log.info { "Location header was $it" } } + assertThat(locationHeader).isNotEmpty + assertEquals(json, resp.body(), true) + + val id = locationHeader.substringAfterLast("/") + assertThat(blobStore.exists(id)).isTrue + } + + private fun validateGet(request: (blobId: String) -> String) { + val id = type1UUIDJsonBlobHandler.generate() + assertThat(blobStore.store(id, json, gzipBlobCompressor)).isTrue + assertThat(blobStore.exists(id)).isTrue + val body = request.invoke(id) + assertEquals(json, body, true) + + assertThat(blobStore.exists(id)).isTrue + } + + @Test + fun `blob is retrieved for standard API GET`() { + validateGet { + client.toBlocking().retrieve("/api/jsonBlob/$it") + } + } + + @Test + fun `blob is retrieved for custom API GET`() { + validateGet { + val request = HttpRequest.GET("/api/company/$it/employees/engineers") + client.toBlocking().retrieve(request) + } + } + + @Test + fun `blob is retrieved for custom API GET with X-jsonblob header`() { + validateGet { + val request = HttpRequest.GET("/api/company/employees/engineers").header("X-jsonblob", it) + client.toBlocking().retrieve(request) + } + } + + private fun validateUpdate(request: (blobId: String) -> HttpResponse) { + val id = type1UUIDJsonBlobHandler.generate() + assertThat(blobStore.store(id, json, gzipBlobCompressor)).isTrue + assertThat(blobStore.exists(id)).isTrue + + val resp = request.invoke(id) + + assertThat(resp.code()).isEqualTo(200) + assertEquals(updateJson, resp.body(), true) + + assertThat(blobStore.exists(id)).isTrue + assertEquals(updateJson, blobStore.read(id)!!.json, true) + } + + @Test + fun `blob is updated on API PUT`() { + validateUpdate { + client + .toBlocking() + .exchange(PUT("/api/jsonBlob/$it", updateJson).contentType(MediaType.APPLICATION_JSON_TYPE), String::class.java) + } + } + + @Test + fun `blob is created on API PUT`() { + val resp = client + .toBlocking() + .exchange(PUT("/api/jsonBlob/${type1UUIDJsonBlobHandler.generate()}", json).contentType(MediaType.APPLICATION_JSON_TYPE), String::class.java) + assertThat(resp.code()).isEqualTo(200) + } + + @Test + fun `blob is not created on bad API PUT`() { + assertThatThrownBy { + client + .toBlocking() + .exchange(PUT("/api/jsonBlob/${UUID.randomUUID()}", json).contentType(MediaType.APPLICATION_JSON_TYPE), String::class.java) + + }.isInstanceOf(HttpClientResponseException::class.java) + } + + @Test + fun `blob is updated on custom API PUT`() { + validateUpdate { + client + .toBlocking() + .exchange(PUT("/api/company/$it/employees/engineers", updateJson).contentType(MediaType.APPLICATION_JSON_TYPE), String::class.java) + } + } + + @Test + fun `blob is updated on custom API PUT with X-jsonblob header`() { + validateUpdate { + client + .toBlocking() + .exchange(PUT("/api/company/employees/engineers", updateJson).contentType(MediaType.APPLICATION_JSON_TYPE).header("X-jsonblob", it), String::class.java) + } + } + + private fun validateFsDelete(request: (blobId: String) -> HttpResponse) { + val id = type1UUIDJsonBlobHandler.generate() + assertThat(blobStore.store(id, json, gzipBlobCompressor)).isTrue + assertThat(blobStore.exists(id)).isTrue + + val resp = request.invoke(id) + + assertThat(resp.code()).isEqualTo(200) + assertThat(blobStore.exists(id)).isFalse + } + + private fun validateS3Delete(request: (blobId: String) -> HttpResponse) { + val id = type1UUIDJsonBlobHandler.generate() + assertThat(blobStore.store(id, json, gzipBlobCompressor)).isTrue + assertThat(blobStore.exists(id)).isTrue + + val resp = request.invoke(id) + + assertThat(resp.code()).isEqualTo(200) + assertThat(blobStore.exists(id)).isFalse + } + + @Test + fun `blob is deleted on API DETELE`() { + validateFsDelete { + client + .toBlocking() + .exchange(DELETE("/api/jsonBlob/$it"), Any::class.java) + } + } + + @Test + fun `blob is deleted on custom API DETELE`() { + validateS3Delete { + client + .toBlocking() + .exchange(DELETE("/api/company/$it/employees/engineers"), Any::class.java) + } + } + + @Test + fun `blob is deleted on custom API DETELE with X-jsonblob header`() { + validateS3Delete { + client + .toBlocking() + .exchange(DELETE("/api/company/employees/engineers").header("X-jsonblob", it), Any::class.java) + } + } + + override fun getProperties(): MutableMap { + return mutableMapOf( + "file-system-blob-store.base-path" to tempDir.absolutePath, + ) + } +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/api/view/ViewTests.kt b/src/test/kotlin/jsonblob/api/view/ViewTests.kt new file mode 100644 index 0000000..edc2ec9 --- /dev/null +++ b/src/test/kotlin/jsonblob/api/view/ViewTests.kt @@ -0,0 +1,66 @@ +package jsonblob.api.view + +import io.micronaut.http.client.HttpClient +import io.micronaut.http.client.annotation.Client +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.config.S3ClientBuilderListener +import mu.KotlinLogging +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.testcontainers.containers.localstack.LocalStackContainer +import org.testcontainers.shaded.com.google.common.io.Files +import org.testcontainers.utility.DockerImageName +import javax.inject.Inject + +private val log = KotlinLogging.logger {} + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class ApiMidMigrationTest: TestPropertyProvider { + private val tempDir = Files.createTempDir().apply { deleteOnExit() } + + companion object { + private val localstack = LocalStackContainer(DockerImageName.parse("localstack/localstack:0.11.3")) + .withServices(LocalStackContainer.Service.S3) + .apply { start() } + } + + @AfterAll + fun stopLocalStack() { + localstack.stop() + } + + @Inject + @field:Client("/") + lateinit var client: HttpClient + + @Test + fun `api view`() { + val html = client.toBlocking().retrieve("/api") + log.info { "html=$html" } + assertThat(html).isNotBlank + } + + @Test + fun `about view`() { + val html = client.toBlocking().retrieve("/about") + log.info { "html=$html" } + assertThat(html).isNotBlank + } + + override fun getProperties() = mutableMapOf( + "file-system-blob-store.base-path" to tempDir.absolutePath, + "s3-blob-store.bucket" to "fubar", + S3ClientBuilderListener.endpointProp to localstack.getEndpointOverride( + LocalStackContainer.Service.S3 + ).toString(), + "aws.accessKeyId" to localstack.accessKey, + "aws.secretAccessKey" to localstack.secretKey, + "aws.region" to localstack.region, + "blob-migrator.enabled" to "false", + "micronaut.http.client.read-timeout" to "1m" + ) +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/core/SnowflakeTest.kt b/src/test/kotlin/jsonblob/core/SnowflakeTest.kt new file mode 100644 index 0000000..d0f169b --- /dev/null +++ b/src/test/kotlin/jsonblob/core/SnowflakeTest.kt @@ -0,0 +1,18 @@ +package jsonblob.core + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import javax.inject.Inject + +@MicronautTest +internal class SnowflakeTest { + + @Inject + lateinit var snowflake: Snowflake + + @Test + fun testNoArgs() { + assertNotNull(snowflake) + } +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/core/compression/BlobCompressorPickerTest.kt b/src/test/kotlin/jsonblob/core/compression/BlobCompressorPickerTest.kt new file mode 100644 index 0000000..c682e27 --- /dev/null +++ b/src/test/kotlin/jsonblob/core/compression/BlobCompressorPickerTest.kt @@ -0,0 +1,50 @@ +package jsonblob.core.compression + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.core.compression.compressor.BrotliBlobCompressor +import jsonblob.core.compression.compressor.NoCompressionJsonBlobCompressor +import mu.KotlinLogging +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import javax.inject.Inject +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.absolutePathString +import kotlin.io.path.createTempDirectory + + +private val log = KotlinLogging.logger {} + +@ExperimentalPathApi +@MicronautTest +class BlobCompressorPickerTest : TestPropertyProvider { + var tempDir = createTempDirectory(javaClass.simpleName) + + @Inject + lateinit var compressorPicker: BlobCompressorPicker + + @Test + fun test() { + val json = """ + { + "name" : "bob", + "age": 38 + } + """.trimIndent() + val compressor = compress(json) + assertThat(compressor).isInstanceOf(NoCompressionJsonBlobCompressor::class.java) + } + + @Test + fun largeFile() { + val largeFile = javaClass.classLoader.getResourceAsStream("large-file.json") + val compressor = compress(String(largeFile.readBytes())) + assertThat(compressor).isInstanceOf(BrotliBlobCompressor::class.java) + } + + private fun compress(json: String) = compressorPicker.bestCompressor(json) + + override fun getProperties(): MutableMap = mutableMapOf( + "file-system-blob-store.base-path" to tempDir.absolutePathString() + ) +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/core/store/file/FileSystemBlobPrunerTest.kt b/src/test/kotlin/jsonblob/core/store/file/FileSystemBlobPrunerTest.kt new file mode 100644 index 0000000..a236c40 --- /dev/null +++ b/src/test/kotlin/jsonblob/core/store/file/FileSystemBlobPrunerTest.kt @@ -0,0 +1,75 @@ +package jsonblob.core.store.file + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.config.FileSystemJsonBlobStoreConfig +import jsonblob.core.id.IdHandler +import jsonblob.model.JsonBlob +import mu.KotlinLogging +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.io.TempDir +import java.io.File +import javax.inject.Inject + + +private val log = KotlinLogging.logger {} + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +internal class FileSystemBlobPrunerTest: TestPropertyProvider { + companion object { + @TempDir + lateinit var tempDir: File + } + + @Inject + lateinit var config: FileSystemJsonBlobStoreConfig + + @Inject + lateinit var store : FileSystemJsonBlobStore + + @Inject + lateinit var idGenerator: IdHandler<*> + + @Inject + lateinit var fileSystemBlobPruner: FileSystemBlobPruner + + @Test + fun testPurging() { + val jsonBlobOne = JsonBlob( + id = idGenerator.generate(), + json = String(javaClass.classLoader.getResourceAsStream("large-file.json").readBytes()).trimIndent() + ) + val jsonBlobTwo = JsonBlob( + id = idGenerator.generate(), + json = """ + { + "name" : "bob", + "age" : 1 + } + """.trimIndent() + ) + store.write(jsonBlobOne) + store.write(jsonBlobTwo) + Thread.sleep(6000) + store.read(jsonBlobOne.id) + fileSystemBlobPruner.removeUnAccessedFilesSince() + val remainingFiles = File(config.basePath) + .walkBottomUp() + .filterNot{ it.isDirectory } + .asSequence() + .toList() + log.info { "Remaining files are $remainingFiles" } + assertThat(remainingFiles.size).isEqualTo(1) + assertThat(remainingFiles.map { it.blobId() }).contains(jsonBlobOne.id) + } + + override fun getProperties(): MutableMap = mutableMapOf( + "file-system-blob-store.base-path" to tempDir.absolutePath, + "json-blob.prune-enabled" to "false", + "json-blob.delete-after" to "5s" + ) + +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/core/store/file/FileSystemJsonBlobBaseTest.kt b/src/test/kotlin/jsonblob/core/store/file/FileSystemJsonBlobBaseTest.kt new file mode 100644 index 0000000..7833016 --- /dev/null +++ b/src/test/kotlin/jsonblob/core/store/file/FileSystemJsonBlobBaseTest.kt @@ -0,0 +1,62 @@ +package jsonblob.core.store.file + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.core.id.IdHandler +import jsonblob.model.JsonBlob +import org.assertj.core.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.io.TempDir +import org.skyscreamer.jsonassert.JSONAssert +import java.io.File +import java.time.Instant +import javax.inject.Inject +import kotlin.io.path.ExperimentalPathApi + + +@ExperimentalPathApi +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +internal class FileSystemJsonBlobBaseTest : TestPropertyProvider { + companion object { + @TempDir + lateinit var tempDir: File + } + + @Inject + lateinit var store : FileSystemJsonBlobStore + + @Inject + lateinit var idGenerator: IdHandler<*> + + @Test + fun testBasicFunctionality() { + val jsonBlob = JsonBlob( + id = idGenerator.generate(), + json = """ + { + "name" : "bob", + "bob" : "name", + "foo" : "age", + "age": 38 + } + """.trimIndent(), + created = Instant.now() + ) + Assertions.assertThat(store.path(jsonBlob.id)).isNull() + val writtenBlob = store.write(jsonBlob) + Assertions.assertThat(store.path(jsonBlob.id)).isNotNull + Assertions.assertThat(writtenBlob.id).isEqualTo(jsonBlob.id) + JSONAssert.assertEquals(jsonBlob.json, writtenBlob.json, true) + val receivedBlob = store.read(jsonBlob.id) ?: Assertions.fail("Didn't retrieve blob") + Assertions.assertThat(receivedBlob.id).isEqualTo(jsonBlob.id) + JSONAssert.assertEquals(jsonBlob.json, receivedBlob.json, true) + Assertions.assertThat(store.remove(jsonBlob.id)).isTrue + Assertions.assertThat(store.path(jsonBlob.id)).isNull() + } + + override fun getProperties(): MutableMap = mutableMapOf( + "file-system-blob-store.base-path" to tempDir.absolutePath + ) +} \ No newline at end of file diff --git a/src/test/kotlin/jsonblob/core/store/s3/S3JsonBlobStoreTest.kt b/src/test/kotlin/jsonblob/core/store/s3/S3JsonBlobStoreTest.kt new file mode 100644 index 0000000..912b555 --- /dev/null +++ b/src/test/kotlin/jsonblob/core/store/s3/S3JsonBlobStoreTest.kt @@ -0,0 +1,91 @@ +package jsonblob.core.store.s3 + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jsonblob.config.S3ClientBuilderListener +import jsonblob.core.id.IdHandler +import jsonblob.model.JsonBlob +import mu.KotlinLogging +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.io.TempDir +import org.skyscreamer.jsonassert.JSONAssert +import org.testcontainers.containers.localstack.LocalStackContainer +import org.testcontainers.containers.localstack.LocalStackContainer.Service.S3 +import org.testcontainers.utility.DockerImageName +import software.amazon.awssdk.services.s3.S3Client +import java.io.File +import javax.inject.Inject + + +private val log = KotlinLogging.logger {} + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +internal class S3JsonBlobStoreTest : TestPropertyProvider { + companion object { + private val localstack = LocalStackContainer(DockerImageName.parse("localstack/localstack:0.11.3")) + .withServices(S3) + .apply { start() } + + @TempDir + lateinit var tempDir: File + + private const val bucket = "test-bucket" + } + + private val json = """ + { + "name" : "bob", + "age": 38 + } + """.trimIndent() + + @AfterAll + fun stopLocalStack() { + localstack.stop() + } + + @BeforeAll + fun createBucket(s3Client: S3Client) { + log.info { "Creating bucket named $bucket" } + s3Client.createBucket { + it.bucket(bucket) + } + } + + @Inject + lateinit var s3Store : S3JsonBlobStore + + @Inject + lateinit var idHandler : IdHandler<*> + + @Test + fun test() { + val id = idHandler.generate() + assertThat(s3Store.path(id)).isNull() + + s3Store.write(JsonBlob( + id, + json + )) + + val retrievedJson = s3Store.read(id) + JSONAssert.assertEquals(json, retrievedJson!!.json, true) + } + + override fun getProperties() = mutableMapOf( + S3ClientBuilderListener.endpointProp to localstack.getEndpointOverride(S3).toString(), + "aws.accessKeyId" to localstack.accessKey, + "aws.secretAccessKey" to localstack.secretKey, + "aws.region" to localstack.region, + "file-system-blob-store.base-path" to tempDir.absolutePath, + "s3-blob-store.bucket" to bucket, + ).apply { + log.info { "Test Properties are $this " } + } + +} diff --git a/src/test/kotlin/jsonblob/util/JsonCleanerTest.kt b/src/test/kotlin/jsonblob/util/JsonCleanerTest.kt new file mode 100644 index 0000000..6cb743b --- /dev/null +++ b/src/test/kotlin/jsonblob/util/JsonCleanerTest.kt @@ -0,0 +1,22 @@ +package jsonblob.util + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.skyscreamer.jsonassert.JSONAssert.assertEquals + + +internal class JsonCleanerTest { + @Test + fun test() { + val json = """ + { + "name": "Tristan", + "age": 21 + } + """.trimIndent() + val cleaned = JsonCleaner.removeWhiteSpace(json) + assertThat(json).isNotEqualTo(cleaned) + assertEquals(json, cleaned, true) + + } +} \ No newline at end of file diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml new file mode 100644 index 0000000..d42caaa --- /dev/null +++ b/src/test/resources/application-test.yml @@ -0,0 +1,11 @@ +micronaut: + application: + name: jsonblob + metrics: + enabled: false + +ga: + web-property-id: foo-bar + custom-tracking-codes: + - 123 + - xyz \ No newline at end of file diff --git a/src/test/resources/data/2021/03/16/02bca75e-866c-11eb-bcea-e76852e708e7.json.gz b/src/test/resources/data/2021/03/16/02bca75e-866c-11eb-bcea-e76852e708e7.json.gz new file mode 100644 index 0000000..934b0d6 Binary files /dev/null and b/src/test/resources/data/2021/03/16/02bca75e-866c-11eb-bcea-e76852e708e7.json.gz differ diff --git a/src/test/resources/data/2021/03/16/1fdd3de2-865f-11eb-bcea-c981c0c75220.json.gz b/src/test/resources/data/2021/03/16/1fdd3de2-865f-11eb-bcea-c981c0c75220.json.gz new file mode 100644 index 0000000..6680fea Binary files /dev/null and b/src/test/resources/data/2021/03/16/1fdd3de2-865f-11eb-bcea-c981c0c75220.json.gz differ diff --git a/src/test/resources/data/2021/03/16/3bf19449-8622-11eb-bcea-a142cad69ee8.json.gz b/src/test/resources/data/2021/03/16/3bf19449-8622-11eb-bcea-a142cad69ee8.json.gz new file mode 100644 index 0000000..e8124d5 Binary files /dev/null and b/src/test/resources/data/2021/03/16/3bf19449-8622-11eb-bcea-a142cad69ee8.json.gz differ diff --git a/src/test/resources/data/2021/03/16/5e9c7e56-864d-11eb-bcea-9d4766ff9f6e.json.gz b/src/test/resources/data/2021/03/16/5e9c7e56-864d-11eb-bcea-9d4766ff9f6e.json.gz new file mode 100644 index 0000000..1fe3065 Binary files /dev/null and b/src/test/resources/data/2021/03/16/5e9c7e56-864d-11eb-bcea-9d4766ff9f6e.json.gz differ diff --git a/src/test/resources/data/2021/03/16/7d3e02a7-8630-11eb-bcea-1dce92222d6c.json.gz b/src/test/resources/data/2021/03/16/7d3e02a7-8630-11eb-bcea-1dce92222d6c.json.gz new file mode 100644 index 0000000..9d27591 Binary files /dev/null and b/src/test/resources/data/2021/03/16/7d3e02a7-8630-11eb-bcea-1dce92222d6c.json.gz differ diff --git a/src/test/resources/data/2021/03/16/blobMetadata.json.gz b/src/test/resources/data/2021/03/16/blobMetadata.json.gz new file mode 100644 index 0000000..44315a9 Binary files /dev/null and b/src/test/resources/data/2021/03/16/blobMetadata.json.gz differ diff --git a/src/test/resources/data/2021/03/17/0a6295b6-86ce-11eb-8336-b940a9970164.json.gz b/src/test/resources/data/2021/03/17/0a6295b6-86ce-11eb-8336-b940a9970164.json.gz new file mode 100644 index 0000000..a2cdbf6 Binary files /dev/null and b/src/test/resources/data/2021/03/17/0a6295b6-86ce-11eb-8336-b940a9970164.json.gz differ diff --git a/src/test/resources/data/2021/03/17/0acf1d1b-86b5-11eb-8336-fb2a3ba18e45.json.gz b/src/test/resources/data/2021/03/17/0acf1d1b-86b5-11eb-8336-fb2a3ba18e45.json.gz new file mode 100644 index 0000000..f970240 Binary files /dev/null and b/src/test/resources/data/2021/03/17/0acf1d1b-86b5-11eb-8336-fb2a3ba18e45.json.gz differ diff --git a/src/test/resources/data/2021/03/17/0b37c861-86c0-11eb-8336-516ae34a8a52.json.gz b/src/test/resources/data/2021/03/17/0b37c861-86c0-11eb-8336-516ae34a8a52.json.gz new file mode 100644 index 0000000..28ae1ca Binary files /dev/null and b/src/test/resources/data/2021/03/17/0b37c861-86c0-11eb-8336-516ae34a8a52.json.gz differ diff --git a/src/test/resources/data/2021/03/17/0d9680d7-86bd-11eb-8336-655767a02606.json.gz b/src/test/resources/data/2021/03/17/0d9680d7-86bd-11eb-8336-655767a02606.json.gz new file mode 100644 index 0000000..47288b5 Binary files /dev/null and b/src/test/resources/data/2021/03/17/0d9680d7-86bd-11eb-8336-655767a02606.json.gz differ diff --git a/src/test/resources/data/2021/03/17/0d98bb8c-86c3-11eb-8336-7b67acd8a763.json.gz b/src/test/resources/data/2021/03/17/0d98bb8c-86c3-11eb-8336-7b67acd8a763.json.gz new file mode 100644 index 0000000..63b4b6d Binary files /dev/null and b/src/test/resources/data/2021/03/17/0d98bb8c-86c3-11eb-8336-7b67acd8a763.json.gz differ diff --git a/src/test/resources/large-file.json b/src/test/resources/large-file.json new file mode 100644 index 0000000..48f2fd0 --- /dev/null +++ b/src/test/resources/large-file.json @@ -0,0 +1,88 @@ +{"web-app": { + "servlet": [ + { + "servlet-name": "cofaxCDS", + "servlet-class": "org.cofax.cds.CDSServlet", + "init-param": { + "configGlossary:installationAt": "Philadelphia, PA", + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templatePath": "templates", + "templateOverridePath": "", + "defaultListTemplate": "listTemplate.htm", + "defaultFileTemplate": "articleTemplate.htm", + "useJSP": false, + "jspListTemplate": "listTemplate.jsp", + "jspFileTemplate": "articleTemplate.jsp", + "cachePackageTagsTrack": 200, + "cachePackageTagsStore": 200, + "cachePackageTagsRefresh": 60, + "cacheTemplatesTrack": 100, + "cacheTemplatesStore": 50, + "cacheTemplatesRefresh": 15, + "cachePagesTrack": 200, + "cachePagesStore": 100, + "cachePagesRefresh": 10, + "cachePagesDirtyRead": 10, + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "useDataStore": true, + "dataStoreClass": "org.cofax.SqlDataStore", + "redirectionClass": "org.cofax.SqlRedirection", + "dataStoreName": "cofax", + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreInitConns": 10, + "dataStoreMaxConns": 100, + "dataStoreConnUsageLimit": 100, + "dataStoreLogLevel": "debug", + "maxUrlLength": 500}}, + { + "servlet-name": "cofaxEmail", + "servlet-class": "org.cofax.cds.EmailServlet", + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2"}}, + { + "servlet-name": "cofaxAdmin", + "servlet-class": "org.cofax.cds.AdminServlet"}, + + { + "servlet-name": "fileServlet", + "servlet-class": "org.cofax.cds.FileServlet"}, + { + "servlet-name": "cofaxTools", + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "init-param": { + "templatePath": "toolstemplates/", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "lookInContext": 1, + "adminGroupID": 4, + "betaServer": true}}], + "servlet-mapping": { + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxAdmin": "/admin/*", + "fileServlet": "/static/*", + "cofaxTools": "/tools/*"}, + + "taglib": { + "taglib-uri": "cofax.tld", + "taglib-location": "/WEB-INF/tlds/cofax.tld"}}} \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..b85032d --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,17 @@ + + + + true + + + %cyan(%d{HH:mm:ss.SSS}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n + + + + + + + + + diff --git a/system.properties b/system.properties deleted file mode 100644 index 916c446..0000000 --- a/system.properties +++ /dev/null @@ -1 +0,0 @@ -java.runtime.version=1.8 \ No newline at end of file