NIX Validator

Dockerfile

@@ -1,4 +1,4 @@
-# BUILDER IMAGE
+# SERVICE BUILDER IMAGE
 FROM golang:alpine AS binbuilder
 
 # Build package deps
@@ -27,6 +27,32 @@ RUN go build ./cmd/ginvalid
 
 ### ============================ ###
 
+# NIX BUILDER IMAGE
+FROM alpine:latest as nixbuilder
+
+# HDF5 is in the 'testing' repository
+RUN echo http://dl-2.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories
+RUN apk --no-cache --no-progress add \
+    git \
+    openssh \
+    cmake \
+    doxygen \
+    git \
+    build-base \
+    boost-dev \
+    boost-static \
+    cppunit-dev \
+    hdf5-dev \
+    hdf5-static
+
+RUN git clone https://github.com/G-Node/nix /nix
+WORKDIR /nix
+RUN git checkout master
+RUN mkdir build
+WORKDIR build
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=Yes -DBUILD_STATIC=on ..
+RUN make all
+
 # RUNNER IMAGE
 FROM alpine:latest
 
@@ -39,13 +65,17 @@ RUN echo http://dl-2.alpinelinux.org/alpine/edge/community/ >> /etc/apk/reposito
         npm \
         openssh
 
+# Install the BIDS validator
+RUN npm install -g bids-validator
+
 # Copy git-annex from builder image
 COPY --from=binbuilder /git-annex /git-annex
 ENV PATH="${PATH}:/git-annex/git-annex.linux"
 
-# Install the BIDS validator
-RUN npm install -g bids-validator
+# Copy nixio-tool from nixbuilder image
+COPY --from=nixbuilder /nix/build/nixio-tool /bin
 
+RUN nixio-tool
 RUN mkdir -p /gin-valid/results/
 RUN mkdir -p /gin-valid/tmp/
 RUN mkdir -p /gin-valid/config

README.md

@@ -5,4 +5,10 @@
 
 gin-valid is the G-Node Infrastructure data validation service. It is a microservice server written in go that is meant to be run together with a GIN repository server.
 
-Repositories on a GIN server can trigger validation of data files via this service. Currently validation of the [BIDS](bids.neuroimaging.io) fMRI data format is supported.
+Repositories on a GIN server can trigger validation of data files via this service. Currently there are two validators supported:
+- The [BIDS](https://bids.neuroimaging.io) fMRI data format.
+- The [NIX](http://g-node.org/nix) (Neuroscience Information Exchange) format.
+
+## Contributing
+
+For instructions on how to add more validators, see the [adding validators](docs/adding-validators.md) contribution guide.

docs/adding-validators.md

@@ -0,0 +1,55 @@
+# Contribution guide: Adding new validator
+
+The following is a list of everything needed to add a new validator to the service.  The placeholder name `V` should be replaced with the name of the validator in the example function and variable names.  More detailed descriptions of each requirement can be found in the sections below.
+- A validation function, `validateV()`, in the `internal/web/validate.go` file.
+- A `v_results.go` file that contains a template to render the results of the validation.  The template should be stored in a const string called `VResults`.
+- A `renderVResults()` function, in the `internal/web/results.go` file, that uses the `VResults` template to render the results.
+- Configuration settings for the new validator:
+    - `ServerCfg.Executables.V` should point to the executable that runs the validation.
+    - `ServerCfg.Settings.Validators` should include the (all lowercase) name of the validator.
+- The executable should be included in the Dockerfile.
+
+
+## Validation function
+
+Validation functions are named `validateV()`, where `V` is the name of the validator.  This function should take two arguments and return an `error` type.
+The two arguments are:
+- `valroot`: The location of the repository that will be validated.  Use this directory as the starting point for the validation command.
+- `resdir`: The results directory where the results of the validation should be stored.  The validator function should create two files in this directory: a badge and a file with the results.
+
+For the name of the badge file name, use `srvcfg.Label.ResultsBadge`.  Depending on the results of the validation, the contents of this file should be one of the const strings found in `internal/resources/svg.go`.
+
+The format of the results is different for each validator.  These results will be processed by the `VResults` function to render the `v_results.go` template, so the results should be stored in a way that will make this most convenient.  The name of the file should be `srvcfg.Label.ResultsFile`.
+
+Once the validation function has been written, a `switch` case should be added for it at the bottom of the `runValidator()` function.
+
+
+## Results template
+
+The template should contain a header with the badge and name of the repository.  The main body should be the rendered contents of the results.
+
+See the existing templates for examples on what this should look like.
+
+The name of this template should be `VResults`.
+
+
+## Results rendering function
+
+The `renderVResults()` function should use the data stored in the results file (`resdir/srvcfg.Label.ResultsFile`) to render the results page.  This function should take 6 arguments.
+The arguments are:
+- `w` and `r`: The `http.ResponseWriter` and `http.Request` coming from the web request.  Use these to render the resulting page.
+- `badge`: A byte slice containing the badge contents.  The template should use the data in this slice to render the badge.
+- `content`: A byte slice containing the contents of the results file that you stored in the [Validation function](#validation-function).
+- `user` and `repo`: The user and repository names as strings.  Use these to render the repository name in the header and for error reporting.
+
+This function should load the main layout template (found in `templates.Layout`), then parse the validator template (found in `templates.VResults`), add the data it requires and execute it.
+
+
+## Configuration settings
+
+These settings can be added at runtime, but they can also be added to the default configuration for simplicity.
+
+
+## Dockerfile
+
+The executable and any required dependencies should be included in the `RUNNER IMAGE` part of the Dockerfile.  Like with NIX, binaries that are built from source can be built in separate images then copied to the main Docker runner image.  See the `NIX BUILDER IMAGE` section as well as the [Multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) Docker documentation.

internal/config/config.go

@@ -8,6 +8,7 @@ import (
 // Executables used by the server.
 type Executables struct {
 	BIDS string `json:"bids"`
+	NIX  string `json:"nix"`
 }
 
 // Directories used by the server for temporary and long term storage.
@@ -71,6 +72,7 @@ var defaultCfg = ServerCfg{
 	},
 	Executables{
 		BIDS: "bids-validator",
+		NIX:  "nixio-tool",
 	},
 	Directories{
 		Temp:   filepath.Join(os.Getenv("GINVALIDHOME"), "tmp"),

internal/resources/svg.go

@@ -1,13 +1,13 @@
 package resources
 
-// BidsSuccess contains the svg corresponding to a BIDS validation success
-const BidsSuccess = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#4c1" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">success</text><text x="1150" y="140" transform="scale(.1)" textLength="550">success</text></g></svg>`
+// SuccessBadge contains the svg corresponding to a validation success
+const SuccessBadge = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#4c1" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">success</text><text x="1150" y="140" transform="scale(.1)" textLength="550">success</text></g></svg>`
 
-// BidsWarning contains the svg corresponding to a BIDS validation with warnings
-const BidsWarning = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#dfb317" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">warning</text><text x="1150" y="140" transform="scale(.1)" textLength="550">warning</text></g></svg>`
+// WarningBadge contains the svg corresponding to a validation with warnings
+const WarningBadge = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#dfb317" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">warning</text><text x="1150" y="140" transform="scale(.1)" textLength="550">warning</text></g></svg>`
 
-// BidsFailure contains the svg corresponding to a BIDS validation with errors
-const BidsFailure = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#cb2431" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">failure</text><text x="1150" y="140" transform="scale(.1)" textLength="550">failure</text></g></svg>`
+// FailureBadge contains the svg corresponding to a validation with errors
+const FailureBadge = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="150" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="150" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#cb2431" d="M83 0 h67 v20 H83 z"/><path fill="url(#b)" d="M0 0 h150 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1150" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="550">failure</text><text x="1150" y="140" transform="scale(.1)" textLength="550">failure</text></g></svg>`
 
-// BidsUnavailable contains the svg corresponding to a non available BIDS validation
-const BidsUnavailable = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="170" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="170" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#9f9f9f" d="M83 0 h87 v20 H83 z"/><path fill="url(#b)" d="M0 0 h170 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1250" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">unavailable</text><text x="1250" y="140" transform="scale(.1)" textLength="750">unavailable</text></g></svg>`
+// UnavailableBadge contains the svg corresponding to a non available validation
+const UnavailableBadge = `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="170" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="170" height="20" rx="4" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0 h83 v20 H0 z"/><path fill="#9f9f9f" d="M83 0 h87 v20 H83 z"/><path fill="url(#b)" d="M0 0 h170 v20 H0 z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="115"><text x="400" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">validation</text><text x="400" y="140" transform="scale(.1)" textLength="750">validation</text><text x="1250" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="750">unavailable</text><text x="1250" y="140" transform="scale(.1)" textLength="750">unavailable</text></g></svg>`

internal/resources/templates/nix_results.go

@@ -0,0 +1,32 @@
+package templates
+
+const NIXResults = `
+{{define "content"}}
+	<div class="repository file list">
+		<div class="header-wrapper">
+			<div class="ui container">
+				<div class="ui vertically padded grid head">
+					<div class="column">
+						<div class="ui header">
+							<div class="ui huge breadcrumb">
+								<i class="mega-octicon octicon-repo"></i>
+								{{.Header}}
+								{{.Badge}}
+							</div>
+						</div>
+					</div>
+				</div>
+			</div>
+			<div class="ui tabs container">
+			</div>
+			<div class="ui tabs divider"></div>
+		</div>
+		<div class="ui container">
+	<hr>
+	<div>
+		<pre>
+			{{.Content}}
+		</pre>
+	</div>
+{{end}}
+`

internal/web/results.go

@@ -29,7 +29,7 @@ type BidsResultStruct struct {
 				Key  string `json:"key"`
 				Code int    `json:"code"`
 				File struct {
-					Name         string `json:"name"`
+					string       `json:"name"`
 					Path         string `json:"path"`
 					RelativePath string `json:"relativePath"`
 				} `json:"file"`
@@ -95,7 +95,7 @@ type BidsResultStruct struct {
 	} `json:"summary"`
 }
 
-// Results returns the results of a previously run BIDS validation.
+// Results returns the results of a previously run validation.
 func Results(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	user := vars["user"]
@@ -125,9 +125,22 @@ func Results(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	switch validator {
+	case "bids":
+		renderBIDSResults(w, r, badge, content, user, repo)
+	case "nix":
+		renderNIXResults(w, r, badge, content, user, repo)
+	default:
+		log.ShowWrite("[Error] Validator %q is supported but no render result function is set up", validator)
+		http.ServeContent(w, r, "unavailable", time.Now(), bytes.NewReader([]byte("404 Validator results missing")))
+	}
+	return
+}
+
+func renderBIDSResults(w http.ResponseWriter, r *http.Request, badge []byte, content []byte, user, repo string) {
 	// Parse results file
 	var resBIDS BidsResultStruct
-	err = json.Unmarshal(content, &resBIDS)
+	err := json.Unmarshal(content, &resBIDS)
 	if err != nil {
 		log.ShowWrite("[Error] unmarshalling '%s/%s' result: %s\n", user, repo, err.Error())
 		http.ServeContent(w, r, "unavailable", time.Now(), bytes.NewReader([]byte("500 Something went wrong...")))
@@ -150,7 +163,7 @@ func Results(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Parse results into html template and serve it
-	head := fmt.Sprintf("%s validation for %s/%s", strings.ToUpper(validator), user, repo)
+	head := fmt.Sprintf("BIDS validation for %s/%s", user, repo)
 	info := struct {
 		Badge  template.HTML
 		Header string
@@ -164,3 +177,36 @@ func Results(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+func renderNIXResults(w http.ResponseWriter, r *http.Request, badge []byte, content []byte, user, repo string) {
+	// Parse results file
+	// Parse html template
+	tmpl := template.New("layout")
+	tmpl, err := tmpl.Parse(templates.Layout)
+	if err != nil {
+		log.ShowWrite("[Error] '%s/%s' result: %s\n", user, repo, err.Error())
+		http.ServeContent(w, r, "unavailable", time.Now(), bytes.NewReader([]byte("500 Something went wrong...")))
+		return
+	}
+	tmpl, err = tmpl.Parse(templates.NIXResults)
+	if err != nil {
+		log.ShowWrite("[Error] '%s/%s' result: %s\n", user, repo, err.Error())
+		http.ServeContent(w, r, "unavailable", time.Now(), bytes.NewReader([]byte("500 Something went wrong...")))
+		return
+	}
+
+	// Parse results into html template and serve it
+	head := fmt.Sprintf("NIX validation for %s/%s", user, repo)
+	info := struct {
+		Badge   template.HTML
+		Header  string
+		Content string
+	}{template.HTML(badge), head, string(content)}
+
+	err = tmpl.ExecuteTemplate(w, "layout", info)
+	if err != nil {
+		log.ShowWrite("[Error] '%s/%s' result: %s\n", user, repo, err.Error())
+		http.ServeContent(w, r, "unavailable", time.Now(), bytes.NewReader([]byte("500 Something went wrong...")))
+		return
+	}
+}

internal/web/status.go

@@ -33,7 +33,7 @@ func Status(w http.ResponseWriter, r *http.Request) {
 	content, err := ioutil.ReadFile(fp)
 	if err != nil {
 		log.Write("[Error] serving '%s/%s' status: %s\n", user, repo, err.Error())
-		http.ServeContent(w, r, "unavailable.svg", time.Now(), bytes.NewReader([]byte(resources.BidsUnavailable)))
+		http.ServeContent(w, r, "unavailable.svg", time.Now(), bytes.NewReader([]byte(resources.UnavailableBadge)))
 		return
 	}
 	http.ServeContent(w, r, srvcfg.Label.ResultsBadge, time.Now(), bytes.NewReader(content))