s3 support

cmd/mort.go

@@ -3,18 +3,24 @@ package main
 import (
 	"net/http"
 	"time"
+	"flag"
+	"fmt"
 
 	"github.com/labstack/echo"
 	"mort"
 	"mort/config"
 	"mort/object"
 
-	"mort/response"
 )
 
 func main() {
+	configPath := flag.String("config", "configuration/config.yml", "Path to configuration")
+	listenAddr := flag.String("listen", ":8080", "Listen addr")
+	flag.Parse()
+	fmt.Println(*configPath, *listenAddr)
+
 	imgConfig := config.GetInstance()
-	imgConfig.Load("configuration/config.yml")
+	imgConfig.Load(*configPath)
 	// Echo instance
 	e := echo.New()
 
@@ -31,14 +37,14 @@ func main() {
 		res.WriteHeaders(ctx.Response())
 		defer res.Close()
 
-		return ctx.Stream(res.StatusCode, res.Headers[response.ContentType], res.Stream)
+		return ctx.Stream(res.StatusCode, res.ContentType, res.Stream)
 	})
 
 
 	s := &http.Server{
-		Addr:         ":8080",
-		ReadTimeout:  1 * time.Minute,
-		WriteTimeout: 1 * time.Minute,
+		Addr:        *listenAddr,
+		ReadTimeout:  2 * time.Minute,
+		WriteTimeout: 2 * time.Minute,
 	}
 
 	e.Logger.Fatal(e.StartServer(s))

config/config.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"regexp"
 	"sync"
-	"fmt"
 )
 
 type Config struct {
@@ -23,6 +22,14 @@ func GetInstance() *Config {
 	return instance
 }
 
+//func (c *Config) validate() {
+//	for name, bucket := range c.Buckets {
+//		//if bucket.Storages.Basic() == nil {
+//		//	panic("No basic storage for " + name)
+//		//}
+//	}
+//}
+
 func (self *Config) Load(filePath string) {
 	data, err := ioutil.ReadFile(filePath)
 	if err != nil {
@@ -32,12 +39,18 @@ func (self *Config) Load(filePath string) {
 	errYaml := yaml.Unmarshal([]byte(data), self)
 
 	for name, bucket := range self.Buckets {
-		if bucket.Transform != nil && bucket.Transform.Path != "" {
-			bucket.Transform.PathRegexp = regexp.MustCompile(bucket.Transform.Path)
-			self.Buckets[name] = bucket
+		if bucket.Transform != nil {
+			if bucket.Transform.Path != "" {
+				bucket.Transform.PathRegexp = regexp.MustCompile(bucket.Transform.Path)
+			}
+
+			if bucket.Transform.ParentStorage == "" {
+				bucket.Transform.ParentStorage = "basic"
+			}
 		}
+		self.Buckets[name] = bucket
 	}
-	fmt.Println(self)
+
 	if errYaml != nil {
 		panic(errYaml)
 	}

config/yaml.go

@@ -27,26 +27,41 @@ type PresetsYaml struct {
 }
 
 type TransformYaml struct {
-	Path       string `yaml:"path"`
-	PathRegexp *regexp.Regexp
-	Kind       string                 `yaml:"kind"`
-	Presets    map[string]PresetsYaml `yaml:"presets"`
-	Order      struct {
+	Path          string `yaml:"path"`
+	ParentStorage string `yaml:"parentStorage"`
+	ParentPrefix  string `yaml:"parentPrefix"`
+	PathRegexp    *regexp.Regexp
+	Kind          string                 `yaml:"kind"`
+	Presets       map[string]PresetsYaml `yaml:"presets"`
+	Order         struct {
 		PresetName int `yaml:"presetName"`
 		Parent     int `yaml:"parent"`
 	} `yaml:"order"`
 }
 
 type Storage struct {
-	RootPath string            `yaml:"rootPath", omitempty`
-	Kind     string            `yaml:"kind"`
-	Url      string            `yaml:"url",omitempty`
-	Headers  map[string]string `yaml:"headers",omitempty`
+	RootPath        string            `yaml:"rootPath", omitempty`
+	Kind            string            `yaml:"kind"`
+	Url             string            `yaml:"url",omitempty`
+	Headers         map[string]string `yaml:"headers",omitempty`
+	AccessKey       string            `yaml:"accessKey",omitempty`
+	SecretAccessKey string            `yaml:"secretAccessKey",omitempty`
+	Region          string            `yaml:"region",omitempty`
+	Endpoint        string            `yaml:"endpoint",omitempty`
 }
 
-type StorageTypes struct {
-	Transform Storage `yaml:"transform"`
-	Basic     Storage `yaml:"basic"`
+type StorageTypes map[string]Storage
+
+func (s *StorageTypes) Basic() Storage {
+	return s.Get("basic")
+}
+
+func (s *StorageTypes) Transform() Storage {
+	return s.Get("transform")
+}
+
+func (s *StorageTypes) Get(name string) Storage {
+	return (*s)[name]
 }
 
 type S3Key struct {
@@ -57,7 +72,7 @@ type S3Key struct {
 type Bucket struct {
 	Transform *TransformYaml `yaml:"transform",omitempty`
 	Storages  StorageTypes   `yaml:"storages"`
-	Keys      []S3Key       `yaml:"keys"`
+	Keys      []S3Key        `yaml:"keys"`
 }
 
 type HeaderYaml struct {

configuration/config-s3.yml

@@ -0,0 +1,161 @@
+headers:
+  - statusCodes: [200]
+    values:
+      "cache-control": "max-age=84000, public"
+  - statusCodes: [404, 400]
+    values:
+      "cache-control": "max-age=60, public"
+  - statusCodes: [500, 503]
+    values:
+      "cache-control": "max-age=10, public"
+
+buckets:
+    liip:
+        keys:
+          - accessKey: "acc"
+            secretAccessKey: "sec"
+        transform:
+            path: "\\/+(?:cache|resolve)\\/([a-z0-9_]+)\\/+[a-z]+\\/[a-zA-Z0-9]+\\/+(.*)"
+            kind: "presets"
+            order:
+              presetName: 0
+              parent: 1
+            presets:
+                blog_small:
+                    quality: 75
+                    filters:
+                        thumbnail: { size: [100, 70], mode: outbound }
+                        interlace:
+                            mode: line
+                blog_medium:
+                    quality: 75
+                    filters:
+                        thumbnail: { size: [903, 600], mode: outbound }
+                        crop: { start: [0, 0], size: [900, 320] }
+
+                blog_home:
+                    quality: 75
+                    filters:
+                        entropy_crop: { size: [756, 396], mode: outbound }
+                recent_gallery:
+                    quality: 75
+                    filters:
+                        entropy_crop: { size: [85, 85], mode: outbound }
+
+                promoted:
+                    quality: 75
+                    filters:
+                        entropy_crop: { size: [896, 465], mode: outbound }
+
+                relatedposts:
+                    quality: 75
+                    filters:
+                        entropy_crop: { size: [504, 369], mode: outbound }
+
+                blog_big:
+                    quality: 75
+                    filters:
+                        auto_rotate: ~
+                        strip: ~
+                gallery_thumb:
+                    quality: 75
+                    filters:
+                        thumbnail: { size: [200], mode: outbound }
+                        auto_rotate: ~
+                        strip: ~
+                photocategory_img:
+                    quality: 75
+                    filters:
+                        entropy_crop: { size: [600, 450], mode: outbound }
+                        auto_rotate: ~
+                        strip: ~
+                        interlace:
+                            mode: line
+        storages:
+            basic:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+            transform:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+    media:
+        keys:
+          - accessKey: "acc"
+            secretAccessKey: "sec"
+        transform:
+            path: "\\/\\w+\\/\\w+\\/\\w+\\/[a-z]+_([a-z0-9-]+)_([a-z0-9_]+).*"
+            kind: "presets"
+            order:
+              parent: 0
+              presetName: 1
+            parentStorage: "api"
+            parentPrefix: "media"
+            presets:
+                default_small:
+                    quality: 95
+                    filters:
+                        thumbnail: {size: [150]}
+                default_medium:
+                    quality: 95
+                    filters:
+                        thumbnail: {size: [450]}
+                default_big:
+                    quality: 95
+                    filters:
+                        thumbnail: {size: [700]}
+                blog_small:
+                    quality: 75
+                    filters:
+                        thumbnail: {size: [150]}
+                blog_big:
+                    quality: 80
+                    filters:
+                        thumbnail: {size: [700]}
+                blog_big1000:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [1000]}
+                blog_big1300:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [1300]}
+                gallery_small:
+                    quality: 75
+                    filters:
+                        thumbnail: {size: [150]}
+                gallery_big:
+                    quality: 80
+                    filters:
+                        thumbnail: {size: [700]}
+                gallery_big1000:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [1000]}
+                gallery_big1300:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [1300]}
+                gallery_big300:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [300]}
+                gallery_big200:
+                    quality: 100
+                    filters:
+                        thumbnail: {size: [200]}
+        storages:
+            api:
+                kind: "http"
+                url: "https://mkaciuba.pl/download<item>"
+                headers:
+                  "x-security-key": "123qwe123qwe"
+            transform:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+            basic:
+                kind: "s3"
+                accessKey: "acc"
+                secretAccessKey: "sec"
+                region: "mort"
+                endpoint: "http://localhost:8080"
+

configuration/config.yml

@@ -88,8 +88,8 @@ buckets:
             order:
               parent: 0
               presetName: 1
-            parent:
-              appendBucket: true
+            parentStorage: "api"
+            parentPrefix: "media"
             presets:
                 default_small:
                     quality: 95
@@ -144,13 +144,14 @@ buckets:
                     filters:
                         thumbnail: {size: [200]}
         storages:
-            basic:
+            api:
                 kind: "http"
-                url: "https://mkaciuba.pl/<container>/download/<item>"
+                url: "https://mkaciuba.pl/download<item>"
                 headers:
-                  "x-auth": "1234"
-#                kind: "local"
-#                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+                  "x-security-key": "123qwe123qwe"
             transform:
                 kind: "local"
                 rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+            basic:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"