diff --git a/cmd/mort.go b/cmd/mort.go index dab44d2..8eff1c8 100644 --- a/cmd/mort.go +++ b/cmd/mort.go @@ -3,18 +3,24 @@ package main import ( "net/http" "time" + "flag" + "fmt" "github.com/labstack/echo" "mort" "mort/config" "mort/object" - "mort/response" ) func main() { + configPath := flag.String("config", "configuration/config.yml", "Path to configuration") + listenAddr := flag.String("listen", ":8080", "Listen addr") + flag.Parse() + fmt.Println(*configPath, *listenAddr) + imgConfig := config.GetInstance() - imgConfig.Load("configuration/config.yml") + imgConfig.Load(*configPath) // Echo instance e := echo.New() @@ -31,14 +37,14 @@ func main() { res.WriteHeaders(ctx.Response()) defer res.Close() - return ctx.Stream(res.StatusCode, res.Headers[response.ContentType], res.Stream) + return ctx.Stream(res.StatusCode, res.ContentType, res.Stream) }) s := &http.Server{ - Addr: ":8080", - ReadTimeout: 1 * time.Minute, - WriteTimeout: 1 * time.Minute, + Addr: *listenAddr, + ReadTimeout: 2 * time.Minute, + WriteTimeout: 2 * time.Minute, } e.Logger.Fatal(e.StartServer(s)) diff --git a/config/config.go b/config/config.go index 2495faf..a62cedb 100644 --- a/config/config.go +++ b/config/config.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "regexp" "sync" - "fmt" ) type Config struct { @@ -23,6 +22,14 @@ func GetInstance() *Config { return instance } +//func (c *Config) validate() { +// for name, bucket := range c.Buckets { +// //if bucket.Storages.Basic() == nil { +// // panic("No basic storage for " + name) +// //} +// } +//} + func (self *Config) Load(filePath string) { data, err := ioutil.ReadFile(filePath) if err != nil { @@ -32,12 +39,18 @@ func (self *Config) Load(filePath string) { errYaml := yaml.Unmarshal([]byte(data), self) for name, bucket := range self.Buckets { - if bucket.Transform != nil && bucket.Transform.Path != "" { - bucket.Transform.PathRegexp = regexp.MustCompile(bucket.Transform.Path) - self.Buckets[name] = bucket + if bucket.Transform != nil { + if bucket.Transform.Path != "" { + bucket.Transform.PathRegexp = regexp.MustCompile(bucket.Transform.Path) + } + + if bucket.Transform.ParentStorage == "" { + bucket.Transform.ParentStorage = "basic" + } } + self.Buckets[name] = bucket } - fmt.Println(self) + if errYaml != nil { panic(errYaml) } diff --git a/config/yaml.go b/config/yaml.go index 8e7d53b..9d557fc 100644 --- a/config/yaml.go +++ b/config/yaml.go @@ -27,26 +27,41 @@ type PresetsYaml struct { } type TransformYaml struct { - Path string `yaml:"path"` - PathRegexp *regexp.Regexp - Kind string `yaml:"kind"` - Presets map[string]PresetsYaml `yaml:"presets"` - Order struct { + Path string `yaml:"path"` + ParentStorage string `yaml:"parentStorage"` + ParentPrefix string `yaml:"parentPrefix"` + PathRegexp *regexp.Regexp + Kind string `yaml:"kind"` + Presets map[string]PresetsYaml `yaml:"presets"` + Order struct { PresetName int `yaml:"presetName"` Parent int `yaml:"parent"` } `yaml:"order"` } type Storage struct { - RootPath string `yaml:"rootPath", omitempty` - Kind string `yaml:"kind"` - Url string `yaml:"url",omitempty` - Headers map[string]string `yaml:"headers",omitempty` + RootPath string `yaml:"rootPath", omitempty` + Kind string `yaml:"kind"` + Url string `yaml:"url",omitempty` + Headers map[string]string `yaml:"headers",omitempty` + AccessKey string `yaml:"accessKey",omitempty` + SecretAccessKey string `yaml:"secretAccessKey",omitempty` + Region string `yaml:"region",omitempty` + Endpoint string `yaml:"endpoint",omitempty` } -type StorageTypes struct { - Transform Storage `yaml:"transform"` - Basic Storage `yaml:"basic"` +type StorageTypes map[string]Storage + +func (s *StorageTypes) Basic() Storage { + return s.Get("basic") +} + +func (s *StorageTypes) Transform() Storage { + return s.Get("transform") +} + +func (s *StorageTypes) Get(name string) Storage { + return (*s)[name] } type S3Key struct { @@ -57,7 +72,7 @@ type S3Key struct { type Bucket struct { Transform *TransformYaml `yaml:"transform",omitempty` Storages StorageTypes `yaml:"storages"` - Keys []S3Key `yaml:"keys"` + Keys []S3Key `yaml:"keys"` } type HeaderYaml struct { diff --git a/configuration/config-s3.yml b/configuration/config-s3.yml new file mode 100644 index 0000000..c4a3338 --- /dev/null +++ b/configuration/config-s3.yml @@ -0,0 +1,161 @@ +headers: + - statusCodes: [200] + values: + "cache-control": "max-age=84000, public" + - statusCodes: [404, 400] + values: + "cache-control": "max-age=60, public" + - statusCodes: [500, 503] + values: + "cache-control": "max-age=10, public" + +buckets: + liip: + keys: + - accessKey: "acc" + secretAccessKey: "sec" + transform: + path: "\\/+(?:cache|resolve)\\/([a-z0-9_]+)\\/+[a-z]+\\/[a-zA-Z0-9]+\\/+(.*)" + kind: "presets" + order: + presetName: 0 + parent: 1 + presets: + blog_small: + quality: 75 + filters: + thumbnail: { size: [100, 70], mode: outbound } + interlace: + mode: line + blog_medium: + quality: 75 + filters: + thumbnail: { size: [903, 600], mode: outbound } + crop: { start: [0, 0], size: [900, 320] } + + blog_home: + quality: 75 + filters: + entropy_crop: { size: [756, 396], mode: outbound } + recent_gallery: + quality: 75 + filters: + entropy_crop: { size: [85, 85], mode: outbound } + + promoted: + quality: 75 + filters: + entropy_crop: { size: [896, 465], mode: outbound } + + relatedposts: + quality: 75 + filters: + entropy_crop: { size: [504, 369], mode: outbound } + + blog_big: + quality: 75 + filters: + auto_rotate: ~ + strip: ~ + gallery_thumb: + quality: 75 + filters: + thumbnail: { size: [200], mode: outbound } + auto_rotate: ~ + strip: ~ + photocategory_img: + quality: 75 + filters: + entropy_crop: { size: [600, 450], mode: outbound } + auto_rotate: ~ + strip: ~ + interlace: + mode: line + storages: + basic: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" + transform: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" + media: + keys: + - accessKey: "acc" + secretAccessKey: "sec" + transform: + path: "\\/\\w+\\/\\w+\\/\\w+\\/[a-z]+_([a-z0-9-]+)_([a-z0-9_]+).*" + kind: "presets" + order: + parent: 0 + presetName: 1 + parentStorage: "api" + parentPrefix: "media" + presets: + default_small: + quality: 95 + filters: + thumbnail: {size: [150]} + default_medium: + quality: 95 + filters: + thumbnail: {size: [450]} + default_big: + quality: 95 + filters: + thumbnail: {size: [700]} + blog_small: + quality: 75 + filters: + thumbnail: {size: [150]} + blog_big: + quality: 80 + filters: + thumbnail: {size: [700]} + blog_big1000: + quality: 100 + filters: + thumbnail: {size: [1000]} + blog_big1300: + quality: 100 + filters: + thumbnail: {size: [1300]} + gallery_small: + quality: 75 + filters: + thumbnail: {size: [150]} + gallery_big: + quality: 80 + filters: + thumbnail: {size: [700]} + gallery_big1000: + quality: 100 + filters: + thumbnail: {size: [1000]} + gallery_big1300: + quality: 100 + filters: + thumbnail: {size: [1300]} + gallery_big300: + quality: 100 + filters: + thumbnail: {size: [300]} + gallery_big200: + quality: 100 + filters: + thumbnail: {size: [200]} + storages: + api: + kind: "http" + url: "https://mkaciuba.pl/download" + headers: + "x-security-key": "123qwe123qwe" + transform: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" + basic: + kind: "s3" + accessKey: "acc" + secretAccessKey: "sec" + region: "mort" + endpoint: "http://localhost:8080" + diff --git a/configuration/config.yml b/configuration/config.yml index f112ea0..c5cab59 100644 --- a/configuration/config.yml +++ b/configuration/config.yml @@ -88,8 +88,8 @@ buckets: order: parent: 0 presetName: 1 - parent: - appendBucket: true + parentStorage: "api" + parentPrefix: "media" presets: default_small: quality: 95 @@ -144,13 +144,14 @@ buckets: filters: thumbnail: {size: [200]} storages: - basic: + api: kind: "http" - url: "https://mkaciuba.pl//download/" + url: "https://mkaciuba.pl/download" headers: - "x-auth": "1234" -# kind: "local" -# rootPath: "/Users/aldor/workspace/mkaciubacom/web" + "x-security-key": "123qwe123qwe" transform: kind: "local" rootPath: "/Users/aldor/workspace/mkaciubacom/web" + basic: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" diff --git a/object/file_object.go b/object/file_object.go index e3aadee..2a0d2ce 100644 --- a/object/file_object.go +++ b/object/file_object.go @@ -3,7 +3,7 @@ package object import ( "errors" "strings" - "fmt" + "path" Logger "github.com/labstack/gommon/log" "mort/config" @@ -40,30 +40,29 @@ type FileObject struct { Parent *FileObject } -func NewFileObject(path string, mortConfig *config.Config) (*FileObject, error) { +func NewFileObject(uri string, mortConfig *config.Config) (*FileObject, error) { obj := FileObject{} - obj.Uri = path + obj.Uri = uri err := obj.decode(mortConfig) - fmt.Println(obj.Storage) - Logger.Infof("path = %s key = %s bucket = %s storage = %s transforms = %s hasParent = %s \n",path, obj.Key, obj.Bucket, obj.Storage.Kind, obj.HasTransform(), obj.HasParent()) + Logger.Infof("path = %s key = %s bucket = %s storage = %s transforms = %s hasParent = %s \n", uri, obj.Key, obj.Bucket, obj.Storage.Kind, obj.HasTransform(), obj.HasParent()) return &obj, err } func (self *FileObject) decode(mortConfig *config.Config) error { elements := strings.Split(self.Uri, "/") - if len(elements) < 3 { - return errors.New("Invalid path") - } self.Bucket = elements[1] - self.Key = "/" + strings.Join(elements[2:], "/") + if len(elements) > 2 { + self.Key = "/" + strings.Join(elements[2:], "/") + } + if bucket, ok := mortConfig.Buckets[self.Bucket]; ok { self.decodeKey(bucket, mortConfig) if self.HasTransform() { - self.Storage = bucket.Storages.Transform + self.Storage = bucket.Storages.Transform() } else { - self.Storage = bucket.Storages.Basic + self.Storage = bucket.Storages.Basic() } } else { @@ -88,7 +87,9 @@ func (self *FileObject) decodeKey(bucket config.Bucket, mortConfig *config.Confi parent := "/" + string(matches[trans.Order.Parent+1]) self.Transforms = presetToTransform(bucket.Transform.Presets[presetName]) + parent = "/" + path.Join(bucket.Transform.ParentPrefix, parent) parentObj, _ := NewFileObject(parent, mortConfig) + parentObj.Storage = bucket.Storages.Get(bucket.Transform.ParentStorage) self.Parent = parentObj return nil } diff --git a/processor.go b/processor.go index 0d76e8d..b977b16 100644 --- a/processor.go +++ b/processor.go @@ -12,11 +12,12 @@ import ( "mort/transforms" "github.com/labstack/echo" ) +const S3_LOCATION_STR = "EU" func Process(ctx echo.Context, obj *object.FileObject) *response.Response { switch ctx.Request().Method { case "GET": - return hanldeGET(obj) + return hanldeGET(ctx, obj) case "PUT": return handlePUT(ctx, obj) @@ -29,7 +30,10 @@ func handlePUT(ctx echo.Context, obj *object.FileObject) *response.Response { return storage.Set(obj, ctx.Request().Header, ctx.Request().ContentLength, ctx.Request().Body) } -func hanldeGET(obj *object.FileObject) *response.Response { +func hanldeGET(ctx echo.Context, obj *object.FileObject) *response.Response { + if obj.Key == "" { + return handleS3Get(ctx, obj); + } var currObj *object.FileObject = obj var parentObj *object.FileObject = nil var transforms []transforms.Transforms @@ -49,10 +53,10 @@ func hanldeGET(obj *object.FileObject) *response.Response { // get parent from storage if parentObj != nil { res = updateHeaders(storage.Get(parentObj)) - } - if res.StatusCode != 200 { - return res + if res.StatusCode != 200 { + return res + } } // check if object is on storage @@ -61,27 +65,39 @@ func hanldeGET(obj *object.FileObject) *response.Response { return res } - if strings.Contains(res.Headers[response.ContentType], "image/") { + if strings.Contains(res.ContentType, "image/") { // revers order of transforms for i := 0; i < len(transforms)/2; i++ { j := len(transforms) - i - 1 transforms[i], transforms[j] = transforms[j], transforms[i] } - return updateHeaders(processImage(res, transforms)) + return updateHeaders(processImage(obj, res, transforms)) } return updateHeaders(storage.Get(obj)) } -func processImage(parent *response.Response, transforms []transforms.Transforms) *response.Response { +func handleS3Get(ctx echo.Context, obj *object.FileObject) *response.Response { + req := ctx.Request() + query := req.URL.Query() + + if _, ok := query["location"]; ok { + return response.NewBuf(200, []byte(S3_LOCATION_STR)) + } + + return response.NewBuf(405, []byte("")) +} + +func processImage(obj *object.FileObject, parent *response.Response, transforms []transforms.Transforms) *response.Response { engine := engine.NewImageEngine(parent) - result, err := engine.Process(transforms) + res, err := engine.Process(transforms) if err != nil { return response.NewError(400, err) } - return result + storage.Set(obj, res.Headers, res.ContentLength, res.Stream) + return res } diff --git a/response/response.go b/response/response.go index 2480a6c..ce8a92e 100644 --- a/response/response.go +++ b/response/response.go @@ -13,14 +13,16 @@ const ( ) type Response struct { - StatusCode int - Stream io.ReadCloser - Headers map[string]string + StatusCode int + Stream io.ReadCloser + Headers http.Header + ContentLength int64 + ContentType string } func New(statusCode int, body io.ReadCloser) *Response { res := Response{StatusCode: statusCode, Stream: body} - res.Headers = make(map[string]string) + res.Headers = make(http.Header) if body == nil { res.SetContentType("application/octet-stream") } else { @@ -30,7 +32,8 @@ func New(statusCode int, body io.ReadCloser) *Response { } func NewBuf(statusCode int, body []byte) *Response { res := Response{StatusCode: statusCode, Stream: ioutil.NopCloser(bytes.NewReader(body))} - res.Headers = make(map[string]string) + res.ContentLength = int64(len(body)) + res.Headers = make(http.Header) if body == nil { res.SetContentType("application/octet-stream") } else { @@ -43,24 +46,27 @@ func NewError(statusCode int, err error) *Response { body := map[string]string{"message": err.Error()} jsonBody, _ := json.Marshal(body) res := Response{StatusCode: statusCode, Stream: ioutil.NopCloser(bytes.NewReader(jsonBody))} - res.Headers = make(map[string]string) + res.ContentLength = int64(len(jsonBody)) + res.Headers = make(http.Header) res.SetContentType("application/json") return &res } func (r *Response) SetContentType(contentType string) *Response { - r.Headers[ContentType] = contentType + r.Headers.Set(ContentType, contentType) + r.ContentType = ContentType return r } func (r *Response) Set(headerName string, headerValue string) { - r.Headers[headerName] = headerValue + r.Headers.Set(headerName, headerValue) } func (r *Response) WriteHeaders(writer http.ResponseWriter) { for headerName, headerValue := range r.Headers { - writer.Header().Set(headerName, headerValue) + writer.Header().Set(headerName, headerValue[0]) } + writer.Header().Set(ContentType, r.ContentType) } func (r *Response) ReadBody() ([]byte, error) { diff --git a/s3.go b/s3.go index b713bfc..9936e27 100644 --- a/s3.go +++ b/s3.go @@ -24,24 +24,33 @@ func S3Middleware(config *config.Config) echo.MiddlewareFunc { req := c.Request() path := req.URL.Path pathSlice := strings.Split(path, "/") + pathSliceLen := len(pathSlice) //fmt.Printf("slice = %s path = %s len = %d path= %s \n", pathSlice, path, len(pathSlice), pathSlice[0]) - if len(pathSlice) < 3 { + if pathSliceLen < 2 { return echo.NewHTTPError(400, "invalid path") } bucketName := pathSlice[1] - realPath := strings.Join(pathSlice[2:], "/") + //realPath := "" + //if pathSliceLen > 2 { + // realPath = strings.Join(pathSlice[2:], "/") + //} + bucket, ok := config.Buckets[bucketName] + if !ok { - return echo.NewHTTPError(400, "unknown bucket") + return echo.NewHTTPError(404, "unknown bucket") } // TODO: auth for get request - if req.Method == "GET" && realPath != "/" { - return next(c) - } auth := req.Header.Get(echo.HeaderAuthorization) + + + if req.Method == "GET" && auth == "" { + return next(c) + } + matches := AutHeaderRegexpv4.FindStringSubmatch(auth) if len(matches) == 5 { alg := matches[1] diff --git a/storage/http/container.go b/storage/http/container.go index f1e09e9..556aa9c 100644 --- a/storage/http/container.go +++ b/storage/http/container.go @@ -7,7 +7,7 @@ import ( "github.com/aldor007/stow" "github.com/pkg/errors" - +"fmt" ) type container struct { @@ -57,6 +57,7 @@ func (c *container) Put(name string, r io.Reader, size int64, metadata map[strin // for this if so. func (c *container) getItem(id string) (*item, error) { endpoint := strings.Replace(c.endpoint, "", id, 1) + fmt.Println("cont", endpoint) req, err := http.NewRequest("HEAD", endpoint, nil) if err != nil { return nil, err diff --git a/storage/http/http.go b/storage/http/http.go index 38ee88a..04ebd42 100644 --- a/storage/http/http.go +++ b/storage/http/http.go @@ -7,7 +7,6 @@ import ( "time" "errors" "encoding/json" - "fmt" "github.com/aldor007/stow" ) @@ -26,7 +25,6 @@ const ( ) func init() { - fmt.Println("AAAAAAAa init") makefn := func(config stow.Config) (stow.Location, error) { url, ok := config.Config(ConfigUrl) diff --git a/storage/storage.go b/storage/storage.go index 5db206f..e2f8cfc 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,15 +1,16 @@ package storage import ( + "encoding/json" "io" "mime" - "path" "net/http" - "encoding/json" + "path" + "fmt" "github.com/aldor007/stow" fileStorage "github.com/aldor007/stow/local" - _ "github.com/aldor007/stow/s3" + s3Storage "github.com/aldor007/stow/s3" httpStorage "mort/storage/http" "mort/object" @@ -42,7 +43,7 @@ func Get(obj *object.FileObject) *response.Response { return prepareResponse(obj, reader) } -func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCloser) *response.Response{ +func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCloser) *response.Response { client, err := getClient(obj) if err != nil { return response.NewError(503, err) @@ -59,7 +60,6 @@ func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCl return res } - func getClient(obj *object.FileObject) (stow.Container, error) { storageCfg := obj.Storage var config stow.Config @@ -73,12 +73,19 @@ func getClient(obj *object.FileObject) (stow.Container, error) { case "http": headers, _ := json.Marshal(storageCfg.Headers) config = stow.ConfigMap{ - httpStorage.ConfigUrl: storageCfg.Url, + httpStorage.ConfigUrl: storageCfg.Url, httpStorage.ConfigHeader: string(headers), } + case "s3": + config = stow.ConfigMap{ + s3Storage.ConfigAccessKeyID: storageCfg.AccessKey, + s3Storage.ConfigSecretKey: storageCfg.SecretAccessKey, + s3Storage.ConfigRegion: storageCfg.Region, + s3Storage.ConfigEndpoint: storageCfg.Endpoint, + } } - + fmt.Println(config) client, err := stow.Dial(storageCfg.Kind, config) if err != nil { return nil, err