Merge pull request #9 from darsnack/darsnack/initial-impl

Move all optimizers to Optimisers.jl

Author: DhairyaLGandhi

src/Optimisers.jl

@@ -5,7 +5,8 @@ using Functors: functor, fmap, isleaf
 include("interface.jl")
 include("rules.jl")
 
-export Descent, Momentum, Nesterov, RMSProp,
-       ADAM
+export Descent, ADAM, Momentum, Nesterov, RMSProp,
+       ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM, ADAMW, RADAM, OADAM, AdaBelief,
+       WeightDecay, OptimiserChain
 
 end # module

src/interface.jl

@@ -1,13 +1,11 @@
-function patch(x, x̄)
-  return x .- x̄
-end
+patch(x, x̄) = x .- x̄
 
 function state(o, x)
   if isleaf(x)
     return init(o, x)
   else
     x, _ = functor(x)
-    map(x -> state(o, x), x)
+    return map(x -> state(o, x), x)
   end
 end
 
@@ -20,11 +18,11 @@ function update(o, x::T, x̄, state) where T
   if x̄ === nothing
     return x, state
   elseif isleaf(x)
-    _update(o, x, x̄, state)
+    return _update(o, x, x̄, state)
   else
     x̄, _  = functor(typeof(x), x̄)
-    x, re = functor(typeof(x), x)
+    x, restructure = functor(typeof(x), x)
     xstate = map((x, x̄, state) -> update(o, x, x̄, state), x, x̄, state)
-    re(map(first, xstate)), map(x -> x[2], xstate)
+    return restructure(map(first, xstate)), map(x -> x[2], xstate)
   end
 end