# dlib C++ Library - solvers.h

    // Copyright (C) 2015  Davis E. King (davis@dlib.net)
    // License: Boost Software License   See LICENSE.txt for the full license.
    #ifndef DLIB_DNn_SOLVERS_H_
    #define DLIB_DNn_SOLVERS_H_

    #include "solvers_abstract.h"
    #include "../cuda/tensor.h"
    #include <iostream>
    #include "layers.h"

    namespace dlib
    {
        class sgd
        {
        public:

            explicit sgd(
                float weight_decay_,
                float momentum_ = 0.9
            )
            {
                weight_decay = weight_decay_;
                momentum = momentum_;
            }

            sgd(
            ) : sgd(0.0005f, 0.9f)
            {
            }

            float get_momentum (
            ) const { return momentum; }

            float get_weight_decay (
            ) const { return weight_decay; }

            template <typename layer_type>
            const tensor& operator() (
                const float learning_rate,
                const layer_type& l,
                const tensor& params_grad
            )
            {
                const tensor& params = l.get_layer_params();

                DLIB_CASSERT(params.size() != 0);
                if (v.size() == 0)
                {
                    v.copy_size(params_grad);
                    v = 0;
                }

                const double lr = learning_rate*get_learning_rate_multiplier(l);
                const double wd = weight_decay*get_weight_decay_multiplier(l);

                //perform: v = momentum*mat(v) - wd*lr*mat(params) - lr*mat(params_grad);
                tt::affine_transform(v, v, params, params_grad, momentum, -wd*lr, -lr);

                return v;
            }

            template <unsigned long N>
            const tensor& operator() (
                const float learning_rate,
                const fc_<N,FC_HAS_BIAS>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.get_num_outputs());
                return v;
            }

            template <
                long _num_filters,
                long _nr,
                long _nc,
                int _stride_y,
                int _stride_x,
                int _padding_y,
                int _padding_x
                >
            const tensor& operator() (
                const float learning_rate,
                const con_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
                return v;
            }

            template <
                long _num_filters,
                long _nr,
                long _nc,
                int _stride_y,
                int _stride_x,
                int _padding_y,
                int _padding_x
                >
            const tensor& operator() (
                const float learning_rate,
                const cont_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
                return v;
            }

            template < layer_mode mode >
            const tensor& operator() (
                const float learning_rate,
                const bn_<mode>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()/2);
                return v;
            }

            friend void serialize(const sgd& item, std::ostream& out)
            {
                serialize("sgd2", out);
                serialize(item.v, out);
                serialize(item.weight_decay, out);
                serialize(item.momentum, out);
            }

            friend void deserialize(sgd& item, std::istream& in)
            {
                std::string version;
                deserialize(version, in);
                if (version != "sgd2")
                    throw serialization_error("Unexpected version found while deserializing dlib::sgd.");
                deserialize(item.v, in);
                deserialize(item.weight_decay, in);
                deserialize(item.momentum, in);
            }

            friend std::ostream& operator<< (std::ostream& out, const sgd& item)
            {
                out << "sgd: weight_decay="<<item.get_weight_decay() << ", momentum="<<item.get_momentum();
                return out;
            }

        private:

            template <typename layer_type>
            void update_considering_bias(
                const float learning_rate,
                const layer_type& l,
                const tensor& params_grad,
                unsigned long bias_offset
            )
            {
                const tensor& params = l.get_layer_params();

                DLIB_CASSERT(params.size() != 0);
                if (v.size() == 0)
                {
                    v.copy_size(params_grad);
                    v = 0;
                }

                double lr = learning_rate*get_learning_rate_multiplier(l);
                double wd = weight_decay*get_weight_decay_multiplier(l);

                //perform: v = momentum*mat(v) - wd*lr*mat(params) - lr*mat(params_grad);

                if (l.get_bias_learning_rate_multiplier() == 1 && l.get_bias_weight_decay_multiplier() == 1)
                {
                    tt::affine_transform(v, v, params, params_grad, momentum, -wd*lr, -lr);
                }
                else
                {

                    tt::affine_transform_range(0, bias_offset, v, v, params, params_grad, momentum, -wd*lr, -lr);

                    // now update the biases but apply their multipliers
                    lr *= l.get_bias_learning_rate_multiplier();
                    wd *= l.get_bias_weight_decay_multiplier();
                    tt::affine_transform_range(bias_offset, v.size(), v, v, params, params_grad, momentum, -wd*lr, -lr);
                }
            }

            resizable_tensor v;
            float weight_decay;
            float momentum;

        };

    // ----------------------------------------------------------------------------------------

        class adam
        {
        public:

            adam(
                float weight_decay_,
                float momentum1_,
                float momentum2_
            )
            {
                weight_decay = weight_decay_;
                momentum1 = momentum1_;
                momentum2 = momentum2_;
                t = 0;
            }

            adam(
            ) : adam(0.0005f, 0.9f, 0.999f)
            {}

            float get_momentum1 (
            ) const { return momentum1; }

            float get_momentum2 (
            ) const { return momentum2; }

            float get_weight_decay (
            ) const { return weight_decay; }

            template <typename layer_type>
            const tensor& operator() (
                const float learning_rate,
                const layer_type& l,
                const tensor& params_grad
            )
            {
                const tensor& params = l.get_layer_params();
                DLIB_CASSERT(params.size() != 0);
                if (v.size() == 0)
                {
                    m.copy_size(params_grad);
                    m = 0;
                    v.copy_size(params_grad);
                    v = 0;
                    s.copy_size(params_grad);
                }

                ++t;

                tt::compute_adam_update(0, params.size(), s, m, v, t,
                    learning_rate*get_learning_rate_multiplier(l),
                    weight_decay*get_weight_decay_multiplier(l),
                    momentum1, momentum2, params, params_grad);

                return s;
            }

            template <unsigned long N>
            const tensor& operator() (
                const float learning_rate,
                const fc_<N,FC_HAS_BIAS>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.get_num_outputs());
                return s;
            }

            template <
                long _num_filters,
                long _nr,
                long _nc,
                int _stride_y,
                int _stride_x,
                int _padding_y,
                int _padding_x
                >
            const tensor& operator() (
                const float learning_rate,
                const con_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
                return s;
            }

            template <
                long _num_filters,
                long _nr,
                long _nc,
                int _stride_y,
                int _stride_x,
                int _padding_y,
                int _padding_x
                >
            const tensor& operator() (
                const float learning_rate,
                const cont_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
                return s;
            }

            template < layer_mode mode >
            const tensor& operator() (
                const float learning_rate,
                const bn_<mode>& l,
                const tensor& params_grad
            )
            {
                update_considering_bias(learning_rate, l, params_grad, params_grad.size()/2);
                return s;
            }

            friend void serialize(const adam& item, std::ostream& out)
            {
                serialize("adam2", out);
                serialize(item.m, out);
                serialize(item.v, out);
                serialize(item.s, out);
                serialize(item.weight_decay, out);
                serialize(item.momentum1, out);
                serialize(item.momentum2, out);
                serialize(item.t, out);
            }

            friend void deserialize(adam& item, std::istream& in)
            {
                std::string version;
                deserialize(version, in);
                if (version != "adam2")
                    throw serialization_error("Unexpected version found while deserializing dlib::adam.");
                deserialize(item.m, in);
                deserialize(item.v, in);
                deserialize(item.s, in);
                deserialize(item.weight_decay, in);
                deserialize(item.momentum1, in);
                deserialize(item.momentum2, in);
                deserialize(item.t, in);
            }

            friend std::ostream& operator<< (std::ostream& out, const adam& item)
            {
                out << "adam: weight_decay="<<item.get_weight_decay() << ", momentum1="<<item.get_momentum1() << ", momentum2="<<item.get_momentum2();
                return out;
            }

        private:

            template <typename layer_type>
            void update_considering_bias(
                const float learning_rate,
                const layer_type& l,
                const tensor& params_grad,
                unsigned long bias_offset
            )
            {
                const tensor& params = l.get_layer_params();
                DLIB_CASSERT(params.size() != 0);
                if (v.size() == 0)
                {
                    m.copy_size(params_grad);
                    m = 0;
                    v.copy_size(params_grad);
                    v = 0;
                    s.copy_size(params_grad);
                }

                ++t;

                if (l.get_bias_learning_rate_multiplier() == 1 && l.get_bias_weight_decay_multiplier() == 1)
                {
                    tt::compute_adam_update(0, params.size(), s, m, v, t,
                        learning_rate*get_learning_rate_multiplier(l),
                        weight_decay*get_weight_decay_multiplier(l),
                        momentum1, momentum2, params, params_grad);
                }
                else
                {
                    tt::compute_adam_update(0, bias_offset, s, m, v, t,
                        learning_rate*get_learning_rate_multiplier(l),
                        weight_decay*get_weight_decay_multiplier(l),
                        momentum1, momentum2, params, params_grad);

                    tt::compute_adam_update(bias_offset, params.size(), s, m, v, t,
                        learning_rate*get_learning_rate_multiplier(l)*l.get_bias_learning_rate_multiplier(),
                        weight_decay*get_weight_decay_multiplier(l)*l.get_bias_weight_decay_multiplier(),
                        momentum1, momentum2, params, params_grad);
                }
            }
            resizable_tensor m;
            resizable_tensor v;
            resizable_tensor s;
            float weight_decay;
            float momentum1;
            float momentum2;
            float t;
        };

    // ----------------------------------------------------------------------------------------

    }

    #endif // DLIB_DNn_SOLVERS_H_