Resuming an optimization in scipy.optimize?
<p>scipy.optimize presents many different methods for local and global optimization of multivariate systems.  However, I have a very long optimization run needed that may be interrupted (and in some cases I may want to interrupt it deliberately).  Is there any way to restart... well, any of them?  I mean, clearly one can provide the last, most optimized set of parameters found as the initial guess, but that's not the only parameter in play - for example, there are also gradients (jacobians, for example), populations in differential evolution, etc.  I obviously don't want these to have to start over as well.</p>

<p>I see little way to prove these to scipy, nor to save its state.  For functions that take a jacobian for example, there is a jacobian argument ("jac"), but it's either a boolean (indicating that your evaluation function returns a jacobian, which mine doesn't), or a callable function (I would only have the single result from the last run to provide).  Nothing takes just an array of the last jacobian available.  And with differential evolution, loss of the population would be horrible for performance and convergence.</p>

<p>Are there any solutions to this?  Any way to resume optimizations at all?</p>

<p>The following can save and restart from a previous <code>x</code>,
but I gather you want to save and restart more state, e.g. gradients, too; can you clarify that ?</p>

<p>See also <a href="http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.basinhopping.html" rel="nofollow">basinhopping</a>,
which has a nice-looking gui, <a href="https://github.com/pele-python/pele" rel="nofollow">pele-python</a> .</p>

<pre><code>#!/usr/bin/env python
""" Funcgradmn: wrap f() and grad(), save all x[] f[] grad[] to plot or restart """

from __future__ import division
import numpy as np

__version__ = "2016-10-18 oct denis"


class Funcgradmon(object):
    """ Funcgradmn: wrap f() and grad(), save all x[] f[] grad[] to plot or restart

    Example: minimize, save, restart --

    fg = Funcgradmon( func, gradfunc, verbose=1 )
        # fg(x): f(x), g(x)  for minimize( jac=True )

        # run 100 iter (if linesearch, 200-300 calls of fg()) --
    options = dict( maxiter=100 )  # ... 
    min0 = minimize( fg, x0, jac=True, options=options )
    fg.savez( "0.npz", paramstr="..." )  # to plot or restart

        # restart from x[50] --
        # (won't repeat the previous path from 50
        # unless you save and restore the whole state of the optimizer)
    x0 = fg.restart( 50 )
    # change params ...
    min50 = minimize( fg, x0, jac=True, options=options )
    """

    def __init__( self, func, gradfunc, verbose=1 ):
        self.func = func
        self.gradfunc = gradfunc
        self.verbose = verbose
        self.x, self.f, self.g = [], [], []  # growing lists
        self.t = 0

    def __call__( self, x ):
        """ f, g = func(x), gradfunc(x); save them; return f, g """
        x = np.asarray_chkfinite( x )  # always
        f = self.func(x)
        g = self.gradfunc(x)
        g = np.asarray_chkfinite( g )
        self.x.append( np.copy(x) )
        self.f.append( _copy( f ))
        self.g.append( np.copy(g) )
        if self.verbose:
            print "%3d:" % self.t ,
            fmt = "%-12g" if np.isscalar(f)  else "%s\t"
            print fmt % f ,
            print "x: %s" % x ,  # with user's np.set_printoptions
            print "\tgrad: %s" % g
                # better df dx dg
        # callback: plot
        self.t += 1
        return f, g

    def restart( self, n ):
        """ x0 = fg.restart( n )  returns x[n] to minimize( fg, x0 )
        """
        x0 = self.x[n]  # minimize from here
        del self.x[:n]
        del self.f[:n]
        del self.g[:n]
        self.t = n
        if self.verbose:
            print "Funcgradmon: restart from x[%d] %s" % (n, x0)
        return x0

    def savez( self, npzfile, **kw ):
        """ np.savez( npzfile, x= f= g= ) """
        x, f, g = map( np.array, [self.x, self.f, self.g] )
        if self.verbose:
            asum = "f: %s \nx: %s \ng: %s" % (
                _asum(f), _asum(x), _asum(g) )
            print "Funcgradmon: saving to %s: \n%s \n" % (npzfile, asum)
        np.savez( npzfile, x=x, f=f, g=g, **kw )

    def load( self, npzfile ):
        load = np.load( npzfile )
        x, f, g = load["x"], load["f"], load["g"]
        if self.verbose:
            asum = "f: %s \nx: %s \ng: %s" % (
                _asum(f), _asum(x), _asum(g) )
            print "Funcgradmon: load %s: \n%s \n" % (npzfile, asum)
        self.x = list( x )
        self.f = list( f )
        self.g = list( g )
        self.loaddict = load
        return self.restart( len(x) - 1 )


def _asum( X ):
    """ one-line array summary: "shape type min av max" """
    if not hasattr( X, "dtype" ):
        return str(X)
    return "%s %s  min av max %.3g %.3g %.3g" % (
            X.shape, X.dtype, X.min(), X.mean(), X.max() )

def _copy( x ):
    return x if x is None  or np.isscalar(x) \
        else np.copy( x )

#...............................................................................
if __name__ == "__main__":
    import sys
    from scipy.optimize import minimize, rosen, rosen_der

    np.set_printoptions( threshold=20, edgeitems=10, linewidth=140,
            formatter = dict( float = lambda x: "%.3g" % x ))  # float arrays %.3g

    dim = 3
    method = "cg"
    maxiter = 10  # 1 linesearch -&gt; 2-3 calls of fg

    # to change these params, run this.py a=1 b=None 'c = ...'  in sh or ipython
    for arg in sys.argv[1:]:
        exec( arg )

    print "\n", 80 * "-"
    print "Funcgradmon: dim %d  method %s  maxiter %d \n" % (
            dim, method, maxiter )
    x0 = np.zeros( dim )

    #...........................................................................
    fg = Funcgradmon( rosen, rosen_der, verbose=1 )
    options = dict( maxiter=maxiter )  # ... 

    min0 = minimize( fg, x0, jac=True, method=method, options=options )
    fg.savez( "0.npz", paramstr="..." )  # to plot or restart

    x0 = fg.restart( 5 )  # = fg.x[5]
    # change params, print them all
    min5 = minimize( fg, x0, jac=True, method=method, options=options )

    fg.savez( "5.npz", paramstr="..." )
</code></pre>

python scipy mathematical-optimization