-
Notifications
You must be signed in to change notification settings - Fork 16
/
fista.py
45 lines (34 loc) · 1.39 KB
/
fista.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import torch
from torch.optim import Optimizer
class FISTA(Optimizer):
def __init__(self, params, lr=1e-2, gamma=0.1):
defaults = dict(lr=lr, gamma=gamma)
super(FISTA, self).__init__(params, defaults)
def step(self, decay=1, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
state = self.state[p]
if 'alpha' not in state or decay:
state['alpha'] = torch.ones_like(p.data)
state['data'] = p.data
y = p.data
else:
alpha = state['alpha']
data = state['data']
state['alpha'] = (1 + (1 + 4 * alpha**2).sqrt()) / 2
y = p.data + ((alpha - 1) / state['alpha']) * (p.data - data)
state['data'] = p.data
mom = y - group['lr'] * grad
p.data = self._prox(mom, group['lr'] * group['gamma'])
# no-negative
p.data = torch.max(p.data, torch.zeros_like(p.data))
return loss
def _prox(self, x, gamma):
y = torch.max(torch.abs(x) - gamma, torch.zeros_like(x))
return torch.sign(x) * y