Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for transform=None #128

Merged
merged 3 commits into from
Mar 4, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Release history
- Deprecated
- Removed

3.0.1 (unreleased)
3.1.0 (unreleased)
------------------

**Added**
Expand All @@ -41,6 +41,10 @@ Release history
``nengo_dl.configure_settings(dtype=...)`` config option. Note that this will
override the default precision set in ``nengo.rc``. (`#119`_)
- Minimum Numpy version is now 1.16.0 (required by TensorFlow). (`#119`_)
- Added support for the new ``transform=None`` default in Nengo connections
(see `Nengo#1591`_). Note that this may change the number of trainable
parameters in a network as the scalar default ``transform=1`` weights on
non-Ensemble connections will no longer be present. (`#128`_)

**Fixed**

Expand All @@ -52,8 +56,12 @@ Release history
submodels. (`#119`_)
- Keras Layers inside TensorNodes will be called with the ``training`` argument set
correctly (previously it was always set to the default value). (`#119`_)
- Fixed compatibility with ``progressbar2`` version 3.50.0. (`#136`_)

.. _#119: https://github.com/nengo/nengo-dl/pull/119
.. _#128: https://github.com/nengo/nengo-dl/pull/128
.. _Nengo#1591: https://github.com/nengo/nengo/pull/1591
.. _#136: https://github.com/nengo/nengo-dl/pull/136

3.0.0 (December 17, 2019)
-------------------------
Expand Down Expand Up @@ -417,7 +425,7 @@ details.
``sim.loss``/``sim.train`` data argument, if no input/target data is
required.
- The ``objective`` dict in ``sim.train``/``sim.loss`` can now contain
tuples of probes as the keys, in which case the objective function will be
tuples of probes as the keys, in which case the objective function will be
called with a corresponding tuple of probe/target values as each argument.
- Added the ``sim.run_batch`` function. This exposes all the functionality
that the ``sim.run``/``sim.train``/``sim.loss`` functions are based on,
Expand Down Expand Up @@ -744,7 +752,7 @@ details.
- Fixed a bug where input nodes that were only read as a view were not
feedable
- Updated ``tensorflow-gpu`` installation check
- Improved numerical stability of ``LIFRate`` gradients (`#26
- Improved numerical stability of ``LIFRate`` gradients (`#26
<https://github.com/nengo/nengo-dl/issues/26>`_)
- Added more informative error message when data is provided with fewer items
than ``sim.minibatch_size`` (`#30 <https://github.com/nengo/nengo-dl/issues/30>`_)
Expand Down
7 changes: 6 additions & 1 deletion nengo_dl/callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import tensorflow as tf
from tensorflow.python.eager import context

from nengo_dl import utils
from nengo_dl import compat, utils


class NengoSummaries(tf.keras.callbacks.Callback):
Expand Down Expand Up @@ -61,6 +61,11 @@ def __init__(self, log_dir, sim, objects):
param = "bias"
name = "Ensemble.neurons_%s" % obj.ensemble.label
elif isinstance(obj, nengo.Connection):
if not compat.conn_has_weights(obj):
raise ValidationError(
"Connection '%s' does not have any weights to log" % obj,
"objects",
)
param = "weights"
name = "Connection_%s" % obj.label

Expand Down
18 changes: 18 additions & 0 deletions nengo_dl/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
dependencies.
"""

from distutils.version import LooseVersion

import nengo
from nengo._vendor.scipy.sparse import linalg_interface, linalg_onenormest
import tensorflow as tf

Expand Down Expand Up @@ -75,3 +78,18 @@ def filter(self, record):

# monkeypatch fix for https://github.com/nengo/nengo/pull/1587
linalg_onenormest.aslinearoperator = linalg_interface.aslinearoperator

if LooseVersion(nengo.__version__) < "3.1.0":
default_transform = 1

def conn_has_weights(conn):
"""All connections have weights."""
return True


else:
default_transform = None

def conn_has_weights(conn):
"""Equivalent to conn.has_weights."""
return conn.has_weights
2 changes: 1 addition & 1 deletion nengo_dl/losses.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class Regularize(tf.losses.Loss):
with nengo.Network() as net:
a = nengo.Node([0])
b = nengo.Node(size_in=1)
c = nengo.Connection(a, b)
c = nengo.Connection(a, b, transform=1)
p = nengo.Probe(c, "weights")
...

Expand Down
26 changes: 9 additions & 17 deletions nengo_dl/op_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ def __init__(self, ops, signals, config):
if np.issubdtype(dtype, np.floating):
dtype = signals.dtype.as_numpy_dtype

# unlike other ops, Reset signals might be spread across multiple
# bases, which we need to handle
# Reset signals might be spread across multiple bases, so group them
# by the ones that do share a base
scatters = defaultdict(list)
for op in ops:
scatters[signals[op.dst].key].append(op)
Expand All @@ -73,7 +73,7 @@ def __init__(self, ops, signals, config):
],
axis=1,
)
self.scatters += [(signals.combine([x.dst for x in group]), value)]
self.scatters.append((signals.combine([x.dst for x in group]), value))

logger.debug("scatters")
logger.debug("\n".join([str(x) for x in self.scatters]))
Expand Down Expand Up @@ -101,24 +101,16 @@ def __init__(self, ops, signals, config):
logger.debug("dst %s", [op.dst for op in ops])
logger.debug("dst_slice %s", [getattr(op, "dst_slice", None) for op in ops])

srcs = []
dsts = []
for op in ops:
srcs += [signals[op.src][op.src_slice]]
dsts += [signals[op.dst][op.dst_slice]]
self.src_data = signals.combine([signals[op.src][op.src_slice] for op in ops])
self.dst_data = signals.combine([signals[op.dst][op.dst_slice] for op in ops])

self.mode = "inc" if ops[0].inc else "update"

self.src_data = signals.combine(srcs)
self.dst_data = signals.combine(dsts)

if not self.src_data.minibatched and self.dst_data.minibatched:
# broadcast indices so that the un-minibatched src data gets
# copied to each minibatch dimension in dst
self.src_data = self.src_data.broadcast(signals.minibatch_size)

def build_step(self, signals):
signals.scatter(self.dst_data, signals.gather(self.src_data), mode=self.mode)
src = signals.gather(self.src_data)
if not self.src_data.minibatched and self.dst_data.minibatched:
src = tf.broadcast_to(src, self.dst_data.full_shape)
signals.scatter(self.dst_data, src, mode=self.mode)

@staticmethod
def mergeable(x, y):
Expand Down
28 changes: 0 additions & 28 deletions nengo_dl/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,34 +171,6 @@ def reshape(self, shape):
label=self.label + ".reshape(%s)" % (shape,),
)

def broadcast(self, length):
"""
Add a new dimension by broadcasting this signal along the first axis
for the given length.

Parameters
----------
length : int
The number of times to duplicate signal along the first dimension.

Returns
-------
sig : `.signals.TensorSignal`
TensorSignal with new broadcasted shape
"""

# this only works on vectors
assert self.ndim == 1 and not self.minibatched

return TensorSignal(
self.slices * length,
self.key,
self.dtype,
(length,) + self.shape,
self.minibatch_size,
label=self.label + ".broadcast(%d)" % length,
)

@property
def tf_shape(self):
"""
Expand Down
17 changes: 13 additions & 4 deletions nengo_dl/simulator.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import tensorflow as tf
from tensorflow.python.keras import backend

from nengo_dl import utils, config, callbacks
from nengo_dl import callbacks, compat, config, utils
from nengo_dl.builder import NengoBuilder, NengoModel
from nengo_dl.tensor_graph import TensorGraph

Expand Down Expand Up @@ -1346,7 +1346,8 @@ def get_nengo_params(self, nengo_objs, as_dict=False):
fetches = []
for obj in nengo_objs:
if isinstance(obj, Connection):
fetches.append((obj, "weights"))
if compat.conn_has_weights(obj):
fetches.append((obj, "weights"))
elif isinstance(obj, Ensemble):
if isinstance(obj.neuron_type, Direct):
# we cannot transfer direct ensemble parameters, because
Expand All @@ -1372,6 +1373,10 @@ def get_nengo_params(self, nengo_objs, as_dict=False):
idx = 0
for obj in nengo_objs:
if isinstance(obj, Connection):
if not compat.conn_has_weights(obj):
params.append({"transform": None})
continue

weights = data[idx]
idx += 1
if isinstance(obj.pre_obj, Ensemble):
Expand All @@ -1381,7 +1386,7 @@ def get_nengo_params(self, nengo_objs, as_dict=False):
"function": lambda x, weights=weights: np.zeros(
weights.shape[0]
),
"transform": 1,
"transform": compat.default_transform,
}
)
elif isinstance(obj.transform, Convolution):
Expand Down Expand Up @@ -2104,7 +2109,11 @@ def __getitem__(self, obj):
)
elif isinstance(obj, Connection):
# get the live simulation values
weights = self.get_params((obj, "weights"))[0]
weights = (
self.get_params((obj, "weights"))[0]
if compat.conn_has_weights(obj)
else None
)

# impossible to recover transform
transform = None
Expand Down
46 changes: 30 additions & 16 deletions nengo_dl/tensor_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import warnings

from nengo import Connection, Process
from nengo.builder.operator import SimPyFunc, Reset
from nengo.builder.operator import Reset, SimPyFunc
from nengo.builder.processes import SimProcess
from nengo.config import ConfigError
from nengo.exceptions import BuildError
Expand All @@ -19,7 +19,15 @@
from tensorflow.python.eager import context
from tensorflow.python.training.tracking import base as trackable

from nengo_dl import builder, graph_optimizer, signals, utils, tensor_node, config
from nengo_dl import (
builder,
config,
compat,
graph_optimizer,
tensor_node,
signals,
utils,
)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -856,10 +864,11 @@ def mark_network(parent_configs, net):
for conn in net.connections:
# note: this doesn't include probe connections, since they
# aren't added to the network
self.model.sig[conn]["weights"].trainable = get_trainable(
parent_configs, conn
)
self.model.sig[conn]["weights"].minibatched = False
if compat.conn_has_weights(conn):
self.model.sig[conn]["weights"].trainable = get_trainable(
parent_configs, conn
)
self.model.sig[conn]["weights"].minibatched = False

# parameters can't be modified by an online Nengo learning rule
# and offline training at the same time. (it is possible in
Expand Down Expand Up @@ -909,8 +918,9 @@ def mark_network(parent_configs, net):
probe_seeds = [self.model.seeds[p] for p in self.model.probes]
for obj, seed in self.model.seeds.items():
if isinstance(obj, Connection) and seed in probe_seeds:
self.model.sig[obj]["weights"].trainable = False
self.model.sig[obj]["weights"].minibatched = False
if compat.conn_has_weights(obj):
self.model.sig[obj]["weights"].trainable = False
self.model.sig[obj]["weights"].minibatched = False

# time/step are not minibatched and not trainable
self.model.step.trainable = False
Expand Down Expand Up @@ -956,14 +966,18 @@ def create_signals(self, sigs):
breaks = []
diff = defaultdict(int)
for ops in self.plan:
# note: we don't include Resets, otherwise the big reset block
# overrides most of the partitioning
if not isinstance(ops[0], Reset):
for i in range(len(ops[0].all_signals)):
op_sigs = [op.all_signals[i].base for op in ops]
idxs = [sig_idxs[s] for s in op_sigs]
diff[op_sigs[np.argmin(idxs)]] += 1
diff[op_sigs[np.argmax(idxs)]] -= 1
if isinstance(ops[0], Reset):
# don't include Resets, otherwise the big reset block
# overrides most of the partitioning
partition_sigs = []
else:
partition_sigs = range(len(ops[0].all_signals))

for i in partition_sigs:
op_sigs = [op.all_signals[i].base for op in ops]
idxs = [sig_idxs[s] for s in op_sigs]
diff[op_sigs[np.argmin(idxs)]] += 1
diff[op_sigs[np.argmax(idxs)]] -= 1

# find the partition points in signal list
open = 0
Expand Down
3 changes: 2 additions & 1 deletion nengo_dl/tensor_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import tensorflow as tf

from nengo_dl.builder import Builder, OpBuilder, NengoBuilder
from nengo_dl.compat import default_transform
from nengo_dl.config import configure_settings


Expand Down Expand Up @@ -403,7 +404,7 @@ def __init__(self, layer_func):
def __call__(
self,
input,
transform=1,
transform=default_transform,
shape_in=None,
synapse=None,
return_conn=False,
Expand Down
2 changes: 1 addition & 1 deletion nengo_dl/tests/dummies.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def linear_net():
with nengo.Network() as net:
a = nengo.Node([1])
b = nengo.Node(size_in=1)
nengo.Connection(a, b, synapse=None)
nengo.Connection(a, b, synapse=None, transform=1)
p = nengo.Probe(b)

return net, a, p
13 changes: 6 additions & 7 deletions nengo_dl/tests/test_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,10 +190,10 @@ def test_lmu(Simulator, native_nengo, pytestconfig):
@pytest.mark.parametrize(
"net, train, minibatch_size, min, max",
[
(benchmarks.cconv(128, 64, nengo.RectifiedLinear()), False, 64, 0.7, 0.85),
(benchmarks.cconv(128, 64, nengo.RectifiedLinear()), False, 64, 0.65, 0.8),
(benchmarks.cconv(128, 64, nengo.LIF()), False, 64, 1.45, 1.65),
(benchmarks.integrator(128, 32, nengo.RectifiedLinear()), True, 64, 0.5, 0.75),
(benchmarks.integrator(128, 32, nengo.LIF()), True, 64, 0.9, 1.2),
(benchmarks.integrator(128, 32, nengo.RectifiedLinear()), True, 64, 0.6, 1.0),
(benchmarks.integrator(128, 32, nengo.LIF()), True, 64, 1.1, 1.4),
(
benchmarks.random_network(
64,
Expand All @@ -208,8 +208,7 @@ def test_lmu(Simulator, native_nengo, pytestconfig):
0.35,
0.55,
),
(benchmarks.lmu(1000, 1, native_nengo=True), True, 100, 0.75, 1.05),
# (benchmarks.spaun(1), False, None, 8.02, 9.52),
(benchmarks.lmu(1000, 1, native_nengo=True), True, 100, 0.85, 1.15),
],
)
def test_performance(net, train, minibatch_size, min, max):
Expand All @@ -218,8 +217,8 @@ def test_performance(net, train, minibatch_size, min, max):
# GPU: GeForce GTX Titan X
# Python version: 3.6.8
# TensorFlow GPU version: 2.0.0
# Nengo version: 3.0.0
# NengoDL version: 3.0.0
# Nengo version: 3.1.0
# NengoDL version: 3.1.0

time = benchmarks.run_profile(
net,
Expand Down
Loading