From 895409e7203e3f959ece5eb53c49be3a3f03e3a0 Mon Sep 17 00:00:00 2001 From: Daniel Rasmussen Date: Wed, 8 Jan 2020 12:11:25 -0400 Subject: [PATCH] Support for transform=None --- CHANGES.rst | 12 +++++++++--- nengo_dl/callbacks.py | 7 ++++++- nengo_dl/compat.py | 18 ++++++++++++++++++ nengo_dl/losses.py | 2 +- nengo_dl/simulator.py | 17 +++++++++++++---- nengo_dl/tensor_graph.py | 24 +++++++++++++++++------- nengo_dl/tensor_node.py | 3 ++- nengo_dl/tests/dummies.py | 2 +- nengo_dl/tests/test_benchmarks.py | 11 +++++------ nengo_dl/tests/test_keras.py | 4 ++-- nengo_dl/tests/test_simulator.py | 26 ++++++++++++++++---------- nengo_dl/tests/test_tensor_node.py | 12 ++++++++++-- nengo_dl/version.py | 4 ++-- 13 files changed, 102 insertions(+), 40 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index dd355b039..78ca68fe2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,7 +18,7 @@ Release history - Deprecated - Removed -3.0.1 (unreleased) +3.1.0 (unreleased) ------------------ **Added** @@ -41,6 +41,10 @@ Release history ``nengo_dl.configure_settings(dtype=...)`` config option. Note that this will override the default precision set in ``nengo.rc``. (`#119`_) - Minimum Numpy version is now 1.16.0 (required by TensorFlow). (`#119`_) +- Added support for the new ``transform=None`` default in Nengo connections + (see `#1591`_). Note that this may change the number of trainable parameters in a + network as the scalar default ``transform=1`` weights on non-Ensemble connections + will no longer be present. (`#128`_) **Fixed** @@ -54,6 +58,8 @@ Release history correctly (previously it was always set to the default value). (`#119`_) .. _#119: https://github.com/nengo/nengo-dl/pull/119 +.. _#1591: https://github.com/nengo/nengo/pull/1591 +.. _#128: https://github.com/nengo/nengo-dl/pull/128 3.0.0 (December 17, 2019) ------------------------- @@ -417,7 +423,7 @@ details. ``sim.loss``/``sim.train`` data argument, if no input/target data is required. - The ``objective`` dict in ``sim.train``/``sim.loss`` can now contain - tuples of probes as the keys, in which case the objective function will be + tuples of probes as the keys, in which case the objective function will be called with a corresponding tuple of probe/target values as each argument. - Added the ``sim.run_batch`` function. This exposes all the functionality that the ``sim.run``/``sim.train``/``sim.loss`` functions are based on, @@ -744,7 +750,7 @@ details. - Fixed a bug where input nodes that were only read as a view were not feedable - Updated ``tensorflow-gpu`` installation check -- Improved numerical stability of ``LIFRate`` gradients (`#26 +- Improved numerical stability of ``LIFRate`` gradients (`#26 `_) - Added more informative error message when data is provided with fewer items than ``sim.minibatch_size`` (`#30 `_) diff --git a/nengo_dl/callbacks.py b/nengo_dl/callbacks.py index 61f579400..b0844d339 100644 --- a/nengo_dl/callbacks.py +++ b/nengo_dl/callbacks.py @@ -17,7 +17,7 @@ import tensorflow as tf from tensorflow.python.eager import context -from nengo_dl import utils +from nengo_dl import compat, utils class NengoSummaries(tf.keras.callbacks.Callback): @@ -61,6 +61,11 @@ def __init__(self, log_dir, sim, objects): param = "bias" name = "Ensemble.neurons_%s" % obj.ensemble.label elif isinstance(obj, nengo.Connection): + if not compat.conn_has_weights(obj): + raise ValidationError( + "Connection '%s' does not have any weights to log" % obj, + "objects", + ) param = "weights" name = "Connection_%s" % obj.label diff --git a/nengo_dl/compat.py b/nengo_dl/compat.py index 618199cb8..213214dcd 100644 --- a/nengo_dl/compat.py +++ b/nengo_dl/compat.py @@ -5,6 +5,9 @@ dependencies. """ +from distutils.version import LooseVersion + +import nengo from nengo._vendor.scipy.sparse import linalg_interface, linalg_onenormest import tensorflow as tf @@ -75,3 +78,18 @@ def filter(self, record): # monkeypatch fix for https://github.com/nengo/nengo/pull/1587 linalg_onenormest.aslinearoperator = linalg_interface.aslinearoperator + +if LooseVersion(nengo.__version__) < "3.1.0": + default_transform = 1 + + def conn_has_weights(conn): + """All connections have weights.""" + return True + + +else: + default_transform = None + + def conn_has_weights(conn): + """Equivalent to conn.has_weights.""" + return conn.has_weights diff --git a/nengo_dl/losses.py b/nengo_dl/losses.py index db54dd2f9..6c80ca5f0 100644 --- a/nengo_dl/losses.py +++ b/nengo_dl/losses.py @@ -41,7 +41,7 @@ class Regularize(tf.losses.Loss): with nengo.Network() as net: a = nengo.Node([0]) b = nengo.Node(size_in=1) - c = nengo.Connection(a, b) + c = nengo.Connection(a, b, transform=1) p = nengo.Probe(c, "weights") ... diff --git a/nengo_dl/simulator.py b/nengo_dl/simulator.py index bc1d2327a..d95b0b971 100644 --- a/nengo_dl/simulator.py +++ b/nengo_dl/simulator.py @@ -30,7 +30,7 @@ import tensorflow as tf from tensorflow.python.keras import backend -from nengo_dl import utils, config, callbacks +from nengo_dl import callbacks, compat, config, utils from nengo_dl.builder import NengoBuilder, NengoModel from nengo_dl.tensor_graph import TensorGraph @@ -1346,7 +1346,8 @@ def get_nengo_params(self, nengo_objs, as_dict=False): fetches = [] for obj in nengo_objs: if isinstance(obj, Connection): - fetches.append((obj, "weights")) + if compat.conn_has_weights(obj): + fetches.append((obj, "weights")) elif isinstance(obj, Ensemble): if isinstance(obj.neuron_type, Direct): # we cannot transfer direct ensemble parameters, because @@ -1372,6 +1373,10 @@ def get_nengo_params(self, nengo_objs, as_dict=False): idx = 0 for obj in nengo_objs: if isinstance(obj, Connection): + if not compat.conn_has_weights(obj): + params.append({"transform": None}) + continue + weights = data[idx] idx += 1 if isinstance(obj.pre_obj, Ensemble): @@ -1381,7 +1386,7 @@ def get_nengo_params(self, nengo_objs, as_dict=False): "function": lambda x, weights=weights: np.zeros( weights.shape[0] ), - "transform": 1, + "transform": compat.default_transform, } ) elif isinstance(obj.transform, Convolution): @@ -2104,7 +2109,11 @@ def __getitem__(self, obj): ) elif isinstance(obj, Connection): # get the live simulation values - weights = self.get_params((obj, "weights"))[0] + weights = ( + self.get_params((obj, "weights"))[0] + if compat.conn_has_weights(obj) + else None + ) # impossible to recover transform transform = None diff --git a/nengo_dl/tensor_graph.py b/nengo_dl/tensor_graph.py index 453856a5a..e604c087d 100644 --- a/nengo_dl/tensor_graph.py +++ b/nengo_dl/tensor_graph.py @@ -19,7 +19,15 @@ from tensorflow.python.eager import context from tensorflow.python.training.tracking import base as trackable -from nengo_dl import builder, graph_optimizer, signals, utils, tensor_node, config +from nengo_dl import ( + builder, + config, + compat, + graph_optimizer, + tensor_node, + signals, + utils, +) logger = logging.getLogger(__name__) @@ -856,10 +864,11 @@ def mark_network(parent_configs, net): for conn in net.connections: # note: this doesn't include probe connections, since they # aren't added to the network - self.model.sig[conn]["weights"].trainable = get_trainable( - parent_configs, conn - ) - self.model.sig[conn]["weights"].minibatched = False + if compat.conn_has_weights(conn): + self.model.sig[conn]["weights"].trainable = get_trainable( + parent_configs, conn + ) + self.model.sig[conn]["weights"].minibatched = False # parameters can't be modified by an online Nengo learning rule # and offline training at the same time. (it is possible in @@ -909,8 +918,9 @@ def mark_network(parent_configs, net): probe_seeds = [self.model.seeds[p] for p in self.model.probes] for obj, seed in self.model.seeds.items(): if isinstance(obj, Connection) and seed in probe_seeds: - self.model.sig[obj]["weights"].trainable = False - self.model.sig[obj]["weights"].minibatched = False + if compat.conn_has_weights(obj): + self.model.sig[obj]["weights"].trainable = False + self.model.sig[obj]["weights"].minibatched = False # time/step are not minibatched and not trainable self.model.step.trainable = False diff --git a/nengo_dl/tensor_node.py b/nengo_dl/tensor_node.py index 2567fda2d..b94acc228 100644 --- a/nengo_dl/tensor_node.py +++ b/nengo_dl/tensor_node.py @@ -20,6 +20,7 @@ import tensorflow as tf from nengo_dl.builder import Builder, OpBuilder, NengoBuilder +from nengo_dl.compat import default_transform from nengo_dl.config import configure_settings @@ -403,7 +404,7 @@ def __init__(self, layer_func): def __call__( self, input, - transform=1, + transform=default_transform, shape_in=None, synapse=None, return_conn=False, diff --git a/nengo_dl/tests/dummies.py b/nengo_dl/tests/dummies.py index 379072029..72f261819 100644 --- a/nengo_dl/tests/dummies.py +++ b/nengo_dl/tests/dummies.py @@ -157,7 +157,7 @@ def linear_net(): with nengo.Network() as net: a = nengo.Node([1]) b = nengo.Node(size_in=1) - nengo.Connection(a, b, synapse=None) + nengo.Connection(a, b, synapse=None, transform=1) p = nengo.Probe(b) return net, a, p diff --git a/nengo_dl/tests/test_benchmarks.py b/nengo_dl/tests/test_benchmarks.py index 70c86a31d..98a28d26c 100644 --- a/nengo_dl/tests/test_benchmarks.py +++ b/nengo_dl/tests/test_benchmarks.py @@ -192,8 +192,8 @@ def test_lmu(Simulator, native_nengo, pytestconfig): [ (benchmarks.cconv(128, 64, nengo.RectifiedLinear()), False, 64, 0.65, 0.8), (benchmarks.cconv(128, 64, nengo.LIF()), False, 64, 1.45, 1.65), - (benchmarks.integrator(128, 32, nengo.RectifiedLinear()), True, 64, 0.5, 0.75), - (benchmarks.integrator(128, 32, nengo.LIF()), True, 64, 0.9, 1.2), + (benchmarks.integrator(128, 32, nengo.RectifiedLinear()), True, 64, 0.6, 1.0), + (benchmarks.integrator(128, 32, nengo.LIF()), True, 64, 1.1, 1.4), ( benchmarks.random_network( 64, @@ -208,8 +208,7 @@ def test_lmu(Simulator, native_nengo, pytestconfig): 0.35, 0.55, ), - (benchmarks.lmu(1000, 1, native_nengo=True), True, 100, 0.75, 1.05), - # (benchmarks.spaun(1), False, None, 8.02, 9.52), + (benchmarks.lmu(1000, 1, native_nengo=True), True, 100, 0.85, 1.15), ], ) def test_performance(net, train, minibatch_size, min, max): @@ -218,8 +217,8 @@ def test_performance(net, train, minibatch_size, min, max): # GPU: GeForce GTX Titan X # Python version: 3.6.8 # TensorFlow GPU version: 2.0.0 - # Nengo version: 3.0.0 - # NengoDL version: 3.0.0 + # Nengo version: 3.1.0 + # NengoDL version: 3.1.0 time = benchmarks.run_profile( net, diff --git a/nengo_dl/tests/test_keras.py b/nengo_dl/tests/test_keras.py index 3b5b5fd87..4ba225774 100644 --- a/nengo_dl/tests/test_keras.py +++ b/nengo_dl/tests/test_keras.py @@ -236,8 +236,8 @@ def test_fit(Simulator, seed): inp_a = nengo.Node([0]) inp_b = nengo.Node([0]) inp = nengo.Node(size_in=2) - nengo.Connection(inp_a, inp[0]) - nengo.Connection(inp_b, inp[1]) + nengo.Connection(inp_a, inp[0], transform=1) + nengo.Connection(inp_b, inp[1], transform=1) ens = nengo.Ensemble( n_hidden + 1, n_hidden, neuron_type=nengo.Sigmoid(tau_ref=1) diff --git a/nengo_dl/tests/test_simulator.py b/nengo_dl/tests/test_simulator.py index f66918fa8..eb2f9773e 100644 --- a/nengo_dl/tests/test_simulator.py +++ b/nengo_dl/tests/test_simulator.py @@ -1,6 +1,7 @@ # pylint: disable=missing-docstring from collections import OrderedDict +from distutils.version import LooseVersion import logging import os import pickle @@ -20,7 +21,7 @@ from tensorflow.core.util import event_pb2 from nengo_dl import Layer, TensorNode, callbacks, configure_settings, dists, utils -from nengo_dl.compat import TFLogFilter +from nengo_dl.compat import TFLogFilter, default_transform from nengo_dl.simulator import SimulationData from nengo_dl.tests import dummies @@ -167,8 +168,8 @@ def test_train_ff(Simulator, neurons, use_loop, seed): inp_a = nengo.Node([0]) inp_b = nengo.Node([0]) inp = nengo.Node(size_in=2) - nengo.Connection(inp_a, inp[0]) - nengo.Connection(inp_b, inp[1]) + nengo.Connection(inp_a, inp[0], transform=1) + nengo.Connection(inp_b, inp[1], transform=1) ens = nengo.Ensemble( n_hidden + 1, n_hidden, neuron_type=nengo.Sigmoid(tau_ref=1) @@ -300,11 +301,11 @@ def test_train_objective(Simulator, unroll, seed): inp = nengo.Node([1]) ens = nengo.Ensemble(n_hidden, 1, neuron_type=nengo.RectifiedLinear()) - nengo.Connection(inp, ens, synapse=0.01) + nengo.Connection(inp, ens, synapse=0.01, transform=1) p = nengo.Probe(ens) ens2 = nengo.Ensemble(n_hidden, 1, neuron_type=nengo.RectifiedLinear()) - nengo.Connection(inp, ens2, synapse=0.01) + nengo.Connection(inp, ens2, synapse=0.01, transform=1) p2 = nengo.Probe(ens2) with Simulator( @@ -688,7 +689,8 @@ def test_tensorboard(Simulator, tmpdir): with nengo.Network() as net: a = nengo.Node([0]) b = nengo.Ensemble(10, 1, neuron_type=nengo.LIFRate()) - c = nengo.Connection(a, b) + c = nengo.Connection(a, b, transform=1) + c0 = nengo.Connection(a, b) p = nengo.Probe(b) p2 = nengo.Probe(c) @@ -768,6 +770,10 @@ def test_tensorboard(Simulator, tmpdir): with pytest.raises(ValidationError, match="Unknown summary object"): callbacks.NengoSummaries(log_dir=log_dir + "/nengo", sim=sim, objects=[a]) + if LooseVersion(nengo.__version__) >= "3.1.0": + with pytest.raises(ValidationError, match="does not have any weights"): + callbacks.NengoSummaries(log_dir=log_dir + "/nengo", sim=sim, objects=[c0]) + @pytest.mark.parametrize("mode", ("predict", "train")) @pytest.mark.training @@ -1259,7 +1265,7 @@ def test_get_nengo_params(Simulator, seed): # check that single objects are returned as single dicts params = sim.get_nengo_params(d) - assert params["transform"] == 1 + assert params["transform"] is default_transform fetches = [a.neurons, b, c, d, e, h] @@ -1290,7 +1296,7 @@ def test_direct_grads(Simulator, mixed): if mixed: with net: c = nengo.Node(size_in=1) - nengo.Connection(a, c, synapse=None) + nengo.Connection(a, c, synapse=None, transform=1) p2 = nengo.Probe(c) with Simulator(net, minibatch_size=1) as sim: @@ -1328,7 +1334,7 @@ def test_non_differentiable(Simulator): with nengo.Network() as net: a = nengo.Node([0]) b = nengo.Node(lambda t, x: x, size_in=1) - c = nengo.Connection(a, b) + c = nengo.Connection(a, b, transform=1) p = nengo.Probe(b) with Simulator(net) as sim: @@ -1429,7 +1435,7 @@ def test_inference_only(Simulator, neuron_type, seed): a = nengo.Node([0]) b = nengo.Ensemble(10, 1, neuron_type=neuron_type) - c = nengo.Connection(a, b, synapse=None) + c = nengo.Connection(a, b, synapse=None, transform=1) p = nengo.Probe(b) with Simulator(net) as sim: diff --git a/nengo_dl/tests/test_tensor_node.py b/nengo_dl/tests/test_tensor_node.py index aeb4a2cee..d97e4b0f8 100644 --- a/nengo_dl/tests/test_tensor_node.py +++ b/nengo_dl/tests/test_tensor_node.py @@ -1,5 +1,6 @@ # pylint: disable=missing-docstring +from distutils.version import LooseVersion from functools import partial import nengo @@ -261,12 +262,19 @@ def call(self, x): assert np.allclose(sim.data[p2], 3) # note: when inference-only=True the weights will be marked as non-trainable + + default_conn_params = 2 if LooseVersion(nengo.__version__) < "3.1.0" else 0 + if sim.tensor_graph.inference_only: - assert len(sim.keras_model.non_trainable_variables) == 10 + assert ( + len(sim.keras_model.non_trainable_variables) == 8 + default_conn_params + ) assert len(sim.keras_model.trainable_variables) == 0 vars = sim.keras_model.non_trainable_variables[-2:] else: - assert len(sim.keras_model.non_trainable_variables) == 8 + assert ( + len(sim.keras_model.non_trainable_variables) == 6 + default_conn_params + ) assert len(sim.keras_model.trainable_variables) == 2 vars = sim.keras_model.trainable_variables diff --git a/nengo_dl/version.py b/nengo_dl/version.py index 2146cba12..e544fcd5f 100644 --- a/nengo_dl/version.py +++ b/nengo_dl/version.py @@ -8,7 +8,7 @@ import warnings name = "nengo-dl" -version_info = (3, 0, 1) # (major, minor, patch) +version_info = (3, 1, 0) # (major, minor, patch) dev = 0 # set to None for releases version = "{v}{dev}".format( @@ -28,7 +28,7 @@ # for release versions of nengo-dl, this should be the latest released # nengo version. for dev versions of nengo-dl, this should be the current # nengo dev version. - latest_nengo_version = (3, 0, 1) + latest_nengo_version = (3, 1, 0) if nengo.version.version_info < minimum_nengo_version: # pragma: no cover raise ValueError(