UnusedInputError

[uniwf2016]

Hi,

Thanks for reading this post.

Currently, I am trying to create my own network for reinforcement learning. To this end, I have adapted the Q network from
Playing Atari with Deep Reinforcement Learning
Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis
Antonoglou, Daan Wierstra, Martin Riedmiller
and
Mnih, Volodymyr, et al. "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533.

When theano tries to compile function for loss and q_val (),

self._train = theano.function([], [loss, q_vals], updates=updates,
givens=givens_train)
self._q_vals = theano.function([], q_vals,
givens=givens_q_val)

it keeps returning
UnusedInputError: theano.function was asked to create a function computing outputs given certain inputs, but the provided input variable at index 0 is not part of the computational graph needed to compute the outputs: <CudaNdarrayType(float32, 4D)>.
To make this error into a warning, you can pass the parameter on_unused_input='warn' to theano.function. To disable it completely, use on_unused_input='ignore'.

I have been debugging the code many many times, but I cannot understand why the inputs (from givens) are not used as part of the function/ calculation.

Many thanks in advance for your explanation.

Here is my full source code for the network:

"""
import lasagne
import numpy as np
import theano
import theano.tensor as T
from updates import deepmind_rmsprop
import logging

class DeepQLearner:
"""
Deep Q-learning network using Lasagne.
"""
def init(self, width_img,
height_img,
width_loc,
height_loc,
width_his,
height_his,
target_dis_size,
num_actions, num_frames, discount, learning_rate, rho,
rms_epsilon, momentum, clip_delta, freeze_interval,
batch_size, network_type, update_rule,
batch_accumulator, rng, input_scale=8.0):

    self.width_img = width_img
    self.height_img = height_img
    self.width_loc = width_loc
    self.height_loc = height_loc
    self.width_his = width_his
    self.height_his = height_his
    self.target_dis_size = target_dis_size

    self.num_actions = num_actions
    self.num_frames = num_frames
    self.batch_size = batch_size
    self.discount = discount
    self.rho = rho
    self.lr = learning_rate
    self.rms_epsilon = rms_epsilon
    self.momentum = momentum
    self.clip_delta = clip_delta
    self.freeze_interval = freeze_interval
    self.rng = rng

    self.logger = logging.getLogger(__name__)
    if not getattr(self.logger, 'handler_set', None):

        self.logger.setLevel(logging.DEBUG)
        # create a file handler

        handler = logging.FileHandler('toy.log', mode='a')
        handler.setLevel(logging.DEBUG)

        # create a logging format

        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        handler.setFormatter(formatter)

        # add the handlers to the logger

        self.logger.addHandler(handler)
        self.logger.handler_set = True
    self.logger.info('initialise a Q network.')

    lasagne.random.set_rng(self.rng)

    self.update_counter = 0

    self.l_out = self.build_network(network_type, num_actions, num_frames, batch_size)
    if self.freeze_interval > 0:
        self.next_l_out = self.build_network(network_type, num_actions,
                                             num_frames, batch_size)
        self.reset_q_hat()

    #states = T.tensor4('states')
    #next_states = T.tensor4('next_states')
    imgs = T.tensor4('imgs')
    next_imgs = T.tensor4('next_imgs')
    locs = T.tensor4('locs')
    next_locs = T.tensor4('next_locs')
    hiss = T.tensor4('hiss')
    next_hiss = T.tensor4('next_hiss')

target_distribution = T.tensor('target_distribution')

next_target_distribution = T.tensor('next_target_distribution')

    sds = T.icol('sds')
    next_sds  = T.icol('next_sds')

    rewards = T.col('rewards')
    actions = T.icol('actions')
    terminals = T.icol('terminals')

self.states_shared = theano.shared(

np.zeros((batch_size, num_frames, input_height, input_width),

dtype=theano.config.floatX))

self.next_states_shared = theano.shared(

np.zeros((batch_size, num_frames, input_height, input_width),

dtype=theano.config.floatX))

    self.imgs_shared = theano.shared(
            np.zeros((batch_size, num_frames, width_img, height_img),
                     dtype=theano.config.floatX))
    self.next_imgs_shared = theano.shared(
            np.zeros((batch_size, num_frames, width_img, height_img),
                     dtype=theano.config.floatX))

    self.locs_shared = theano.shared(
    np.zeros((batch_size, num_frames, width_loc, height_loc),
             dtype=theano.config.floatX))
    self.next_locs_shared = theano.shared(
    np.zeros((batch_size, num_frames, width_loc, height_loc),
             dtype=theano.config.floatX))
    self.hiss_shared = theano.shared(
    np.zeros((batch_size, num_frames, width_his, height_his),
             dtype=theano.config.floatX))
    self.next_hiss_shared = theano.shared(
    np.zeros((batch_size, num_frames, width_his, height_his),
             dtype=theano.config.floatX))
    self.sds_shared = theano.shared(
        np.zeros((batch_size, 1), dtype='int32'),
        broadcastable=(False, True))
    self.next_sds_shared= theano.shared(
        np.zeros((batch_size, 1), dtype='int32'),
        broadcastable=(False, True))

    self.rewards_shared = theano.shared(
        np.zeros((batch_size, 1), dtype=theano.config.floatX),
        broadcastable=(False, True))

    self.actions_shared = theano.shared(
        np.zeros((batch_size, 1), dtype='int32'),
        broadcastable=(False, True))

    self.terminals_shared = theano.shared(
        np.zeros((batch_size, 1), dtype='int32'),
        broadcastable=(False, True))

q_vals = lasagne.layers.get_output(self.l_out, states / input_scale)

    # massage/ unpack states into the right form for multi input network


    q_vals = lasagne.layers.get_output(self.l_out, {'l_in':imgs,
              'l_loc1':locs,  'l_his':hiss,
              'l_dis': sds})

    if self.freeze_interval > 0:

next_q_vals = lasagne.layers.get_output(self.next_l_out,

next_states / input_scale)

        next_q_vals = lasagne.layers.get_output(self.next_l_out,
                                                {'l_in':next_imgs,
                              'l_loc1':next_locs,  'l_his':next_hiss,
                              'l_dis': next_sds})
    else:
        next_q_vals = lasagne.layers.get_output(self.l_out,
                                                {'l_in':next_imgs,
                              'l_loc1':next_locs,  'l_his':next_hiss,
                              'l_dis': next_sds})

        next_q_vals = theano.gradient.disconnected_grad(next_q_vals)

    target = (rewards +
              (T.ones_like(terminals) - terminals) *
              self.discount * T.max(next_q_vals, axis=1, keepdims=True))
    diff = target - q_vals[T.arange(batch_size),
                           actions.reshape((-1,))].reshape((-1, 1))

    if self.clip_delta > 0:
        # If we simply take the squared clipped diff as our loss,
        # then the gradient will be zero whenever the diff exceeds
        # the clip bounds. To avoid this, we extend the loss
        # linearly past the clip point to keep the gradient constant
        # in that regime.
        # 
        # This is equivalent to declaring d loss/d q_vals to be
        # equal to the clipped diff, then backpropagating from
        # there, which is what the DeepMind implementation does.
        quadratic_part = T.minimum(abs(diff), self.clip_delta)
        linear_part = abs(diff) - quadratic_part
        loss = 0.5 * quadratic_part ** 2 + self.clip_delta * linear_part
    else:
        loss = 0.5 * diff ** 2

    if batch_accumulator == 'sum':
        loss = T.sum(loss)
    elif batch_accumulator == 'mean':
        loss = T.mean(loss)
    else:
        raise ValueError("Bad accumulator: {}".format(batch_accumulator))

    params = lasagne.layers.helper.get_all_params(self.l_out)  
    givens_train = {

states: self.states_shared,

next_states: self.next_states_shared,

        imgs :self.imgs_shared,
        next_imgs :self.next_imgs_shared, 
        locs: self.locs_shared ,
        next_locs :self.next_locs_shared,
        hiss :self.hiss_shared,
        next_hiss :self.next_hiss_shared ,
        sds : self.sds_shared ,
        next_sds : self.next_sds_shared ,
        rewards: self.rewards_shared,
        actions: self.actions_shared,
        terminals: self.terminals_shared
    }
    givens_q_val = {

states: self.states_shared,

next_states: self.next_states_shared,

        imgs :self.imgs_shared,
        locs: self.locs_shared ,
        hiss :self.hiss_shared,
        sds : self.sds_shared

rewards: self.rewards_shared,

actions: self.actions_shared,

terminals: self.terminals_shared

    }
    if update_rule == 'deepmind_rmsprop':
        updates = deepmind_rmsprop(loss, params, self.lr, self.rho,
                                   self.rms_epsilon)
    elif update_rule == 'rmsprop':
        updates = lasagne.updates.rmsprop(loss, params, self.lr, self.rho,
                                          self.rms_epsilon)
    elif update_rule == 'sgd':
        updates = lasagne.updates.sgd(loss, params, self.lr)
    else:
        raise ValueError("Unrecognized update: {}".format(update_rule))

    if self.momentum > 0:
        updates = lasagne.updates.apply_momentum(updates, None,
                                                 self.momentum)

    self._train = theano.function([], [loss, q_vals], updates=updates,
                                  givens=givens_train)
    self._q_vals = theano.function([], q_vals,
                                   givens=givens_q_val)

def build_network(self, network_type, output_dim, num_frames, batch_size):
    if network_type == "myOwn":
        return self.build_myNetwork(output_dim, num_frames, batch_size)
    else:
        raise ValueError("Unrecognized network: {}".format(network_type))


def build_myNetwork(self, output_dim, num_frames, batch_size):

    from lasagne.layers import dnn
    l_in = lasagne.layers.InputLayer(
        shape=(batch_size, num_frames, self.width_img, self.height_img)
    )

    l_conv1 = dnn.Conv2DDNNLayer(
        l_in,
        num_filters=32,
        filter_size=(8, 8),
        stride=(3, 3),
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.HeUniform(),
        b=lasagne.init.Constant(.1)
    )

    l_conv2 = dnn.Conv2DDNNLayer(
        l_conv1,
        num_filters=64,
        filter_size=(4, 4),
        stride=(1, 1),
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.HeUniform(),
        b=lasagne.init.Constant(.1)
    )

    l_conv3 = dnn.Conv2DDNNLayer(
        l_conv2,
        num_filters=64,
        filter_size=(3, 3),
        stride=(1, 1),
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.HeUniform(),
        b=lasagne.init.Constant(.1)
    )

    l_loc1 = lasagne.layers.InputLayer(
        shape=(batch_size, num_frames, self.width_loc, self.height_loc)
    )

    n = 64
    l_loc2 = lasagne.layers.DenseLayer(l_loc1, num_units=n)



    #history = np.zeros((batch_size, num_frames, 4, 24*24), dtype=int)


    l_his = lasagne.layers.InputLayer(
        shape=(batch_size, num_frames, self.width_his, self.height_his)
    )

    l_his2 = lasagne.layers.DenseLayer(l_his, num_units=n)


    l_dis =  lasagne.layers.InputLayer(
        shape=(batch_size, num_frames, self.target_dis_size)
    )

    l_dis2 =  lasagne.layers.DenseLayer(l_dis, num_units=n)

    l_conv4 = lasagne.layers.ReshapeLayer(l_conv3, (batch_size, 1, -1))
    l_loc2 = lasagne.layers.ReshapeLayer(l_loc2, (batch_size,1,-1))
    l_his2 = lasagne.layers.ReshapeLayer(l_his2, (batch_size,1,-1))
    l_dis2 = lasagne.layers.ReshapeLayer(l_dis2, (batch_size,1,-1))
    l_merge = lasagne.layers.ElemwiseSumLayer((l_conv4,l_loc2, l_his2, l_dis2 ))

    print (l_conv4.output_shape)
    print l_loc2.output_shape
    print l_his2.output_shape
    print l_dis2.output_shape
    print l_merge.output_shape

    l_hidden1 = lasagne.layers.DenseLayer(
        l_merge,
        num_units=320,
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.HeUniform(),
        b=lasagne.init.Constant(.1)
    )
    #
    l_out = lasagne.layers.DenseLayer(
        l_hidden1,
        num_units=output_dim,
        nonlinearity=None,
        W=lasagne.init.HeUniform(),
        b=lasagne.init.Constant(.1)
    )
    return l_out
def train(self,  imgs ,next_imgs ,  locs,  next_locs ,hiss,
                 next_hiss, sds ,
                 next_sds,
                    actions, rewards, terminals):
    """
    Train one batch.

    Arguments:

    states - b x f x h x w numpy array, where b is batch size,
             f is num frames, h is height and w is width.
    actions - b x 1 numpy array of integers
    rewards - b x 1 numpy array
    next_states - b x f x h x w numpy array
    terminals - b x 1 numpy boolean array (currently ignored)

    Returns: average loss
    """
    self.imgs_shared.set_value(imgs)
    self.next_imgs_shared.set_value(next_imgs)
    self.locs_shared.set_value(locs)
    self.next_locs_share.set_value(next_locs)
    self.hiss_shared.set_value(hiss)
    self.next_hiss_shared.set_value(next_hiss)
    self.sds_shared.set_value(sds)
    self.next_sds_shared.set_value(next_sds)

self.states_shared.set_value(states)

self.next_states_shared.set_value(next_states)

    self.actions_shared.set_value(actions)
    self.rewards_shared.set_value(rewards)
    self.terminals_shared.set_value(terminals)
    if (self.freeze_interval > 0 and
        self.update_counter % self.freeze_interval == 0):
        self.reset_q_hat()
    loss, _ = self._train()
    self.update_counter += 1
    return np.sqrt(loss)

def q_vals(self, img , loc,  his, sd):

states = np.zeros((self.batch_size, self.num_frames, self.input_height,

self.input_width), dtype=theano.config.floatX)

states[0, ...] = state

self.states_shared.set_value(states)

    imgs = np.zeros((self.batch_size, self.num_frames, self.height_img, 
                     self.width_img), dtype=theano.config.floatX)
    imgs[0, ...] = img
    locs = np.zeros((self.batch_size, self.num_frames, self.height_loc, 
                     self.width_loc), dtype=theano.config.floatX)
    locs[0, ...] = loc

    hiss = np.zeros((self.batch_size, self.num_frames, self.height_his, 
                     self.width_his), dtype=theano.config.floatX)

    hiss[0,...] = his

    sds = np.zeros((self.batch_size, self.num_frames, self.target_dis_size),
                    dtype='int32')

    sds[0, ...] = sd

    self.imgs_shared.set_value(imgs)
    self.locs_shared.set_value(locs)
    self.hiss_shared.set_value(hiss)
    self.sds_shared.set_value(sds)

    return self._q_vals()[0]

def choose_action(self, img , loc,  his, sd, epsilon):
    if self.rng.rand() < epsilon:
        return self.rng.randint(0, self.num_actions)
    q_vals = self.q_vals(img , loc,  his, sd)
    return np.argmax(q_vals)

def reset_q_hat(self):
    all_params = lasagne.layers.helper.get_all_param_values(self.l_out)
    lasagne.layers.helper.set_all_param_values(self.next_l_out, all_params)

def main():

net = DeepQLearner(84, 84, 16, 4, .99, .00025, .95, .95, 10000,

#32, 'nature_cuda')

width_img = 24
height_img = 24
width_loc = 1
height_loc = 3
width_his = width_img *height_img
height_his = 4
target_dis_size = 1
num_actions = 9
num_frames = 1
discount = 0.99
learning_rate = .00025
rho = 0.95
rms_epsilon = 0.95
momentum = 0.95
clip_delta = 1
freeze_interval = 100
batch_size = 100
network_type = 'myOwn'
update_rule = 'deepmind_rmsprop' 
batch_accumulator ='sum'
rng = np.random.RandomState(123456)

net = DeepQLearner(width_img,
             height_img,
             width_loc,
             height_loc,
             width_his,
             height_his,
             target_dis_size,
            num_actions, num_frames, discount, learning_rate, rho,
             rms_epsilon, momentum, clip_delta, freeze_interval,
             batch_size, network_type, update_rule,
             batch_accumulator, rng)

if name == 'main':
main()