diff --git a/spider_control/__pycache__/knowledge_transfer.cpython-36.pyc b/spider_control/__pycache__/knowledge_transfer.cpython-36.pyc index db188800b78c2e9e5d23d4543e5fa36327b92b06..0ddd2f193207d611e02411ce4917d72763b6b04a 100644 Binary files a/spider_control/__pycache__/knowledge_transfer.cpython-36.pyc and b/spider_control/__pycache__/knowledge_transfer.cpython-36.pyc differ diff --git a/spider_control/__pycache__/leg_dynamics.cpython-36.pyc b/spider_control/__pycache__/leg_dynamics.cpython-36.pyc index ad33e58cdb0f4b55e3658926f86a7f47fa4f48de..9d5ebba92260dd5953528556cd67848afbcadde4 100644 Binary files a/spider_control/__pycache__/leg_dynamics.cpython-36.pyc and b/spider_control/__pycache__/leg_dynamics.cpython-36.pyc differ diff --git a/spider_control/__pycache__/my_network.cpython-36.pyc b/spider_control/__pycache__/my_network.cpython-36.pyc index eed87b38b4403fe9c137f7359e9ed47455bf8153..758a61426096542dedd40e8289d623e9b9defbff 100644 Binary files a/spider_control/__pycache__/my_network.cpython-36.pyc and b/spider_control/__pycache__/my_network.cpython-36.pyc differ diff --git a/spider_control/control1.py b/spider_control/control1.py index fefb63333f7a5e83b8aba61c02a4a4f6f9401e1a..908ca78b28be7c7b4915ec93dc0060331eee006f 100755 --- a/spider_control/control1.py +++ b/spider_control/control1.py @@ -31,13 +31,14 @@ from sklearn.svm import LinearSVC g_joint_states = None g_positions = None g_pos1 = None -tactile_output = None +#tactile_output = None def tactile_callback(msg): global tactile_output tactile_output = msg.data + leg.update_tactile(tactile_output) def joint_callback(data, args): global g_positions @@ -71,25 +72,21 @@ def joint_callback(data, args): print("printing new position") print(new_position)#debuggings pub_msg.position = new_position - model1.nn_learn(reward) elif(model_num == 1): print("printing new position") new_position = model2.nn_run(g_positions) print(new_position)#debugging pub_msg.position = new_position - model2.nn_learn(reward) elif(model_num == 2): print("printing new position") new_position = model3.nn_run(g_positions) print(new_position)#degingbug pub_msg.position = new_position - model3.nn_learn(reward) elif(model_num == 3): new_position = model4.nn_run(g_positions) print("printing new position") print(new_position)#debugging pub_msg.position = new_position - model4.nn_learn(reward) elif(simulation_mode == 'train'): knowledge.learn(out_tensor, ou) if(model_number == 1): @@ -97,37 +94,42 @@ def joint_callback(data, args): new_position = model1.nn_run(g_positions) print(new_position)#debugging pub_msg.position = new_position - model1.nn_learn(reward) elif(model_number == 2): print("printing new position") new_position = model2.nn_run(g_positions) print(new_position)#debugging pub_msg.position = new_position - model2.nn_learn(reward) elif(model_number == 3): print("printing new position") new_position = model3.nn_run(g_positions) print(new_position)#debugging pub_msg.position = new_position - model3.nn_learn(reward) elif(model_number == 4): print("printing new position") new_position = model4.nn_run(g_positions) print(new_position)#debugging pub_msg.position = new_position - model4.nn_learn(reward) leg.update_angles(new_position) leg.update_effort(effort) leg.update_velocity(velocity) - leg.update_tactile(tactile_output) joint_pub.publish(pub_msg) if __name__=='__main__': - model1 = nn.Architecture(3, 24, 1) - model2 = nn.Architecture(3, 24, 2) - model3 = nn.Architecture(3, 24, 3) - model4 = nn.Architecture(3, 24, 4) - knowledge = kt.MultiClassLogistic(8, 3) + graph1 = tf.Graph() + with graph1.as_default(): + model1 = nn.Architecture(4, 24, 1) + graph2 = tf.Graph() + with graph2.as_default(): + model2 = nn.Architecture(4, 24, 2) + graph3 = tf.Graph() + with graph3.as_default(): + model3 = nn.Architecture(4, 24, 3) + graph4 = tf.Graph() + with graph4.as_default(): + model4 = nn.Architecture(4, 24, 4) + graph5 = tf.Graph() + with graph5.as_default(): + knowledge = kt.MultiClassLogistic(8, 3) pos = np.random.uniform(low = 0, high =2.5, size=24) vel = 10 eff = 100 diff --git a/spider_control/knowledge_transfer.py b/spider_control/knowledge_transfer.py index 6313235b9d7ece8450fb04953e2b72077b7cb7b7..d1f5df99cd080a8872d112e0bd787ca616361a97 100755 --- a/spider_control/knowledge_transfer.py +++ b/spider_control/knowledge_transfer.py @@ -44,55 +44,13 @@ class MultiClassLogistic: with tf.Session() as sess: sess.run(init_l) sess.run(optimizer) - return - - def get_neuron(self, a, b): - #'while a being the name of the neuron and b being the value of that neuron' - with tf.Session() as sess_n: - sess_n.run(a, feed_dict={a: b}) - 'a function to findout the input of the neurn giving its weights and biases, can only be used in neurons in hidden and output layer' - def input_method(self, a=[], b=[], c=[]): - multiplication = tf.multiply(a, b) - d = tf.reduce_sum(multiplication, 0) - add = tf.add(d, c) - with tf.Session() as sess: - out = sess.run(add) - return out def normalization(self, inputs): a_min = min(inputs) a_max = max(inputs) for i in range(len(inputs)): inputs[i] = (inputs[i]-a_min)/(a_max-a_min) - #def input_weight(self): - #with tf.compat.v1.variable_scope("weight_in", reuse=tf.compat.v1.AUTO_REUSE): - #v = tf.compat.v1.get_variable("weight_input", dtype=tf.float64, shape=[self.neurons, 1], initializer=self.weight_initer) - #return v - - #def hid_weight(self): - #with tf.compat.v1.variable_scope("weight_hid", reuse=tf.compat.v1.AUTO_REUSE): - #v = tf.compat.v1.get_variable("weight_hidden", dtype=tf.float64, shape=[self.neurons, 1], initializer=self.weight_initer1) - #return v - - #def out_weight(self): - #with tf.compat.v1.variable_scope("weight_out", reuse=tf.compat.v1.AUTO_REUSE): - #v = tf.compat.v1.get_variable("weight_input", dtype=tf.float64, shape=[4, 2], initializer=self.weight_initer2) - #return v - - #def bias_in(self): - # with tf.compat.v1.variable_scope("bias_in", reuse=tf.compat.v1.AUTO_REUSE): - # v = tf.compat.v1.get_variable("bias_input", dtype=tf.float64, shape=[self.neurons, 1], initializer=self.bias_initer) - # return v - - #def bias_hid(self): - # with tf.compat.v1.variable_scope("bias_hidd", reuse=tf.compat.v1.AUTO_REUSE): - # v = tf.compat.v1.get_variable("bias_hidden", dtype=tf.float64, shape=[self.neurons, 1], initializer=self.bias_initer1) - # return v - - #def bias_out(self): - # with tf.compat.v1.variable_scope("bias_outt", reuse=tf.compat.v1.AUTO_REUSE): - # v = tf.compat.v1.get_variable("bias_out", dtype=tf.float64, shape=[1, 4], initializer=self.bias_initer2) - # return v + def run(self, carollis_input): c_in=normalize(carollis_input, axis = 0) print('normalized carollis input is \n') @@ -125,15 +83,9 @@ class MultiClassLogistic: bias_o = tf.compat.v1.get_variable("bias_out", dtype=tf.float64, shape=[1, 4], initializer=self.bias_initer2) init_o_b = tf.compat.v1.global_variables_initializer() sess.run(init_o_b) - #weight_i = self.input_weight() - #weight_h = self.hid_weight() - #weight_o = self.out_weight() - #bias_i = self.bias_in() - #bias_h = self.bias_hid() - #bias_o = self.bias_out() uninitialized_variables = [] - print("printing the names of all variables") - print(tf.all_variables()) + #print("printing the names of all variables") + #print(tf.all_variables()) for var in tf.all_variables(): try: sess.run(var) @@ -142,8 +94,8 @@ class MultiClassLogistic: init = tf.variables_initializer(uninitialized_variables) sess.run(init) - print("priniting uninitialized variables") - print(sess.run(tf.report_uninitialized_variables(uninitialized_variables))) + #print("priniting uninitialized variables") + #print(sess.run(tf.report_uninitialized_variables(uninitialized_variables))) knowledge_input = tf.compat.v1.add(tf.compat.v1.multiply(c_input, weight_i), bias_i) #sess.run(knowledge_input) knowledge_hidden = tf.compat.v1.nn.leaky_relu(knowledge_input, alpha=0.01) @@ -174,6 +126,7 @@ class MultiClassLogistic: #just testing #softmax_output = tf.nn.softmax(in_out) #output = sess.run(softmax_output) + sess.close() return output, in_out def learn(self, preferred_out, soft_out): self.kt_learning(0.1, soft_out, preferred_out) diff --git a/spider_control/leg_dynamics.py b/spider_control/leg_dynamics.py index d6e21b681ecc1d29e480d9133623cfdd4f5bfbfc..c265c045a78d4cf4f85df45a2e3523ebf308589d 100755 --- a/spider_control/leg_dynamics.py +++ b/spider_control/leg_dynamics.py @@ -40,6 +40,8 @@ class Leg_attribute: self.tactile = tac def update_effort(self, ef): self.j_effort = ef + def give_angles(self): + return self.j_angles @@ -90,11 +92,14 @@ class Leg_attribute: def tactile_run(self): score = 0 total = 0 - for element in range(0, len(self.tactile)): - if(self.tactile[element]>0.5): - total +=1 - if(total>3): - score = total + if(sum(self.tactile) != 0): + for element in range(0, len(self.tactile)): + if(self.tactile[element]>0.5): + total +=1 + if(total>3): + score = total + else: + score = 0 else: score = 0 return score @@ -132,7 +137,7 @@ class Leg_attribute: term6 = term_6_1+term_6_2+term_6_3 term7 = term_7_1+term_7_2+term_7_3 term8 = term_8_1+term_8_2+term_8_3 - term = np.array([[term1], [term2], [term3], [term4], [term5], [term6], [term7], [term8]], dtype=np.float32) + term = np.array([[term1], [term2], [term3], [term4], [term5], [term6], [term7], [term8]]) return term def leg_run(self): x_l_com = self.x_left_com() diff --git a/spider_control/my_network.py b/spider_control/my_network.py index e0aa889598bbaec5d67e2c759ace8c1f39d6a4be..128babe3319a511f0486d5ddbb41b1b0a02c0273 100755 --- a/spider_control/my_network.py +++ b/spider_control/my_network.py @@ -16,54 +16,51 @@ class Architecture: self.layers = layers self.neurons = neurons self.number = number#number of the model - self.weight_in = 'weight_in'+str(self.number) - self.weight_out = 'weight_out'+str(self.number) - self.weight_hid = 'weight_hid'+str(self.number) - self.weight_hid_cont = 'weight_hid_cont'+str(self.number) - self.bias_in = 'bias_in'+str(self.number) - self.bias_out = 'bias_out'+str(self.number) - self.bias_hid = 'bias_hid'+str(self.number) - self.bias_hid_con = 'bias_hid_con'+str(self.number) - self.neuron_i = tf.compat.v1.placeholder(tf.float32, shape=(neurons, 1)) # _input layer neurons - self.neuron_o = tf.compat.v1.placeholder(tf.float32, shape=(neurons, 1)) # output layer neurons - self.neuron_h = tf.compat.v1.placeholder(tf.float32, shape=(neurons/3, 1)) # hidden layer neurons, only 8 neurons because there are eight legs - self.neuron_h_c = tf.compat.v1.placeholder(tf.float32, shape=(neurons/3, 1)) # context layer neurons, only 8 neurons because there are eight in hidden layer + #self.neurons_hidden_context = tf.compat.v1.placeholder(dtype = tf.float64, shape=[8, 1], name="hidden_context") + #self.is_filled = tf.compat.v1.placeholder(tf.bool) + self.neuron_hidden_context = None self.weight_initer = tf.truncated_normal_initializer(mean=1.0, stddev=0.01) self.weight_initer1 = tf.truncated_normal_initializer(mean=1.0, stddev=0.05) self.weight_initer2 = tf.truncated_normal_initializer(mean=1.0, stddev=0.1) self.weight_initer3 = tf.truncated_normal_initializer(mean=1.0, stddev=0.15) - self.weights_i = tf.get_variable(name=self.weight_in ,dtype=tf.float32, shape=[self.neurons, 1], initializer=self.weight_initer) # weights of input layer neurons - self.weights_o = tf.get_variable(name=self.weight_out, dtype=tf.float32, shape=[self.neurons, 1], initializer=self.weight_initer1) # weights of output layer neurons - self.weights_h = tf.get_variable(name=self.weight_hid, dtype=tf.float32, shape=[self.neurons/3, self.layers], initializer=self.weight_initer2) # weights of hidden layer neurons - self.weights_h_c = tf.get_variable(name=self.weight_hid_cont, dtype=tf.float32, shape=[self.neurons/3, self.layers], initializer=self.weight_initer3) # weights of context layer neurons self.bias_initer =tf.truncated_normal_initializer(mean=0.1, stddev=0.01) self.bias_initer1 =tf.truncated_normal_initializer(mean=0.1, stddev=0.01) self.bias_initer2 =tf.truncated_normal_initializer(mean=0.1, stddev=0.01) self.bias_initer3 =tf.truncated_normal_initializer(mean=0.1, stddev=0.01) - self.bias_i = tf.get_variable(name=self.bias_in, dtype=tf.float32, shape=[self.neurons, 1], initializer=self.bias_initer) #biases of each neurons - self.bias_o = tf.get_variable(name=self.bias_out, dtype=tf.float32, shape=[self.neurons, 1], initializer=self.bias_initer1) - self.bias_h = tf.get_variable(name=self.bias_hid, dtype=tf.float32, shape=[self.neurons/3, 1], initializer=self.bias_initer2) - self.bias_h_c = tf.get_variable(name=self.bias_hid_con, dtype=tf.float32, shape=[self.neurons/3, 1], initializer=self.bias_initer3) # get neuron value method helps us to write the joint angles to the individual neurons which we can later use for learning - def get_neuron_value(self, a, b): - with tf.Session() as sess_n: - sess_n.run(a, feed_dict={a: b}) - return - def input_method(self, neurons = [], weights = [], bias = []): - #input function of the main neural network - multi = tf.matmul(neurons, weights, transpose_a = False, transpose_b=False)#use False as tf.False if this does not work - add1 = tf.add(multi, bias) - ses = tf.Session() - neuron_input = ses.run(add1) - return neuron_input - def transfer_functions(self, a = []): - b = tf.nn.softmax(a, name='sigmoid') - #0.5 being the threshold that is required to activate the neuron - if (b >= 0.5): - out = b*a - else: - out = 0 - return out + def reward_method(self, j_angles): + mask = tf.compat.v1.constant([0.12, 0.33, 0.5]) + mask = tf.compat.v1.tile(mask, [2]) + mask_1 = tf.compat.v1.constant([0.12, 0.33, 0.5]) + mask_1 = tf.compat.v1.tile(mask_1, [8]) + mask_1 = tf.compat.v1.reshape(mask_1, shape=[8, 3]) + j_angles_0 = tf.compat.v1.reshape(j_angles, shape=[4, 6]) + j_angles_1 = tf.compat.v1.reshape(j_angles, shape=[8, 3]) + com_1 = tf.compat.v1.multiply(mask, j_angles_0[0, :]) + com_2 = tf.compat.v1.multiply(mask, j_angles_0[1, :]) + com_3 = tf.compat.v1.multiply(mask, j_angles_0[2, :]) + com_4 = tf.compat.v1.multiply(mask, j_angles_0[3, :]) + x_com_1 = tf.compat.v1.math.cos(com_1) + x_com_2 = tf.compat.v1.math.cos(com_2) + x_com_3 = tf.compat.v1.math.cos(com_3) + x_com_4 = tf.compat.v1.math.cos(com_4) + y_com_1 = tf.compat.v1.math.sin(com_1) + y_com_2 = tf.compat.v1.math.sin(com_2) + y_com_3 = tf.compat.v1.math.sin(com_3) + y_com_4 = tf.compat.v1.math.sin(com_4) + x_com = tf.add(tf.add(tf.add(x_com_1, x_com_2), x_com_3), x_com_4) + y_com = tf.add(tf.add(tf.add(y_com_1, y_com_2), y_com_3), y_com_4) + m_com = tf.compat.v1.multiply(mask_1, j_angles_1) + m_com_x = tf.compat.v1.math.cos(m_com) + m_com_x = tf.compat.v1.math.reduce_sum(m_com_x) + m_com_y = tf.compat.v1.math.sin(m_com) + m_com_y = tf.compat.v1.math.reduce_sum(m_com_y) + x_mid = 0.5*(tf.add(x_com[0:2], x_com[3:5])) + y_mid = 0.5*(tf.add(y_com[0:2], y_com[3:5])) + x_mid = tf.compat.v1.math.reduce_sum(x_mid) + y_mid = tf.compat.v1.math.reduce_sum(y_mid) + m = 0.017*(tf.compat.v1.math.divide(tf.compat.v1.math.subtract(m_com_y, y_mid),tf.compat.v1.math.subtract(m_com_x, x_mid))) + return m def NN_learningPunishment(self, LearningRate, punishment): #a method to minimize the rewards and alot the weights based on that optimizer = tf.train.GradientDescentOptimizer(LearningRate).minimize(punishment) @@ -72,41 +69,110 @@ class Architecture: return def NN_learningReward(self, LearningRate, reward): #a method to maximize reward and alot the weights based on that - optimizer = tf.train.GradientDescentOptimizer(LearningRate).minimize((-1*reward)) + reward = -1*reward + optimizer = tf.train.GradientDescentOptimizer(LearningRate).minimize(reward) + init_m = tf.compat.v1.global_variables_initializer() with tf.Session as session: + session.run(init_m) session.run(optimizer) def output_function(self, a = []): b = tf.nn.softmax(a, name='sigmoid') out = b*a return out - def nn_run(self, joint_angles = []): + def nn_run(self, joint_angles): #function that directly runs the neural network and returns the prediction - output_layer1 = np.array(24) - input_layerhid = np.array(8) - output_layerhid = np.array(8) - input_layerhidcont = np.array(8) - output_layerhidcont = np.array(8) - input_layerout = np.array(24) - output_layerout = np.array(24) - self.get_neuron_value(self.neuron_i, joint_angles) - for i in range(0, 24, 1): - output_layer1[i] = self.input_method(self.neuron_i[i, :], self.weights_i[:, i]) # finding the ouput of the input layer - for j in range(0, 8, 1): - output_layerh = output_layer1[0, 3 * j:3 * j + 3] - output_layerh.append(output_layerhidcont[j]) # adding output from context layer from previous iteration as input for hidden layer - input_layerhid[j] = self.input_method(output_layer1, self.weights_h[j, :]) - self.get_neuron_value(self.neuron_h, input_layerhid) # feeding the neuron value of hidden layer - output_layerhid = self.transfer_functions(self.neuron_h) - for i in range(0, 8, 1): - input_layerhidcont[i] = self.input_method(output_layerhid[i], self.weights_h_c[i]) # input function for hidden context layer, context layr only has one input to each neuron and one weight associated to each neuron - self.get_neuron_value(self.neuron_h_c, input_layerhidcont) # passing the input to the hidden context - output_layerhidcont = self.transfer_frunctions(self.neuron_h_c) # output of the neurons from context layer - input_layerout = tf.multiply(output_layerhid, self.weights_o[0:8]) - input_layerout.append(tf.multiply(output_layerhid, self.weights_o[8:16])) - input_layerout.append(tf.multiply(output_layerhid, self.weights_o[16:24])) # element wise multiplication - self.get_neuron_value(self.neuron_o, input_layerout) - output_layerout = self.output_function(self.neuron_o) # output of the neural network - return output_layerout + joint_angles = tf.compat.v1.convert_to_tensor(joint_angles, dtype=tf.float32) + joint_angles = tf.compat.v1.reshape(joint_angles, shape=[24, 1]) + sessi = tf.InteractiveSession() + with tf.compat.v1.variable_scope("weight_in_main", reuse=tf.compat.v1.AUTO_REUSE): + weight_i_m = tf.compat.v1.get_variable("weight_input_main", dtype=tf.float32, shape=[self.neurons, 1], initializer=self.weight_initer) + init_i_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_i_m) + with tf.compat.v1.variable_scope("weight_hid_main", reuse=tf.compat.v1.AUTO_REUSE): + weights_h_m = tf.compat.v1.get_variable("weight_hidden_main", dtype=tf.float32, shape=[int(self.neurons/3), int(self.layers)], initializer=self.weight_initer1) + init_h_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_h_m) + with tf.compat.v1.variable_scope("weight_hid_cont_main", reuse=tf.compat.v1.AUTO_REUSE): + weights_h_c_m = tf.compat.v1.get_variable("weight_hid_cont_main", dtype=tf.float32, shape=[int(self.neurons/3), 1], initializer=self.weight_initer2) + init_h_c_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_h_c_m) + with tf.compat.v1.variable_scope("weight_out_main", reuse=tf.compat.v1.AUTO_REUSE): + weights_o_m = tf.compat.v1.get_variable("weight_input_main", dtype=tf.float32, shape=[int(self.neurons), 1], initializer=self.weight_initer3) + init_o_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_o_m) + with tf.compat.v1.variable_scope("bias_in_main", reuse=tf.compat.v1.AUTO_REUSE): + bias_i_m = tf.compat.v1.get_variable("bias_input_main", dtype=tf.float32, shape=[self.neurons, 1], initializer=self.bias_initer) + init_i_b_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_i_b_m) + with tf.compat.v1.variable_scope("bias_hid_main", reuse=tf.compat.v1.AUTO_REUSE): + bias_h_m = tf.compat.v1.get_variable("bias_hidden_main", dtype=tf.float32, shape=[int(self.neurons/3), 1], initializer=self.bias_initer1) + init_h_b_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_h_b_m) + with tf.compat.v1.variable_scope("bias_hid_cont_main", reuse=tf.compat.v1.AUTO_REUSE): + bias_h_c_m = tf.compat.v1.get_variable("bias_hidden_cont_main", dtype=tf.float32, shape=[int(self.neurons/3), 1], initializer=self.bias_initer2) + init_h_c_b_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_h_c_b_m) + with tf.compat.v1.variable_scope("bias_out_main", reuse=tf.compat.v1.AUTO_REUSE): + bias_o_m = tf.compat.v1.get_variable("bias_out_main", dtype=tf.float32, shape=[int(self.neurons), 1], initializer=self.bias_initer3) + init_o_b_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_o_b_m) + uninitialized_variables_main = [] + print("printing the names of all variables") + print(tf.all_variables()) + for var in tf.all_variables(): + try: + sessi.run(var) + except tf.errors.FailedPreconditionError: + uninitialized_variables_main.append(var) + + init_m = tf.variables_initializer(uninitialized_variables_main) + sessi.run(init_m) + input_layer_activation = tf.add(tf.multiply(joint_angles, weight_i_m), bias_i_m) + input_layer_output = tf.compat.v1.nn.leaky_relu(input_layer_activation, alpha=0.05) + print("printing joint angles fr main network") + print(tf.compat.v1.shape(input_layer_output)) + input_layer_output = tf.compat.v1.reshape(input_layer_output, shape=[8, 3]) + if(self.neuron_hidden_context != None): + n_h_c = tf.compat.v1.convert_to_tensor(self.neuron_hidden_context, dtype=tf.float32) + input_layer_out = tf.compat.v1.concat([input_layer_output, n_h_c], 1) + hidden_layer_activation_multiply = tf.compat.v1.multiply(input_layer_out, weights_h_m) + hidden_layer_activation_input = tf.compat.v1.add(hidden_layer_activation_multiply[:,0], hidden_layer_activation_multiply[:, 1]) + hidden_layer_activation_input = tf.compat.v1.add(hidden_layer_activation_input, hidden_layer_activation_multiply[:, 2]) + hidden_layer_activation_input = tf.compat.v1.add(hidden_layer_activation_input, hidden_layer_activation_multiply[:,3]) + hidden_layer_activation_input = tf.compat.v1.reshape(hidden_layer_activation_input, [8, 1]) + hidden_layer_activation = tf.compat.v1.add(hidden_layer_activation_input, bias_h_m) + elif(self.neuron_hidden_context == None): + hidden_layer_activation_multiply = tf.compat.v1.multiply(input_layer_output, weights_h_m[:,0:3]) + hidden_layer_activation_input = tf.compat.v1.add(hidden_layer_activation_multiply[:,0], hidden_layer_activation_multiply[:, 1]) + hidden_layer_activation_input = tf.compat.v1.add(hidden_layer_activation_input, hidden_layer_activation_multiply[:, 2]) + hidden_layer_activation_input = tf.compat.v1.reshape(hidden_layer_activation_input, [8, 1]) + hidden_layer_activation = tf.compat.v1.add(hidden_layer_activation_input, bias_h_m) + hidden_layer_output = tf.compat.v1.nn.leaky_relu(hidden_layer_activation, alpha=0.05) + print("printing hidden_layer_out") + print(sessi.run(hidden_layer_output)) + neurons_hidden_context = tf.compat.v1.add(tf.compat.v1.multiply(hidden_layer_output, weights_h_c_m), bias_h_c_m) + print("printing hidden_layer_out") + print(sessi.run(neurons_hidden_context)) + self.neuron_hidden_context = tf.compat.v1.nn.leaky_relu(neurons_hidden_context, alpha=0.05) + output_layer_mask = tf.compat.v1.constant([0.25, 0.25, 0.5]) + output_layer_mask = tf.compat.v1.tile(output_layer_mask, [8]) + output_layer_mask = tf.compat.v1.reshape(output_layer_mask, [8, 3]) + output_layer_mask = tf.compat.v1.cast(output_layer_mask, dtype=tf.float32) + output_layer_input = tf.compat.v1.multiply(hidden_layer_output, output_layer_mask) + output_layer_input = tf.compat.v1.reshape(output_layer_input, [24, 1]) + output_layer_activation = tf.compat.v1.add(tf.compat.v1.multiply(output_layer_input, weights_o_m), bias_o_m) + output_layerout = tf.compat.v1.nn.leaky_relu(output_layer_activation, alpha=0.05) + reward = self.reward_method(output_layerout) + learningrate = 0.01 + reward = -1*reward + optimizer = tf.train.GradientDescentOptimizer(learningrate).minimize(reward) + init_m = tf.compat.v1.global_variables_initializer() + sessi.run(init_m) + sessi.run(optimizer) + out = sessi.run(output_layerout) + print("printing the shape of input of output layer") + print(out) + return out def nn_learn(self, rew): Learningrate = 0.1 self.NN_learningReward(Learningrate, rew)