def average_gradients(self, tower_grads):
        average_grads = []
        for grad_and_vars in zip(*tower_grads):
            grads = []
            for g, _ in grad_and_vars:
                expanded_g = tf.expand_dims(g, 0)
                grads.append(expanded_g)
            grad = tf.concat(axis=0, values=grads)
            grad = tf.reduce_mean(grad, 0)
            v = grad_and_vars[0][1]
            grad_and_var = (grad, v)
            average_grads.append(grad_and_var)
        return average_grads

with tf.Graph().as_default(), tf.device('/cpu:0'):
    定义输入占位符tf.placeholder()
    定义变量tf.get_variable()
    使用tf.split()函数对各个placeholder进行分割，意思是整体的batch会拆分为几个小的batch分别喂给不同的GPU
    如self.encoder_inputs_split = tf.split(value=self.encoder_inputs, num_or_size_splits=self.gpu_num, axis=0)
    定义优化器
    定义全局的梯度的list：multi_grads = []
 
    with tf.variable_scope(tf.get_variable_scope()):
        for gpu_id in self.gpu_list:#这里也可以使用range(gpu_num)，反正就是一个GPU编号的list
            with tf.device('/gpu:%d'%gpu_id), tf.variable_scope(name_or_scope=tf.get_variable_scope(), reuse= gpu_id > 0):
                with tf.name_scope('%s_%d' % ('GPU', gpu_id)) as scope:
                    当前GPU使用的placeholdert：self.encoder_inputs_split[gpu_id]
                    self.loss = 得到你的loss
                    # tensorboard显示各GPU的loss
                    tf.summary.scalar('loss_{}'.format(i), self.loss)
                    # 重用变量
                    tf.get_variable_scope().reuse_variables()
                    self.summary_op = tf.summary.merge_all()
                    # 计算梯度
                    gradients = optimizer.compute_gradients(self.loss)
                    multi_grads.append(gradients)

    # 计算平均梯度
    grads = self.average_gradients(multi_grads)
    self.train_op = optimizer.apply_gradients(grads)
    # 保存模型
    self.saver = tf.train.Saver(tf.global_variables())                

def train(self, sess, batch):
        #对于训练阶段，需要执行self.train_op, self.loss, self.summary_op三个op，并传入相应的数据
        feed_dict = {self.encoder_inputs: batch.encoder_inputs,
                      self.encoder_inputs_length: batch.encoder_inputs_length,
                      self.decoder_targets: batch.decoder_targets,
                      self.decoder_targets_length: batch.decoder_targets_length,
                      self.keep_prob_placeholder: 0.5,
                      self.batch_size: len(batch.encoder_inputs) / len(self.gpu_num),
                      self.lr_pl: self.learing_rate,
                      self.mask : batch.masks}
        _, loss, summary = sess.run([self.train_op, self.loss, self.summary_op], feed_dict=feed_dict)
        return loss, summary
# 实例化模型对象
model = Seq2SeqModel(FLAGS.rnn_size,
                         FLAGS.num_layers,
                         FLAGS.embedding_size,
                         FLAGS.learning_rate,
                         word2id,
                         mode='train',
                         use_attention=True,
                         beam_search=False,
                         beam_size=5,
                         max_gradient_norm=5.0,
                         gpu_list = gpu_list
                         )
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True), allow_soft_placement=True)) as sess:
    sess.run(tf.global_variables_initializer())  
    current_step = 0   
    # tensorboard用writer
    summary_writer = tf.summary.FileWriter(os.path.join('./tensorboard', FLAGS.attempt), graph=sess.graph)
    for e in range(epoch_num):
        for batch in getNextBatch: # 得到下一个batch的训练数据
            loss, summary = model.train(sess, nextBatch)
            current_step += 1
            summary_writer.add_summary(summary, current_step)   

# decoder的时候的部分代码
training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=decoder_inputs_embedded,
                                    sequence_length=self.decoder_targets_length_split[gpu_id],
                                    time_major=False, name='training_helper')
training_decoder = tf.contrib.seq2seq.BasicDecoder(cell=decoder_cell, 
                                    helper=training_helper,
                                    initial_state=decoder_initial_state, output_layer=output_layer)
decoder_outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder=training_decoder,
                                    impute_finished=True,
                                    maximum_iterations=self.max_target_sequence_length)
# 根据输出计算loss和梯度，并定义进行更新的AdamOptimizer和train_op
self.decoder_logits_train = tf.identity(decoder_outputs.rnn_output)
self.decoder_predict_train = tf.argmax(self.decoder_logits_train, axis=-1, name='decoder_pred_train')
# 使用sequence_loss计算loss，这里需要传入之前定义的mask标志
self.loss = tf.contrib.seq2seq.sequence_loss(logits=self.decoder_logits_train,
                                    targets=self.decoder_targets_split[gpu_id], 
                                    weights=self.mask_split[gpu_id])

with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True), allow_soft_placement=True)) as sess: