# -*- coding: utf-8 -*-import numpy as np
from python_speech_features import mfcc  #需要pip install 
import scipy.io.wavfile as wav
import os'''读取wav文件对应的label'''
def get_wavs_lables1(wav_path):#获得训练用的wav文件路径列表wav_files = []labels = []for (dirpath, dirnames, filenames) in os.walk(wav_path):for filename in filenames:if filename.endswith('.wav') or filename.endswith('.WAV'):filename_path = os.sep.join([dirpath, filename])if os.stat(filename_path).st_size < 2400:  # 剔除掉一些小文件continuewav_files.append(filename_path)name = filename.split('.')[0]label = name.split('-')[2]labels.append(label)return wav_files, labels# Constants
SPACE_TOKEN = '<space>'
SPACE_INDEX = 2
FIRST_INDEX = ord('a') - 1  # 0 is reserved to space#n_input = 26 计算美尔倒谱系数的个数# n_context = 9 对于每个时间点，要包含上下文样本的个数
def get_audio_and_transcriptch(txt_files, wav_files, n_input, n_context,word_num_map,txt_labels=None):audio = []audio_len = []transcript = []transcript_len = []if txt_files!=None:txt_labels = txt_filesfor txt_obj, wav_file in zip(txt_labels, wav_files):# load audio and convert to featuresaudio_data = audiofile_to_input_vector(wav_file, n_input, n_context)audio_data = audio_data.astype('float32')audio.append(audio_data)audio_len.append(np.int32(len(audio_data)))# load text transcription and convert to numerical arraytarget = []if txt_files!=None:#txt_obj是文件target = get_ch_lable_v(txt_obj,word_num_map)else:target = get_ch_lable_v(None,word_num_map,txt_obj)#txt_obj是labels#target = text_to_char_array(target)transcript.append(target)transcript_len.append(len(target))audio = np.asarray(audio)audio_len = np.asarray(audio_len)transcript = np.asarray(transcript)transcript_len = np.asarray(transcript_len)return audio, audio_len, transcript, transcript_len#优先转文件里的字符到向量
def get_ch_lable_v(txt_file,word_num_map,txt_label=None):words_size = len(word_num_map)# dict.get(key, default=None)  key -- 字典中要查找的键。default -- 如果指定键的值不存在时，返回该默认值值。to_num = lambda word: word_num_map.get(word, words_size) if txt_file!= None:txt_label = get_ch_lable(txt_file)#print(txt_label)labels_vector = list(map(to_num, txt_label)) #print(labels_vector)return labels_vector  def get_ch_lable(txt_file):  labels= ""with open(txt_file, 'rb') as f:for label in f: #labels =label.decode('utf-8')labels =labels+label.decode('gb2312')#labels.append(label.decode('gb2312'))return  labels# numcep参数是 n_input = 26 计算美尔倒谱系数的个数# numcontext参数是 n_context = 9 对于每个时间点，要包含上下文样本的个数
def audiofile_to_input_vector(audio_filename, numcep, numcontext):# Load wav files 读取.wav音频文件，返回一个元组，第一项为音频的采样率，第二项为音频数据的numpy数组。fs, audio = wav.read(audio_filename)# Get mfcc coefficientsorig_inputs = mfcc(audio, samplerate=fs, numcep=numcep)#print(np.shape(orig_inputs))#(277, 26) list[start:end:step] start:起始位置 end:结束位置 step:步长orig_inputs = orig_inputs[::2]#(139, 26)train_inputs = np.array([], np.float32)#ValueError: cannot resize an array that references or is referenced by another array in this way.  Use the resize function#train_inputs.resize((orig_inputs.shape[0], numcep + 2 * numcep * numcontext))train_inputs = np.resize(train_inputs,(orig_inputs.shape[0], numcep + 2 * numcep * numcontext))# or  a.resize(new_shape, refcheck=False)#print(np.shape(train_inputs))#)(139, 494)# Prepare pre-fix post fix contextempty_mfcc = np.array([])#empty_mfcc.resize((numcep))empty_mfcc = np.resize(empty_mfcc,(numcep))# Prepare train_inputs with past and future contextstime_slices = range(train_inputs.shape[0])#139个切片context_past_min = time_slices[0] + numcontextcontext_future_max = time_slices[-1] - numcontext#[9,1,2...,137,129]for time_slice in time_slices:# 前9个补0，mfcc featuresneed_empty_past = max(0, (context_past_min - time_slice))empty_source_past = list(empty_mfcc for empty_slots in range(need_empty_past))data_source_past = orig_inputs[max(0, time_slice - numcontext):time_slice]assert(len(empty_source_past) + len(data_source_past) == numcontext)# 后9个补0，mfcc featuresneed_empty_future = max(0, (time_slice - context_future_max))empty_source_future = list(empty_mfcc for empty_slots in range(need_empty_future))data_source_future = orig_inputs[time_slice + 1:time_slice + numcontext + 1]assert(len(empty_source_future) + len(data_source_future) == numcontext)#np.concatenate联结函数，两个数组连接在一起if need_empty_past:past = np.concatenate((empty_source_past, data_source_past))else:past = data_source_pastif need_empty_future:future = np.concatenate((data_source_future, empty_source_future))else:future = data_source_futurepast = np.reshape(past, numcontext * numcep)now = orig_inputs[time_slice]future = np.reshape(future, numcontext * numcep)train_inputs[time_slice] = np.concatenate((past, now, future))assert(len(train_inputs[time_slice]) == numcep + 2 * numcep * numcontext)# 将数据使用正太分布标准化，减去均值然后再除以方差train_inputs = (train_inputs - np.mean(train_inputs)) / np.std(train_inputs)return train_inputsdef pad_sequences(sequences, maxlen=None, dtype=np.float32,padding='post', truncating='post', value=0.):lengths = np.asarray([len(s) for s in sequences], dtype=np.int64)nb_samples = len(sequences)if maxlen is None:maxlen = np.max(lengths)# take the sample shape from the first non empty sequence# checking for consistency in the main loop below.sample_shape = tuple()for s in sequences:if len(s) > 0:sample_shape = np.asarray(s).shape[1:]breakx = (np.ones((nb_samples, maxlen) + sample_shape) * value).astype(dtype)for idx, s in enumerate(sequences):if len(s) == 0:continue  # empty list was foundif truncating == 'pre':trunc = s[-maxlen:]elif truncating == 'post':trunc = s[:maxlen]else:raise ValueError('Truncating type "%s" not understood' % truncating)# check `trunc` has expected shapetrunc = np.asarray(trunc, dtype=dtype)if trunc.shape[1:] != sample_shape:raise ValueError('Shape of sample %s of sequence at position %s is different from expected shape %s' %(trunc.shape[1:], idx, sample_shape))if padding == 'post':x[idx, :len(trunc)] = truncelif padding == 'pre':x[idx, -len(trunc):] = truncelse:raise ValueError('Padding type "%s" not understood' % padding)return x, lengths

# -*- coding: utf-8 -*-import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
## 自定义
speechutils = __import__("speechutils")
sparse_tuple_to_texts_ch = speechutils.sparse_tuple_to_texts_ch
ndarray_to_text_ch = speechutils.ndarray_to_text_ch
get_audio_and_transcriptch = speechutils.get_audio_and_transcriptch
pad_sequences = speechutils.pad_sequences
sparse_tuple_from = speechutils.sparse_tuple_from
get_wavs_lables1 = speechutils.get_wavs_lables1tf.reset_default_graph()b_stddev = 0.046875
h_stddev = 0.046875n_hidden = 1024
n_hidden_1 = 1024
n_hidden_2 =1024
n_hidden_5 = 1024
n_cell_dim = 1024
n_hidden_3 = 2 * 1024keep_dropout_rate=0.95
relu_clip = 20def BiRNN_model( batch_x, seq_length, n_input, n_context,n_character ,keep_dropout):# batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]batch_x_shape = tf.shape(batch_x)# 将输入转成时间序列优先,[1, 0, 2]将第一维和第二维交换batch_x = tf.transpose(batch_x, [1, 0, 2])# 再转成2维传入第一层batch_x = tf.reshape(batch_x,[-1, n_input + 2 * n_input * n_context])  # (n_steps*batch_size, n_input + 2*n_input*n_context)# 使用clipped RELU activation and dropout.# 1st layerwith tf.name_scope('fc1'):b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer(stddev=b_stddev))h1 = variable_on_cpu('h1', [n_input + 2 * n_input * n_context, n_hidden_1],tf.random_normal_initializer(stddev=h_stddev))#tf.multiply（）两个矩阵中对应元素各自相乘；tf.matmul（）将矩阵a乘以矩阵b，生成a * b。#tf.minimum(a,b)返回a，b之间的最小值layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)layer_1 = tf.nn.dropout(layer_1, keep_dropout)# 2nd layerwith tf.name_scope('fc2'):b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer(stddev=b_stddev))h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer(stddev=h_stddev))layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)layer_2 = tf.nn.dropout(layer_2, keep_dropout)# 3rd layerwith tf.name_scope('fc3'):b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer(stddev=b_stddev))h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer(stddev=h_stddev))layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)layer_3 = tf.nn.dropout(layer_3, keep_dropout)# 双向rnnwith tf.name_scope('lstm'):# Forward direction cell:lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(lstm_fw_cell,input_keep_prob=keep_dropout)# Backward direction cell:lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(lstm_bw_cell,input_keep_prob=keep_dropout)# `layer_3`  `[n_steps, batch_size, 2*n_cell_dim]`layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,inputs=layer_3,dtype=tf.float32,time_major=True,sequence_length=seq_length)# 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]outputs = tf.concat(outputs, 2)# to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]        outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])with tf.name_scope('fc5'):b5 = variable_on_cpu('b5', [n_hidden_5], tf.random_normal_initializer(stddev=b_stddev))h5 = variable_on_cpu('h5', [(2 * n_cell_dim), n_hidden_5], tf.random_normal_initializer(stddev=h_stddev))layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)layer_5 = tf.nn.dropout(layer_5, keep_dropout)with tf.name_scope('fc6'):# 全连接层用于softmax分类b6 = variable_on_cpu('b6', [n_character], tf.random_normal_initializer(stddev=b_stddev))h6 = variable_on_cpu('h6', [n_hidden_5, n_character], tf.random_normal_initializer(stddev=h_stddev))layer_6 = tf.add(tf.matmul(layer_5, h6), b6)# 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].print(layer_6.get_shape().as_list())#[None, 2]layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])print(layer_6.get_shape().as_list())#[None, None, 2]layer_6 = tf.reduce_sum(layer_6, 0)#0按列相加，1按行相加print(layer_6.get_shape().as_list())#[None, 2]return layer_6"""
used to create a variable in CPU memory.
"""    
def variable_on_cpu(name, shape, initializer):# Use the /cpu:0 device for scoped operationswith tf.device('/cpu:0'):# Create or get apropos variablevar = tf.get_variable(name=name, shape=shape, initializer=initializer)return varwav_path='train'
test_path='test'
wav_files, labels = get_wavs_lables1(wav_path)
test_files, test_labels = get_wavs_lables1(test_path)
print(wav_files[0], labels[0])  print("wav:",len(wav_files),"label",len(labels))# 字表 
all_words = []  
for label in labels:  #print(label)    all_words += [word for word in label]  
counter = Counter(all_words) 
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size))) print('字表大小:', words_size) n_input = 26#计算美尔倒谱系数的个数
n_context = 9#对于每个时间点，要包含上下文样本的个数
batch_size =8
def next_batch(labels, start_idx = 0,batch_size=1,wav_files = wav_files):filesize = len(labels)end_idx = min(filesize, start_idx + batch_size)idx_list = range(start_idx, end_idx)txt_labels = [labels[i] for i in idx_list]wav_files = [wav_files[i] for i in idx_list](source, audio_len, target, transcript_len) = get_audio_and_transcriptch(None,wav_files,n_input,n_context,word_num_map,txt_labels)start_idx += batch_size# Verify that the start_idx is not larger than total available sample sizeif start_idx >= filesize:start_idx = -1# Pad input to max_time_step of this batchsource, source_lengths = pad_sequences(source)#如果多个文件将长度统一，支持按最大截断或补0sparse_labels = [label[0] for label in target]return start_idx,source, source_lengths, sparse_labels
#sparse_lab 为文字转化成向量后并生成的稀疏矩阵，所以长度为3，补0对齐后的音频数据的shape为（8，1168，494），
# 8代表batchsize；1168代表时序的总个数。494是组合好的MFCC特征数：取前9个时序的MFCC，当前MFCC再加上后9个
#MFCC，每个MFCC由26个数字组成。#在矩阵中，若数值为0的元素数目远远多于非0元素的数目，并且非0元素分布没有规律时，则称该矩阵为稀疏矩阵；
# 与之相反，若非0元素数目占大多数时，则称该矩阵为稠密矩阵。
next_idx,source,source_len,sparse_lab = next_batch(labels,0,batch_size)
print(len(sparse_lab))
print(np.shape(source))
#print(sparse_lab)
#t = sparse_tuple_to_texts_ch(sparse_lab,words)
#print(t[0])
#source为具体的样本,每条样本的内容为19个时间序列，source已经将变为前9（不够补空）+本身+后9，
#每个时间序列有26个美尔倒谱系数。第一条的样本是从第10个时间序列开始# shape = [batch_size, max_stepsize, n_input + (2 * n_input * n_context)]
# the batch_size and max_stepsize每步都是变长的。
input_tensor = tf.placeholder(tf.float32, [None, None, n_input + (2 * n_input * n_context)], name='input')#语音log filter bank or MFCC features
# Use sparse_placeholder; will generate a SparseTensor, required by ctc_loss op.
targets = tf.placeholder(tf.int64, shape=[None], name='targets')#文本
# 1d array of size [batch_size]
seq_length = tf.placeholder(tf.int32, [None], name='seq_length')#序列长
keep_dropout= tf.placeholder(tf.float32)logits = BiRNN_model( input_tensor, tf.to_int64(seq_length), n_input, n_context,words_size,keep_dropout)avg_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=targets, logits=logits))#[optimizer]
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(avg_loss)with tf.name_scope("decode"):    decoded=tf.nn.softmax(logits)with tf.name_scope("accuracy"):ler = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), targets), tf.float32))epochs = 20
savedir = "train"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# create the session
sess = tf.Session()
# 没有模型的话，就重新初始化
sess.run(tf.global_variables_initializer())kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:saver.restore(sess, kpt) ind = kpt.find("-")startepo = int(kpt[ind+1:])startepo +=1print(startepo)# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('Run training epoch'))train_start = time.time()
for epoch in range(epochs):#样本集迭代次数epoch_start = time.time()if epoch<startepo:continueprint("epoch start:",epoch,"total epochs= ",epochs)
#######################run batch####n_batches_per_epoch = int(np.ceil(len(labels) / batch_size))print("total loop ",n_batches_per_epoch,"in one epoch，",batch_size,"items in one loop") train_cost = 0train_ler = 0next_idx =0test_next_idx=0for batch in range(n_batches_per_epoch):#一次batch_size，取多少次#取数据next_idx,source,source_lengths,sparse_labels = \next_batch(labels,next_idx ,batch_size)feed = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:keep_dropout_rate}#计算 avg_loss optimizer ;batch_cost, _ = sess.run([avg_loss, optimizer],  feed_dict=feed )train_cost += batch_cost if (batch +1)%2 == 0:print('Epoch/Epochs:',epoch,'/',epochs,'loop:',batch, 'Train cost: ', train_cost/(batch+1))feed2 = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:1.0}d,train_ler = sess.run([decoded,ler], feed_dict=feed2)dense_decoded = tf.argmax( d, 1).eval(session=sess)counter =0print('Label right rate: ', train_ler)for orig, decoded_arr in zip(sparse_labels, dense_decoded):print(' file {}'.format( counter))print('Original: {}'.format(orig))print('Decoded:  {}'.format(decoded_arr))counter=counter+1# breakif (batch +1)%5 == 0:print('Epoch/Epochs:',epoch,'/',epochs,'loop:',batch, 'Train cost: ', train_cost/(batch+1))#测试数据test_next_idx,test_source,test_source_lengths,test_sparse_labels = \next_batch(test_labels,test_next_idx ,batch_size,wav_files=test_files)feed2 = {input_tensor: test_source, targets: test_sparse_labels,seq_length: test_source_lengths,keep_dropout:1.0}d,test_ler = sess.run([decoded,ler], feed_dict=feed2)dense_decoded = tf.argmax( d, 1).eval(session=sess)counter =0print('test Label right rate: ', test_ler)for orig, decoded_arr in zip(test_sparse_labels, dense_decoded):print(' test file {}'.format( counter))print('test Original: {}'.format(orig))print('test Decoded:  {}'.format(decoded_arr))counter=counter+1# breakepoch_duration = time.time() - epoch_startlog = 'Epoch {}/{}, train_cost: {:.3f}, train_ler: {:.3f}, time: {:.2f} sec'print(log.format(epoch ,epochs, train_cost,train_ler,epoch_duration))saver.save(sess, savedir+"speechclassify.gd", global_step=epoch)train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))sess.close()