1 # Copyright (c) 2019 Guo Yejun
3 # This file is part of FFmpeg.
5 # FFmpeg is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License, or (at your option) any later version.
10 # FFmpeg is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with FFmpeg; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 # ==============================================================================
20 import tensorflow as tf
23 import convert_header as header
25 __all__ = ['convert_from_tensorflow']
27 class Operand(object):
30 IOTYPE_INTERMEDIATE = IOTYPE_INPUT | IOTYPE_OUTPUT
34 def __init__(self, name, dtype, dims):
40 self.index = Operand.index
41 Operand.index = Operand.index + 1
42 self.iotype2str = {Operand.IOTYPE_INPUT: 'in', Operand.IOTYPE_OUTPUT: 'out', Operand.IOTYPE_INTERMEDIATE: 'inout'}
43 self.dtype2str = {Operand.DTYPE_FLOAT: 'DT_FLOAT', Operand.DTYPE_UINT8: 'DT_UINT8'}
45 def add_iotype(self, iotype):
46 self.iotype = self.iotype | iotype
47 if iotype == Operand.IOTYPE_INPUT:
48 self.used_count = self.used_count + 1
51 return "{}: (name: {}, iotype: {}, dtype: {}, dims: ({},{},{},{}) used_count: {})".format(self.index,
52 self.name, self.iotype2str[self.iotype], self.dtype2str[self.dtype],
53 self.dims[0], self.dims[1], self.dims[2], self.dims[3], self.used_count)
55 def __lt__(self, other):
56 return self.index < other.index
59 def __init__(self, graph_def, nodes, outfile, dump4tb):
60 self.graph_def = graph_def
62 self.outfile = outfile
63 self.dump4tb = dump4tb
65 self.output_names = []
66 self.name_node_dict = {}
68 self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'None':3, 'LeakyRelu':4}
69 self.conv_paddings = {'VALID':0, 'SAME':1}
70 self.converted_nodes = set()
71 self.conv2d_scope_names = set()
72 self.conv2d_scopename_inputname_dict = {}
73 self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5}
74 self.mathbin2code = {'Sub':0, 'Add':1, 'Mul':2, 'RealDiv':3}
75 self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
76 self.name_operand_dict = {}
79 def add_operand(self, name, type):
80 node = self.name_node_dict[name]
81 if name not in self.name_operand_dict:
82 dtype = node.attr['dtype'].type
84 dtype = node.attr['T'].type
86 if 'shape' in node.attr:
87 dims[0] = node.attr['shape'].shape.dim[0].size
88 dims[1] = node.attr['shape'].shape.dim[1].size
89 dims[2] = node.attr['shape'].shape.dim[2].size
90 dims[3] = node.attr['shape'].shape.dim[3].size
91 operand = Operand(name, dtype, dims)
92 self.name_operand_dict[name] = operand;
93 self.name_operand_dict[name].add_iotype(type)
94 return self.name_operand_dict[name].index
97 def dump_for_tensorboard(self):
98 graph = tf.get_default_graph()
99 tf.import_graph_def(self.graph_def, name="")
100 tf.summary.FileWriter('/tmp/graph', graph)
101 print('graph saved, run "tensorboard --logdir=/tmp/graph" to see it')
104 def get_conv2d_params(self, conv2d_scope_name):
105 knode = self.name_node_dict[conv2d_scope_name + '/kernel']
106 bnode = self.name_node_dict[conv2d_scope_name + '/bias']
108 if conv2d_scope_name + '/dilation_rate' in self.name_node_dict:
109 dnode = self.name_node_dict[conv2d_scope_name + '/dilation_rate']
113 # the BiasAdd name is possible be changed into the output name,
114 # if activation is None, and BiasAdd.next is the last op which is Identity
115 if conv2d_scope_name + '/BiasAdd' in self.edges:
116 anode = self.edges[conv2d_scope_name + '/BiasAdd'][0]
117 if anode.op not in self.conv_activations:
121 return knode, bnode, dnode, anode
124 def dump_complex_conv2d_to_file(self, node, f):
125 assert(node.op == 'Conv2D')
126 self.layer_number = self.layer_number + 1
127 self.converted_nodes.add(node.name)
129 scope_name = TFConverter.get_scope_name(node.name)
130 #knode for kernel, bnode for bias, dnode for dilation, anode for activation
131 knode, bnode, dnode, anode = self.get_conv2d_params(scope_name)
133 if dnode is not None:
134 dilation = struct.unpack('i', dnode.attr['value'].tensor.tensor_content[0:4])[0]
138 if anode is not None:
139 activation = anode.op
143 padding = node.attr['padding'].s.decode("utf-8")
144 # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use this tricky method.
145 if dilation > 1 and scope_name + '/stack' in self.name_node_dict:
146 if self.name_node_dict[scope_name + '/stack'].op == "Const":
148 padding = self.conv_paddings[padding]
150 ktensor = knode.attr['value'].tensor
151 filter_height = ktensor.tensor_shape.dim[0].size
152 filter_width = ktensor.tensor_shape.dim[1].size
153 in_channels = ktensor.tensor_shape.dim[2].size
154 out_channels = ktensor.tensor_shape.dim[3].size
155 kernel = np.frombuffer(ktensor.tensor_content, dtype=np.float32)
156 kernel = kernel.reshape(filter_height, filter_width, in_channels, out_channels)
157 kernel = np.transpose(kernel, [3, 0, 1, 2])
160 np.array([self.op2code[node.op], dilation, padding, self.conv_activations[activation], in_channels, out_channels, filter_height, has_bias], dtype=np.uint32).tofile(f)
163 btensor = bnode.attr['value'].tensor
164 if btensor.tensor_shape.dim[0].size == 1:
165 bias = struct.pack("f", btensor.float_val[0])
167 bias = btensor.tensor_content
170 input_name = self.conv2d_scopename_inputname_dict[scope_name]
171 input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
173 if anode is not None:
174 output_operand_index = self.add_operand(anode.name, Operand.IOTYPE_OUTPUT)
176 output_operand_index = self.add_operand(self.edges[bnode.name][0].name, Operand.IOTYPE_OUTPUT)
177 np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
180 def dump_simple_conv2d_to_file(self, node, f):
181 assert(node.op == 'Conv2D')
182 self.layer_number = self.layer_number + 1
183 self.converted_nodes.add(node.name)
185 node0 = self.name_node_dict[node.input[0]]
186 node1 = self.name_node_dict[node.input[1]]
187 if node0.op == 'Const':
189 input_name = node.input[1]
192 input_name = node.input[0]
194 ktensor = knode.attr['value'].tensor
195 filter_height = ktensor.tensor_shape.dim[0].size
196 filter_width = ktensor.tensor_shape.dim[1].size
197 in_channels = ktensor.tensor_shape.dim[2].size
198 out_channels = ktensor.tensor_shape.dim[3].size
199 if filter_height * filter_width * in_channels * out_channels == 1:
200 kernel = np.float32(ktensor.float_val[0])
202 kernel = np.frombuffer(ktensor.tensor_content, dtype=np.float32)
203 kernel = kernel.reshape(filter_height, filter_width, in_channels, out_channels)
204 kernel = np.transpose(kernel, [3, 0, 1, 2])
208 padding = node.attr['padding'].s.decode("utf-8")
209 np.array([self.op2code[node.op], dilation, self.conv_paddings[padding], self.conv_activations['None'],
210 in_channels, out_channels, filter_height, has_bias], dtype=np.uint32).tofile(f)
213 input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
214 output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
215 np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
218 def dump_depth2space_to_file(self, node, f):
219 assert(node.op == 'DepthToSpace')
220 self.layer_number = self.layer_number + 1
221 block_size = node.attr['block_size'].i
222 np.array([self.op2code[node.op], block_size], dtype=np.uint32).tofile(f)
223 self.converted_nodes.add(node.name)
224 input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
225 output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
226 np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
229 def dump_mirrorpad_to_file(self, node, f):
230 assert(node.op == 'MirrorPad')
231 self.layer_number = self.layer_number + 1
232 mode = node.attr['mode'].s
233 mode = self.mirrorpad_mode[mode.decode("utf-8")]
234 np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f)
235 pnode = self.name_node_dict[node.input[1]]
236 self.converted_nodes.add(pnode.name)
237 paddings = pnode.attr['value'].tensor.tensor_content
239 self.converted_nodes.add(node.name)
240 input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
241 output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
242 np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
245 def dump_maximum_to_file(self, node, f):
246 assert(node.op == 'Maximum')
247 self.layer_number = self.layer_number + 1
248 ynode = self.name_node_dict[node.input[1]]
249 y = ynode.attr['value'].tensor.float_val[0]
250 np.array([self.op2code[node.op]], dtype=np.uint32).tofile(f)
251 np.array([y], dtype=np.float32).tofile(f)
252 self.converted_nodes.add(node.name)
253 input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
254 output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
255 np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
258 def dump_mathbinary_to_file(self, node, f):
259 self.layer_number = self.layer_number + 1
260 self.converted_nodes.add(node.name)
261 i0_node = self.name_node_dict[node.input[0]]
262 i1_node = self.name_node_dict[node.input[1]]
263 np.array([self.op2code['MathBinary'], self.mathbin2code[node.op]], dtype=np.uint32).tofile(f)
264 if i0_node.op == 'Const':
265 scalar = i0_node.attr['value'].tensor.float_val[0]
266 np.array([1], dtype=np.uint32).tofile(f) # broadcast: 1
267 np.array([scalar], dtype=np.float32).tofile(f)
268 np.array([0], dtype=np.uint32).tofile(f) # broadcast: 0
269 input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT)
270 np.array([input_operand_index], dtype=np.uint32).tofile(f)
271 elif i1_node.op == 'Const':
272 scalar = i1_node.attr['value'].tensor.float_val[0]
273 np.array([0], dtype=np.uint32).tofile(f)
274 input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
275 np.array([input_operand_index], dtype=np.uint32).tofile(f)
276 np.array([1], dtype=np.uint32).tofile(f)
277 np.array([scalar], dtype=np.float32).tofile(f)
279 np.array([0], dtype=np.uint32).tofile(f)
280 input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
281 np.array([input_operand_index], dtype=np.uint32).tofile(f)
282 np.array([0], dtype=np.uint32).tofile(f)
283 input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT)
284 np.array([input_operand_index], dtype=np.uint32).tofile(f)
285 output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
286 np.array([output_operand_index], dtype=np.uint32).tofile(f)
289 def dump_layers_to_file(self, f):
290 for node in self.nodes:
291 if node.name in self.converted_nodes:
294 # conv2d with dilation generates very complex nodes, so handle it in special
295 if self.in_conv2d_scope(node.name):
296 if node.op == 'Conv2D':
297 self.dump_complex_conv2d_to_file(node, f)
300 if node.op == 'Conv2D':
301 self.dump_simple_conv2d_to_file(node, f)
302 elif node.op == 'DepthToSpace':
303 self.dump_depth2space_to_file(node, f)
304 elif node.op == 'MirrorPad':
305 self.dump_mirrorpad_to_file(node, f)
306 elif node.op == 'Maximum':
307 self.dump_maximum_to_file(node, f)
308 elif node.op == 'Sub':
309 self.dump_mathbinary_to_file(node, f)
310 elif node.op == 'Add':
311 self.dump_mathbinary_to_file(node, f)
312 elif node.op == 'Mul':
313 self.dump_mathbinary_to_file(node, f)
314 elif node.op == 'RealDiv':
315 self.dump_mathbinary_to_file(node, f)
317 def dump_operands_to_file(self, f):
318 operands = sorted(self.name_operand_dict.values())
319 for operand in operands:
320 #print('{}'.format(operand))
321 np.array([operand.index, len(operand.name)], dtype=np.uint32).tofile(f)
322 f.write(operand.name.encode('utf-8'))
323 np.array([operand.iotype, operand.dtype], dtype=np.uint32).tofile(f)
324 np.array([operand.dims[0], operand.dims[1], operand.dims[2], operand.dims[3]], dtype=np.uint32).tofile(f)
327 def dump_to_file(self):
328 with open(self.outfile, 'wb') as f:
329 f.write(header.str.encode('utf-8'))
330 np.array([header.major, header.minor], dtype=np.uint32).tofile(f)
331 self.dump_layers_to_file(f)
332 self.dump_operands_to_file(f)
333 np.array([self.layer_number, len(self.name_operand_dict)], dtype=np.uint32).tofile(f)
336 def generate_name_node_dict(self):
337 for node in self.nodes:
338 self.name_node_dict[node.name] = node
341 def generate_output_names(self):
343 for node in self.nodes:
344 for input in node.input:
345 used_names.append(input)
347 for node in self.nodes:
348 if node.name not in used_names:
349 self.output_names.append(node.name)
352 def remove_identity(self):
355 for node in self.nodes:
356 if node.op == 'Identity':
358 input = node.input[0]
359 id_nodes.append(node)
360 # do not change the output name
361 if name in self.output_names:
362 self.name_node_dict[input].name = name
363 self.name_node_dict[name] = self.name_node_dict[input]
364 del self.name_node_dict[input]
366 id_dict[name] = input
368 for idnode in id_nodes:
369 self.nodes.remove(idnode)
371 for node in self.nodes:
372 for i in range(len(node.input)):
373 input = node.input[i]
375 node.input[i] = id_dict[input]
378 def generate_edges(self):
379 for node in self.nodes:
380 for input in node.input:
381 if input in self.edges:
382 self.edges[input].append(node)
384 self.edges[input] = [node]
388 def get_scope_name(name):
389 index = name.rfind('/')
395 def in_conv2d_scope(self, name):
396 inner_scope = TFConverter.get_scope_name(name)
397 if inner_scope == "":
399 for scope in self.conv2d_scope_names:
400 index = inner_scope.find(scope)
406 def generate_conv2d_scope_info(self):
407 # mostly, conv2d is a sub block in graph, get the scope name
408 for node in self.nodes:
409 if node.op == 'Conv2D':
410 scope = TFConverter.get_scope_name(node.name)
411 # for the case tf.nn.conv2d is called directly
414 # for the case tf.nn.conv2d is called within a scope
415 if scope + '/kernel' not in self.name_node_dict:
417 self.conv2d_scope_names.add(scope)
419 # get the input name to the conv2d sub block
420 for node in self.nodes:
421 scope = TFConverter.get_scope_name(node.name)
422 if scope in self.conv2d_scope_names:
423 if node.op == 'Conv2D' or node.op == 'Shape':
424 for inp in node.input:
425 if TFConverter.get_scope_name(inp) != scope:
426 self.conv2d_scopename_inputname_dict[scope] = inp
430 self.generate_name_node_dict()
431 self.generate_output_names()
432 self.remove_identity()
433 self.generate_edges()
434 self.generate_conv2d_scope_info()
437 self.dump_for_tensorboard()
442 def convert_from_tensorflow(infile, outfile, dump4tb):
443 with open(infile, 'rb') as f:
444 # read the file in .proto format
445 graph_def = tf.GraphDef()
446 graph_def.ParseFromString(f.read())
447 nodes = graph_def.node
449 converter = TFConverter(graph_def, nodes, outfile, dump4tb)