Skip to content
This repository was archived by the owner on Jan 3, 2023. It is now read-only.

Commit 9111b29

Browse files
Adam Rogowiecpostrational
Adam Rogowiec
authored and
postrational
committed
Update ONNX to 1.5.0 (#386)
* Bump up ONNX version to 1.5.0 * Xfail failing backend tests. * Refactor old slice ut to use slice in opset 1. * Update model zoo tests. * Xfail yolov3 and skip bidaf big models.
1 parent 2f2fbb4 commit 9111b29

File tree

5 files changed

+143
-37
lines changed

5 files changed

+143
-37
lines changed

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
numpy==1.16.3
2-
onnx==1.4.1
2+
onnx==1.5.0
33
cachetools==3.1.0

‎tests/test_backend.py

Lines changed: 82 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -160,18 +160,93 @@
160160
# Non zero -> NGONNX-472
161161
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonzero_example_cpu)
162162

163+
# ConvInteger NGONNX-410
164+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_basic_convinteger_cpu)
165+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_convinteger_with_padding_cpu)
166+
167+
# Quantized NGONNX-595
168+
# Scale / zero point not a scalar
169+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dequantizelinear_cpu)
170+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_qlinearconv_cpu)
171+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_qlinearmatmul_2D_cpu)
172+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_qlinearmatmul_3D_cpu)
173+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_quantizelinear_cpu)
174+
175+
# MatmulInteger NGONNX-410
176+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_matmulinteger_cpu)
177+
178+
# IsInf - NGONNX-528
179+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_isinf_cpu)
180+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_isinf_negative_cpu)
181+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_isinf_positive_cpu)
182+
183+
# Pooling ops NGONNX-597
184+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_maxpool_2d_ceil_cpu)
185+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_maxpool_2d_dilations_cpu)
186+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_averagepool_2d_ceil_cpu)
187+
188+
# Modulus - NGONNX-527
189+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_mod_bcast_cpu)
190+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_mod_float_mixed_sign_example_cpu)
191+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_mod_fmod_mixed_sign_example_cpu)
192+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_mod_int64_mixed_sign_example_cpu)
193+
194+
# NonMaxSuppression - NGONNX-526
195+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_center_point_box_format_cpu)
196+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_flipped_coordinates_cpu)
197+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_identical_boxes_cpu)
198+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_limit_output_size_cpu)
199+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_single_box_cpu)
200+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu)
201+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_cpu)
202+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_two_batches_cpu)
203+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu)
204+
205+
# Resize NGONNX-598
206+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_resize_downsample_linear_cpu)
207+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_resize_downsample_nearest_cpu)
208+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_resize_nearest_cpu)
209+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_resize_upsample_linear_cpu)
210+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_resize_upsample_nearest_cpu)
211+
212+
# ReverseSequence - NGONNX-525
213+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_reversesequence_batch_cpu)
214+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_reversesequence_time_cpu)
215+
216+
# Dynamic Slice NGONNX-522, 599
217+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_cpu)
218+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_default_axes_cpu)
219+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_default_steps_cpu)
220+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_end_out_of_bounds_cpu)
221+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_neg_cpu)
222+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_neg_steps_cpu)
223+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_slice_start_out_of_bounds_cpu)
224+
225+
# StrNormalizer NGONNX-600
226+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_lower_cpu)
227+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_nochangecase_cpu)
228+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_upper_cpu)
229+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_export_monday_empty_output_cpu)
230+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_export_monday_insensintive_upper_twodim_cpu)
231+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_strnormalizer_nostopwords_nochangecase_cpu)
232+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_lower_cpu)
233+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_nochangecase_cpu)
234+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_upper_cpu)
235+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_monday_empty_output_cpu)
236+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_monday_insensintive_upper_twodim_cpu)
237+
pytest.mark.xfail(OnnxBackendSimpleModelTest.test_strnorm_model_nostopwords_nochangecase_cpu)
238+
239+
# RoiAlign NGONNX-601
240+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_roialign_cpu)
241+
242+
# NGONNX-521
243+
pytest.mark.xfail(OnnxBackendNodeModelTest.test_top_k_cpu)
244+
163245
# Other tests
164246
pytest.mark.xfail(OnnxBackendNodeModelTest.test_instancenorm_epsilon_cpu)
165247
pytest.mark.xfail(OnnxBackendNodeModelTest.test_instancenorm_example_cpu)
166248
pytest.mark.xfail(OnnxBackendNodeModelTest.test_upsample_nearest_cpu)
167249

168-
# Dynamic Slice -> EXPERIMENTAL https://github.com/onnx/onnx/blob/master/docs/Operators.md#DynamicSlice
169-
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_cpu)
170-
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_default_axes_cpu)
171-
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_end_out_of_bounds_cpu)
172-
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_neg_cpu)
173-
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_start_out_of_bounds_cpu)
174-
175250
# Tests which fail on the INTELGPU backend
176251
if selected_backend_name == 'INTELGPU':
177252
pytest.mark.xfail(OnnxBackendNodeModelTest.test_dynamic_slice_start_out_of_bounds_cpu)

‎tests/test_reshape.py

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import onnx
2121
import pytest
2222

23-
from tests.utils import all_arrays_equal, run_node, get_runtime
23+
from tests.utils import all_arrays_equal, run_model, run_node, get_node_model, get_runtime
2424
from onnx.helper import make_node, make_graph, make_tensor_value_info, make_model
2525
from ngraph_onnx.onnx_importer.importer import import_onnx_model
2626

@@ -115,58 +115,51 @@ def test_transpose():
115115
assert np.array_equal(ng_results, [expected_output])
116116

117117

118-
def test_slice():
118+
def test_slice_opset1():
119119
data = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
120120

121121
expected_output = np.array([[5, 6, 7]])
122-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'],
123-
axes=[0, 1], starts=[1, 0], ends=[2, 3])
124-
ng_results = run_node(node, [data])
122+
model = get_node_model('Slice', data, axes=[0, 1], starts=[1, 0], ends=[2, 3])
123+
ng_results = run_model(model, [data])
125124
assert np.array_equal(ng_results, [expected_output])
126125

127126
expected_output = np.array([[2, 3, 4]])
128-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], starts=[0, 1],
129-
ends=[-1, 1000])
130-
ng_results = run_node(node, [data])
127+
model = get_node_model('Slice', data, starts=[0, 1], ends=[-1, 1000])
128+
ng_results = run_model(model, [data])
131129
assert np.array_equal(ng_results, [expected_output])
132130

133-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], axes=[0, 1],
134-
starts=[0, 0], ends=[3, 10])
135131
data = np.random.randn(20, 10, 5).astype(np.float32)
136132
expected_output = data[0:3, 0:10]
137-
ng_results = run_node(node, [data])
133+
model = get_node_model('Slice', data, axes=[0, 1], starts=[0, 0], ends=[3, 10])
134+
ng_results = run_model(model, [data])
138135
assert np.array_equal(ng_results, [expected_output])
139136

140137
# default axes
141-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], starts=[0, 0, 3],
142-
ends=[20, 10, 4])
143138
data = np.random.randn(20, 10, 5).astype(np.float32)
144139
expected_output = data[:, :, 3:4]
145-
ng_results = run_node(node, [data])
140+
model = get_node_model('Slice', data, starts=[0, 0, 3], ends=[20, 10, 4])
141+
ng_results = run_model(model, [data])
146142
assert np.array_equal(ng_results, [expected_output])
147143

148144
# end out of bounds
149-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], axes=[1], starts=[1],
150-
ends=[1000])
151145
data = np.random.randn(20, 10, 5).astype(np.float32)
152146
expected_output = data[:, 1:1000]
153-
ng_results = run_node(node, [data])
147+
model = get_node_model('Slice', data, axes=[1], starts=[1], ends=[1000])
148+
ng_results = run_model(model, [data])
154149
assert np.array_equal(ng_results, [expected_output])
155150

156151
# negative value
157-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], axes=[1], starts=[0],
158-
ends=[-1])
159152
data = np.random.randn(20, 10, 5).astype(np.float32)
160153
expected_output = data[:, 0:-1]
161-
ng_results = run_node(node, [data])
154+
model = get_node_model('Slice', data, axes=[1], starts=[0], ends=[-1])
155+
ng_results = run_model(model, [data])
162156
assert np.array_equal(ng_results, [expected_output])
163157

164158
# start ouf of bounds
165-
node = onnx.helper.make_node('Slice', inputs=['x'], outputs=['y'], axes=[1], starts=[1000],
166-
ends=[1000])
167159
data = np.random.randn(20, 10, 5).astype(np.float32)
168160
expected_output = data[:, 1000:1000]
169-
ng_results = run_node(node, [data])
161+
model = get_node_model('Slice', data, axes=[1], starts=[1000], ends=[1000])
162+
ng_results = run_model(model, [data])
170163
assert np.array_equal(ng_results, [expected_output])
171164

172165

‎tests/test_zoo_models.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@
3434
'url': _S3_MODEL_ZOO + 'arcface/resnet100/resnet100.tar.gz',
3535
},
3636

37+
# BiDAF
38+
{
39+
'model_name': 'bidaf_opset9',
40+
'atol': 1e-07,
41+
'rtol': 0.001,
42+
'url': _WINDOWS_NET + 'opset_9/bidaf/bidaf.tar.gz',
43+
},
44+
3745
# BVLC AlexNet
3846
{
3947
'model_name': 'bvlc_alexnet_opset3',
@@ -362,6 +370,21 @@
362370
'rtol': 0.001,
363371
'url': _S3_DOWNLOAD_ONNX + 'opset_9/squeezenet.tar.gz',
364372
},
373+
{
374+
'model_name': 'squeezenet1.1_opset7',
375+
'atol': 1e-07,
376+
'rtol': 0.001,
377+
'url': _S3_MODEL_ZOO + 'squeezenet/squeezenet1.1/squeezenet1.1.tar.gz',
378+
379+
},
380+
381+
# SSD
382+
{
383+
'model_name': 'ssd_opset10',
384+
'atol': 1e-07,
385+
'rtol': 0.001,
386+
'url': _WINDOWS_NET + 'opset_10/ssd/ssd.tar.gz',
387+
},
365388

366389
# Tiny-YOLOv2
367390
{
@@ -384,6 +407,14 @@
384407
{'model_name': 'vgg19_opset8', 'url': _S3_DOWNLOAD_ONNX + 'opset_8/vgg19.tar.gz'},
385408
{'model_name': 'vgg19_opset9', 'url': _S3_DOWNLOAD_ONNX + 'opset_9/vgg19.tar.gz'},
386409

410+
# YOLOv3
411+
{
412+
'model_name': 'yolov3_opset10',
413+
'atol': 1e-07,
414+
'rtol': 0.001,
415+
'url': _WINDOWS_NET + 'opset_10/yolov3/yolov3.tar.gz',
416+
},
417+
387418
# ZFNet-512
388419
{
389420
'model_name': 'zfnet512_opset3',
@@ -430,6 +461,13 @@
430461
# ONNX ValidationError
431462
backend_test.exclude('test_mnist_opset1')
432463
backend_test.exclude('test_tiny_yolov2_opset1')
464+
backend_test.exclude('test_yolov3_opset10')
465+
466+
# Use of unsupported domain: ai.onnx.ml
467+
pytest.mark.skip(test_cases.test_bidaf_opset9_cpu)
468+
469+
# Not yet supported
470+
backend_test.exclude('test_ssd_opset10')
433471

434472
# Tests which fail on the INTELGPU backend
435473
if tests.utils.BACKEND_NAME == 'INTELGPU':

‎tests/utils/model_zoo_tester.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,9 @@ def run(test_self, device): # type: (Any, Text) -> None
171171
inputs = list(test_data['inputs'])
172172
outputs = list(prepared_model.run(inputs))
173173
ref_outputs = test_data['outputs']
174-
self._assert_similar_outputs(ref_outputs, outputs,
175-
rtol=model_test.rtol,
176-
atol=model_test.atol)
174+
self.assert_similar_outputs(ref_outputs, outputs,
175+
rtol=model_test.rtol,
176+
atol=model_test.atol)
177177

178178
for test_data_dir in glob.glob(
179179
os.path.join(model_dir, 'test_data_set*')):
@@ -194,8 +194,8 @@ def run(test_self, device): # type: (Any, Text) -> None
194194
tensor.ParseFromString(f.read())
195195
ref_outputs.append(numpy_helper.to_array(tensor))
196196
outputs = list(prepared_model.run(inputs))
197-
self._assert_similar_outputs(ref_outputs, outputs,
198-
rtol=model_test.rtol,
199-
atol=model_test.atol)
197+
self.assert_similar_outputs(ref_outputs, outputs,
198+
rtol=model_test.rtol,
199+
atol=model_test.atol)
200200

201201
self._add_test(kind + 'Model', model_test.name, run, model_marker)

0 commit comments

Comments
 (0)