Skip to content

Commit 2e9db81

Browse files
lukasrothenbergerlukasrothenberger
committed
end-to-end tests: improved structure
1 parent dbdd915 commit 2e9db81

File tree

58 files changed

+862
-462
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+862
-462
lines changed

‎test/end_to_end/do_all/backwards_array_access/test.py

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
1+
import copy
12
import os
23
import pathlib
34
import unittest
45

56
import jsonpickle
67

78
from discopop_library.result_classes.DetectionResult import DetectionResult
9+
from test.utils.existence.existence_utils import check_patterns_for_FP, check_patterns_for_FN
810
from test.utils.subprocess_wrapper.command_execution_wrapper import run_cmd
911
from test.utils.validator_classes.DoAllInfoForValidation import DoAllInfoForValidation
1012
from discopop_library.ConfigProvider.config_provider import run as run_config_provider
1113
from discopop_library.ConfigProvider.ConfigProviderArguments import ConfigProviderArguments
1214

1315

1416
class TestMethods(unittest.TestCase):
15-
def setUp(self):
17+
@classmethod
18+
def setUpClass(self):
1619
current_dir = pathlib.Path(__file__).parent.resolve()
1720
dp_build_dir = run_config_provider(
1821
ConfigProviderArguments(
@@ -36,6 +39,7 @@ def setUp(self):
3639
# build
3740
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3841
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
42+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
3943
cmd = "make"
4044
run_cmd(cmd, src_dir, env_vars)
4145
# execute instrumented program
@@ -48,20 +52,26 @@ def setUp(self):
4852
self.src_dir = src_dir
4953
self.env_vars = env_vars
5054

51-
def tearDown(self):
52-
run_cmd("make veryclean", self.src_dir, self.env_vars)
53-
54-
def test(self):
55-
"""Check that exactly one do-all is suggested"""
5655
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5756
# load detection results
5857
with open(test_output_file, "r") as f:
5958
tmp_str = f.read()
60-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
59+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6160

62-
for pattern_type in test_output.patterns.__dict__:
63-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
61+
@classmethod
62+
def tearDownClass(self):
63+
run_cmd("make veryclean", self.src_dir, self.env_vars)
64+
65+
def test(self):
66+
for pattern_type in self.test_output.patterns.__dict__:
67+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6468
if pattern_type == "do_all":
65-
self.assertEqual(amount_of_identified_patterns, 1)
69+
expected_lines = ["1:10"]
70+
with self.subTest("check for FP"):
71+
res, msg = check_patterns_for_FP(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
72+
self.assertTrue(res, msg)
73+
with self.subTest("check for FN"):
74+
res, msg = check_patterns_for_FN(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
75+
self.assertTrue(res, msg)
6676
else:
67-
self.assertEqual(amount_of_identified_patterns, 0)
77+
self.assertEqual(amount_of_identified_patterns, 0)

‎test/end_to_end/do_all/calls/LULESH_proxy/test.py

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414

1515

1616
class TestMethods(unittest.TestCase):
17-
def setUp(self):
17+
@classmethod
18+
def setUpClass(self):
1819
current_dir = pathlib.Path(__file__).parent.resolve()
1920
dp_build_dir = run_config_provider(
2021
ConfigProviderArguments(
@@ -37,6 +38,7 @@ def setUp(self):
3738
# build
3839
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3940
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
41+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
4042
cmd = "make"
4143
run_cmd(cmd, src_dir, env_vars)
4244
# execute instrumented program
@@ -50,23 +52,24 @@ def setUp(self):
5052
self.src_dir = src_dir
5153
self.env_vars = env_vars
5254

53-
def tearDown(self):
54-
run_cmd("make veryclean", self.src_dir, self.env_vars)
55-
56-
def test(self):
57-
"""Check that main loop do-all is suggested"""
5855
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5956
# load detection results
6057
with open(test_output_file, "r") as f:
6158
tmp_str = f.read()
62-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
59+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6360

61+
@classmethod
62+
def tearDownClass(self):
63+
run_cmd("make veryclean", self.src_dir, self.env_vars)
64+
65+
def test(self):
66+
"""Check that main loop do-all is suggested"""
6467
found_main_loop = False
6568

66-
for pattern_type in test_output.patterns.__dict__:
67-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
69+
for pattern_type in self.test_output.patterns.__dict__:
70+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6871
if pattern_type == "do_all":
69-
do_all_patterns = test_output.patterns.__dict__[pattern_type]
72+
do_all_patterns = self.test_output.patterns.__dict__[pattern_type]
7073

7174
for pattern in do_all_patterns:
7275
if cast(DoAllInfo, pattern).start_line == "1:271":

‎test/end_to_end/do_all/calls/above_nesting_level_3/test.py

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
1+
import copy
12
import os
23
import pathlib
34
import unittest
45

56
import jsonpickle
67

78
from discopop_library.result_classes.DetectionResult import DetectionResult
9+
from test.utils.existence.existence_utils import check_patterns_for_FN, check_patterns_for_FP
810
from test.utils.subprocess_wrapper.command_execution_wrapper import run_cmd
911
from test.utils.validator_classes.DoAllInfoForValidation import DoAllInfoForValidation
1012
from discopop_library.ConfigProvider.config_provider import run as run_config_provider
1113
from discopop_library.ConfigProvider.ConfigProviderArguments import ConfigProviderArguments
1214

1315

1416
class TestMethods(unittest.TestCase):
15-
def setUp(self):
17+
@classmethod
18+
def setUpClass(self):
1619
current_dir = pathlib.Path(__file__).parent.resolve()
1720
dp_build_dir = run_config_provider(
1821
ConfigProviderArguments(
@@ -35,6 +38,7 @@ def setUp(self):
3538
# build
3639
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3740
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
41+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
3842
cmd = "make"
3943
run_cmd(cmd, src_dir, env_vars)
4044
# execute instrumented program
@@ -48,21 +52,26 @@ def setUp(self):
4852
self.src_dir = src_dir
4953
self.env_vars = env_vars
5054

51-
def tearDown(self):
52-
run_cmd("make veryclean", self.src_dir, self.env_vars)
53-
54-
55-
def test(self):
56-
"""Check that exactly one do-all is suggested"""
5755
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5856
# load detection results
5957
with open(test_output_file, "r") as f:
6058
tmp_str = f.read()
61-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
59+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6260

63-
for pattern_type in test_output.patterns.__dict__:
64-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
61+
@classmethod
62+
def tearDownClass(self):
63+
run_cmd("make veryclean", self.src_dir, self.env_vars)
64+
65+
def test(self):
66+
for pattern_type in self.test_output.patterns.__dict__:
67+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6568
if pattern_type == "do_all":
66-
self.assertEqual(amount_of_identified_patterns, 5)
69+
expected_lines = ["1:6", "1:17", "1:20", "1:21", "1:22"]
70+
with self.subTest("check for FP"):
71+
res, msg = check_patterns_for_FP(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
72+
self.assertTrue(res, msg)
73+
with self.subTest("check for FN"):
74+
res, msg = check_patterns_for_FN(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
75+
self.assertTrue(res, msg)
6776
else:
68-
self.assertEqual(amount_of_identified_patterns, 0)
77+
self.assertEqual(amount_of_identified_patterns, 0)

‎test/end_to_end/do_all/calls/allowing/test.py

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
1+
import copy
12
import os
23
import pathlib
34
import unittest
45

56
import jsonpickle
67

78
from discopop_library.result_classes.DetectionResult import DetectionResult
9+
from test.utils.existence.existence_utils import check_patterns_for_FN, check_patterns_for_FP
810
from test.utils.subprocess_wrapper.command_execution_wrapper import run_cmd
911
from test.utils.validator_classes.DoAllInfoForValidation import DoAllInfoForValidation
1012
from discopop_library.ConfigProvider.config_provider import run as run_config_provider
1113
from discopop_library.ConfigProvider.ConfigProviderArguments import ConfigProviderArguments
1214

1315

1416
class TestMethods(unittest.TestCase):
15-
def setUp(self):
17+
@classmethod
18+
def setUpClass(self):
1619
current_dir = pathlib.Path(__file__).parent.resolve()
1720
dp_build_dir = run_config_provider(
1821
ConfigProviderArguments(
@@ -35,6 +38,7 @@ def setUp(self):
3538
# build
3639
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3740
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
41+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
3842
cmd = "make"
3943
run_cmd(cmd, src_dir, env_vars)
4044
# execute instrumented program
@@ -48,20 +52,26 @@ def setUp(self):
4852
self.src_dir = src_dir
4953
self.env_vars = env_vars
5054

51-
def tearDown(self):
52-
run_cmd("make veryclean", self.src_dir, self.env_vars)
53-
54-
def test(self):
55-
"""Check that exactly one do-all is suggested"""
5655
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5756
# load detection results
5857
with open(test_output_file, "r") as f:
5958
tmp_str = f.read()
60-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
59+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6160

62-
for pattern_type in test_output.patterns.__dict__:
63-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
61+
@classmethod
62+
def tearDownClass(self):
63+
run_cmd("make veryclean", self.src_dir, self.env_vars)
64+
65+
def test(self):
66+
for pattern_type in self.test_output.patterns.__dict__:
67+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6468
if pattern_type == "do_all":
65-
self.assertEqual(amount_of_identified_patterns, 1)
69+
expected_lines = ["1:14"]
70+
with self.subTest("check for FP"):
71+
res, msg = check_patterns_for_FP(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
72+
self.assertTrue(res, msg)
73+
with self.subTest("check for FN"):
74+
res, msg = check_patterns_for_FN(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
75+
self.assertTrue(res, msg)
6676
else:
6777
self.assertEqual(amount_of_identified_patterns, 0)

‎test/end_to_end/do_all/calls/allowing_2/test.py

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
1+
import copy
12
import os
23
import pathlib
34
import unittest
45

56
import jsonpickle
67

78
from discopop_library.result_classes.DetectionResult import DetectionResult
9+
from test.utils.existence.existence_utils import check_patterns_for_FN, check_patterns_for_FP
810
from test.utils.subprocess_wrapper.command_execution_wrapper import run_cmd
911
from test.utils.validator_classes.DoAllInfoForValidation import DoAllInfoForValidation
1012
from discopop_library.ConfigProvider.config_provider import run as run_config_provider
1113
from discopop_library.ConfigProvider.ConfigProviderArguments import ConfigProviderArguments
1214

1315

1416
class TestMethods(unittest.TestCase):
15-
def setUp(self):
17+
@classmethod
18+
def setUpClass(self):
1619
current_dir = pathlib.Path(__file__).parent.resolve()
1720
dp_build_dir = run_config_provider(
1821
ConfigProviderArguments(
@@ -35,6 +38,7 @@ def setUp(self):
3538
# build
3639
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3740
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
41+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
3842
cmd = "make"
3943
run_cmd(cmd, src_dir, env_vars)
4044
# execute instrumented program
@@ -48,20 +52,26 @@ def setUp(self):
4852
self.src_dir = src_dir
4953
self.env_vars = env_vars
5054

51-
def tearDown(self):
52-
run_cmd("make veryclean", self.src_dir, self.env_vars)
53-
54-
def test(self):
55-
"""Check that exactly one do-all is suggested"""
5655
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5756
# load detection results
5857
with open(test_output_file, "r") as f:
5958
tmp_str = f.read()
60-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
59+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6160

62-
for pattern_type in test_output.patterns.__dict__:
63-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
61+
@classmethod
62+
def tearDownClass(self):
63+
run_cmd("make veryclean", self.src_dir, self.env_vars)
64+
65+
def test(self):
66+
for pattern_type in self.test_output.patterns.__dict__:
67+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6468
if pattern_type == "do_all":
65-
self.assertEqual(amount_of_identified_patterns, 2)
69+
expected_lines = ["1:17", "1:6"]
70+
with self.subTest("check for FP"):
71+
res, msg = check_patterns_for_FP(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
72+
self.assertTrue(res, msg)
73+
with self.subTest("check for FN"):
74+
res, msg = check_patterns_for_FN(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
75+
self.assertTrue(res, msg)
6676
else:
6777
self.assertEqual(amount_of_identified_patterns, 0)

‎test/end_to_end/do_all/calls/complex/test.py

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import copy
12
import os
23
import pathlib
34
import subprocess
@@ -6,6 +7,7 @@
67
import jsonpickle
78

89
from discopop_library.result_classes.DetectionResult import DetectionResult
10+
from test.utils.existence.existence_utils import check_patterns_for_FN, check_patterns_for_FP
911
from test.utils.subprocess_wrapper.command_execution_wrapper import run_cmd
1012
from test.utils.validator_classes.DoAllInfoForValidation import DoAllInfoForValidation
1113
from subprocess import DEVNULL
@@ -14,7 +16,8 @@
1416

1517

1618
class TestMethods(unittest.TestCase):
17-
def setUp(self):
19+
@classmethod
20+
def setUpClass(self):
1821
current_dir = pathlib.Path(__file__).parent.resolve()
1922
dp_build_dir = run_config_provider(
2023
ConfigProviderArguments(
@@ -36,6 +39,7 @@ def setUp(self):
3639
# build
3740
env_vars["CC"] = os.path.join(dp_build_dir, "scripts", "CC_wrapper.sh")
3841
env_vars["CXX"] = os.path.join(dp_build_dir, "scripts", "CXX_wrapper.sh")
42+
env_vars["DP_PROJECT_ROOT_DIR"] = src_dir
3943
cmd = "make"
4044
run_cmd(cmd, src_dir, env_vars)
4145

@@ -49,23 +53,26 @@ def setUp(self):
4953
self.src_dir = src_dir
5054
self.env_vars = env_vars
5155

52-
def tearDown(self):
53-
run_cmd("make veryclean", self.src_dir, self.env_vars)
54-
55-
def test(self):
56-
"""Check that exactly one do-all is suggested"""
5756
test_output_file = os.path.join(self.src_dir, ".discopop", "explorer", "detection_result_dump.json")
5857
# load detection results
5958
with open(test_output_file, "r") as f:
6059
tmp_str = f.read()
61-
test_output: DetectionResult = jsonpickle.decode(tmp_str)
60+
self.test_output: DetectionResult = jsonpickle.decode(tmp_str)
6261

63-
for pattern_type in test_output.patterns.__dict__:
64-
amount_of_identified_patterns = len(test_output.patterns.__dict__[pattern_type])
62+
@classmethod
63+
def tearDownClass(self):
64+
run_cmd("make veryclean", self.src_dir, self.env_vars)
65+
66+
def test(self):
67+
for pattern_type in self.test_output.patterns.__dict__:
68+
amount_of_identified_patterns = len(self.test_output.patterns.__dict__[pattern_type])
6569
if pattern_type == "do_all":
66-
expected = ["1:22", "1:27"]
67-
for pattern in test_output.patterns.__dict__[pattern_type]:
68-
self.assertTrue(pattern.start_line in expected, "False positive: Pattern at " + pattern.start_line + " not in expected result: " + str(expected))
69-
self.assertTrue(len(test_output.patterns.__dict__[pattern_type]) == 2, "False negative: Missed pattern. \nFound: " + " ".join([p.start_line for p in test_output.patterns.__dict__[pattern_type]])+"\nExpected: " + " ".join(expected))
70+
expected_lines = ["1:22", "1:27"]
71+
with self.subTest("check for FP"):
72+
res, msg = check_patterns_for_FP(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
73+
self.assertTrue(res, msg)
74+
with self.subTest("check for FN"):
75+
res, msg = check_patterns_for_FN(self, pattern_type, copy.deepcopy(expected_lines), self.test_output.patterns.__dict__[pattern_type])
76+
self.assertTrue(res, msg)
7077
else:
7178
self.assertEqual(amount_of_identified_patterns, 0)

0 commit comments

Comments
 (0)