Minor reliability improvement to pattern detection and additional end-to-end tests

CMakeLists.txt

@@ -152,6 +152,9 @@ else()
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_optimizer)
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_sanity_checker)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_project_manager)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_configuration_manager)
     message(STATUS "--> discopop_auto_tuner")
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_auto_tuner ${DP_LOCAL_BIN_DIR}/discopop_auto_tuner)
     message(STATUS "--> discopop_config_provider")
@@ -164,5 +167,13 @@ else()
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_applicator ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
     message(STATUS "--> discopop_patch_generator")
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_generator ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
+    message(STATUS "--> discopop_sanity_checker")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_sanity_checker ${DP_LOCAL_BIN_DIR}/discopop_sanity_checker)
+    message(STATUS "--> discopop_preprocessor")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_preprocessor ${DP_LOCAL_BIN_DIR}/discopop_preprocessor)
+    message(STATUS "--> discopop_project_manager")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_project_manager ${DP_LOCAL_BIN_DIR}/discopop_project_manager)
+    message(STATUS "--> discopop_configuration_manager")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_configuration_manager ${DP_LOCAL_BIN_DIR}/discopop_configuration_manager)
 
 endif()

DEBIAN/postinst

@@ -46,10 +46,12 @@ su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_explorer"
 su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_optimizer"
 su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_applicator"
 su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_generator"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_sanity_checker"
 
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_auto_tuner ~/.local/bin/discopop_auto_tuner"
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_config_provider ~/.local/bin/discopop_config_provider"
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_explorer ~/.local/bin/discopop_explorer"
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_optimizer ~/.local/bin/discopop_optimizer"
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_patch_applicator ~/.local/bin/discopop_patch_applicator"
 su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_patch_generator ~/.local/bin/discopop_patch_generator"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_sanity_checker ~/.local/bin/discopop_sanity_checker"

discopop_explorer/__main__.py

@@ -108,6 +108,10 @@ def parse_args() -> ExplorerArguments:
     experimental_parser.add_argument(
         "--enable-statistics", action="store_true", help="Disable the calculation and storing of statistics for code and generated suggestions."
     )
+    experimental_parser.add_argument(
+        "--plot-pet", type=str, nargs="?", default=None, const="explorer/pet_plot.gexf",
+        help="Plots PET as a GEXF file. If a path is given (file extension has to be .gexf), the PET Graph is written to the given file, otherwise to pet_plot.gexf"
+    )
     # fmt: on
 
     arguments = parser.parse_args()
@@ -135,6 +139,7 @@ def parse_args() -> ExplorerArguments:
     arguments.dump_pet = get_path_or_none(arguments.path, arguments.dump_pet)
     arguments.dump_detection_result = get_path_or_none(arguments.path, arguments.dump_detection_result)
     arguments.microbench_file = get_path_or_none(arguments.path, arguments.microbench_file)
+    arguments.plot_pet = get_path_or_none(arguments.path, arguments.plot_pet)
 
     return ExplorerArguments(
         discopop_build_path=arguments.dp_build_path,
@@ -162,6 +167,7 @@ def parse_args() -> ExplorerArguments:
         load_existing_doall_and_reduction_patterns=arguments.load_existing_doall_and_reduction_patterns,
         collect_statistics=arguments.enable_statistics,
         jobs=arguments.jobs,
+        enable_pet_plot_file=arguments.plot_pet,
     )
 
 

discopop_explorer/classes/PEGraph/Node.py

@@ -87,6 +87,12 @@ def contains_line(self, other_line: str) -> bool:
             return True
         return False
 
+    def get_contained_line_ids(self) -> List[LineID]:
+        result: List[LineID] = []
+        for i in range(self.start_line, self.end_line + 1):
+            result.append(LineID(str(self.file_id) + ":" + str(i)))
+        return result
+
     def __str__(self) -> str:
         return self.id
 

discopop_explorer/classes/patterns/PatternBase.py

@@ -8,6 +8,7 @@
 
 import json
 import os
+from typing import List, Optional
 
 from filelock import FileLock  # type: ignore
 from discopop_explorer.classes.PEGraph.Node import Node
@@ -19,11 +20,13 @@ class PatternBase(object):
     """Base class for pattern info"""
 
     pattern_id: int
+    pattern_tag: str
     _node: Node
     node_id: NodeID
     start_line: LineID
     end_line: LineID
     applicable_pattern: bool
+    affected_cu_ids: List[NodeID]
 
     def __init__(self, node: Node):
         # create a file lock to synchronize processes
@@ -45,6 +48,8 @@ def __init__(self, node: Node):
         self.start_line = node.start_position()
         self.end_line = node.end_position()
         self.applicable_pattern = True
+        self.pattern_tag = ""
+        self.affected_cu_ids = []
 
     def to_json(self) -> str:
         dic = self.__dict__
@@ -54,3 +59,8 @@ def to_json(self) -> str:
                 del dic[key]
 
         return json.dumps(dic, indent=2, default=lambda o: o.toJSON())  # , default=lambda o: "<not serializable>")
+
+    def get_tag(self) -> str:
+        result = "" + self.__class__.__name__
+        result += "@" + self.start_line + "-" + self.end_line
+        return result

discopop_explorer/discopop_explorer.py

@@ -21,6 +21,7 @@
 import pstats2  # type:ignore
 from pluginbase import PluginBase  # type: ignore
 from discopop_explorer.functions.PEGraph.output.json import dump_to_pickled_json
+from discopop_explorer.functions.PEGraph.output.gephi import dump_to_gephi_file
 from discopop_explorer.utilities.statistics.collect_statistics import collect_statistics
 from discopop_library.ArgumentClasses.GeneralArguments import GeneralArguments  # type: ignore
 from discopop_library.FolderStructure.setup import setup_explorer
@@ -74,6 +75,7 @@ class ExplorerArguments(GeneralArguments):
     microbench_file: Optional[str]
     load_existing_doall_and_reduction_patterns: bool
     collect_statistics: bool
+    enable_pet_plot_file: Optional[str]  # None means no dump, otherwise the path
 
     def __post_init__(self) -> None:
         self.__validate()
@@ -258,6 +260,11 @@ def run(arguments: ExplorerArguments) -> None:
                 f.flush()
                 f.close()
 
+        # experimental
+        # dumps gexf plot of pet
+        if arguments.enable_pet_plot_file is not None:
+            dump_to_gephi_file(res.pet, arguments.enable_pet_plot_file)
+
         if arguments.enable_detection_result_dump_file is not None:
             with open(arguments.enable_detection_result_dump_file, "w+") as f:
                 f.write(res.dump_to_pickled_json())

discopop_explorer/functions/PEGraph/output/gephi.py

@@ -12,7 +12,7 @@
 import networkx as nx  # type: ignore
 
 
-def dump_to_gephi_file(pet: PEGraphX, name: str = "pet.gexf") -> None:
+def dump_to_gephi_file(pet: PEGraphX, name: str) -> None:
     """Note: Destroys the PETGraph!"""
     # replace node data with label
     for node_id in pet.g.nodes:

discopop_explorer/functions/PEGraph/traversal/parent.py

@@ -6,7 +6,7 @@
 # the 3-Clause BSD License.  See the LICENSE file in the package base
 # directory for details.
 from __future__ import annotations
-from typing import TYPE_CHECKING, Optional, cast
+from typing import TYPE_CHECKING, List, Optional, cast
 from discopop_explorer.classes.PEGraph.FunctionNode import FunctionNode
 from discopop_explorer.classes.PEGraph.Node import Node
 
@@ -41,3 +41,19 @@ def get_parent_function(pet: PEGraphX, node: Node) -> FunctionNode:
 
     assert node.parent_function_id
     return cast(FunctionNode, pet.node_at(node.parent_function_id))
+
+
+def get_all_parent_functions(pet: PEGraphX, node: Node) -> List[FunctionNode]:
+    queue: List[Node] = [node]
+    result: set[FunctionNode] = set()
+    while queue:
+        current = queue.pop()
+        current_parent = get_parent_function(pet, current)
+        if current_parent not in queue and current_parent not in result:
+            queue.append(current_parent)
+        if current_parent not in result:
+            result.add(current_parent)
+        for in_edge in in_edges(pet, current.id, EdgeType.CALLSNODE):
+            queue.append(pet.node_at(in_edge[0]))
+
+    return list(result)

discopop_explorer/pattern_detectors/do_all_detector.py

@@ -17,7 +17,7 @@
 from discopop_explorer.functions.PEGraph.queries.nodes import all_nodes
 from discopop_explorer.functions.PEGraph.queries.subtree import subtree_of_type
 from discopop_explorer.functions.PEGraph.queries.variables import get_variables
-from discopop_explorer.functions.PEGraph.traversal.parent import get_parent_function
+from discopop_explorer.functions.PEGraph.traversal.parent import get_all_parent_functions, get_parent_function
 from discopop_library.HostpotLoader.HotspotNodeType import HotspotNodeType
 from discopop_library.HostpotLoader.HotspotType import HotspotType  # type: ignore
 
@@ -57,6 +57,19 @@ def __init__(self, pet: PEGraphX, node: Node):
         self.reduction = r
         self.scheduling_clause = "static"
         self.collapse_level = 1
+        self.pattern_tag = self.get_tag()
+
+        # determine affected cu and line ids
+        affected_cus = subtree_of_type(pet, node)
+        self.affected_cu_ids = [n.id for n in affected_cus]
+        self.affected_line_ids: List[LineID] = []
+        for node in affected_cus:
+            self.affected_line_ids += [
+                lid for lid in node.get_contained_line_ids() if lid not in self.affected_line_ids
+            ]
+        self.affected_functions = [
+            str(f.start_position()) + ":" + f.name for f in [get_parent_function(pet, node)]
+        ]  # get_all_parent_functions(pet, node)]
 
     def __str__(self) -> str:
         return (
@@ -76,6 +89,15 @@ def __str__(self) -> str:
             f"scheduling clause: {self.scheduling_clause}"
         )
 
+    def get_tag(self) -> str:
+        result = super().get_tag() + "_"
+        result += f"p({[v.name for v in self.private]})_"
+        result += f"s({[v.name for v in self.shared]})_"
+        result += f"fp({[v.name for v in self.first_private]})_"
+        result += f"r({[v.name for v in self.reduction]})_"
+        result += f"lp({[v.name for v in self.last_private]})"
+        return result
+
 
 global_pet = None
 
@@ -396,8 +418,25 @@ def __check_loop_dependencies(
                         return True
         elif dep.dtype == DepType.WAW:
             # check WAW dependencies
-            # handled by variable classification
-            pass
+            if (
+                dep.metadata_intra_iteration_dep is None
+                or dep.metadata_inter_iteration_dep is None
+                or dep.metadata_intra_call_dep is None
+                or dep.metadata_inter_call_dep is None
+            ):
+                # no metadata created
+                # handled by variable classification
+                pass
+            else:
+                # metadata exists
+
+                cond_2 = len([cf for cf in called_functions_lineids if cf in dep.metadata_inter_call_dep]) > 0
+                cond_4 = root_loop.start_position() in dep.metadata_inter_iteration_dep
+                # if cond_1 or cond_2 or cond_3:
+                if cond_2 or cond_4:
+                    return True
+                # handled by variable classification
+                pass
         else:
             raise ValueError("Unsupported dependency type: ", dep.dtype)
 

discopop_explorer/pattern_detectors/reduction_detector.py

@@ -17,7 +17,7 @@
 from discopop_explorer.functions.PEGraph.queries.nodes import all_nodes
 from discopop_explorer.functions.PEGraph.queries.subtree import subtree_of_type
 from discopop_explorer.functions.PEGraph.queries.variables import get_variables
-from discopop_explorer.functions.PEGraph.traversal.parent import get_parent_function
+from discopop_explorer.functions.PEGraph.traversal.parent import get_all_parent_functions, get_parent_function
 from discopop_library.HostpotLoader.HotspotNodeType import HotspotNodeType
 from discopop_library.HostpotLoader.HotspotType import HotspotType  # type: ignore
 
@@ -56,6 +56,19 @@ def __init__(self, pet: PEGraphX, node: Node):
         self.last_private = lp
         self.shared = s
         self.reduction = r
+        self.pattern_tag = self.get_tag()
+
+        # determine affected cu and line ids
+        affected_cus = subtree_of_type(pet, node)
+        self.affected_cu_ids = [n.id for n in affected_cus]
+        self.affected_line_ids: List[LineID] = []
+        for node in affected_cus:
+            self.affected_line_ids += [
+                lid for lid in node.get_contained_line_ids() if lid not in self.affected_line_ids
+            ]
+        self.affected_functions = [
+            str(f.start_position()) + ":" + f.name for f in [get_parent_function(pet, node)]
+        ]  # get_all_parent_functions(pet, node)]
 
     def __str__(self) -> str:
         return (
@@ -70,6 +83,15 @@ def __str__(self) -> str:
             f"last private: {[v.name for v in self.last_private]}"
         )
 
+    def get_tag(self) -> str:
+        result = super().get_tag() + "_"
+        result += f"p({[v.name for v in self.private]})_"
+        result += f"s({[v.name for v in self.shared]})_"
+        result += f"fp({[v.name for v in self.first_private]})_"
+        result += f'r({[str(v.operation) + ":" + v.name for v in self.reduction]})_'
+        result += f"lp({[v.name for v in self.last_private]})"
+        return result
+
 
 global_pet = None
 
@@ -331,8 +353,25 @@ def __check_loop_dependencies(
                             return True
         elif dep.dtype == DepType.WAW:
             # check WAW dependencies
-            # handled by variable classification
-            pass
+            if (
+                dep.metadata_intra_iteration_dep is None
+                or dep.metadata_inter_iteration_dep is None
+                or dep.metadata_intra_call_dep is None
+                or dep.metadata_inter_call_dep is None
+            ):
+                # no metadata created
+                # handled by variable classification
+                pass
+            else:
+                # metadata exists
+
+                cond_2 = len([cf for cf in called_functions_lineids if cf in dep.metadata_inter_call_dep]) > 0
+                cond_4 = root_loop.start_position() in dep.metadata_inter_iteration_dep
+                # if cond_1 or cond_2 or cond_3:
+                if cond_2 or cond_4:
+                    return True
+                # handled by variable classification
+                pass
         else:
             raise ValueError("Unsupported dependency type: ", dep.dtype)
 

discopop_explorer/utilities/statistics/collect_statistics.py

@@ -37,18 +37,22 @@
 
 def collect_statistics(arguments: ExplorerArguments, res: DetectionResult) -> None:
     logger.info("Collecting code statistics...")
+
     maximum_call_path_depth = get_maximum_call_path_depth(res.pet)
     logger.debug("--> maximum_call_path_depth: " + str(maximum_call_path_depth))
+
     suggestion_call_path_depths = get_suggestion_call_path_depths(res)
     logger.debug(
         "--> suggestion_call_path_depths: "
         + str([str(key) + " => " + str(suggestion_call_path_depths[key]) for key in suggestion_call_path_depths])
     )
+
     suggestion_num_function_calls = get_suggestion_num_function_calls(res)
     logger.debug(
         "--> suggestion_num_function_calls: "
         + str([str(key) + " => " + str(suggestion_num_function_calls[key]) for key in suggestion_num_function_calls])
     )
+
     suggestion_immediate_lines_of_code = get_suggestion_immediate_lines_of_code(res)
     logger.debug(
         "--> suggestion_immediate_lines_of_code: "

discopop_explorer/utilities/statistics/cyclomatic_complexity/boxplot.py

@@ -16,25 +16,31 @@
 # define aliases for readability
 from subprocess import check_output
 from discopop_library.PathManagement.PathManagement import load_file_mapping
-
+import logging
 
 MIN = int
 MAX = int
 AVG = int
 LOWER_QUART = int
 UPPER_QUART = int
 
+logger = logging.getLogger("statistics").getChild("CC").getChild("boxplot")
+
 
 def get_cyclomatic_complexities_for_boxplot(
     arguments: ExplorerArguments, res: DetectionResult
 ) -> Tuple[MIN, MAX, AVG, LOWER_QUART, UPPER_QUART]:
     file_mapping = load_file_mapping(arguments.file_mapping_file)
     # get summed cyclomatic complexity for all functions in all files
-    cmd = ["pmccabe", "-C"]
-    for file_id in file_mapping:
-        file_path = file_mapping[file_id]
-        cmd.append(str(file_path))
-    out = check_output(cmd).decode("utf-8")
+    try:
+        cmd = ["pmccabe", "-C"]
+        for file_id in file_mapping:
+            file_path = file_mapping[file_id]
+            cmd.append(str(file_path))
+        out = check_output(cmd).decode("utf-8")
+    except FileNotFoundError as fnfe:
+        logger.error(str(fnfe))
+        return (0, 0, 0, 0, 0)
 
     # unpack and store results temporarily
     cyclomatic_complexities: List[int] = []