Merge pull request #1331 from ait-aecid/development

Release 2.7.0

Author: landauermax

.deepsource.toml

@@ -13,6 +13,7 @@ enabled = true
   [analyzers.meta]
   runtime_version = "3.x.x"
   max_line_length = 140
+  cyclomatic_complexity_threshold = "critical"
 
 [[analyzers]]
 name = "test-coverage"

.pre-commit-config.yaml

@@ -20,6 +20,7 @@ repos:
     hooks:
     - id: flake8
       args: [--max-line-length=140]
+      exclude: ^aecid-testsuite/
 
 -   repo: https://github.com/pre-commit/mirrors-autopep8
     rev: v1.7.0

Dockerfile

@@ -10,11 +10,14 @@
 #
 
 # Pull base image.
-FROM debian:bullseye
+FROM debian:bookworm
 ARG UNAME=aminer
 ARG UID=1000
 ARG GID=1000
 
+# allow the system to use two package managers (apt and pip), as we do it intentionally (needed since Debain Bookworm - see PEP 668
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
 # Set local timezone
 ENV TZ=Europe/Vienna
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@@ -23,6 +26,7 @@ LABEL maintainer="wolfgang.hotwagner@ait.ac.at"
 
 # Install necessary debian packages
 ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
 RUN apt-get update && apt-get install -y \
     supervisor \
     python3 \
@@ -40,7 +44,11 @@ RUN apt-get update && apt-get install -y \
     python3-pylibacl \
     python3-urllib3 \
     python3-statsmodels \
-    libacl1-dev
+    python3-patsy \
+    python3-numpy \
+    python3-defusedxml \
+    libacl1-dev \
+    rsyslog
 
 # Docs
 RUN apt-get update && apt-get install -y \
@@ -61,8 +69,10 @@ ADD source/root/usr/lib/logdata-anomaly-miner /usr/lib/logdata-anomaly-miner
 
 # copy these files instead as symlinks would need absolute paths.
 ADD source/root/etc/aminer/conf-available/ait-lds/* /etc/aminer/conf-enabled/
+ADD source/root/etc/aminer/conf-available/ait-lds2/* /etc/aminer/conf-enabled/
 ADD source/root/etc/aminer/conf-available/generic/* /etc/aminer/conf-enabled/
 ADD source/root/etc/aminer/conf-available/ait-lds /etc/aminer/conf-available/ait-lds
+ADD source/root/etc/aminer/conf-available/ait-lds2 /etc/aminer/conf-available/ait-lds2
 ADD source/root/etc/aminer/conf-available/generic /etc/aminer/conf-available/generic
 
 # Entrypoint-wrapper
@@ -85,6 +95,9 @@ RUN ln -s /usr/lib/logdata-anomaly-miner/aminerremotecontrol.py /usr/bin/aminerr
 	&& ln -s /usr/lib/python3/dist-packages/six.py /usr/lib/logdata-anomaly-miner/six.py \
 	&& ln -s /usr/lib/python3/dist-packages/urllib3 /usr/lib/logdata-anomaly-miner/urllib3 \
 	&& ln -s /usr/lib/python3/dist-packages/statsmodels /usr/lib/logdata-anomaly-miner/statsmodels \
+	&& ln -s /usr/lib/python3/dist-packages/packaging /usr/lib/logdata-anomaly-miner/packaging \
+	&& ln -s /usr/lib/python3/dist-packages/patsy /etc/aminer/conf-enabled/patsy \
+	&& ln -s /usr/lib/python3/dist-packages/defusedxml /etc/aminer/conf-enabled/defusedxml \
 	&& groupadd -g $GID -o $UNAME && useradd -u $UID -g $GID -ms /usr/sbin/nologin $UNAME && mkdir -p /var/lib/aminer/logs \
     && chown $UID.$GID -R /var/lib/aminer \
     && chown $UID.$GID -R /docs \

Jenkinsfile

@@ -6,7 +6,9 @@ This tool parses log data and allows to define analysis pipelines for anomaly de
 
 ## Requirements
 
-In order to install logdata-anomaly-miner a **Linux system** with **python >= 3.6** is required. **Debian-based** distributions are currently recommended. 
+In order to install logdata-anomaly-miner a **Linux system** with **python >= 3.6** is required. All **Ubuntu** and **Debian** versions that we have in the tests are currently recommended.
+There is only experimental support for **Fedora**.
+More specifically the tested systems include Debian Buster, Debian Bullseye, Debian Bookworm, Ubuntu 20.04, Ubuntu 22.04, Fedora (docker image fedora:latest), and RedHat (docker image redhat/ubi9).
 
 _See [requirements.txt](https://github.com/ait-aecid/logdata-anomaly-miner/requirements.txt) for further module dependencies_
 
@@ -15,8 +17,7 @@ _See [requirements.txt](https://github.com/ait-aecid/logdata-anomaly-miner/requi
 
 ### Debian
 
-There are Debian packages for logdata-anomaly-miner in the official Debian/Ubuntu
-repositories.
+There are Debian packages for logdata-anomaly-miner in the official Debian/Ubuntu repositories.
 
 ```
 apt-get update && apt-get install logdata-anomaly-miner
@@ -49,9 +50,10 @@ Here are some resources to read in order to get started with configurations:
 
 Publications and talks:
 
-* Wurzenberger M., Skopik F., Settanni G., Fiedler R. (2018): [AECID: A Self-learning Anomaly Detection Approach Based on Light-weight Log Parser Models](http://www.scitepress.org/DigitalLibrary/Link.aspx?doi=10.5220/0006643003860397). [4th International Conference on Information Systems Security and Privacy (ICISSP 2018)](http://www.icissp.org/), January 22-24, 2018, Funchal, Madeira - Portugal. INSTICC. \[[PDF](https://www.markuswurzenberger.com/wp-content/uploads/2020/05/2018_icissp.pdf)\]
-* Wurzenberger M., Landauer M., Skopik F., Kastner W. (2019): AECID-PG: [AECID-PG: A Tree-Based Log Parser Generator To Enable Log Analysis](https://ieeexplore.ieee.org/document/8717887). [4th IEEE/IFIP International Workshop on Analytics for Network and Service Management (AnNet 2019)](https://annet2019.moogsoft.com/) in conjunction with the [IFIP/IEEE International Symposium on Integrated Network Management (IM)](https://im2019.ieee-im.org/), April 8, 2019, Washington D.C., USA. IEEE. \[[PDF](https://www.markuswurzenberger.com/wp-content/uploads/2020/05/2019_annet.pdf)\]
-* Landauer M., Skopik F., Wurzenberger M., Hotwagner W., Rauber A. (2019): [A Framework for Cyber Threat Intelligence Extraction from Raw Log Data](https://ieeexplore.ieee.org/document/9006328). [International Workshop on Big Data Analytics for Cyber Threat Hunting (CyberHunt 2019)](https://securitylab.no/cyberhunt2019/) in conjunction with the [IEEE International Conference on Big Data 2019](http://bigdataieee.org/BigData2019/), December 9-12, 2019, Los Angeles, CA, USA. IEEE. \[[PDF](https://www.markuswurzenberger.com/wp-content/uploads/2020/05/2019_cyberhunt.pdf)\]
+* Landauer M., Wurzenberger M., Skopik F., Hotwagner W., Höld G. (2023): [AMiner: A Modular Log Data Analysis Pipeline for Anomaly-based Intrusion Detection](https://dl.acm.org/doi/full/10.1145/3567675). [Digital Threats: Research and Practice](https://dl.acm.org/toc/dtrap/2023/4/1), Volume 4, Issue 1. March 2023, pp. 1–16, ACM. \[[PDF](https://dl.acm.org/doi/pdf/10.1145/3567675)\]
+* Wurzenberger M., Skopik F., Settanni G., Fiedler R. (2018): [AECID: A Self-learning Anomaly Detection Approach Based on Light-weight Log Parser Models](http://www.scitepress.org/DigitalLibrary/Link.aspx?doi=10.5220/0006643003860397). [4th International Conference on Information Systems Security and Privacy (ICISSP 2018)](http://www.icissp.org/), January 22-24, 2018, Funchal, Madeira - Portugal. INSTICC. \[[PDF](https://pdfs.semanticscholar.org/cd58/8e51d7a1d7f02f95ef2127623b21e2cd02c6.pdf)\]
+* Wurzenberger M., Landauer M., Skopik F., Kastner W. (2019): AECID-PG: [AECID-PG: A Tree-Based Log Parser Generator To Enable Log Analysis](https://ieeexplore.ieee.org/document/8717887). [4th IEEE/IFIP International Workshop on Analytics for Network and Service Management (AnNet 2019)](https://annet2019.moogsoft.com/) in conjunction with the [IFIP/IEEE International Symposium on Integrated Network Management (IM)](https://im2019.ieee-im.org/), April 8, 2019, Washington D.C., USA. IEEE. \[[PDF](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8717887)\]
+* Landauer M., Skopik F., Wurzenberger M., Hotwagner W., Rauber A. (2019): [A Framework for Cyber Threat Intelligence Extraction from Raw Log Data](https://ieeexplore.ieee.org/document/9006328). [International Workshop on Big Data Analytics for Cyber Threat Hunting (CyberHunt 2019)](https://securitylab.no/cyberhunt2019/) in conjunction with the [IEEE International Conference on Big Data 2019](http://bigdataieee.org/BigData2019/), December 9-12, 2019, Los Angeles, CA, USA. IEEE. \[[PDF](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9006328)\]
 
 A complete list of publications can be found at [https://aecid.ait.ac.at/further-information/](https://aecid.ait.ac.at/further-information/).
 

README.md

@@ -18,7 +18,10 @@
 #
 
 # Pull base image.
-FROM debian:bullseye
+FROM debian:bookworm
+
+# allow the system to use two package managers (apt and pip), as we do it intentionally (needed since Debain Bookworm - see PEP 668
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
 # Set local timezone
 ENV TZ=Europe/Vienna
@@ -28,23 +31,25 @@ LABEL maintainer="wolfgang.hotwagner@ait.ac.at"
 
 # Install necessary debian packages
 ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
 RUN apt-get update && apt-get install -y \
     python3 \
     python3-pip \
-    python3-pip \
     python3-tz \
     python3-scipy \
     python3-pkg-resources \
     python3-setuptools \
     python3-dateutil \
     python3-six \
-    python3-scipy \
     python3-kafka \
     python3-cerberus \
     python3-yaml \
     python3-pylibacl \
     python3-urllib3 \
     python3-statsmodels \
+    python3-pandas \
+    python3-patsy \
+    python3-defusedxml \
     libacl1-dev \
     postfix \
     procps \
@@ -53,7 +58,7 @@ RUN apt-get update && apt-get install -y \
     curl \
     vim \
     postfix \
-    openjdk-11-jre \
+    openjdk-17-jre \
     locales \
     locales-all \
     rsyslog \
@@ -78,8 +83,10 @@ ADD source/root/usr/lib/logdata-anomaly-miner /usr/lib/logdata-anomaly-miner
 
 # copy these files instead as symlinks would need absolute paths.
 ADD source/root/etc/aminer/conf-available/ait-lds/* /etc/aminer/conf-enabled/
+ADD source/root/etc/aminer/conf-available/ait-lds2/* /etc/aminer/conf-enabled/
 ADD source/root/etc/aminer/conf-available/generic/* /etc/aminer/conf-enabled/
 ADD source/root/etc/aminer/conf-available/ait-lds /etc/aminer/conf-available/ait-lds
+ADD source/root/etc/aminer/conf-available/ait-lds2 /etc/aminer/conf-available/ait-lds2
 ADD source/root/etc/aminer/conf-available/generic /etc/aminer/conf-available/generic
 
 # Entrypoint-wrapper
@@ -102,6 +109,10 @@ RUN ln -s /usr/lib/logdata-anomaly-miner/aminerremotecontrol.py /usr/bin/aminerr
 	&& ln -s /usr/lib/python3/dist-packages/six.py /etc/aminer/conf-enabled/six.py \
 	&& ln -s /usr/lib/python3/dist-packages/urllib3 /etc/aminer/conf-enabled/urllib3 \
 	&& ln -s /usr/lib/python3/dist-packages/statsmodels /etc/aminer/conf-enabled/statsmodels \
+	&& ln -s /usr/lib/python3/dist-packages/packaging /etc/aminer/conf-enabled/packaging \
+	&& ln -s /usr/lib/python3/dist-packages/pandas /etc/aminer/conf-enabled/pandas \
+	&& ln -s /usr/lib/python3/dist-packages/patsy /etc/aminer/conf-enabled/patsy \
+	&& ln -s /usr/lib/python3/dist-packages/defusedxml /etc/aminer/conf-enabled/defusedxml \
 	&& useradd -ms /usr/sbin/nologin aminer && mkdir -p /var/lib/aminer/log && chmod 0755 /aminerwrapper.sh \
     && chown aminer.aminer -R /var/lib/aminer && chmod 0755 /etc/aminer
 

aecid-testsuite/Dockerfile

@@ -1706,7 +1706,7 @@ Parser:
     name: 'EximParsingModel'
 
   - id: SyslogParsingModel
-    type: SyslogParsingModel
+    type: SyslogParsingModelAIT-LDSv2
     name: 'SyslogParsingModel'
 
   - id: SuricataFastParsingModel