128-revival-bugfixes-and-enhancements (#140)

* Fix express function

* Fix allow to create/use expressions for objects that are not yet interfaced

* Add missing tests on test_pipeline part I

* Add stages for pipeline class

* WIP debugging field names

* formatting

* Factorizing generate_enum_member

* Fix generate_enum_member and some buggy enum member names

* Refactored generate_enum_member

* Cleaning

* Fix project stage

* Fix SortByCount

* Fix sort

* Fix match

* WIP Debugging lookup

* Fixed lookup stage

* Fixed SearchBase

* bump version 0.21.0 -> 0.22.0

* Fix sort

* Update CI

* Update CI

Author: VianneyMI

.github/workflows/python-app-ci.yml

@@ -23,12 +23,14 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: "3.10"
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        version: "latest"
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if ${{ -f requirements.txt }}; then pip install -r requirements.txt; fi
-        python -m pip install -e .
+        uv sync --all-extras
+        uv pip install -e .
     #- name: Lint with flake8
     #  run: |
     #    # stop the build if there are Python syntax errors or undefined names
@@ -39,7 +41,7 @@ jobs:
       env:
         MONGODB_PASSWORD: ${{secrets.MONGODB_PASSWORD}}
       run: |
-        pytest
+        uv run pytest
 
   lint-and-format-backend:
     continue-on-error: true
@@ -56,6 +58,10 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
 
       - name: Check Black formatting
         uses: reviewdog/action-black@v3
@@ -76,8 +82,7 @@ jobs:
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           cd monggregate
-          pip3 install ruff
-          ruff check . \
+          uv run --with ruff ruff check . \
             | reviewdog -name=Ruff -reporter=local -reporter=github-pr-review -efm="%f:%l:%c: %m" -filter-mode=added -fail-on-error=true
 
       - name: Check Mypy linting
@@ -87,7 +92,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           reporter: github-pr-review
           filter_mode: added
-          setup_command: pip3 install --no-cache-dir --upgrade -r ../requirements/all.txt
+          setup_command: uv sync --all-extras
           setup_method: install
           level: warning
           workdir: mongreggate

.github/workflows/python-app.yml

@@ -23,17 +23,19 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: "3.10"
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        version: "latest"
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .
-        pip install -r requirements/core.txt
-        pip install -r requirements/testing.txt
+        uv sync --all-extras
+        uv pip install -e .
     - name: Test with pytest
       env:
           MONGODB_PASSWORD: ${{secrets.MONGODB_PASSWORD}}
       run: |
-        pytest
+        uv run pytest
 
   lint-and-format-backend:
     continue-on-error: true
@@ -50,6 +52,10 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
 
       - name: Check Black formatting
         uses: reviewdog/action-black@v3
@@ -68,8 +74,7 @@ jobs:
         env:
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          pip3 install ruff
-          ruff check . \
+          uv run --with ruff ruff check . \
             | reviewdog -name=Ruff -reporter=local -reporter=github-pr-review -efm="%f:%l:%c: %m" -filter-mode=added -fail-on-error=false
 
       - name: Check Mypy linting
@@ -79,7 +84,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           reporter: github-pr-review
           filter_mode: added
-          setup_command: pip3 install --no-cache-dir --upgrade -r requirements/all.txt
+          setup_command: uv sync --all-extras
           setup_method: install
           level: warning
           mypy_flags: --python-version 3.10

.gitignore

@@ -17,4 +17,5 @@ dist
 test.py
 site/
 .pypirc
-clean_pycache.ps1
+clean_pycache.ps1
+test_deployments

pyproject.toml

@@ -2,7 +2,7 @@
 
 [project]
 name = "monggregate"
-version = "0.21.0"
+version = "0.22.0"
 description = "MongoDB aggregation pipelines made easy. Joins, grouping, counting and much more..."
 readme = "README.md"
 authors = [{ name = "Vianney Mixtur", email = "vianney.mixtur@outlook.fr" }]
@@ -22,12 +22,18 @@ dependencies = [
     "typing-extensions>=4.0",
 ]
 
+[project.optional-dependencies]
+mongodb = [
+    "pymongo>=3.0.0",
+    "motor>=3.0.0",
+]
+
 [project.urls]
 Homepage = "https://github.com/VianneyMI/monggregate"
 documentation = "https://vianneymi.github.io/monggregate/"
 
 [tool.bumpver]
-current_version = "0.21.0"
+current_version = "0.22.0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true
@@ -41,11 +47,37 @@ push = false
 ]
 "src/monggregate/__init__.py" = ['__version__ = "{version}"']
 
+[tool.setuptools]
+packages = ["monggregate", "tests"]
+
+[tool.setuptools.package-dir]
+monggregate = "src/monggregate"
+tests = "tests"
+
 [dependency-groups]
-dev = [
+test = [
     "pytest>=8.3.5",
+    "pytest-html>=3.2.0",
+    "python-dotenv>=1.0.0",
+    "certifi",
+]
+lint = [
+    "mypy>=1.6.1",
+    "ruff",
+    "black",
 ]
 doc = [
     "mkdocs==1.5.2",
     "mkdocs-material==9.2.7",
 ]
+dev = [
+    "pytest>=8.3.5",
+    "pytest-html>=3.2.0",
+    "python-dotenv>=1.0.0",
+    "certifi",
+    "mypy>=1.6.1",
+    "ruff",
+    "black",
+    "mkdocs==1.5.2",
+    "mkdocs-material==9.2.7",
+]

pytest.ini

@@ -1,5 +1,6 @@
 [pytest]
 # Be careful when registering the mark the semi-colon (:) should be right after the mark without any space before it.
+pythonpath = . tests
 markers =
     unit: unit tests
     functional: functional tests

src/monggregate/__init__.py

@@ -5,7 +5,7 @@
 
 __all__ = ["Pipeline", "S", "SS"]
 
-__version__ = "0.21.0"
+__version__ = "0.22.0"
 __author__ = "Vianney Mixtur"
 __contact__ = "prenom.nom@outlook.fr"
 __copyright__ = "Copyright © 2022-2024 Vianney Mixtur"

src/monggregate/base.py

@@ -72,6 +72,21 @@ class Config(pyd.BaseConfig):
         alias_generator = camelize
 
 
+class ExpressionWrapper(BaseModel):
+    """Wrapper for an expression.
+
+    To be used for Stage, Operator or other MongoDB object that hasn't been interfaced yet in `monggregate`.
+    """
+
+    _expression: Expression
+
+    @property
+    def expression(self) -> Expression:
+        """Expression property"""
+
+        return self.expression
+
+
 def isbasemodel(instance: Any) -> TypeGuard[BaseModel]:
     """Returns true if instance is an instance of BaseModel"""
 
@@ -82,24 +97,20 @@ def express(obj: Any) -> dict | list[dict]:
     """Resolves an expression encapsulated in an object from a class inheriting from BaseModel"""
 
     if isbasemodel(obj):
+        # If it's a BaseModel instance, get its expression
         output: dict | list = obj.expression
-    elif isinstance(obj, list) and any(map(isbasemodel, obj)):
+    elif isinstance(obj, list):
+        # Always process lists recursively - they might contain nested BaseModel instances
         output = []
         for element in obj:
-            if isinstance(element, BaseModel):
-                output.append(
-                    element.expression
-                )  # probably should call express(element)
-            else:
-                output.append(element)
+            output.append(express(element))
     elif isinstance(obj, dict):
+        # Always process dictionaries recursively - they might contain nested BaseModel instances
         output = {}
         for key, value in obj.items():
-            if isinstance(value, BaseModel):
-                output[key] = value.expression  # probably should call express(value)
-            else:
-                output[key] = express(value)
+            output[key] = express(value)
     else:
+        # For primitive types (int, str, bool, None, etc.), return as-is
         output = obj
 
     return output

src/monggregate/operators/operator.py

@@ -1,21 +1,22 @@
 """Operator Module"""
 
 # Standard Library imports
-#----------------------------
+# ----------------------------
 from abc import ABC
 
 # Package imports
 # ---------------------------
 from monggregate.base import BaseModel
 from monggregate.utils import StrEnum
 
+
 class Operator(BaseModel, ABC):
     """MongoDB operator abstract base class"""
 
+
 class OperatorEnum(StrEnum):
     """Enumeration of available operators"""
 
-
     ABS = "$abs"
     ACCUMULATOR = "$accumulator"
     ACOS = "$acos"
@@ -34,7 +35,7 @@ class OperatorEnum(StrEnum):
     ATANH = "$atanh"
     AVG = "$avg"
     BINARY_SIZE = "$binarySize"
-    BSON_SIZE ="$bsonSize"
+    BSON_SIZE = "$bsonSize"
     CEIL = "$ceil"
     CMP = "$cmp"
     CONCAT = "$concat"
@@ -47,15 +48,15 @@ class OperatorEnum(StrEnum):
     DATE_FROM_STRING = "$dateFromString"
     DATE_TO_PARTS = "$dateToParts"
     DATE_TO_STRING = "$dateToString"
-    DAY_OF_MONTH ="$dayOfMonth"
+    DAY_OF_MONTH = "$dayOfMonth"
     DAY_OF_WEEK = "$dayOfWeek"
     DAY_OF_YEAR = "$dayOfYear"
     DEGREES_TO_RADIANS = "$degreesToRadians"
     DIVIDE = "$divide"
     EQ = "$eq"
     EXP = "$exp"
     FILTER = "$filter"
-    FIRST = "$first" # two operators one for array one for accumulator
+    FIRST = "$first"  # two operators one for array one for accumulator
     FLOOR = "$floor"
     FUNCTION = "$function"
     GET_FIELD = "$getField"
@@ -71,9 +72,9 @@ class OperatorEnum(StrEnum):
     IS_NUMBER = "$isNumber"
     ISO_DAY_OF_WEEK = "$isoDayOfWeek"
     ISO_WEEK = "$isoWeek"
-    ISO_WEEK_YEAR ="$isoWeekYear"
+    ISO_WEEK_YEAR = "$isoWeekYear"
     LAST = "$last"  # two operators one for array one for accumulator
-    LET ="$let"
+    LET = "$let"
     LITERAL = "$literal"
     LN = "$ln"
     LOG = "$log"
@@ -85,26 +86,26 @@ class OperatorEnum(StrEnum):
     MAX = "$max"
     MERGE_OBJECTS = "$mergeObjects"
     META = "$meta"
-    MILLI_SECOND = "$millisecond"
+    MILLISECOND = "$millisecond"
     MIN = "$min"
-    MINUTE ="$minute"
-    MOD ="$mod"
+    MINUTE = "$minute"
+    MOD = "$mod"
     MONTH = "$month"
-    MULTIPLY ="$multiply"
-    NE ="$ne"
-    NOT ="$not"
-    OBJECT_TO_ARRAY ="$objectToArray"
+    MULTIPLY = "$multiply"
+    NE = "$ne"
+    NOT = "$not"
+    OBJECT_TO_ARRAY = "$objectToArray"
     OR = "$or"
     POW = "$pow"
     PUSH = "$push"
     RADIANS_TO_DEGREES = "$radiansToDegrees"
     RAND = "$rand"
     RANGE = "$range"
     REDUCE = "$reduce"
-    REGEX_FIND ="$regexFind"
+    REGEX_FIND = "$regexFind"
     REGEX_FIND_ALL = "$regexFindAll"
     REGEX_MATCH = "$regexMatch"
-    REPLACE_ONE ="$replaceOne"
+    REPLACE_ONE = "$replaceOne"
     REPLACE_ALL = "$replaceAll"
     REVERSE_ARRAY = "$reverseArray"
     ROUND = "$round"
@@ -126,16 +127,16 @@ class OperatorEnum(StrEnum):
     STD_DEV_SAMP = "$stdDevSamp"
     STR_LEN_BYTES = "$strLenBytes"
     STR_LEN_CP = "$strLenCP"
-    STR_CASE_CMP = "$strcasecmp"
+    STRCASECMP = "$strcasecmp"
     SUBSTR = "$substr"
     SUBSTR_BYTES = "$substrBytes"
     SUBSTR_CP = "$substrCP"
-    SUBSTRACT = "$subtract"
+    SUBTRACT = "$subtract"
     SUM = "$sum"
     SWITCH = "$switch"
     TAN = "$tan"
     TANH = "$tanh"
-    TO_BOOL ="$toBool"
+    TO_BOOL = "$toBool"
     TO_DATE = "$toDate"
     TO_DECIMAL = "$toDecimal"
     TO_DOUBLE = "$toDouble"
@@ -150,4 +151,4 @@ class OperatorEnum(StrEnum):
     TYPE = "$type"
     WEEK = "$week"
     YEAR = "$year"
-    ZIP = "$zip"
+    ZIP = "$zip"