Python: Introduce python configuration with pyproject.toml

`pyproject.toml` was introduced with PEP 518 as an effort
to standardize packaging and dependency management in python projects.

This commit sets up `pyproject.toml` with following:
1. Basic project information (Name, author, etc)
2. List of runtime dependencies for building TF-M
3. List of optional dependencies (for Docs) for TF-M
5. Minimum python version requirement for TF-M

This commit also includes:
1. `uv.lock` file that aids with reproducibility and consistency of
    python build/dev environment required for building the project.
2. Update docs to use `pyproject.toml`  instead of `requirements.txt`
3. Docs for use UV to manage different python versions
4. Docs for writing python scripts and modules and including capturing
   related metadata in pyproject.toml.

We also add `.venv` and `trusted_firmware_m.egg-info/` in `.gitignore`.
These folders contain data related to virtual environment and setuptools
respectively.

Change-Id: I4b3e7dfab41690dc7a079074d4da891250e616ce
Signed-off-by: Mudit Sharma <mudit.sharma@arm.com>

Author: muditsharmaa-arm

.gitignore

@@ -24,3 +24,6 @@ localrepos.cmake
 
 # Python venv
 venv
+.venv
+
+trusted_firmware_m.egg-info/

.readthedocs.yaml

@@ -1,6 +1,5 @@
 #-------------------------------------------------------------------------------
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
+# SPDX-FileCopyrightText: Copyright The TrustedFirmware-M Contributors
 #
 # SPDX-License-Identifier: BSD-3-Clause
 #
@@ -19,6 +18,14 @@ build:
     python: "3.11"
   apt_packages:
     - plantuml
+  jobs:
+      create_environment:
+         - asdf plugin add uv
+         - asdf install uv latest
+         - asdf global uv latest
+         - uv venv
+      install:
+         - uv pip install ".[docs]"
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:
@@ -28,8 +35,3 @@ sphinx:
 formats:
    - pdf
    - htmlzip
-
-# Configuration of the Python environment
-python:
-   install:
-   - requirements: docs/requirements.txt

docs/building/documentation_generation.rst

@@ -9,6 +9,8 @@ Two documents are available for generation:
 
 The documentation build is independent from building the binary artifacts.
 
+.. _build-environment-setup:
+
 ******************************
 Tools and building environment
 ******************************
@@ -19,7 +21,7 @@ These tools are used to generate TF-M documentation:
     - Graphviz dot v2.38.0 or later
     - PlantUML v1.2018.11 or later
     - Java runtime environment v1.8 or later (for running PlantUML)
-    - Sphinx and other python modules, listed in ``docs/requirements.txt``
+   -  Optionally, `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>`
 
 Additionally, for PDFs format:
 
@@ -49,10 +51,18 @@ To prepare your building environment execute the following steps:
             # For PDF generation
             sudo apt-get install -y doxygen-latex librsvg2-bin
 
-            # Install the required Python modules
-            pip3 install --upgrade pip
-            cd <TF-M base folder>
-            pip3 install -r docs/requirements.txt
+            # Setup python venv for the project
+            python3 -m venv .venv
+
+            # NOTE: If your system python install version is <3.10 you can use `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>` to setup your .venv
+            uv venv --python 3.12
+
+            source .venv/bin/activate
+            pip install ".[docs]"
+
+
+            # NOTE: If you've used `uv` to setup your `.venv`, prepend the `pip` commands with `uv`
+            uv pip install ".[docs]"
 
         Set the environment variables:
 
@@ -82,23 +92,39 @@ To prepare your building environment execute the following steps:
             set PATH=$PATH;<ARM_DS_PATH>\sw\java\bin
 
             # Install the required Python modules
-            pip3 install --upgrade pip
-            cd trusted-firmware-m
-            pip3 install -r docs\requirements.txt
+
+            # Setup python venv for the project
+            python3 -m venv .venv
+
+            # NOTE: If your system python install version is <3.10 you can use `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>` to setup your .venv
+            uv venv --python 3.12
+
+            source .venv/bin/activate
+            pip install ".[docs]"
+            # If uv is installed
+            uv pip install ".[docs]"
+
+            # NOTE: If you've used `uv` to setup your `.venv`, prepend the `pip` commands with `uv`
+            uv pip sync pylock.toml
 
 ***************************
 Build TF-M Reference Manual
 ***************************
 
 The Reference Manual will be generated in the ``build_docs/reference_manual``.
 
+.. Note::
+   Please make sure to follow :ref:`Tools and building environment <build-environment-setup>`
+   before proceeding
+
 .. tabs::
 
     .. group-tab:: Linux
 
         .. code-block:: bash
 
             cd <TF-M base folder>
+            source .venv/bin/activate
             cmake -S docs -B build_docs
             cmake --build build_docs -- tfm_docs_refman_html tfm_docs_refman_pdf
 
@@ -107,6 +133,10 @@ The Reference Manual will be generated in the ``build_docs/reference_manual``.
         .. code-block:: bash
 
             cd <TF-M base folder>
+            source .venv/bin/activate
+            pip install ".[docs]"
+            # If uv is installed
+            uv pip install ".[docs]"
             cmake -S docs -B build_docs -G"Unix Makefiles"
             cmake --build build_docs -- tfm_docs_refman_html tfm_docs_refman_pdf
 
@@ -123,6 +153,7 @@ The User Manual will be available under the directory ``build_docs/user_guide``.
         .. code-block:: bash
 
             cd <TF-M base folder>
+            source .venv/bin/activate
             cmake -S docs -B build_docs
             cmake --build build_docs -- tfm_docs_userguide_html tfm_docs_userguide_pdf
 
@@ -131,6 +162,7 @@ The User Manual will be available under the directory ``build_docs/user_guide``.
         .. code-block:: bash
 
             cd <TF-M base folder>
+            source .venv/bin/activate
             cmake -S docs -B build_docs -G"Unix Makefiles"
             cmake --build build_docs -- tfm_docs_userguide_html tfm_docs_userguide_pdf
 
@@ -150,6 +182,7 @@ The direct build will build both user_guide and reference_manual.
 
             # Build the documentation from build_docs directory
             cd <TF-M base folder>
+            source .venv/bin/activate
             mkdir build_docs
             cp docs/conf.py build_docs/conf.py
             cd build_docs

docs/contributing/index.rst

@@ -13,6 +13,7 @@ Contribution Guidelines
     Documentation <doc_guidelines>
     Design proposal <tfm_design_proposal_guideline>
     MISRA-C <standards/misra>
+    Python scripts and modules <python_scripting>
 
 --------------
 

docs/contributing/python_scripting.rst

@@ -0,0 +1,93 @@
+##########################
+Python Scripts and Modules
+##########################
+
+Overview
+--------
+
+This document describes the guidelines for adding Python scripts and modules to TF-M. TF-M supports
+multiple platforms, each potentially requiring platform-specific scripts or modules. Scripts and
+modules are organized as `Namespace Packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages/>`_.
+
+Adding Platform-Specific Modules
+--------------------------------
+
+Suppose a module is located at
+``path/to/module/``
+and its package name is ``foo``:
+
+- Add the module path to ``pyproject.toml``:
+
+  .. code-block:: toml
+
+      [tool.setuptools.package-dir]
+      foo = "path/to/module/"
+
+- Register the package in ``pyproject.toml``:
+
+  .. code-block:: toml
+
+      [tool.setuptools]
+      packages = ["foo"]
+
+- Install the module:
+
+  .. code-block:: bash
+
+      pip install .
+
+  Or, for editable installation:
+
+  .. code-block:: bash
+
+      source .venv/bin/activate
+      pip install -e .
+
+- Import the installed module in other Python scripts/modules:
+
+  .. code-block:: python
+
+      import foo.<MODULE_NAME> as <MODULE_SHORT_NAME>
+
+Adding Platform-Specific Scripts
+-------------------------------
+
+- Every script should define a ``main()`` entry point.
+- Add the script’s entry point to ``pyproject.toml``. The script executable name must be prefixed
+  with the platform name.
+
+For example, for a script ``bar`` in package ``foo``:
+
+- Register the script in ``pyproject.toml``:
+
+  .. code-block:: toml
+
+      [project.scripts]
+      foo_bar = "foo.bar:main"
+
+- Install scripts in a virtual environment:
+
+  .. code-block:: bash
+      # Subsequent edits made to scripts after install **WONT** be reflected immediately and will need to be installed again
+      source .venv/bin/activate
+      pip install .
+
+  Or, for editable installation:
+
+  .. code-block:: bash
+      # Edits made to the scripts will be reflected immediately
+      source .venv/bin/activate
+      pip install -e .
+
+- Run the script by calling the executable:
+
+  .. code-block:: bash
+
+      (.venv) ± foo_bar
+      foo_bar help text
+
+--------------
+
+*SPDX-License-Identifier: BSD-3-Clause*
+
+*SPDX-FileCopyrightText: Copyright The TrustedFirmware-M Contributors*

docs/getting_started/tfm_getting_started.rst

@@ -119,14 +119,24 @@ dependencies.
 
             git clone https://git.trustedfirmware.org/TF-M/trusted-firmware-m.git
 
-        2. TF-M's ``tools/requirements.txt`` file declares additional Python
-           dependencies. Install them with ``pip3``:
+        2. TF-M recommends installing dependencies in a venv
 
         .. code-block:: bash
 
-            pip3 install --upgrade pip
-            cd trusted-firmware-m
-            pip3 install -r tools/requirements.txt
+            # Setup python venv for the project
+            python3 -m venv .venv
+
+            # NOTE: If your system python install version is <3.10 you can use `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>` to setup your .venv
+            uv venv --python 3.12
+
+            source .venv/bin/activate
+            # `-e` installs modules and scripts in editable/development mode
+            pip install -e .
+
+            # NOTE: If you've used `uv` to setup your `.venv`, prepend the `pip` commands with `uv`
+            # `-e` installs modules and scripts in editable/development mode
+            uv pip install -e .
+
 
     .. group-tab:: Windows
 
@@ -136,13 +146,23 @@ dependencies.
 
             git clone https://git.trustedfirmware.org/TF-M/trusted-firmware-m.git
 
-        2. TF-M's ``tools/requirements.txt`` file declares additional Python
-           dependencies. Install them with ``pip3``:
+        2. TF-M recommends installing dependencies in a venv
 
         .. code-block:: bash
 
-            cd trusted-firmware-m
-            pip3 install -r tools\requirements.txt
+            # Setup python venv for the project
+            python3 -m venv .venv
+
+            # NOTE: If your system python install version is <3.10 you can use `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>` to setup your .venv
+            uv venv --python 3.12
+
+            source .venv/bin/activate
+            # `-e` installs modules and scripts in editable/development mode
+            pip install -e .
+
+            # NOTE: If you've used `uv` to setup your `.venv`, prepend the `pip` commands with `uv`
+            # `-e` installs modules and scripts in editable/development mode
+            uv pip install -e .
 
 ###################
 Install a toolchain
@@ -434,8 +454,8 @@ To build the TF-M firmware the following tools are needed:
    - CMake version 3.21 or later
    - Git
    - gmake, aka GNU Make
-   - Python v3.x
-   - a set of python modules listed in ``tools/requirements.txt``
+   - Python >=v3.11
+   - [Optionally] `uv <https://docs.astral.sh/uv/getting-started/installation/#standalone-installer>`
 
 ****************
 Dependency chain

pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "trusted-firmware-m"
+description = "Trusted Firmware-M (TF-M) Python dependencies."
+authors = [{ name = "Arm Ltd." }]
+license = "BSD-3-Clause"
+readme = "readme.rst"
+version = "2.2.0"
+
+requires-python = ">=3.10"
+
+dependencies = [
+    "pyhsslms (>=2.0.0,<2.1.0)",
+    "cbor2 (>=5.6.5,<6.0.0)",
+    "click (>=8.1.8,<9.0.0)",
+    "cryptography (>=36.0.1)",
+    "pyasn1 (>=0.6.1,<0.7.0)",
+    "imgtool (>=2.1.0,<3.0.0)",
+    "jinja2 (>=3.1.6,<4.0.0)",
+    "jsonschema (>=4.23.0,<5.0.0)",
+    "pyyaml (>=6.0.2,<7.0.0)",
+    "ecdsa (>=0.19.1,<0.20.0)",
+    "kconfiglib (>=14.1.0,<15.0.0)",
+    "networkx (>=3.1,<4.0)",
+    "pyelftools (>=0.32,<0.33)",
+    "rich (>=13.9.4,<14.0.0)",
+    "libclang (>=18.1.1,<19.0.0)",
+]
+
+[project.optional-dependencies]
+docs = [
+  "sphinx==5.0.0",
+  "sphinx-rtd-theme>=1.0.0",
+  "sphinxcontrib-plantuml==0.30",
+  "sphinxcontrib-svg2pdfconverter==1.3.0",
+  "sphinx-tabs==3.4.7",
+  "jinja2==3.1.6",
+  "graphviz==0.20.3",
+  "latex==0.7.0",
+  "m2r2<0.3.3.post2",
+  "setuptools==78.1.0"
+]
+
+[tool.setuptools]
+package-dir = {"" = "tools/modules"}
+
+[tool.setuptools.packages.find]
+where = ["tools/modules"]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"