Remove the need of a --creds-path (#153)

If --creds-path is not provided it will not default to `~/.credentials` any more.
Instead, it will assume that env vars want to be used and it will try so.

Also, the documentation has been updated to reflect this and provide a way to use 1Password CLI as a credentials loader.

Author: cletomartin

CHANGES.md

@@ -1,5 +1,7 @@
-# [1.24.1](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.24.1)
+# [1.25.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.25.0)
 
+- [ADDED] Documentation on how to use it with 1Password CLI.
+- [CHANGED] "--creds-path" does not default to "~/.credentials".
 - [FIXED] Number of errors/warnings shown correctly for single checks.
 
 # [1.24.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.24.0)

compliance/__init__.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Compliance automation package."""
 
-__version__ = "1.24.1"
+__version__ = "1.25.0"

compliance/config.py

@@ -68,11 +68,9 @@ def __init__(self):
     @property
     def creds(self):
         """Credentials used for locker management and running fetchers."""
-        if self.creds_path is None:
-            raise ValueError("Path to credentials file not provided")
-
-        if self._creds is None:
-            self._creds = Config(self.creds_path)
+        if not self._creds:
+            path = None if self.creds_path is None else str(self.creds_path)
+            self._creds = Config(path)
         return self._creds
 
     @property

compliance/runners.py

@@ -91,11 +91,15 @@ def get_test_candidates(self, suite):
                 yield test
 
     def _load_compliance_config(self):
-        creds_path = Path(self.opts.creds_path).expanduser()
-        if not creds_path.is_file():
-            raise ValueError(f"{creds_path} file does not exist.")
         self.config = get_config()
-        self.config.creds_path = str(creds_path)
+        creds_path = None
+        if self.opts.creds_path is not None:
+            creds_path = Path(self.opts.creds_path).expanduser()
+            if not creds_path.is_file():
+                raise ValueError(
+                    f"Invalid path to credentials file '{str(creds_path)}'"
+                )
+        self.config.creds_path = creds_path
         self.config.load(self.opts.compliance_config)
 
     def _init_dirs(self):

compliance/scripts/compliance_cli.py

@@ -83,7 +83,7 @@ def _init_arguments(self):
                 "Defaults to %(default)s."
             ),
             metavar="/path/to/creds.ini",
-            default="~/.credentials",
+            default=None,
         )
         notify_options = [k for k in get_notifiers().keys() if k != "stdout"]
         self.add_argument(

compliance/utils/credentials.py

@@ -36,8 +36,9 @@ def __init__(self, cfg_file="~/.credentials"):
         :param cfg_file: The path to the RawConfigParser compatible config file
         """
         self._cfg = RawConfigParser()
-        self._cfg.read(str(Path(cfg_file).expanduser()))
         self._cfg_file = cfg_file
+        if cfg_file is not None:
+            self._cfg.read(str(Path(cfg_file).expanduser()))
 
     def __getitem__(self, section):
         """
@@ -60,13 +61,16 @@ def _getattr_wrapper(t, attr):
             try:
                 return t.__getattribute__(attr)
             except AttributeError as exc:
-                exc.args = (
-                    (
+                if self._cfg_file:
+                    msg = (
                         f'Unable to locate attribute "{attr}" '
                         f'in section "{type(t).__name__}" '
                         f'at config file "{self._cfg_file}"'
-                    ),
-                )
+                    )
+                else:
+                    env_var_name = f"{type(t).__name__}_{attr}".upper()
+                    msg = f"Unable to find the env var: {env_var_name}"
+                exc.args = (msg,)
                 raise exc
 
         env_vars = [k for k in environ.keys() if k.startswith(f"{section.upper()}_")]

doc-source/design-principles.rst

@@ -750,9 +750,51 @@ levels:
 Credentials
 ~~~~~~~~~~~
 
-If you want to configure your credentials locally, the framework
-will look for a credentials file at ``~/.credentials`` by default. This
-file should be similar to this:
+There are 2 ways for providing credentials:
 
-.. include:: credentials-example.cfg
-   :literal:
+1. *Local file*: if you want to configure your credentials in a local file,
+   you will have to provide the the framework using ``--creds-path`` option.
+   This file should be similar to this:
+
+   .. include:: credentials-example.cfg
+      :literal:
+
+1. *Environment variables*: each section and field of the local file can be
+   rendered as an environment variable.
+   For instance, suppose your code requires ``creds['github'].token`` or ``creds['slack'].webhook``.
+   You just need to export:
+
+   * ``GITHUB_TOKEN = XXX``
+
+   * ``MY_SERVICE_API_KEY = YYY``
+
+   This is equivalent to the credentials file::
+
+     [github]
+     token=XXX
+
+     [my_service]
+     api_key=YYY
+
+Creds with ``.env`` files and 1Password
++++++++++++++++++++++++++++++++++++++++
+
+Combining the method based on passing env vars to Auditree and `1Password CLI <https://developer.1password.com/docs/cli/>`_,
+it is possible to grab the secrets from 1Password and inject them into Auditree.
+Here it is how to do it:
+
+1. Create the following alias::
+
+     alias compliance="op run --env-file .env -- compliance"
+
+1. In your fetchers/checks project, create an ``.env`` file with the following schema::
+
+     <SECTION>_<ATTRIBUTE>="op://<VAULT>/<ITEM>/<FIELD>"
+
+   For example::
+
+     GITHUB_TOKEN="op://Private/github/token"
+     MY_SERVICE_ORG="the-org-id"
+     MY_SERVICE_API_KEY="op://Shared/my_service/api_key"
+
+1. Now running ``compliance`` will pull credentials from 1Password vaults.

doc-source/notifiers.rst

@@ -134,7 +134,9 @@ You can also use a Slack app token (recommended if you need to post
 messages to private channels)::
 
   [slack]
-  slack=XXX
+  token=XXX
+
+Note that you can do the same thing using env vars ``SLACK_WEBHOOK`` and ``SLACK_TOKEN``.
 
 In case you need private channels as part of the list, you have to
 specify the channel ID::

doc-source/running-on-travis.rst

@@ -57,104 +57,22 @@ This is a typical `.travis.yml` file:
      - "3.7"
    install:
      - pip install -r requirements.txt
-     - ./travis/gen-credentials.py > ~/.credentials
    script:
      - make clean
      - ./travis/run.sh
-   after_script:
-     - rm  ~/.credentials
 
 Basically, this will firstly install the dependencies through
 ``pip install -r requirements.txt`` and then generate the credentials file from
 using Travis environment variables.
 
-Credentials generation
-~~~~~~~~~~~~~~~~~~~~~~
+Credentials
+~~~~~~~~~~~
 
-This is an implementation you might want to use for your project of
-``gen-credentials.py``:
+The recommended way to use credentials in a CI job is to export them as environment variables.
+Auditree will automatically parsed the environment variables available to the process and make them available to the fetchers if they follow a specific structure.
 
-.. code-block:: python
+For more information on how to do this, have a look to the :ref:`credentials` section.
 
-   #!/usr/bin/env python
-   # -*- coding:utf-8; mode:python -*-
-
-   '''This script generates a config file suitable to be used by
-   `utilitarian.credentials.Config` from envvars. This is useful for
-   Travis CI that allows to deploy credentials safely using envvars.
-
-   Any new supported credential must be added to SUPPORTED_SECTIONS which
-   includes a list of sections of `Config` supported by the script. For
-   example, adding 'github' will make the script to generate
-   `github.username` from GITHUB_USERNAME and `github.password` from
-   GITHUB_PASSWORD, if both envvars are defined.
-   '''
-
-   import os
-   import sys
-   import ConfigParser
-
-
-   SUPPORTED_SECTIONS = ['github', 'slack']
-
-
-   def main():
-       matched_keys = filter(
-           lambda k: any([k.lower().startswith(x) for x in SUPPORTED_SECTIONS]),
-           os.environ.keys()
-       )
-       if not matched_keys:
-           return 0
-
-       cfg_parser = ConfigParser.ConfigParser()
-       for k in matched_keys:
-           # split the section name and option from this env var (max()
-           # to ensure the longest match)
-           section = max(
-               [s for s in SUPPORTED_SECTIONS if k.lower().startswith(s)],
-               key=len
-           )
-           option = k.split(section.upper())[1][1:].lower()
-
-           # add to the config
-           if not cfg_parser.has_section(section):
-               cfg_parser.add_section(section)
-           cfg_parser.set(section, option, os.environ[k])
-
-       cfg_parser.write(sys.stdout)
-
-       return 0
-
-
-   if __name__ == '__main__':
-       exit(main())
-
-So, for instance, using the previous script you will be able to create
-the credentials required for ``github`` and ``slack`` by
-defining the following environment variables in Travis:
-
-* ``GITHUB_TOKEN = XXX``
-
-* ``SLACK_WEBHOOK = YYY``
-
-Using those variables, ``./travis/gen-credentials.py >
-~/.credentials`` will generate::
-
-  [github]
-  token=XXX
-
-  [slack]
-  webhook=YYY
-
-This method has a few limitation:
-
-* Do not use ``$`` as part of the value of any variable as they will
-  be evaluated by bash.
-
-* You will need to add a new service into the
-  ``SUPPORTED_SECTIONS``. This is actually good since a manual
-  addition requires a code change (so new credentials are
-  tracked).
 
 ``travis/run.sh``
 ~~~~~~~~~~~~~~~~~