Merge branch 'add_user_agent' into 'master'

Add user agent

See merge request ix.ai/csp!14

Author: tlex

.gitlab-ci.yml

@@ -2,6 +2,7 @@
 variables:
   DOCKERHUB_REPO_NAME: csp
   GITHUB_REPO_NAME: ix-ai/csp
+  ENABLE_AMD64: 'true'
   ENABLE_ARM64: 'true'
   ENABLE_ARMv7: 'true'
   ENABLE_ARMv6: 'true'

.pylintrc

@@ -63,7 +63,7 @@ confidence=
 disable=logging-fstring-interpolation,
 #        too-few-public-methods,
         invalid-name,
-        no-self-use,
+#        no-self-use,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -255,8 +255,8 @@ max-module-lines=1000
 # separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
 # `trailing-comma` allows a space between comma and closing bracket: (a, ).
 # `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
+#no-space-check=trailing-comma,
+#               dict-separator
 
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.

Dockerfile

@@ -1,6 +1,7 @@
 FROM alpine:latest
 LABEL maintainer="docker@ix.ai" \
-      ai.ix.repository="ix.ai/csp"
+      ai.ix.repository="ix.ai/csp" \
+      org.opencontianers.image.description="A basic Content Security Policy processor running in docker"
 
 COPY csp/requirements.txt /csp/requirements.txt
 

README.md

@@ -8,47 +8,55 @@
 A basic Content Security Policy processor running in docker
 
 ## WIP Warning
+
 This is still work in progress
 
 ## What does it do?
+
 It logs to STDOUT (LOGLEVEL `INFO`) and, optionally, to a GELF capable host, the received CSP violation.
 
 The request must go to the path `/csp` (default) or to the path set in the environment variable `CSP_PATH`.
 
 Just add the header:
-```
+
+```txt
 Content-Security-Policy-Report-Only: upgrade-insecure-requests; default-src 'self'; report-uri https://example.com/csp;
 ```
 
 ### Invalid requests
+
 The following requests are not logged at all, instead a warning is logged:
+
 * Zero-length requests (a HTTP POST containing no payload)
 * Requests larger than `MAX_CONTENT_LENGTH`
 
 All other requests log the underlying WSGI environment to log level `DEBUG`. Non-JSON requests are also logged to the same level.
 
 ## Healthcheck
+
 To enable a healthcheck, just point it to `/healthz` (default) or to the value set for the environment variable `HEALTHZ_PATH`. You can use `ENABLE_HEALTHZ_VERSION` to also have CSP display the version and build information (disabled by default).
 
 ## Usage Examples
 
 ### CLI
+
 ```sh
 docker run --rm -it \
     -p 9999:80 \
     -e PORT=80 \
     -e GELF_HOST=graylog \
     --name csp \
-    registry.gitlab.com/ix.ai/csp:latest
+    ghcr.io/ix-ai/csp:latest
 ```
 
 ### docker-compose
+
 ```yml
 version: "3.7"
 
 services:
   csp:
-    image: registry.gitlab.com/ix.ai/csp:latest
+    image: ghcr.io/ix-ai/csp:latest
     environment:
       PORT: '80'
       MAX_CONTENT_LENGTH: '512'
@@ -57,12 +65,13 @@ services:
 ```
 
 ### docker stack with traefik
+
 ```yml
 version: "3.7"
 
 services:
   csp:
-    image: registry.gitlab.com/ix.ai/csp:latest
+    image: ghcr.io/ix-ai/csp:latest
     deploy:
       labels:
         traefik.enable: 'true'
@@ -89,17 +98,21 @@ services:
 ```
 
 ## Output example
+
 Firefox browser and `LOGLEVEL: INFO`
-```
+
+```txt
 2020-12-06 14:25:42.853 WARNING [__main__.<module>] Starting **csp refactor-225909200**. Listening on *:9180
 2020-12-06 14:28:15.442 INFO [csp.log_csp] {"csp-report": {"blocked-uri": "inline", "document-uri": "https://xxxREDACTEDxxx/", "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp", "referrer": "", "source-file": "https://xxxREDACTEDxxx/", "violated-directive": "default-src"}}
 2020-12-06 14:28:15.711 INFO [csp.log_csp] {"csp-report": {"blocked-uri": "inline", "column-number": 1, "document-uri": "https://xxxREDACTEDxxx/", "line-number": 925, "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp", "referrer": "", "source-file": "https://xxxREDACTEDxxx/", "violated-directive": "script-src"}}
 2020-12-06 14:28:15.724 INFO [csp.log_csp] {"csp-report": {"blocked-uri": "inline", "column-number": 3975, "document-uri": "https://xxxREDACTEDxxx/", "line-number": 3, "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp", "referrer": "", "source-file": "https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js", "violated-directive": "default-src"}}
 2020-12-06 14:28:15.735 INFO [csp.log_csp] {"csp-report": {"blocked-uri": "inline", "column-number": 3975, "document-uri": "https://xxxREDACTEDxxx/", "line-number": 3, "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp", "referrer": "", "source-file": "https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js", "violated-directive": "default-src"}}
 2020-12-06 14:28:15.738 INFO [csp.log_csp] {"csp-report": {"blocked-uri": "inline", "column-number": 14648, "document-uri": "https://xxxREDACTEDxxx/", "line-number": 3, "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp", "referrer": "", "source-file": "https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js", "violated-directive": "default-src"}}
 ```
+
 Google Chrome browser and `LOGLEVEL: DEBUG`
-```
+
+```txt
 2020-12-06 14:38:27.132 DEBUG [csp.log_csp] {'REMOTE_ADDR': '10.0.14.14', 'REMOTE_HOST': '10.0.14.14', 'REMOTE_PORT': '56224', 'REQUEST_METHOD': 'POST', 'SERVER_PORT': '9180', 'SERVER_NAME': '9f02bb970b0b', 'SERVER_SOFTWARE': None, 'SERVER_PROTOCOL': 'HTTP/1.1', 'SCRIPT_NAME': '', 'PATH_INFO': '/csp', 'QUERY_STRING': '', 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), 'wsgi.errors': <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>, 'wsgi.multithread': True, 'wsgi.multiprocess': False, 'wsgi.run_once': False, 'wsgi.input': <_io.BytesIO object at 0x7fb398c89720>, 'wsgi.file_wrapper': <class 'waitress.buffers.ReadOnlyFileBasedBuffer'>, 'wsgi.input_terminated': True, 'HTTP_HOST': 'csp.example.com', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36', 'CONTENT_LENGTH': '548', 'HTTP_ACCEPT': '*/*', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate, br', 'HTTP_ACCEPT_LANGUAGE': 'en-DE,en-GB;q=0.9,en;q=0.8,de-DE;q=0.7,de;q=0.6,ro-RO;q=0.5,ro;q=0.4,en-US;q=0.3', 'CONTENT_TYPE': 'application/csp-report', 'HTTP_DNT': '1', 'HTTP_ORIGIN': 'https://xxxREDACTEDxxx', 'HTTP_REFERER': 'https://xxxREDACTEDxxx/', 'HTTP_SEC_FETCH_DEST': 'report', 'HTTP_SEC_FETCH_MODE': 'no-cors', 'HTTP_SEC_FETCH_SITE': 'cross-site', 'HTTP_X_FORWARDED_FOR': '2001:0DB8::1', 'HTTP_X_FORWARDED_HOST': 'csp.example.com', 'HTTP_X_FORWARDED_PORT': '443', 'HTTP_X_FORWARDED_PROTO': 'https', 'HTTP_X_FORWARDED_SERVER': '2319d1b2d5bf', 'HTTP_X_REAL_IP': '2001:0DB8::1', 'werkzeug.request': <Request 'http://csp.example.com/csp' [POST]>}
 2020-12-06 14:38:27.132 INFO [csp.log_csp] {"csp-report": {"document-uri": "https://xxxREDACTEDxxx/", "referrer": "", "violated-directive": "script-src-elem", "effective-directive": "script-src-elem", "original-policy": "upgrade-insecure-requests; default-src 'self' https://cdnjs.cloudflare.com; script-src 'self' https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp;", "disposition": "report", "blocked-uri": "inline", "line-number": 925, "source-file": "https://xxxREDACTEDxxx/", "status-code": 0, "script-sample": ""}}
 2020-12-06 14:38:27.134 DEBUG [csp.log_csp] {'REMOTE_ADDR': '10.0.14.14', 'REMOTE_HOST': '10.0.14.14', 'REMOTE_PORT': '56220', 'REQUEST_METHOD': 'POST', 'SERVER_PORT': '9180', 'SERVER_NAME': '9f02bb970b0b', 'SERVER_SOFTWARE': None, 'SERVER_PROTOCOL': 'HTTP/1.1', 'SCRIPT_NAME': '', 'PATH_INFO': '/csp', 'QUERY_STRING': '', 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), 'wsgi.errors': <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>, 'wsgi.multithread': True, 'wsgi.multiprocess': False, 'wsgi.run_once': False, 'wsgi.input': <_io.BytesIO object at 0x7fb398c89720>, 'wsgi.file_wrapper': <class 'waitress.buffers.ReadOnlyFileBasedBuffer'>, 'wsgi.input_terminated': True, 'HTTP_HOST': 'csp.example.com', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36', 'CONTENT_LENGTH': '609', 'HTTP_ACCEPT': '*/*', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate, br', 'HTTP_ACCEPT_LANGUAGE': 'en-DE,en-GB;q=0.9,en;q=0.8,de-DE;q=0.7,de;q=0.6,ro-RO;q=0.5,ro;q=0.4,en-US;q=0.3', 'CONTENT_TYPE': 'application/csp-report', 'HTTP_DNT': '1', 'HTTP_ORIGIN': 'https://xxxREDACTEDxxx', 'HTTP_REFERER': 'https://xxxREDACTEDxxx/', 'HTTP_SEC_FETCH_DEST': 'report', 'HTTP_SEC_FETCH_MODE': 'no-cors', 'HTTP_SEC_FETCH_SITE': 'cross-site', 'HTTP_X_FORWARDED_FOR': '2001:0DB8::1', 'HTTP_X_FORWARDED_HOST': 'csp.example.com', 'HTTP_X_FORWARDED_PORT': '443', 'HTTP_X_FORWARDED_PROTO': 'https', 'HTTP_X_FORWARDED_SERVER': '2319d1b2d5bf', 'HTTP_X_REAL_IP': '2001:0DB8::1', 'werkzeug.request': <Request 'http://csp.example.com/csp' [POST]>}
@@ -111,7 +124,8 @@ Google Chrome browser and `LOGLEVEL: DEBUG`
 ```
 
 Various errors (with `LOGLEVEL:DEBUG`):
-```
+
+```txt
 2020-12-06 14:28:15.448 WARNING [csp.log_csp] Content too large (523445). Dropping.
 2020-12-06 14:52:47.747 WARNING [csp.log_csp] Empty content received
 2020-12-06 14:54:07.615 DEBUG [csp.log_csp] {'REMOTE_ADDR': '10.0.14.16', 'REMOTE_HOST': '10.0.14.16', 'REMOTE_PORT': '32772', 'REQUEST_METHOD': 'POST', 'SERVER_PORT': '9180', 'SERVER_NAME': '45d8708af6ab', 'SERVER_SOFTWARE': None, 'SERVER_PROTOCOL': 'HTTP/1.1', 'SCRIPT_NAME': '', 'PATH_INFO': '/csp', 'QUERY_STRING': '', 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), 'wsgi.errors': <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>, 'wsgi.multithread': True, 'wsgi.multiprocess': False, 'wsgi.run_once': False, 'wsgi.input': <_io.BytesIO object at 0x7f8cbd65c0e0>, 'wsgi.file_wrapper': <class 'waitress.buffers.ReadOnlyFileBasedBuffer'>, 'wsgi.input_terminated': True, 'HTTP_HOST': 'csp.example.com', 'HTTP_USER_AGENT': 'curl/7.64.1', 'CONTENT_LENGTH': '10', 'HTTP_ACCEPT': '*/*', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', 'HTTP_X_FORWARDED_FOR': '2001:0DB8::1', 'HTTP_X_FORWARDED_HOST': 'csp.example.com', 'HTTP_X_FORWARDED_PORT': '443', 'HTTP_X_FORWARDED_PROTO': 'https', 'HTTP_X_FORWARDED_SERVER': 'de9e6f88b502', 'HTTP_X_REAL_IP': '2001:0DB8::1', 'HTTP_ACCEPT_ENCODING': 'gzip', 'werkzeug.request': <Request 'http://csp.example.com/csp' [POST]>}
@@ -121,7 +135,8 @@ Various errors (with `LOGLEVEL:DEBUG`):
 ## Metrics
 
 When setting `ENABLE_METRICS=yes`, the following metrics are exposed:
-```
+
+```txt
 # HELP csp_valid_violation_reports_total Counts the number of valid violation reports
 # TYPE csp_valid_violation_reports_total counter
 csp_valid_violation_reports_total{blocked_uri="inline",document_uri="https://xxxREDACTEDxxx/",line_number="925",original_policy="upgrade-insecure-requests; default-src self https://cdnjs.cloudflare.com; script-src self https://cdnjs.cloudflare.com https://s.ytimg.com; font-src https://fonts.gstatic.com https://cdnjs.cloudflare.com; report-uri https://csp.example.com/csp;",violated_directive="script-src-elem"} 3.0
@@ -150,6 +165,7 @@ csp_version_info{version="0.2.0-225909200"} 1.0
 | **Variable**             | **Default** | **Description**                                                        |
 |:-------------------------|:-----------:|:-----------------------------------------------------------------------|
 | `MAX_CONTENT_LENGTH`     | `32768`     | The maximum content length (in bytes) of the HTTP POST content         |
+| `ENABLE_USER_AGENT`      | `no`        | Enable the labels `user_agent_platform`, `user_agent_browser` and `user_agent_version` |
 | `ENABLE_HEALTHZ_VERSION` | `no`        | Set this to `yes` to show the version on the `HEALTHZ_PATH` endpoint   |
 | `ENABLE_METRICS`         | `no`        | Set this to `yes` to enable the Prometheus metrics                     |
 | `CSP_PATH`               | `/csp`      | The path used for the CSP reporting                                    |
@@ -162,29 +178,43 @@ csp_version_info{version="0.2.0-225909200"} 1.0
 | `ADDRESS`                | `*`         | The IP address to bind to                                              |
 
 ## Breaking Changes
+
 Starting with version `v0.1.0`, the log format has changed!
 
 CSP will now parse and format any JSON received (smaller than `MAX_CONTENT_LENGTH`) and log it in form:
-```
+
+```txt
 2020-12-06 14:59:13.855 INFO [csp.log_csp] {"ab": 2}
 ```
 
 Non-JSON content will be logged as follows:
-```
+
+```txt
 2020-12-06 15:15:58.497 DEBUG [csp.log_csp] Content is not JSON: `{"ab": e2}`
 ```
 
+## Contributors
+
+Thank you to the contributors:
+
+* @bgi: ix.ai/csp!13
+
+## Deprecations
+
+**WARNING**: Due to the [introduction of storage usage quotas by GitLab](https://docs.gitlab.com/ee/user/usage_quotas.html), the `registry.gitlab.com` images will **not** be updated anymore and will soon be removed. Please switch to either Docker Hub or GitHub (see below).
+
 ## Tags and Arch
 
 Starting with version `v0.1.0`, the images are multi-arch, with builds for i386, amd64, arm64, armv7 and armv6.
+
 * `vN.N.N` - for example v0.1.0
 * `latest` - always pointing to the latest version
 * `dev-branch` - the last build on a feature/development branch
 * `dev-master` - the last build on the master branch
 
-## Resources:
-* GitLab: https://gitlab.com/ix.ai/csp
-* GitHub: https://github.com/ix-ai/csp
-* GitLab Registry: https://gitlab.com/ix.ai/csp/container_registry
-* GitHub Registry: https://ghcr.io/ix-ai/csp
-* Docker Hub: https://hub.docker.com/r/ixdotai/csp
+## Resources
+
+* GitLab: [gitlab.com/ix.ai/csp](https://gitlab.com/ix.ai/csp)
+* GitHub: [github.com/ix-ai/csp](https://github.com/ix-ai/csp)
+* GitHub Registry: `ghcr.io/ix-ai/csp` - [ghcr.io/ix-ai/csp](https://ghcr.io/ix-ai/csp)
+* Docker Hub: `ixdotai/csp` - [hub.docker.com/r/ixdotai/csp](https://hub.docker.com/r/ixdotai/csp)

csp/__main__.py

@@ -20,6 +20,7 @@
     'csp_path': 'string',
     'healthz_path': 'string',
     'metrics_path': 'string',
+    'enable_user_agent': 'boolean',
 })
 c = csp.CSP(**options)
 

csp/csp.py

@@ -24,6 +24,7 @@ class CSP():
         'csp_path': '/csp',
         'healthz_path': '/healthz',
         'metrics_path': '/metrics',
+        'enable_user_agent': False,
     }
 
     def __init__(self, **kwargs):
@@ -60,7 +61,10 @@ def log_csp(self):
             log.debug(f"{request.environ}")
             content = request.get_data(as_text=True)
             try:
-                log.info(self.__process_csp(json.loads(content)))
+                if self.settings['enable_user_agent']:
+                    log.info(self.__process_csp(json.loads(content), request.user_agent))
+                else:
+                    log.info(self.__process_csp(json.loads(content), None))
             except CSPError:
                 log.debug(f'{W004}: `{content}`')
                 log.warning(W004)
@@ -74,24 +78,37 @@ def log_csp(self):
 
         return result
 
-    def __process_csp(self, content):
+    def __process_csp(self, content, user_agent):
         """ Takes the JSON content and creates the metrics for it """
         try:
             report = content['csp-report']
         except KeyError:
             raise CSPError from KeyError
 
         try:
-            prometheus.PROM_VALID_VIOLATION_REPORTS_COUNTER.labels(
-                blocked_uri=report['blocked-uri'],
-                document_uri=report['document-uri'],
-                original_policy=report['original-policy'],
-                violated_directive=report['violated-directive'],
-                line_number=report.get('line-number', 0),
-                source_file=report.get('source-file'),
-            ).inc(1)
+            labels = {
+                'blocked_uri': report['blocked-uri'],
+                'document_uri': report['document-uri'],
+                'original_policy': report['original-policy'],
+                'violated_directive': report['violated-directive'],
+                'line_number': report.get('line-number', 0),
+                'source_file': report.get('source-file'),
+            }
+
+            if user_agent is None:
+                prometheus.PROM_VALID_VIOLATION_REPORTS_COUNTER.labels(**labels).inc(1)
+            else:
+                labels.update({
+                    'user_agent_platform': user_agent.platform,
+                    'user_agent_browser': user_agent.browser,
+                    'user_agent_version': user_agent.version,
+                })
+                prometheus.PROM_VALID_VIOLATION_REPORTS_COUNTER_AGENT.labels(**labels).inc(1)
+
+            return json.dumps(labels)
         except KeyError:
             raise CSPError from KeyError
+
         return json.dumps(content)
 
     def healthz(self):

csp/lib/helpers.py

@@ -4,7 +4,6 @@
 
 import logging
 import os
-from distutils.util import strtobool
 
 log = logging.getLogger("csp")
 
@@ -40,3 +39,16 @@ def gather_environ(keys) -> dict:
                 environs[key] = {filters[0]: filters[1]}
             log.info(f'{key.upper()} is set')
     return environs
+
+def strtobool(val):
+    """Convert a string representation of truth to true (1) or false (0).
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return 1
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return 0
+    raise ValueError(f"invalid truth value {(val,)}")

csp/lib/prometheus.py

@@ -15,6 +15,20 @@
         'source_file',
     ]
 )
+PROM_VALID_VIOLATION_REPORTS_COUNTER_AGENT = Counter(
+    'csp_valid_violation_reports_by_user_agent',
+    'Counts the number of valid violation reports with UserAgent details', [
+        'blocked_uri',
+        'document_uri',
+        'original_policy',
+        'violated_directive',
+        'line_number',
+        'source_file',
+        'user_agent_platform',
+        'user_agent_browser',
+        'user_agent_version',
+    ]
+)
 PROM_INVALID_VIOLATION_REPORTS_COUNTER = Counter(
     'csp_invalid_violation_reports', 'Counts the number of invalid violation reports', [
         'reason',

csp/requirements.txt

@@ -1,5 +1,4 @@
 waitress==2.1.2
-flask==2.0.2
-prometheus_client==0.12.0
+flask==2.2.2
+prometheus_client==0.14.1
 pygelf==0.4.2
-