feature : add chaos testing example (#20)

- [x] add chaos testing example using toxiproxy 
- [x] create prometheus dashboard 
- [x] create README to explain the setup

Author: cboudereau

examples/chaostesting/.gitignore

@@ -0,0 +1 @@
+.cache

examples/chaostesting/0-client-toxics.png

@@ -0,0 +1,41 @@
+# Chaos testing with toxiproxy
+
+This setup uses toxiproxy as a middleware between client and database to simulate latency, bandwidth and variation problems at TCP level.
+
+The client service uses toxiproxy service as a sybase database which integrates toxic issues and forwards tcp packets to the database service (sybase server).
+
+![Architecture](./docker-compose.png)
+[Docker compose file](./compose.yml)
+
+Prometheus 
+[dashboard](http://localhost:9090/graph?g0.expr=rate(client%5B1m%5D)&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=1h&g1.expr=sybase_test_table_insert&g1.tab=0&g1.stacked=0&g1.show_exemplars=0&g1.range_input=1h) is used to analyze metrics from statsd:
+
+The ``client`` is a counter from the client point of view while ``sybase_test_table_insert`` is a timer from the server point of view.
+
+## Example without Toxics (direct)
+![Client counter without toxics](./1-client-no-toxics.png)
+![Server rates without toxics](./1-server-no-toxics.png)
+
+## Example with Toxics
+We can see that the that from the client point of view the rate is lower than before (1rps > 0.04rps).
+![Client counter with toxics](./0-client-toxics.png)
+![Server rates with toxics](./0-server-toxics.png)
+
+## Tools
+
+### Toxiproxy
+
+[Project Page](https://github.com/Shopify/toxiproxy)
+
+```bash
+toxiproxy-cli create database --listen 0.0.0.0:5000 --upstream database:5000;
+toxiproxy-cli toxic add -t latency -a latency=2000 database;
+toxiproxy-cli toxic add -t bandwidth -a rate=1 database;
+toxiproxy-cli toxic add -t slicer -a average_size=1000 -a size_variation=900 -a delay=10000 database;
+
+toxiproxy-cli toxic remove -n latency_downstream database;
+```
+
+### Prometheus
+
+[Dashboard](http://localhost:9090/graph?g0.expr=rate(client%5B1m%5D)&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=1h&g1.expr=sybase_test_table_insert&g1.tab=0&g1.stacked=0&g1.show_exemplars=0&g1.range_input=1h)

examples/chaostesting/0-server-toxics.png

@@ -0,0 +1,27 @@
+use TESTDB
+go
+
+set statistics time ON
+
+/* Tradeoff : measure delay with datetime (no support for ticks nor timestamp nor true stopwatch, precision is 1/300 second). Ok to measure high latencies ie : > 1s */
+declare @s datetime
+select @s = GETUTCDATE()
+
+insert into dbo.TEST_TABLE(TEST_FIELD1) values ('1')
+insert into dbo.TEST_TABLE(TEST_FIELD1) values ('1')
+insert into dbo.TEST_TABLE(TEST_FIELD1) values ('1')
+insert into dbo.TEST_TABLE(TEST_FIELD1) values ('1')
+insert into dbo.TEST_TABLE(TEST_FIELD1) values ('1')
+
+declare @e datetime
+select @e = GETUTCDATE()
+
+declare @t integer
+select @t = DATEDIFF(MILLISECOND, @s, @e)
+
+declare @msg varchar(255)
+select @msg = "sybase.test_table.insert:" + convert(varchar, @t) + "|ms"
+exec sp_sendmsg "statsd", 8125, @msg
+select @msg as metric, count(*) as total from dbo.TEST_TABLE
+
+go

examples/chaostesting/1-client-no-toxics.png

@@ -0,0 +1,12 @@
+#!/bin/sh
+. /opt/sap/SYBASE.sh
+
+set -u
+
+while true
+do
+    echo running $1
+    isql -Usa -S${SERVER}:5000 -P${SA_PASSWORD} -D${DATABASE} -i $1
+    echo -n "client:1|c" >/dev/udp/statsd/8125
+    sleep 1
+done

examples/chaostesting/1-server-no-toxics.png

@@ -0,0 +1,74 @@
+version: '3.8'
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.41.0
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    ports:
+      - 9090:9090
+    networks:
+      - bridge
+  
+  statsd:
+    image: prom/statsd-exporter:v0.23.0
+    command: "--statsd.listen-udp=:8125 --web.listen-address=:9102 --log.level=debug"
+    networks:
+      - bridge
+    depends_on:
+      - prometheus
+
+  database:
+    image: superbeeeeeee/docker-sybase
+    pull_policy: always
+    environment:
+      - DATABASE=TESTDB
+      - SA_PASSWORD=Sybase1234
+    volumes:
+      - ./database/:/docker-entrypoint-initdb.d/
+    networks:
+      - bridge
+    healthcheck:
+      test: healthcheck
+      interval: 10s
+      retries: 10
+
+  client:
+    image: superbeeeeeee/docker-sybase
+    pull_policy: always
+    entrypoint: ["/schedule.sh", "/data.sql"]
+    environment:
+      - DATABASE=TESTDB
+      - SA_PASSWORD=Sybase1234
+      - SERVER=toxiproxy
+    volumes:
+      - ./client/schedule.sh:/schedule.sh:ro
+      - ./client/data.sql:/data.sql:ro
+    networks:
+      - bridge
+    depends_on:
+      database:
+        condition: service_healthy
+      statsd:
+        condition: service_started
+      toxiproxy:
+        condition: service_started
+
+  toxiproxy:
+    image: shopify/toxiproxy
+    networks:
+      - bridge
+  toxiproxy-config:
+    image: shopify/toxiproxy
+    entrypoint: > 
+      sh -c "/go/bin/toxiproxy-cli -h toxiproxy:8474 create database --listen 0.0.0.0:5000 --upstream database:5000;
+      /go/bin/toxiproxy-cli -h toxiproxy:8474 toxic add -t latency -a latency=2000 database;
+      /go/bin/toxiproxy-cli -h toxiproxy:8474 toxic add -t bandwidth -a rate=1 database;
+      /go/bin/toxiproxy-cli -h toxiproxy:8474 toxic add -t slicer -a average_size=1000 -a size_variation=900 -a delay=10000 database;"
+    networks:
+      - bridge
+    depends_on:
+      - toxiproxy
+
+networks:
+ bridge:

examples/chaostesting/README.md

@@ -0,0 +1,2 @@
+sp_configure 'allow sendmsg', 1
+go