Skip to content

Commit 3aa8c4c

Browse files
author
Den
committed
collectd + Graphite + Grafana + Zabbix + nginx/certbot with SSL cert autorenewal on docker
1 parent ad50943 commit 3aa8c4c

File tree

11 files changed

+321
-11
lines changed

11 files changed

+321
-11
lines changed

‎README.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,13 @@ Build your own team IT infrastructure with ~blackjack~ encrypted private cloud,
2121
* Software distribution server [storage and docker registry](roles/distribution_hub) based on [Nexus Repository Manager 3](https://github.com/sonatype/docker-nexus3)
2222
behind [nginx for SSL termination](roles/nginx)
2323
* Media [server](playbooks/openmediavault.yml) from [openmediavault.org](https://www.openmediavault.org)
24-
* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files) and [collectd](roles/collectd_beacon)
24+
* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files/dockprom)
2525
based on [dockprom](https://github.com/stefanprodan/dockprom)
26+
* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [collectd](roles/collectd_beacon) and [Graphite + Grafana + Zabbix + nginx/certbot with SSL cert autorenewal](roles/monitoring_hub/files/monitoring_hub)
2627
* BIND DNS server bundled with the Webmin UI based on [sameersbn's docker-bind](https://github.com/sameersbn/docker-bind)
2728

2829
###### Privacy
29-
* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/)
30+
* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn.yml) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/)
3031

3132
###### Security
3233
* [SSH users ACL and management](tests/test_deploy_users.sh) with public ssh keys and common sudoer user
@@ -58,7 +59,7 @@ ansible-playbook playbooks/openvpn-server.yml
5859

5960
See example test [test_deploy_openvpn.sh](tests/test_deploy_openvpn.sh)
6061

61-
1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn).
62+
1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn.yml).
6263
Client may reserve static VPN IP or dynamic otherwise.
6364

6465

@@ -82,7 +83,7 @@ ansible-playbook -i environments/test/inventory playbooks/openvpn-client.yml
8283
```
8384

8485
##### Revoke VPN key
85-
1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory).
86+
1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory)
8687
2. Update OpenVPN server:
8788
```bash
8889
ansible-playbook -i environments/test/inventory playbooks/openvpn-server.yml --limit openvpn-server

‎environments/test/group_vars/all.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
monitoring_grafana_public_root_url: http://192.168.10.101/
1+
monitoring_graphite_host: 192.168.10.101
22

33
# extract subdir name from environment name (e.g. environments/test -> vpnkeys/test)
44
vpnkeys_dir: "../.vpnkeys/{{ inventory_dir.split('/')[-1] }}/"

‎roles/collectd_beacon/tasks/main.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,6 @@
1515
name: hddtemp
1616
state: restarted
1717

18-
- name: install collectd-cuda
19-
shell: |
20-
pip3 install collectd-cuda
21-
2218
- name: add hostname to /etc/hosts (FQDN lookup required for collectd to start)
2319
lineinfile:
2420
dest: /etc/hosts
@@ -31,7 +27,7 @@
3127
name: collectd
3228
state: present
3329

34-
- name: configure /etc/collectd/collectd.conf'
30+
- name: configure /etc/collectd/collectd.conf to send metrics to '{{monitoring_graphite_host}}'
3531
template:
3632
src: templates/collectd.conf.j2
3733
dest: /etc/collectd/collectd.conf

‎roles/collectd_beacon/templates/collectd.conf.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1243,7 +1243,7 @@ LoadPlugin write_prometheus
12431243

12441244
<Plugin write_graphite>
12451245
<Node "example">
1246-
Host "monitoring_graphite_host"
1246+
Host "{{monitoring_graphite_host}}"
12471247
Port "2003"
12481248
Protocol "tcp"
12491249
LogSendErrors true

‎roles/monitoring_hub/files/monitoring_hub/docker-compose.yml

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
version: '2'
2+
services:
3+
4+
nginx:
5+
restart: unless-stopped
6+
image: nginx:1.20
7+
command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & /docker-entrypoint.sh nginx -g \"daemon off;\" '"
8+
ports:
9+
- 80:80
10+
- 443:443
11+
volumes:
12+
- ./nginx/conf.d:/etc/nginx/conf.d
13+
- ./nginx/nginx-auth.htpasswd:/etc/nginx/nginx-auth.htpasswd
14+
# certbox
15+
- ./nginx/certbot/conf:/etc/letsencrypt
16+
- ./nginx/certbot/www:/var/www/certbot
17+
18+
certbot:
19+
image: certbot/certbot
20+
restart: unless-stopped
21+
volumes:
22+
- ./nginx/certbot/conf:/etc/letsencrypt
23+
- ./nginx/certbot/www:/var/www/certbot
24+
entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
25+
26+
grafana:
27+
restart: unless-stopped
28+
image: grafana/grafana:7.4.5
29+
ports:
30+
- 83:3000
31+
volumes:
32+
- ./grafana:/var/lib/grafana
33+
#- ./custom.ini:/opt/grafana/conf/custom.ini
34+
# chown -R 472:472 ./grafana
35+
user: "root"
36+
environment:
37+
GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource,alexanderzobnin-zabbix-app,vonage-status-panel
38+
GF_SECURITY_ADMIN_USER: ${ADMIN_USER:-admin}
39+
GF_SECURITY_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin}
40+
GF_USERS_ALLOW_SIGN_UP: "false"
41+
GF_SERVER_ROOT_URL: https://grafana.orgbackbone.org
42+
# if behind nginx with basicauth with different passwords
43+
GF_AUTH_BASIC_ENABLED: "false"
44+
GF_RENDERING_SERVER_URL: http://renderer:8081/render
45+
GF_RENDERING_CALLBACK_URL: http://grafana:3000/
46+
GF_LOG_FILTERS: rendering:debug
47+
48+
renderer:
49+
image: grafana/grafana-image-renderer:2.0.1
50+
ports:
51+
- 8081
52+
environment:
53+
ENABLE_METRICS: 'true'
54+
55+
graphite:
56+
image: graphiteapp/graphite-statsd:1.1.8-1
57+
hostname: graphite
58+
ports:
59+
- 81:80 # nginx-gunicorn-graphite
60+
#- 8080:8080 # gunicorn graphite
61+
- 2003-2004:2003-2004
62+
- 2023-2024:2023-2024
63+
- 8125:8125/udp
64+
- 8126:8126
65+
volumes:
66+
- ./graphite/storage:/opt/graphite/storage
67+
environment:
68+
- COLLECTD=1
69+
- REDIS_TAGDB=1
70+
71+
postgresql:
72+
image: postgres:12
73+
restart: unless-stopped
74+
environment:
75+
POSTGRES_USER: zabbix
76+
POSTGRES_PASSWORD: zabbix
77+
POSTGRES_DB: zabbixdb
78+
volumes:
79+
- ./postgresql/data:/var/lib/postgresql/data
80+
81+
zabbix-server:
82+
image: zabbix/zabbix-server-pgsql:ubuntu-5.2-latest
83+
restart: unless-stopped
84+
hostname: zabbix-server-container
85+
environment:
86+
DB_SERVER_HOST: postgresql
87+
POSTGRES_USER: zabbix
88+
POSTGRES_PASSWORD: zabbix
89+
POSTGRES_DB: zabbixdb
90+
ZBX_HISTORYSTORAGETYPES: log,text
91+
ZBX_DEBUGLEVEL: 1
92+
ZBX_HOUSEKEEPINGFREQUENCY: 1
93+
ZBX_MAXHOUSEKEEPERDELETE: 5000
94+
ZBX_TRAPPERIMEOUT: 300
95+
depends_on:
96+
- postgresql
97+
volumes:
98+
- ./zabbix/alertscripts:/usr/lib/zabbix/alertscripts
99+
- ./zabbix/externalscripts:/usr/lib/zabbix/externalscripts:z
100+
- ./zabbix/modules:/var/lib/zabbix/modules:z
101+
- ./zabbix/enc:/var/lib/zabbix/enc:z
102+
- ./zabbix/ssh_keys:/var/lib/zabbix/ssh_keys:z
103+
- ./zabbix/ssl/certs:/var/lib/zabbix/ssl/certs:z
104+
- ./zabbix/ssl/keys:/var/lib/zabbix/ssl/keys:z
105+
- ./zabbix/ssl/ssl_ca:/var/lib/zabbix/ssl/ssl_ca:z
106+
- ./zabbix/snmptraps:/var/lib/zabbix/snmptraps:z
107+
- ./zabbix/mibs:/var/lib/zabbix/mibs:z
108+
- ./zabbix/export:/var/lib/zabbix/export:z
109+
ports:
110+
- 10051:10051
111+
112+
zabbix-web:
113+
image: zabbix/zabbix-web-nginx-pgsql:ubuntu-5.2-latest
114+
restart: unless-stopped
115+
environment:
116+
DB_SERVER_HOST: postgresql
117+
POSTGRES_USER: zabbix
118+
POSTGRES_PASSWORD: zabbix
119+
POSTGRES_DB: zabbixdb
120+
ZBX_SERVER_HOST: zabbix-server # Zabbix related and Php variables
121+
ZBX_POSTMAXSIZE: 64M
122+
ZBX_MAXEXECUTIONTIME: 500
123+
depends_on:
124+
- postgresql
125+
- zabbix-server
126+
127+
zabbix-agent:
128+
image: zabbix/zabbix-agent:ubuntu-5.2-latest
129+
# privileged: true # access mode for allowing resource access
130+
hostname: zabbix-agent-container
131+
restart: unless-stopped
132+
environment:
133+
- ZBX_SERVER_HOST=zabbix-server
134+
ports:
135+
- 10050:10050

‎roles/monitoring_hub/files/monitoring_hub/init-letsencrypt.sh

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/bin/bash
2+
#
3+
# based on https://github.com/wmnnd/nginx-certbot
4+
# see https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
5+
#
6+
set -xe
7+
8+
9+
if ! [ -x "$(command -v docker-compose)" ]; then
10+
echo 'Error: docker-compose is not installed.' >&2
11+
exit 1
12+
fi
13+
14+
domains=(grafana.orgbackbone.org zabbix.orgbackbone.org)
15+
rsa_key_size=4096
16+
data_path="./nginx/certbot"
17+
email="support@orgbackbone.org" # Adding a valid address is strongly recommended
18+
staging=0 # Set to 1 if you're testing your setup to avoid hitting request limits
19+
20+
if [ -d "$data_path" ]; then
21+
read -p "Existing data found for $domains. Continue and replace existing certificate? (y/N) " decision
22+
if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then
23+
exit
24+
fi
25+
fi
26+
27+
28+
if [ ! -e "$data_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$data_path/conf/ssl-dhparams.pem" ]; then
29+
echo "### Downloading recommended TLS parameters ..."
30+
mkdir -p "$data_path/conf"
31+
curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$data_path/conf/options-ssl-nginx.conf"
32+
curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$data_path/conf/ssl-dhparams.pem"
33+
echo
34+
fi
35+
36+
echo "### Creating dummy certificate for $domains ..."
37+
path="/etc/letsencrypt/live/$domains"
38+
mkdir -p "$data_path/conf/live/$domains"
39+
docker-compose run --rm --entrypoint "\
40+
openssl req -x509 -nodes -newkey rsa:$rsa_key_size -days 1\
41+
-keyout '$path/privkey.pem' \
42+
-out '$path/fullchain.pem' \
43+
-subj '/CN=localhost'" certbot
44+
echo
45+
46+
47+
echo "### Starting nginx ..."
48+
docker-compose up --force-recreate -d nginx
49+
echo
50+
51+
echo "### Deleting dummy certificate for $domains ..."
52+
docker-compose run --rm --entrypoint "\
53+
rm -Rf /etc/letsencrypt/live/$domains && \
54+
rm -Rf /etc/letsencrypt/archive/$domains && \
55+
rm -Rf /etc/letsencrypt/renewal/$domains.conf" certbot
56+
echo
57+
58+
59+
echo "### Requesting Let's Encrypt certificate for $domains ..."
60+
#Join $domains to -d args
61+
domain_args=""
62+
for domain in "${domains[@]}"; do
63+
domain_args="$domain_args -d $domain"
64+
done
65+
66+
# Select appropriate email arg
67+
case "$email" in
68+
"") email_arg="--register-unsafely-without-email" ;;
69+
*) email_arg="--email $email" ;;
70+
esac
71+
72+
# Enable staging mode if needed
73+
if [ $staging != "0" ]; then staging_arg="--staging"; fi
74+
75+
docker-compose run --rm --entrypoint "\
76+
certbot certonly --webroot -w /var/www/certbot \
77+
$staging_arg \
78+
$email_arg \
79+
$domain_args \
80+
--rsa-key-size $rsa_key_size \
81+
--agree-tos \
82+
--no-eff-email \
83+
--force-renewal" certbot
84+
echo
85+
86+
echo "### Reloading nginx ..."
87+
docker-compose exec nginx nginx -s reload

‎roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/certbot-challenge.conf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
server {
2+
listen 80 default_server;
3+
listen [::]:80 default_server;
4+
5+
location /.well-known/acme-challenge/ { root /var/www/certbot; }
6+
location / { return 301 https://$host$request_uri; }
7+
}
8+
9+
# add docker's embedded DNS 127.0.0.11 to route docker services with container name
10+
resolver 127.0.0.11 127.0.0.1 [::1]:5353 valid=5s;

‎roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/grafana.conf

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
server {
2+
listen 443 ssl;
3+
server_name grafana.orgbackbone.org;
4+
5+
ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem;
6+
ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem;
7+
8+
include /etc/letsencrypt/options-ssl-nginx.conf;
9+
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
10+
11+
auth_basic "";
12+
auth_basic_user_file /etc/nginx/nginx-auth.htpasswd;
13+
14+
location / {
15+
proxy_pass http://grafana:3000;
16+
proxy_set_header Host $host;
17+
proxy_set_header Referer "";
18+
proxy_set_header X-Real-IP $remote_addr;
19+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
20+
proxy_set_header Upgrade $http_upgrade;
21+
proxy_set_header Connection "upgrade";
22+
proxy_http_version 1.1;
23+
}
24+
}
25+
26+

‎roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/zabbix.conf

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
server {
2+
listen 443 ssl;
3+
server_name zabbix.orgbackbone.org;
4+
5+
ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem;
6+
ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem;
7+
8+
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
9+
ssl_ciphers HIGH:!aNULL:!MD5;
10+
11+
auth_basic "";
12+
auth_basic_user_file /etc/nginx/nginx-auth.htpasswd;
13+
14+
location / {
15+
proxy_pass http://zabbix-web:8080/;
16+
proxy_set_header Host $host;
17+
proxy_set_header Referer "";
18+
proxy_set_header X-Real-IP $remote_addr;
19+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
20+
proxy_set_header Upgrade $http_upgrade;
21+
proxy_set_header Connection "upgrade";
22+
proxy_http_version 1.1;
23+
}
24+
}
25+
26+

‎roles/monitoring_hub/files/monitoring_hub/nginx/nginx-auth.htpasswd

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
USER:um2j1VlJgPYvw
2+
3+

0 commit comments

Comments
 (0)