migrate events from mysql to clickhouse

Author: tudddorrr

clickhouse/Dockerfile

@@ -0,0 +1,10 @@
+FROM clickhouse/clickhouse-server:24.6-alpine
+
+RUN apk add --no-cache gettext
+
+COPY /clickhouse/migrations /docker-entrypoint-initdb.d/
+COPY /clickhouse/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

clickhouse/entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+INIT_DIR="/docker-entrypoint-initdb.d"
+FLAG_FILE="$INIT_DIR/.migrated"
+
+if [ ! -f "$FLAG_FILE" ]; then
+  for file in ${INIT_DIR}/*.sql; do
+    if [ -f "$file" ]; then
+      envsubst < "$file" > "$file.processed" && mv "$file.processed" "$file"
+      echo "Processed migration $file"
+    fi
+  done
+
+  echo "Creating migrations lock"
+  touch "$FLAG_FILE"
+else
+  echo "Migrations lock set; Skipping envsubst processing"
+fi
+
+exec /entrypoint.sh "$@"

clickhouse/migrations/001CreateEventsTable.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.events
+(
+    id String,
+    name String,
+    game_id UInt32,
+    player_alias_id UInt32,
+    dev_build Boolean,
+    created_at DateTime,
+    updated_at DateTime,
+    PRIMARY KEY (id),
+    INDEX name_idx (name) TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX game_id_idx (game_id) TYPE minmax GRANULARITY 64,
+    INDEX player_alias_id_idx (player_alias_id) TYPE minmax GRANULARITY 64
+)
+ENGINE = MergeTree()
+ORDER BY (id, created_at, game_id, player_alias_id);

clickhouse/migrations/002CreateEventPropsTable.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.event_props (
+    event_id String,
+    prop_key String,
+    prop_value String
+) ENGINE = MergeTree()
+ORDER BY (event_id, prop_key);

docker-compose.ci.yml

@@ -1,45 +1,9 @@
 services:
-  test-db:
-    image: mysql:8.4
-    command: --mysql-native-password=ON
-    environment:
-      - MYSQL_DATABASE=${DB_NAME}
-      - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"]
-      interval: 2s
-      timeout: 2s
-      retries: 10
-    ports:
-      - ${DB_PORT}:3306
-    volumes:
-      - test-data:/var/lib/mysql
-    networks:
-      - test-network
-    
   test-redis:
     image: bitnami/redis:7.2
+    command:
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
-    ports:
-      - ${REDIS_PORT}:6379
-    depends_on:
-      test-db:
-        condition: service_healthy
-    networks:
-      - test-network
 
   stripe-api:
     image: stripe/stripe-mock:latest
-    ports:
-      - 12111:12111
-      - 12112:12112
-    networks:
-      - test-network
-
-volumes:
-  test-data:
-
-networks:
-  test-network:

docker-compose.dev.yml

@@ -4,10 +4,11 @@ services:
       context: .
       target: dev
     image: backend
+    depends_on:
+      - db
     ports:
       - 3000:80
     volumes:
       - ./src:/usr/backend/src
       - ./tests:/usr/backend/tests
-    depends_on:
-      - db
+ 

docker-compose.test.yml

@@ -29,6 +29,20 @@ services:
     networks:
       - test-network
 
+  test-clickhouse:
+    build:
+      context: .
+      dockerfile: ./clickhouse/Dockerfile
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    networks:
+      - test-network
+
   stripe-api:
     image: stripe/stripe-mock:latest-arm64
     ports:

docker-compose.yml

@@ -5,14 +5,15 @@ services:
     depends_on:
       - db
       - redis
+      - clickhouse
 
   db:
     image: mysql:8.4
     command: --mysql-native-password=ON
     environment:
       - MYSQL_DATABASE=${DB_NAME}
       - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
+    restart: unless-stopped
     ports:
       - 3306:3306
     volumes:
@@ -24,5 +25,20 @@ services:
     ports:
       - 6379:6379
 
+  clickhouse:
+    build:
+      context: .
+      dockerfile: ./clickhouse/Dockerfile
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+
 volumes:
   data:
+  clickhouse-data:

envs/.dockerignore

@@ -9,6 +9,12 @@ DB_PASS=password
 
 REDIS_PASSWORD=password
 
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_USER=gs_ch
+CLICKHOUSE_PASSWORD=password
+CLICKHOUSE_DB=gs_ch_dev
+
 DEMO_ORGANISATION_NAME=Talo Demo
 
 SENDGRID_KEY=