migrate events from mysql to clickhouse

Author: tudddorrr

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         run: npm ci --prefer-offline
 
       - name: Setup environment
-        run: cp envs/.env.dev .env
+        run: cp envs/.env.test .env
 
       - name: Run tests
         run: npm test -- --coverage

clickhouse/migrations/20240813213400CreateEventsTable.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS gs_ch.events
+(
+    id String,
+    name String,
+    game_id UInt32,
+    player_alias_id UInt32,
+    dev_build Boolean,
+    created_at DateTime,
+    updated_at DateTime,
+    PRIMARY KEY (id),
+    INDEX name_idx (name) TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX game_id_idx (game_id) TYPE minmax GRANULARITY 64,
+    INDEX player_alias_id_idx (player_alias_id) TYPE minmax GRANULARITY 64
+)
+ENGINE = MergeTree()
+ORDER BY (id, created_at, game_id, player_alias_id);

clickhouse/migrations/20240815105700CreateEventPropsTable.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS gs_ch.event_props (
+    event_id String,
+    prop_key String,
+    prop_value String
+) ENGINE = MergeTree()
+ORDER BY (event_id, prop_key);

clickhouse/migrations/run-migrations.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+MIGRATIONS_DIR="/migrations"
+
+echo "Starting migrations..."
+
+for migration in $(ls ${MIGRATIONS_DIR}/*.sql | sort); do
+  echo "Applying migration: $migration..."
+
+  if [ -z "$CLICKHOUSE_USER" ] && [ -z "$CLICKHOUSE_PASS" ]; then
+    clickhouse-client --query="$(cat $migration)"
+  else
+    clickhouse-client --user="${CLICKHOUSE_USER}" --password="${CLICKHOUSE_PASS}" --query="$(cat $migration)"
+  fi
+done
+
+echo "Migrations complete!"

docker-compose.ci.yml

@@ -1,45 +1,9 @@
 services:
-  test-db:
-    image: mysql:8.4
-    command: --mysql-native-password=ON
-    environment:
-      - MYSQL_DATABASE=${DB_NAME}
-      - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"]
-      interval: 2s
-      timeout: 2s
-      retries: 10
-    ports:
-      - ${DB_PORT}:3306
-    volumes:
-      - test-data:/var/lib/mysql
-    networks:
-      - test-network
-    
   test-redis:
     image: bitnami/redis:7.2
+    command:
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
-    ports:
-      - ${REDIS_PORT}:6379
-    depends_on:
-      test-db:
-        condition: service_healthy
-    networks:
-      - test-network
 
   stripe-api:
     image: stripe/stripe-mock:latest
-    ports:
-      - 12111:12111
-      - 12112:12112
-    networks:
-      - test-network
-
-volumes:
-  test-data:
-
-networks:
-  test-network:

docker-compose.dev.yml

@@ -4,10 +4,22 @@ services:
       context: .
       target: dev
     image: backend
+    depends_on:
+      - db
     ports:
       - 3000:80
     volumes:
       - ./src:/usr/backend/src
       - ./tests:/usr/backend/tests
+  
+  clickhouse-ui:
+    image: ghcr.io/caioricciuti/ch-ui:latest
     depends_on:
-      - db
+      - clickhouse
+    environment:
+      - VITE_CLICKHOUSE_URL=http://localhost:${CLICKHOUSE_PORT}
+      - VITE_CLICKHOUSE_USER=${CLICKHOUSE_USER}
+      - VITE_CLICKHOUSE_PASS=${CLICKHOUSE_PASS}
+    ports:
+      - 5521:5521
+

docker-compose.test.yml

@@ -29,6 +29,24 @@ services:
     networks:
       - test-network
 
+  test-clickhouse:
+    image: clickhouse/clickhouse-server:24-alpine
+    entrypoint: ["/bin/sh", "-c"]
+    command: |
+      "/entrypoint.sh & \
+      sleep 5 && \
+      /migrations/run-migrations.sh && \
+      wait"
+    environment:
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - ./clickhouse/migrations:/migrations
+    networks:
+      - test-network
+
   stripe-api:
     image: stripe/stripe-mock:latest-arm64
     ports:

docker-compose.yml

@@ -5,14 +5,15 @@ services:
     depends_on:
       - db
       - redis
+      - clickhouse
 
   db:
     image: mysql:8.4
     command: --mysql-native-password=ON
     environment:
       - MYSQL_DATABASE=${DB_NAME}
       - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
+    restart: unless-stopped
     ports:
       - 3306:3306
     volumes:
@@ -24,5 +25,25 @@ services:
     ports:
       - 6379:6379
 
+  clickhouse:
+    image: clickhouse/clickhouse-server:24-alpine
+    entrypoint: ["/bin/sh", "-c"]
+    command: |
+      "/entrypoint.sh & \
+      sleep 5 && \
+      /migrations/run-migrations.sh && \
+      wait"
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASS: ${CLICKHOUSE_PASS}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+      - ./clickhouse/migrations:/migrations
+
 volumes:
   data:
+  clickhouse-data:

envs/.env.dev

@@ -9,6 +9,12 @@ DB_PASS=password
 
 REDIS_PASSWORD=password
 
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_USER=gs_ch
+CLICKHOUSE_PASS=password
+CLICKHOUSE_DB=gs_ch
+
 DEMO_ORGANISATION_NAME=Talo Demo
 
 SENDGRID_KEY=

envs/.env.test

@@ -3,6 +3,12 @@ DB_PORT=3307
 DB_NAME=gs_test
 DB_PASS=password
 
+CLICKHOUSE_HOST=127.0.0.1
+CLICKHOUSE_PORT=8124
+CLICKHOUSE_USER=
+CLICKHOUSE_PASS=
+CLICKHOUSE_DB=gs_ch
+
 SENDGRID_KEY=
 SENTRY_DSN=