release v0.9.3

use php-mysql-replication v2.2.0
add once_binlog_collector
add time monitor
add docker test

Author: kobi97

Makefile

@@ -0,0 +1,5 @@
+composer:
+	composer install --optimize-autoloader
+
+update-composer:
+	composer update --no-dev --optimize-autoloader

composer.json

@@ -0,0 +1,28 @@
+{
+  "name": "ridibooks/binlog-collector",
+  "description": "Ridibooks Binlog Collector",
+  "type": "library",
+  "license": "MIT",
+  "authors": [
+    {
+      "name": "오길원",
+      "email": "kobi97@ridi.com"
+    }
+  ],
+  "require": {
+    "illuminate/support": "^5.4",
+    "krowinski/php-mysql-replication": "2.2.0",
+    "monolog/monolog": "^1.18.0",
+    "ridibooks/platform-gnfdb": "^0.1",
+    "vlucas/phpdotenv": "^2.4",
+    "sentry/sentry": "^1.5"
+  },
+  "require-dev": {
+    "phpunit/phpunit": "^6.0"
+  },
+  "autoload": {
+    "psr-4": {
+      "Binlog\\": "src/Binlog"
+    }
+  }
+}

demo/Resources/all_execute.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+#php tools/generate_test_sql.php > mariadb/my.master.init_test.sql
+./init_environment.sh
+./test_execute.sh
+./php test_result_validate.php

demo/Resources/docker/all-docker-rm.sh

@@ -0,0 +1 @@
+docker rm -f db-master;docker rm -f db-slave;docker rm -f db-slave-slave

demo/Resources/docker/all-docker-run.sh

@@ -0,0 +1,3 @@
+docker run -p 3306:3306  --name db-master -e MYSQL_ROOT_PASSWORD=1234 -v /Users/kobi/workspace/php-binlog-collector/demo/Resources/mariadb/my.master.cnf:/etc/mysql/my.cnf -v /Users/kobi/workspace/php-binlog-collector/demo/Resources/mariadb/:/temp -d mariadb:10.0.27
+docker run -p 3307:3306 --name db-slave --link db-master:master -e MYSQL_ROOT_PASSWORD=1234 -v /Users/kobi/workspace/php-binlog-collector/demo//Resources/mariadb/my.slave.cnf:/etc/mysql/my.cnf -v /Users/kobi/workspace/php-binlog-collector/demo/Resources/mariadb/:/temp -d mariadb:10.0.27
+docker run -p 3308:3306  --name db-slave-slave --link db-slave:slave -e MYSQL_ROOT_PASSWORD=1234 -v /Users/kobi/workspace/php-binlog-collector/demo//Resources/mariadb/my.slave-slave.cnf:/etc/mysql/my.cnf -v /Users/kobi/workspace/php-binlog-collector/demo/Resources/mariadb/:/temp -d mariadb:10.0.27

demo/Resources/init_environment.sh

@@ -0,0 +1,17 @@
+echo "1. remove all docker "
+./docker/all-docker-rm.sh
+echo "2. run all docker"
+./docker/all-docker-run.sh
+sleep 12;
+echo "3. start db-master";
+docker exec -it db-master bash -c "mysql -uroot -p1234 < temp/my.master.sql"
+sleep 4;
+echo "4. start db-slave";
+docker exec -it db-slave bash -c "mysql -uroot -p1234 < temp/my.slave.sql"
+sleep 4;
+echo "5. start db-slave-slave";
+docker exec -it db-slave-slave bash -c "mysql -uroot -p1234 < temp/my.slave-slave.sql"
+sleep 2;
+echo "6. start db-master init sql";
+docker exec -it db-master bash -c "mysql -uroot -p1234 < temp/my.master.init_test.sql"
+

demo/Resources/log/test_execute_20170828105528.log

@@ -0,0 +1,53 @@
+kobiui-MacBook-Pro:Resources kobi$ ./binlog_execute.sh
+[2017-08-28 10:55:28] binlog_collector.INFO: Initialize! [] []
+[2017-08-28 10:55:28] binlog_collector.INFO: Limit Extra PartitionMaxCount: 1000 [] []
+[2017-08-28 10:55:29] binlog_collector.INFO: ~ 10 PartitionCount(mariadb-bin.000004/91045965) [] []
+[2017-08-28 10:55:29] binlog_collector.INFO: Previous ParentBinlogOffset: [/] [] []
+[2017-08-28 10:55:29] binlog_collector.INFO: Previous Child GtidOffsetRanges:  [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Master BinlogOffset: [0-1-9558/mariadb-bin.000005/34268330] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Existed PartitionCount: 15 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Initialize! [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Current BinlogOffset: [0-1-9542/mariadb-bin.000005/33572001] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Limit Extra PartitionMaxCount: 985 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Master BinlogOffset: [0-1-9558/mariadb-bin.000005/34268330] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Existed PartitionCount: 16 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Initialize! [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Current BinlogOffset: [0-1-9557/mariadb-bin.000005/34224297] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Limit Extra PartitionMaxCount: 984 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Master BinlogOffset: [0-1-9558/mariadb-bin.000005/34268330] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: Existed PartitionCount: 16 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: TotalGtidPartitions's Count: 16 [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7113): process started, [0-2-7130/mariadb-bin.000004/4] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7114): process started, [0-1-7346/mariadb-bin.000004/8184313] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7117): process started, [0-1-7964/mariadb-bin.000004/35319670] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7116): process started, [0-1-7761/mariadb-bin.000004/26231209] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7115): process started, [0-1-7554/mariadb-bin.000004/17229505] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7118): process started, [0-1-8071/mariadb-bin.000004/44621874] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7119): process started, [0-1-8177/mariadb-bin.000004/53837143] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7122): process started, [0-1-8498/mariadb-bin.000004/81743761] [] []
+[2017-08-28 10:55:30] binlog_collector.INFO: child_index(7120): process started, [0-1-8284/mariadb-bin.000004/63139350] [] []
+[2017-08-28 10:55:31] binlog_collector.INFO: child_index(7121): process started, [0-1-8391/mariadb-bin.000004/72441554] [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7118): process finished([0-1-8071/mariadb-bin.000004/44621874]~[0-1-8177/mariadb-bin.000004/53837143]), processed(26s, gtidCount(106), eventCount(5300), rowCount(5300) [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7117): process finished([0-1-7964/mariadb-bin.000004/35319670]~[0-1-8071/mariadb-bin.000004/44621874]), processed(26s, gtidCount(107), eventCount(5300), rowCount(5300) [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7123): process started, [0-1-8605/mariadb-bin.000004/91045965] [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7124): process started, [0-1-8711/mariadb-bin.000004/100261234] [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7120): process finished([0-1-8284/mariadb-bin.000004/63139350]~[0-1-8391/mariadb-bin.000004/72441554]), processed(26s, gtidCount(107), eventCount(5300), rowCount(5300) [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7125): process started, [0-1-8770/mariadb-bin.000005/423] [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7122): process finished([0-1-8498/mariadb-bin.000004/81743761]~[0-1-8605/mariadb-bin.000004/91045965]), processed(26s, gtidCount(107), eventCount(5300), rowCount(5300) [] []
+[2017-08-28 10:55:56] binlog_collector.INFO: child_index(7121): process finished([0-1-8391/mariadb-bin.000004/72441554]~[0-1-8498/mariadb-bin.000004/81743761]), processed(25s, gtidCount(107), eventCount(5400), rowCount(5400) [] []
+[2017-08-28 10:55:57] binlog_collector.INFO: child_index(7126): process started, [0-1-9148/mariadb-bin.000005/16438320] [] []
+[2017-08-28 10:55:57] binlog_collector.INFO: child_index(7127): process started, [0-1-9355/mariadb-bin.000005/25440024] [] []
+[2017-08-28 10:55:57] binlog_collector.INFO: child_index(7119): process finished([0-1-8177/mariadb-bin.000004/53837143]~[0-1-8284/mariadb-bin.000004/63139350]), processed(27s, gtidCount(107), eventCount(5400), rowCount(5400) [] []
+[2017-08-28 10:55:57] binlog_collector.INFO: child_index(7128): process started, [0-1-9542/mariadb-bin.000005/33572001] [] []
+[2017-08-28 10:56:00] binlog_collector.INFO: child_index(7128): process finished([0-1-9542/mariadb-bin.000005/33572001]~[0-1-9557/mariadb-bin.000005/34224297]), processed(3s, gtidCount(15), eventCount(350), rowCount(700) [] []
+[2017-08-28 10:56:08] binlog_collector.INFO: child_index(7113): process finished([0-2-7130/mariadb-bin.000004/4]~[0-1-7346/mariadb-bin.000004/8184313]), processed(38s, gtidCount(217), eventCount(4750), rowCount(9500) [] []
+[2017-08-28 10:56:10] binlog_collector.INFO: child_index(7114): process finished([0-1-7346/mariadb-bin.000004/8184313]~[0-1-7554/mariadb-bin.000004/17229505]), processed(40s, gtidCount(208), eventCount(5200), rowCount(10400) [] []
+[2017-08-28 10:56:10] binlog_collector.INFO: child_index(7124): process finished([0-1-8711/mariadb-bin.000004/100261234]~[0-1-8770/mariadb-bin.000005/423]), processed(14s, gtidCount(59), eventCount(2650), rowCount(3000) [] []
+[2017-08-28 10:56:10] binlog_collector.INFO: child_index(7116): process finished([0-1-7761/mariadb-bin.000004/26231209]~[0-1-7964/mariadb-bin.000004/35319670]), processed(40s, gtidCount(203), eventCount(5300), rowCount(10200) [] []
+[2017-08-28 10:56:11] binlog_collector.INFO: child_index(7115): process finished([0-1-7554/mariadb-bin.000004/17229505]~[0-1-7761/mariadb-bin.000004/26231209]), processed(41s, gtidCount(207), eventCount(5150), rowCount(10300) [] []
+[2017-08-28 10:56:15] binlog_collector.INFO: child_index(7123): process finished([0-1-8605/mariadb-bin.000004/91045965]~[0-1-8711/mariadb-bin.000004/100261234]), processed(19s, gtidCount(106), eventCount(5300), rowCount(5300) [] []
+[2017-08-28 10:56:19] binlog_collector.INFO: child_index(7127): process finished([0-1-9355/mariadb-bin.000005/25440024]~[0-1-9542/mariadb-bin.000005/33572001]), processed(22s, gtidCount(187), eventCount(4700), rowCount(9400) [] []
+[2017-08-28 10:56:20] binlog_collector.INFO: child_index(7126): process finished([0-1-9148/mariadb-bin.000005/16438320]~[0-1-9355/mariadb-bin.000005/25440024]), processed(23s, gtidCount(207), eventCount(5150), rowCount(10300) [] []
+[2017-08-28 10:56:28] binlog_collector.INFO: child_index(7125): process finished([0-1-8770/mariadb-bin.000005/423]~[0-1-9148/mariadb-bin.000005/16438320]), processed(32s, gtidCount(378), eventCount(9450), rowCount(18900) [] []
+[2017-08-28 10:56:28] binlog_collector.INFO: TotalGtidPartitions's Count: 0 [] []
+[2017-08-28 10:56:28] binlog_collector.INFO: TotalGtidPartitions's Count: 0 [] []

demo/Resources/mariadb/change.txt

@@ -0,0 +1,27 @@
+my.cnf 변경 -> my.master.cnf
+server-id               = 1
+log_bin                 = /var/log/mysql/mariadb-bin
+log_bin_index           = /var/log/mysql/mariadb-bin.index
+binlog-format           = row
+
+my.cnf 변경 -> my.slave.cnf
+server-id               = 2
+log_bin                 = /var/log/mysql/mariadb-bin
+log_bin_index           = /var/log/mysql/mariadb-bin.index
+# slaves
+relay_log               = /var/log/mysql/relay-bin
+relay_log_index = /var/log/mysql/relay-bin.index
+relay_log_info_file     = /var/log/mysql/relay-bin.info
+log_slave_updates
+binlog-format           = row
+
+my.cnf 변경 -> my.slave-slave.cnf
+server-id               = 3
+log_bin                 = /var/log/mysql/mariadb-bin
+log_bin_index           = /var/log/mysql/mariadb-bin.index
+# slaves
+relay_log               = /var/log/mysql/relay-bin
+relay_log_index = /var/log/mysql/relay-bin.index
+relay_log_info_file     = /var/log/mysql/relay-bin.info
+log_slave_updates
+binlog-format           = row

demo/Resources/mariadb/my.cnf.original

@@ -0,0 +1,173 @@
+# MariaDB database server configuration file.
+#
+# You can copy this file to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+# 
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+port		= 3306
+socket		= /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket		= /var/run/mysqld/mysqld.sock
+nice		= 0
+
+[mysqld]
+skip-host-cache
+skip-name-resolve
+#
+# * Basic Settings
+#
+#user		= mysql
+pid-file	= /var/run/mysqld/mysqld.pid
+socket		= /var/run/mysqld/mysqld.sock
+port		= 3306
+basedir		= /usr
+datadir		= /var/lib/mysql
+tmpdir		= /tmp
+lc_messages_dir	= /usr/share/mysql
+lc_messages	= en_US
+skip-external-locking
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+#bind-address		= 127.0.0.1
+#
+# * Fine Tuning
+#
+max_connections		= 100
+connect_timeout		= 5
+wait_timeout		= 600
+max_allowed_packet	= 16M
+thread_cache_size       = 128
+sort_buffer_size	= 4M
+bulk_insert_buffer_size	= 16M
+tmp_table_size		= 32M
+max_heap_table_size	= 32M
+#
+# * MyISAM
+#
+# This replaces the startup script and checks MyISAM tables if needed
+# the first time they are touched. On error, make copy and try a repair.
+myisam_recover          = BACKUP
+key_buffer_size		= 128M
+#open-files-limit	= 2000
+table_open_cache	= 400
+myisam_sort_buffer_size	= 512M
+concurrent_insert	= 2
+read_buffer_size	= 2M
+read_rnd_buffer_size	= 1M
+#
+# * Query Cache Configuration
+#
+# Cache only tiny result sets, so we can fit more in the query cache.
+query_cache_limit		= 128K
+query_cache_size		= 64M
+# for more write intensive setups, set to DEMAND or OFF
+#query_cache_type		= DEMAND
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+# As of 5.1 you can enable the log at runtime!
+#general_log_file        = /var/log/mysql/mysql.log
+#general_log             = 1
+#
+# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf.
+#
+# we do want to know about network errors and such
+#log_warnings		= 2
+#
+# Enable the slow query log to see queries with especially long duration
+#slow_query_log[={0|1}]
+slow_query_log_file	= /var/log/mysql/mariadb-slow.log
+long_query_time = 10
+#log_slow_rate_limit	= 1000
+#log_slow_verbosity	= query_plan
+
+#log-queries-not-using-indexes
+#log_slow_admin_statements
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+#server-id		= 1
+#report_host		= master1
+#auto_increment_increment = 2
+#auto_increment_offset	= 1
+#log_bin			= /var/log/mysql/mariadb-bin
+#log_bin_index		= /var/log/mysql/mariadb-bin.index
+# not fab for performance, but safer
+#sync_binlog		= 1
+expire_logs_days	= 10
+max_binlog_size         = 100M
+# slaves
+#relay_log		= /var/log/mysql/relay-bin
+#relay_log_index	= /var/log/mysql/relay-bin.index
+#relay_log_info_file	= /var/log/mysql/relay-bin.info
+#log_slave_updates
+#read_only
+#
+# If applications support it, this stricter sql_mode prevents some
+# mistakes like inserting invalid dates etc.
+#sql_mode		= NO_ENGINE_SUBSTITUTION,TRADITIONAL
+#
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+default_storage_engine	= InnoDB
+# you can't just change log file size, requires special procedure
+#innodb_log_file_size	= 50M
+innodb_buffer_pool_size	= 256M
+innodb_log_buffer_size	= 8M
+innodb_file_per_table	= 1
+innodb_open_files	= 400
+innodb_io_capacity	= 400
+innodb_flush_method	= O_DIRECT
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet	= 16M
+
+[mysql]
+#no-auto-rehash	# faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer		= 16M
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#   The files must end with '.cnf', otherwise they'll be ignored.
+#
+!includedir /etc/mysql/conf.d/