[WIP] [Experimental] new architecture

.eslintrc.json

@@ -374,44 +374,6 @@
                 "es2023": true
             }
         },
-        {
-            "files": [
-                "plugins/content/frontend/content-blocks/**/*.js",
-                "plugins/journey_engine/frontend/builder/**/*.js",
-                "plugins/content/frontend/content-blocks/**/*.vue",
-                "plugins/journey_engine/frontend/builder/**/*.vue"
-            ],
-            "plugins": [
-                "vue",
-                "@stylistic"
-            ],
-            "extends": [
-                "eslint:recommended",
-                "plugin:vue/vue3-essential",
-                "plugin:vue/vue3-strongly-recommended",
-                "plugin:vue/vue3-recommended"
-            ],
-            "rules": {
-                // override these post initial content release, to make them fit with countly convention
-                "no-console": ["error"],
-                "@stylistic/quotes": ["error", "single"],
-                "@stylistic/quote-props": ["error", "as-needed"],
-                "no-unused-vars": "off",
-                "vue/no-unused-vars": ["error", {
-                    "ignorePattern": "^_"
-                }]
-            },
-            "parserOptions": {
-                "ecmaVersion": 2023,
-                "sourceType": "module",
-                "extraFileExtensions": [".vue"]
-            },
-            "env": {
-                "node": true,
-                "es2023": true,
-                "es6": true
-            }
-        },
         {
             "files": [
                 "frontend/express/public/javascripts/countly/*.js",

api/aggregator.js

@@ -1,4 +1,4 @@
-const countlyConfig = require('./config', 'dont-enclose');
+const countlyConfig = require('./config');
 const plugins = require('../plugins/pluginManager.js');
 const log = require('./utils/log.js')('aggregator-core:api');
 const common = require('./utils/common.js');

api/api.js

@@ -1,6 +1,6 @@
 const http = require('http');
 const formidable = require('formidable');
-const countlyConfig = require('./config', 'dont-enclose');
+const countlyConfig = require('./config');
 const plugins = require('../plugins/pluginManager.js');
 const log = require('./utils/log.js')('core:api');
 const common = require('./utils/common.js');

api/ingestor.js

@@ -1,6 +1,6 @@
 const http = require('http');
 const formidable = require('formidable');
-const countlyConfig = require('./config', 'dont-enclose');
+const countlyConfig = require('./config');
 const plugins = require('../plugins/pluginManager.js');
 const log = require('./utils/log.js')('ingestor-core:api');
 const {processRequest} = require('./ingestor/requestProcessor');
@@ -18,7 +18,13 @@ console.log("Connecting to databases");
 //Overriding function
 plugins.loadConfigs = plugins.loadConfigsIngestor;
 
-plugins.connectToAllDatabases(true).then(function() {
+/**
+ * TODO
+ * temporarily change this false since it fails at
+ * Cannot create uid TypeError: common.db.ObjectID is not a function
+ * at usersApi.getUid (api/parts/mgmt/app_users.js:434:90)
+ */
+plugins.connectToAllDatabases(false).then(function() {
     log.i("Db connections done");
     // common.writeBatcher = new WriteBatcher(common.db);
     common.readBatcher = new Cacher(common.db);
@@ -273,7 +279,7 @@ plugins.connectToAllDatabases(true).then(function() {
             else {
                 common.returnMessage(params, 405, "Method not allowed");
             }
-        }).listen(common.config.ingestor.port, common.config.ingestor.host || '').timeout = common.config.ingestor.timeout || 120000;
+        }).listen(common.config?.ingestor?.port || 3010, common.config?.ingestor?.host || '').timeout = common.config?.ingestor?.timeout || 120000;
     });
 });
 

api/parts/data/changeStreamReader.js

@@ -9,7 +9,7 @@ class changeStreamReader {
     /** 
     * @param {Object} db - Database object
     * @param {Object} options - Options object
-    * @param {function} onData - Finction to call when getting new data from stream
+    * @param {function} onData - Function to call when getting new data from stream
     */
     constructor(db, options, onData,) {
         this.db = db;
@@ -29,8 +29,8 @@ class changeStreamReader {
         this.fallback = options.fallback;
 
         //I give data
-        //Processor function processes. Sends last processed tken from time to time.
-        //Update last processed token to database
+        //Processor function processes. Sends last processed token from time to time.
+        //Update the last processed token to database
         //On startup - read token, resume from that token.
         this.setUp(onData, false);
 

api/parts/jobs/index.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const countlyConfig = require('./../../config', 'dont-enclose');
+const countlyConfig = require('./../../config');
 
 if (require('cluster').isMaster && process.argv[1].endsWith('api/api.js') && !(countlyConfig && countlyConfig.preventJobs)) {
     module.exports = require('./manager.js');

api/parts/mgmt/app_users.js

@@ -435,9 +435,13 @@ usersApi.getUid = async function(app_id, callback) {
             returnDocument: 'after',
             upsert: true
         });
-        if (result && result.seq) {
+        /**
+         * TODO
+         * temporarily use .value since thats what the findOneAndUpdate returns
+         */
+        if (result && result.value.seq) {
             if (callback) {
-                callback(null, common.parseSequence(result.seq));
+                callback(null, common.parseSequence(result.value.seq));
             }
         }
         else if (callback) {

api/tcp_example.js

@@ -1,5 +1,5 @@
 const net = require('net');
-const countlyConfig = require('./config', 'dont-enclose');
+const countlyConfig = require('./config');
 const plugins = require('../plugins/pluginManager.js');
 const log = require('./utils/log.js')('core:tcp');
 const common = require('./utils/common.js');

api/utils/common.js

@@ -10,7 +10,7 @@ var common = {},
     logger = require('./log.js'),
     mcc_mnc_list = require('mcc-mnc-list'),
     plugins = require('../../plugins/pluginManager.js'),
-    countlyConfig = require('./../config', 'dont-enclose'),
+    countlyConfig = require('./../config'),
     argon2 = require('argon2'),
     mongodb = require('mongodb'),
     getRandomValues = require('get-random-values'),

api/utils/log.js

@@ -16,6 +16,17 @@ catch (e) {
     // do nothing
 }
 
+/**
+ * Fallback for performance.now() if not available
+ * @returns {number} Current timestamp in milliseconds
+ */
+const getNow = () => {
+    if (typeof performance !== 'undefined' && performance.now) {
+        return performance.now();
+    }
+    return Date.now();
+};
+
 /**
  * Mapping of short level codes to full level names
  * @type {Object.<string, string>}
@@ -40,7 +51,7 @@ const ACCEPTABLE = {
 
 
 // Initialize configuration with defaults
-let prefs = require('../config.js', 'dont-enclose').logging || {};
+let prefs = require('../config.js').logging || {};
 prefs.default = prefs.default || "warn";
 let deflt = (prefs && prefs.default) ? prefs.default : 'error';
 
@@ -153,6 +164,25 @@ const logLevel = function(name) {
     }
 };
 
+/**
+ * Build a transport config: pretty in dev, JSON in prod.
+ * @returns {Object} Transport config
+ */
+function getTransport() {
+    if (process.env.NODE_ENV === 'development') {
+        return {
+            target: 'pino-pretty',
+            options: {
+                colorize: true,
+                translateTime: 'yyyy-mm-dd HH:MM:ss.l',
+                ignore: 'pid,hostname'
+            }
+        };
+    }
+    return undefined;
+}
+
+
 /**
  * Creates a Pino logger instance with the appropriate configuration
  * @param {string} name - The module name
@@ -168,11 +198,14 @@ const createLogger = (name, level) => {
             level: (label) => {
                 return { level: label.toUpperCase() };
             },
-            log: (object) => {
+            log: (obj) => {
                 const traceContext = getTraceContext();
-                return traceContext ? { ...object, ...traceContext } : object;
+                return traceContext ? { ...obj, ...traceContext } : obj;
             }
-        }
+        },
+        sync: false,
+        browser: false,
+        transport: getTransport()
     });
 };
 
@@ -187,7 +220,7 @@ const createLogFunction = (logger, name, level) => {
     return function(...args) {
         const currentLevel = levels[name] || deflt;
         if (ACCEPTABLE[level].indexOf(currentLevel) !== -1) {
-            const startTime = performance.now();
+            const startTime = getNow();
             const message = args[0];
 
             // Create span for this logging operation
@@ -207,7 +240,7 @@ const createLogFunction = (logger, name, level) => {
 
                 // Record duration
                 if (logDurationHistogram) {
-                    const duration = (performance.now() - startTime) / 1000; // Convert to seconds
+                    const duration = (getNow() - startTime) / 1000; // Convert to seconds
                     logDurationHistogram.record(duration, {
                         module: name,
                         level: LEVELS[level]

api/utils/render.js

@@ -24,7 +24,7 @@ var alternateChrome = true;
 var chromePath = "";
 var countlyFs = require('./countlyFs');
 var log = require('./log.js')('core:render');
-var countlyConfig = require('./../config', 'dont-enclose');
+var countlyConfig = require('./../config');
 
 
 /**

api/utils/utils.js

@@ -3,7 +3,7 @@
 * @module api/utils/utils
 */
 var crypto = require('crypto'),
-    countlyConfig = require('./../config', 'dont-enclose');
+    countlyConfig = require('./../config');
 
 if (!countlyConfig.encryption) {
     countlyConfig.encryption = {};

frontend/express/app.js

@@ -61,7 +61,7 @@ var versionInfo = require('./version.info'),
     preventBruteforce = require('./libs/preventBruteforce.js'),
     plugins = require('../../plugins/pluginManager.js'),
     request = require('countly-request')(plugins.getConfig("security")),
-    countlyConfig = require('./config', 'dont-enclose'),
+    countlyConfig = require('./config'),
     log = require('../../api/utils/log.js')('core:app'),
     url = require('url'),
     authorize = require('../../api/utils/authorizer.js'), //for token validations

frontend/express/libs/members.js

@@ -13,7 +13,7 @@ var authorize = require('./../../../api/utils/authorizer.js'); //for token valid
 var common = require('./../../../api/utils/common.js');
 var plugins = require('./../../../plugins/pluginManager.js');
 var { getUserApps } = require('./../../../api/utils/rights.js');
-var configs = require('./../config', 'dont-enclose');
+var configs = require('./../config');
 var countlyMail = require('./../../../api/parts/mgmt/mail.js');
 var countlyStats = require('./../../../api/parts/data/stats.js');
 var request = require('countly-request')(plugins.getConfig("security"));

frontend/express/public/javascripts/countly/countly.config.sample.js

@@ -11,6 +11,13 @@
     * @type {string} 
     */
     countlyCommon.API_URL = countlyGlobal.path || "";
+
+    /**
+     * Ingestor service URL. This is used for the ingestion of data for /i and /i/bulk endpoints.
+     * @type {string}
+     */
+    countlyCommon.INGESTOR_URL = countlyGlobal.path || "";
+
     /**
     * Main api path to make all ajax requests to. This value is configured in countly.config.js or over written through theme.
     * @property {object} data - contains default read and write endpoints 

package.json

@@ -43,6 +43,7 @@
   },
   "dependencies": {
     "@breejs/later": "^4.2.0",
+    "@clickhouse/client": "^1.11.1",
     "@pulsecron/pulse": "1.6.7",
     "all-the-cities": "3.1.0",
     "argon2": "0.41.1",

plugins/crashes/frontend/app.js

@@ -1,5 +1,5 @@
 var exportedPlugin = {},
-    countlyConfig = require('../../../frontend/express/config', 'dont-enclose');
+    countlyConfig = require('../../../frontend/express/config');
 
 var config;
 try {

plugins/dashboards/frontend/app.js

@@ -1,6 +1,6 @@
 var plugins = require('../../pluginManager.js');
 var exported = {};
-var countlyConfig = require('../../../frontend/express/config', 'dont-enclose');
+var countlyConfig = require('../../../frontend/express/config');
 var countlyFs = require('../../../api/utils/countlyFs.js');
 
 (function(plugin) {

plugins/enterpriseinfo/frontend/app.js

@@ -1,5 +1,5 @@
 var exported = {},
-    countlyConfig = require('../../../frontend/express/config', 'dont-enclose'),
+    countlyConfig = require('../../../frontend/express/config'),
     versionInfo = require('../../../frontend/express/version.info'),
     async = require('async');
 

plugins/locale/frontend/app.js

@@ -1,5 +1,5 @@
 var exported = {},
-    countlyConfig = require('../../../frontend/express/config', 'dont-enclose'),
+    countlyConfig = require('../../../frontend/express/config'),
     langs = require('../api/utils/langs.js');
 
 (function(plugin) {

plugins/pluginManager.js

@@ -1,13 +1,13 @@
 var pluginDependencies = require('./pluginDependencies.js'),
     path = require('path'),
-    plugins = pluginDependencies.getFixedPluginList(require('./plugins.json', 'dont-enclose'), {
+    plugins = pluginDependencies.getFixedPluginList(require('./plugins.json'), {
         "discoveryStrategy": "disableChildren",
         "overwrite": path.resolve(__dirname, './plugins.json')
     }),
     pluginsApis = {},
     mongodb = require('mongodb'),
-    countlyConfig = require('../frontend/express/config', 'dont-enclose'),
-    apiCountlyConfig = require('../api/config', 'dont-enclose'),
+    countlyConfig = require('../frontend/express/config'),
+    apiCountlyConfig = require('../api/config'),
     utils = require('../api/utils/utils.js'),
     fs = require('fs'),
     url = require('url'),
@@ -1194,8 +1194,8 @@ var pluginManager = function pluginManager() {
     * Try to reload cached plugins json file
     **/
     this.reloadPlugins = function() {
-        delete require.cache[require.resolve('./plugins.json', 'dont-enclose')];
-        plugins = pluginDependencies.getFixedPluginList(require('./plugins.json', 'dont-enclose'), {
+        delete require.cache[require.resolve('./plugins.json')];
+        plugins = pluginDependencies.getFixedPluginList(require('./plugins.json'), {
             "discoveryStrategy": "disableChildren",
             "overwrite": path.resolve(__dirname, './plugins.json')
         });
@@ -1838,7 +1838,7 @@ var pluginManager = function pluginManager() {
         }
         else {
             console.log("using separate connection pool");
-            databases = await Promise.all(dbs.map(this.dbConnection.bind(this, return_original)));
+            databases = await Promise.all(dbs.map((db) => this.dbConnection(db, return_original)));
         }
         const [dbCountly, dbOut, dbFs, dbDrill] = databases;
 

plugins/populator/frontend/public/javascripts/countly.models.js

@@ -1178,7 +1178,7 @@
         this.reportConversion = function(uid, campaingId, deviceId) {
             $.ajax({
                 type: "GET",
-                url: countlyCommon.API_URL + "/i",
+                url: countlyCommon.INGESTOR_URL + "/i",
                 data: {
                     campaign_id: uid,
                     campaign_user: campaingId,
@@ -1365,7 +1365,7 @@
             if (generating) {
                 $.ajax({
                     type: "POST",
-                    url: countlyCommon.API_URL + "/i/bulk",
+                    url: countlyCommon.INGESTOR_URL + "/i/bulk",
                     data: {
                         app_key: countlyCommon.ACTIVE_APP_KEY,
                         requests: JSON.stringify(req),

plugins/recaptcha/frontend/app.js

@@ -1,5 +1,5 @@
 var exportedPlugin = {},
-    countlyConfig = require('../../../frontend/express/config', 'dont-enclose'),
+    countlyConfig = require('../../../frontend/express/config'),
     recaptcha = require('express-recaptcha');
 var plugins = require("../../pluginManager.js");
 

plugins/reports/api/reports.js

@@ -15,7 +15,7 @@ var reportsInstance = {},
     log = require('../../../api/utils/log')('reports:reports'),
     versionInfo = require('../../../frontend/express/version.info'),
     countlyConfig = require('../../../frontend/express/config.js'),
-    countlyApiConfig = require('./../../../api/config', 'dont-enclose'),
+    countlyApiConfig = require('./../../../api/config'),
     pdf = require('../../../api/utils/pdf');
 
 countlyConfig.passwordSecret || "";