diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71f1c39272d..990c7eb1282 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ name: CI on: # Triggers the workflow on push or pull request events but only for the master branch pull_request: - branches: [ master, next, release.*, flex ] + branches: [ master, next, release.*, flex, newarchitecture ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/Dockerfile b/Dockerfile index 43df0776347..2f5088e8ee2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ CMD ["/sbin/my_init"] ENV COUNTLY_CONTAINER="both" \ COUNTLY_DEFAULT_PLUGINS="${COUNTLY_PLUGINS}" \ COUNTLY_CONFIG_API_API_HOST="0.0.0.0" \ + COUNTLY_CONFIG_API_INGESTOR_HOST="0.0.0.0" \ COUNTLY_CONFIG_FRONTEND_WEB_HOST="0.0.0.0" \ NODE_OPTIONS="--max-old-space-size=2048" \ INSIDE_DOCKER=1 @@ -37,12 +38,16 @@ RUN apt-get update && apt-get install -y sudo && \ mkdir /etc/service/nginx && \ mkdir /etc/service/countly-api && \ mkdir /etc/service/countly-dashboard && \ + mkdir /etc/service/countly-ingestor && \ + mkdir /etc/service/countly-aggregator && \ echo "" >> /etc/nginx/nginx.conf && \ echo "daemon off;" >> /etc/nginx/nginx.conf && \ \ cp /opt/countly/bin/commands/docker/mongodb.sh /etc/service/mongodb/run && \ cp /opt/countly/bin/commands/docker/nginx.sh /etc/service/nginx/run && \ cp /opt/countly/bin/commands/docker/countly-api.sh /etc/service/countly-api/run && \ + cp /opt/countly/bin/commands/docker/countly-ingestor.sh /etc/service/countly-ingestor/run && \ + cp /opt/countly/bin/commands/docker/countly-aggregator.sh /etc/service/countly-aggregator/run && \ cp /opt/countly/bin/commands/docker/countly-dashboard.sh /etc/service/countly-dashboard/run && \ \ chown mongodb /etc/service/mongodb/run && \ diff --git a/Dockerfile-core b/Dockerfile-core index b3a807d8b5c..394d94f6f71 100644 --- a/Dockerfile-core +++ b/Dockerfile-core @@ -17,6 +17,7 @@ ENV COUNTLY_CONTAINER="both" \ COUNTLY_DEFAULT_PLUGINS="${COUNTLY_PLUGINS}" \ COUNTLY_CONFIG_API_API_HOST="0.0.0.0" \ COUNTLY_CONFIG_FRONTEND_WEB_HOST="0.0.0.0" \ + COUNTLY_CONFIG_API_INGESTOR_HOST="0.0.0.0" \ NODE_OPTIONS="--max-old-space-size=2048" EXPOSE 80 @@ -67,11 +68,15 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \ mkdir -p /etc/my_init.d && cp ./bin/docker/postinstall.sh /etc/my_init.d/ && \ mkdir /etc/service/nginx && \ mkdir /etc/service/countly-api && \ + mkdir /etc/service/countly-ingestor && \ + mkdir /etc/service/countly-aggregator && \ mkdir /etc/service/countly-dashboard && \ echo "" >> /etc/nginx/nginx.conf && \ echo "daemon off;" >> /etc/nginx/nginx.conf && \ cp ./bin/commands/docker/nginx.sh /etc/service/nginx/run && \ cp ./bin/commands/docker/countly-api.sh /etc/service/countly-api/run && \ + cp ./bin/commands/docker/countly-ingestor.sh /etc/service/countly-ingestor/run && \ + cp ./bin/commands/docker/countly-aggregator.sh /etc/service/countly-aggregator/run && \ cp ./bin/commands/docker/countly-dashboard.sh /etc/service/countly-dashboard/run && \ chown -R countly:countly /opt/countly && \ # cleanup diff --git a/Gruntfile.js b/Gruntfile.js index 4c7c1a98986..6d07227c9ac 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -200,6 +200,8 @@ module.exports = function(grunt) { 'frontend/express/public/core/home/javascripts/countly.views.js', 'frontend/express/public/core/notes/javascripts/countly.views.js', 'frontend/express/public/core/version-history/javascripts/countly.views.js', + 'frontend/express/public/core/aggregator-status/javascripts/countly.views.js', + 'frontend/express/public/core/aggregator-status/javascripts/countly.model.js', 'frontend/express/public/core/onboarding/javascripts/countly.models.js', 'frontend/express/public/core/onboarding/javascripts/countly.views.js' ], diff --git a/api/aggregator.js b/api/aggregator.js new file mode 100644 index 00000000000..04663c7259c --- /dev/null +++ b/api/aggregator.js @@ -0,0 +1,364 @@ +const countlyConfig = require('./config'); +const plugins = require('../plugins/pluginManager.js'); +const log = require('./utils/log.js')('aggregator-core:api'); +const common = require('./utils/common.js'); +const {WriteBatcher} = require('./parts/data/batcher.js'); +const {Cacher} = require('./parts/data/cacher.js'); +const {changeStreamReader} = require('./parts/data/changeStreamReader.js'); +const usage = require('./aggregator/usage.js'); +var t = ["countly:", "aggregator"]; +t.push("node"); + +// Finaly set the visible title +process.title = t.join(' '); + +console.log("Connecting to databases"); + +//Overriding function +plugins.loadConfigs = plugins.loadConfigsIngestor; + +plugins.connectToAllDatabases(true).then(function() { + log.i("Db connections done"); + // common.writeBatcher = new WriteBatcher(common.db); + + common.writeBatcher = new WriteBatcher(common.db); + common.secondaryWriteBatcher = new WriteBatcher(common.db); + common.readBatcher = new Cacher(common.db); //Used for Apps info + + common.readBatcher.transformationFunctions = { + "event_object": function(data) { + if (data && data.list) { + data._list = {}; + data._list_length = 0; + for (let i = 0; i < data.list.length; i++) { + data._list[data.list[i]] = true; + data._list_length++; + } + } + if (data && data.segments) { + data._segments = {}; + for (var key in data.segments) { + data._segments[key] = {}; + data._segments[key]._list = {}; + data._segments[key]._list_length = 0; + for (let i = 0; i < data.segments[key].length; i++) { + data._segments[key]._list[data.segments[key][i]] = true; + data._segments[key]._list_length++; + } + } + } + if (data && data.omitted_segments) { + data._omitted_segments = {}; + for (var key3 in data.omitted_segments) { + for (let i = 0; i < data.omitted_segments[key3].length; i++) { + data._omitted_segments[key3] = data._omitted_segments[key3] || {}; + data._omitted_segments[key3][data.omitted_segments[key3][i]] = true; + } + } + } + + if (data && data.whitelisted_segments) { + data._whitelisted_segments = {}; + for (var key4 in data.whitelisted_segments) { + for (let i = 0; i < data.whitelisted_segments[key4].length; i++) { + data._whitelisted_segments[key4] = data._whitelisted_segments[key4] || {}; + data._whitelisted_segments[key4][data.whitelisted_segments[key4][i]] = true; + } + } + } + return data; + } + }; + + + //Events processing + plugins.register("/aggregator", function() { + var changeStream = new changeStreamReader(common.drillDb, { + pipeline: [ + {"$match": {"operationType": "insert", "fullDocument.e": "[CLY]_custom"}}, + {"$project": {"__iid": "$fullDocument._id", "cd": "$fullDocument.cd", "a": "$fullDocument.a", "e": "$fullDocument.e", "n": "$fullDocument.n", "ts": "$fullDocument.ts", "sg": "$fullDocument.sg", "c": "$fullDocument.c", "s": "$fullDocument.s", "dur": "$fullDocument.dur"}} + ], + fallback: { + pipeline: [{ + "$match": {"e": {"$in": ["[CLY]_custom"]}} + }, {"$project": {"__id": "$_id", "cd": "$cd", "a": "$a", "e": "$e", "n": "$n", "ts": "$ts", "sg": "$sg", "c": "$c", "s": "$s", "dur": "$dur"}}], + }, + "name": "event-ingestion" + }, (token, currEvent) => { + if (currEvent && currEvent.a && currEvent.e) { + usage.processEventFromStream(token, currEvent); + } + // process next document + }); + common.writeBatcher.addFlushCallback("events_data", function(token) { + changeStream.acknowledgeToken(token); + }); + }); + + + //Sessions processing + plugins.register("/aggregator", function() { + var changeStream = new changeStreamReader(common.drillDb, { + pipeline: [ + {"$match": {"operationType": "insert", "fullDocument.e": "[CLY]_session"}}, + {"$addFields": {"__id": "$fullDocument._id", "cd": "$fullDocument.cd"}}, + ], + fallback: { + pipeline: [{ + "$match": {"e": {"$in": ["[CLY]_session"]}} + }] + }, + "name": "session-ingestion" + }, (token, next) => { + if (next.fullDocument) { + next = next.fullDocument; + } + var currEvent = next; + if (currEvent && currEvent.a) { + //Record in session data + common.readBatcher.getOne("apps", common.db.ObjectID(currEvent.a), function(err, app) { + //record event totals in aggregated data + if (err) { + log.e("Error getting app data for session", err); + return; + } + if (app) { + usage.processSessionFromStream(token, currEvent, {"app_id": currEvent.a, "app": app, "time": common.initTimeObj(app.timezone, currEvent.ts), "appTimezone": (app.timezone || "UTC")}); + } + }); + } + }); + common.writeBatcher.addFlushCallback("users", function(token) { + changeStream.acknowledgeToken(token); + }); + }); + + plugins.register("/aggregator", function() { + var writeBatcher = new WriteBatcher(common.db); + var changeStream = new changeStreamReader(common.drillDb, { + pipeline: [ + {"$match": {"operationType": "update"}}, + {"$addFields": {"__id": "$fullDocument._id", "cd": "$fullDocument.cd"}} + ], + fallback: { + pipeline: [{"$match": {"e": {"$in": ["[CLY]_session"]}}}], + "timefield": "lu" + }, + "options": {fullDocument: "updateLookup"}, + "name": "session-updates", + "collection": "drill_events", + "onClose": async function(callback) { + await common.writeBatcher.flush("countly", "users"); + if (callback) { + callback(); + } + } + }, (token, fullDoc) => { + var fallback_processing = true; + var next = fullDoc; + if (next.fullDocument) { + fallback_processing = false; + next = fullDoc.fullDocument; + } + if (next && next.a && next.e && next.e === "[CLY]_session" && next.n && next.ts) { + common.readBatcher.getOne("apps", common.db.ObjectID(next.a), function(err, app) { + //record event totals in aggregated data + if (err) { + log.e("Error getting app data for session", err); + return; + } + if (app) { + var dur = 0; + if (fallback_processing) { + dur = next.dur || 0; + } + else { + dur = (fullDoc && fullDoc.updateDescription && fullDoc.updateDescription.updatedFields && fullDoc.updateDescription.updatedFields.dur) || 0; + }//if(dur){ + usage.processSessionDurationRange(writeBatcher, token, dur, next.did, {"app_id": next.a, "app": app, "time": common.initTimeObj(app.timezone, next.ts), "appTimezone": (app.timezone || "UTC")}); + //} + } + }); + } + }); + writeBatcher.addFlushCallback("users", function(token) { + changeStream.acknowledgeToken(token); + }); + }); + + + /** + * Set Plugins APIs Config + */ + //Put in single file outside(all set configs) + plugins.setConfigs("api", { + domain: "", + safe: false, + session_duration_limit: 86400, + country_data: true, + city_data: true, + event_limit: 500, + event_segmentation_limit: 100, + event_segmentation_value_limit: 1000, + array_list_limit: 10, + metric_limit: 1000, + sync_plugins: false, + session_cooldown: 15, + request_threshold: 30, + total_users: true, + export_limit: 10000, + prevent_duplicate_requests: true, + metric_changes: true, + offline_mode: false, + reports_regenerate_interval: 3600, + send_test_email: "", + //data_retention_period: 0, + batch_processing: true, + //batch_on_master: false, + batch_period: 10, + batch_read_processing: true, + //batch_read_on_master: false, + batch_read_ttl: 600, + batch_read_period: 60, + user_merge_paralel: 1, + trim_trailing_ending_spaces: false + }); + + /** + * Set Plugins APPs Config + */ + plugins.setConfigs("apps", { + country: "TR", + timezone: "Europe/Istanbul", + category: "6" + }); + + /** + * Set Plugins Security Config + */ + plugins.setConfigs("security", { + login_tries: 3, + login_wait: 5 * 60, + password_min: 8, + password_char: true, + password_number: true, + password_symbol: true, + password_expiration: 0, + password_rotation: 3, + password_autocomplete: true, + robotstxt: "User-agent: *\nDisallow: /", + dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000 ; includeSubDomains\nX-Content-Type-Options: nosniff", + api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nAccess-Control-Allow-Origin:*", + dashboard_rate_limit_window: 60, + dashboard_rate_limit_requests: 500, + proxy_hostname: "", + proxy_port: "", + proxy_username: "", + proxy_password: "", + proxy_type: "https" + }); + + /** + * Set Plugins Logs Config + */ + plugins.setConfigs('logs', { + debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '', + info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '', + warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '', + error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '', + default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn', + }, undefined, () => { + const cfg = plugins.getConfig('logs'), msg = { + cmd: 'log', + config: cfg + }; + if (process.send) { + process.send(msg); + } + require('./utils/log.js').ipcHandler(msg); + }); + + /** + * Initialize Plugins + */ + + // plugins.init(); - should run new init ingestor + + /** + * Trying to gracefully handle the batch state + * @param {number} code - error code + */ + async function storeBatchedData(code) { + try { + await common.writeBatcher.flushAll(); + await common.secondaryWriteBatcher.flushAll(); + // await common.insertBatcher.flushAll(); + console.log("Successfully stored batch state"); + } + catch (ex) { + console.log("Could not store batch state", ex); + } + process.exit(typeof code === "number" ? code : 1); + } + + /** + * Handle before exit for gracefull close + */ + process.on('beforeExit', (code) => { + console.log('Received exit, trying to save batch state: ', code); + storeBatchedData(code); + }); + + /** + * Handle exit events for gracefull close + */ + ['SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT', + 'SIGBUS', 'SIGFPE', 'SIGSEGV', 'SIGTERM', + ].forEach(function(sig) { + process.on(sig, async function() { + storeBatchedData(sig); + plugins.dispatch("/aggregator/exit"); + console.log('Got signal: ' + sig); + }); + }); + + /** + * Uncaught Exception Handler + */ + process.on('uncaughtException', (err) => { + console.log('Caught exception: %j', err, err.stack); + if (log && log.e) { + log.e('Logging caught exception'); + } + console.trace(); + plugins.dispatch("/aggregator/exit"); + storeBatchedData(1); + }); + + /** + * Unhandled Rejection Handler + */ + process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled rejection for %j with reason %j stack ', p, reason, reason ? reason.stack : undefined); + if (log && log.e) { + log.e('Logging unhandled rejection'); + } + console.trace(); + }); + console.log("Starting aggregator", process.pid); + //since process restarted mark running tasks as errored + + + plugins.init({"skipDependencies": true, "filename": "aggregator"}); + plugins.loadConfigs(common.db, function() { + plugins.dispatch("/aggregator", {common: common}); + }); +}); + + +/** + * On incoming request + * 1)Get App data (Batcher) + * 2)Get overall configs + * + */ \ No newline at end of file diff --git a/api/aggregator/usage.js b/api/aggregator/usage.js new file mode 100644 index 00000000000..d617329a93a --- /dev/null +++ b/api/aggregator/usage.js @@ -0,0 +1,655 @@ +var usage = {}; +var common = require('./../utils/common.js'); +var plugins = require('./../../plugins/pluginManager.js'); +var async = require('async'); +var crypto = require('crypto'); + + +usage.processSessionDurationRange = function(writeBatcher, token, totalSessionDuration, did, params) { + var durationRanges = [ + [0, 10], + [11, 30], + [31, 60], + [61, 180], + [181, 600], + [601, 1800], + [1801, 3600] + ], + durationMax = 3601, + calculatedDurationRange, + updateUsers = {}, + updateUsersZero = {}, + dbDateIds = common.getDateIds(params), + monthObjUpdate = []; + + if (totalSessionDuration >= durationMax) { + calculatedDurationRange = (durationRanges.length) + ''; + } + else { + for (var i = 0; i < durationRanges.length; i++) { + if (totalSessionDuration <= durationRanges[i][1] && totalSessionDuration >= durationRanges[i][0]) { + calculatedDurationRange = i + ''; + break; + } + } + } + if (totalSessionDuration > 0) { + common.fillTimeObjectMonth(params, updateUsers, common.dbMap.duration, totalSessionDuration); + } + monthObjUpdate.push(common.dbMap.durations + '.' + calculatedDurationRange); + common.fillTimeObjectMonth(params, updateUsers, monthObjUpdate); + common.fillTimeObjectZero(params, updateUsersZero, common.dbMap.durations + '.' + calculatedDurationRange); + var postfix = common.crypto.createHash("md5").update(did).digest('base64')[0]; + writeBatcher.add("users", params.app_id + "_" + dbDateIds.month + "_" + postfix, {'$inc': updateUsers}); + var update = { + '$inc': updateUsersZero, + '$set': {} + }; + update.$set['meta_v2.d-ranges.' + calculatedDurationRange] = true; + writeBatcher.add("users", params.app_id + "_" + dbDateIds.zero + "_" + postfix, update, "countly", {token: token}); + +}; + +usage.processSessionFromStream = function(token, currEvent, params) { + currEvent.up = currEvent.up || {}; + var updateUsersZero = {}, + updateUsersMonth = {}, + usersMeta = {}, + sessionFrequency = [ + [0, 24], + [24, 48], + [48, 72], + [72, 96], + [96, 120], + [120, 144], + [144, 168], + [168, 192], + [192, 360], + [360, 744] + ], + sessionFrequencyMax = 744, + calculatedFrequency, + uniqueLevels = [], + uniqueLevelsZero = [], + uniqueLevelsMonth = [], + zeroObjUpdate = [], + monthObjUpdate = [], + dbDateIds = common.getDateIds(params); + + monthObjUpdate.push(common.dbMap.total); + if (currEvent.up.cc) { + monthObjUpdate.push(currEvent.up.cc + '.' + common.dbMap.total); + } + if (currEvent.sg && currEvent.sg.prev_session) { + //user had session before + if (currEvent.sg.prev_start) { + var userLastSeenTimestamp = currEvent.sg.prev_start, + currDate = common.getDate(currEvent.ts, params.appTimezone), + userLastSeenDate = common.getDate(userLastSeenTimestamp, params.appTimezone), + secInMin = (60 * (currDate.minutes())) + currDate.seconds(), + secInHour = (60 * 60 * (currDate.hours())) + secInMin, + secInMonth = (60 * 60 * 24 * (currDate.date() - 1)) + secInHour, + secInYear = (60 * 60 * 24 * (common.getDOY(currEvent.ts, params.appTimezone) - 1)) + secInHour; + + /* if (dbAppUser.cc !== params.user.country) { + monthObjUpdate.push(params.user.country + '.' + common.dbMap.unique); + zeroObjUpdate.push(params.user.country + '.' + common.dbMap.unique); + }*/ + + // Calculate the frequency range of the user + var ts_sec = currEvent.ts / 1000; + if ((ts_sec - userLastSeenTimestamp) >= (sessionFrequencyMax * 60 * 60)) { + calculatedFrequency = sessionFrequency.length + ''; + } + else { + for (let i = 0; i < sessionFrequency.length; i++) { + if ((ts_sec - userLastSeenTimestamp) < (sessionFrequency[i][1] * 60 * 60) && + (ts_sec - userLastSeenTimestamp) >= (sessionFrequency[i][0] * 60 * 60)) { + calculatedFrequency = (i + 1) + ''; + break; + } + } + } + + //if for some reason we received past data lesser than last session timestamp + //we can't calculate frequency for that part + if (typeof calculatedFrequency !== "undefined") { + zeroObjUpdate.push(common.dbMap.frequency + '.' + calculatedFrequency); + monthObjUpdate.push(common.dbMap.frequency + '.' + calculatedFrequency); + usersMeta['meta_v2.f-ranges.' + calculatedFrequency] = true; + } + + if (userLastSeenTimestamp < (ts_sec - secInMin)) { + // We don't need to put hourly fragment to the unique levels array since + // we will store hourly data only in sessions collection + updateUsersMonth['d.' + params.time.day + '.' + params.time.hour + '.' + common.dbMap.unique] = 1; + } + + if (userLastSeenTimestamp < (ts_sec - secInHour)) { + uniqueLevels[uniqueLevels.length] = params.time.daily; + uniqueLevelsMonth.push(params.time.day); + } + + if ((userLastSeenDate.year() + "") === (params.time.yearly + "") && + Math.ceil(userLastSeenDate.format("DDD") / 7) < params.time.weekly) { + uniqueLevels[uniqueLevels.length] = params.time.yearly + ".w" + params.time.weekly; + uniqueLevelsZero.push("w" + params.time.weekly); + } + + if (userLastSeenTimestamp < (ts_sec - secInMonth)) { + uniqueLevels[uniqueLevels.length] = params.time.monthly; + uniqueLevelsZero.push(params.time.month); + } + + if (userLastSeenTimestamp < (ts_sec - secInYear)) { + uniqueLevels[uniqueLevels.length] = params.time.yearly; + uniqueLevelsZero.push("Y"); + } + } + } + else { + zeroObjUpdate.push(common.dbMap.unique); + monthObjUpdate.push(common.dbMap.new); + monthObjUpdate.push(common.dbMap.unique); + if (currEvent.up.cc) { + zeroObjUpdate.push(currEvent.up.cc + '.' + common.dbMap.unique); + monthObjUpdate.push(currEvent.up.cc + '.' + common.dbMap.new); + monthObjUpdate.push(currEvent.up.cc + '.' + common.dbMap.unique); + } + + // First time user. + calculatedFrequency = '0'; + + zeroObjUpdate.push(common.dbMap.frequency + '.' + calculatedFrequency); + monthObjUpdate.push(common.dbMap.frequency + '.' + calculatedFrequency); + + usersMeta['meta_v2.f-ranges.' + calculatedFrequency] = true; + //this was first session for this user + } + usersMeta['meta_v2.countries.' + (currEvent.up.cc || "Unknown")] = true; + common.fillTimeObjectZero(params, updateUsersZero, zeroObjUpdate); + common.fillTimeObjectMonth(params, updateUsersMonth, monthObjUpdate); + + var postfix = common.crypto.createHash("md5").update(currEvent.did).digest('base64')[0]; + if (Object.keys(updateUsersZero).length || Object.keys(usersMeta).length) { + usersMeta.m = dbDateIds.zero; + usersMeta.a = params.app_id + ""; + var updateObjZero = {$set: usersMeta}; + + if (Object.keys(updateUsersZero).length) { + updateObjZero.$inc = updateUsersZero; + } + common.writeBatcher.add("users", params.app_id + "_" + dbDateIds.zero + "_" + postfix, updateObjZero, "countly", {token: token}); + } + if (Object.keys(updateUsersMonth).length) { + common.writeBatcher.add("users", params.app_id + "_" + dbDateIds.month + "_" + postfix, { + $set: { + m: dbDateIds.month, + a: params.app_id + "" + }, + '$inc': updateUsersMonth + }, "countly", {token: token}); + } + usage.processSessionMetricsFromStream(currEvent, uniqueLevelsZero, uniqueLevelsMonth, params); +}; + + +usage.processEventFromStream = function(token, currEvent, writeBatcher) { + writeBatcher = writeBatcher || common.writeBatcher; + var forbiddenSegValues = []; + for (let i = 1; i < 32; i++) { + forbiddenSegValues.push(i + ""); + } + + //Write event totals for aggregated Data + + common.readBatcher.getOne("apps", common.db.ObjectID(currEvent.a), function(err, app) { + if (err || !app) { + return; + } + else { + common.readBatcher.getOne("events", common.db.ObjectID(currEvent.a), {"transformation": "event_object"}, async function(err2, eventColl) { + var tmpEventObj = {}; + var tmpEventColl = {}; + var tmpTotalObj = {}; + var pluginsGetConfig = plugins.getConfig("api", app.plugins, true); + + var time = common.initTimeObj(app.timezone, currEvent.ts); + var params = {time: time, app_id: currEvent.a, app: app, appTimezone: app.timezone || "UTC"}; + + var shortEventName = currEvent.n; + if (currEvent.e !== "[CLY]_custom") { + shortEventName = currEvent.e; + } + var rootUpdate = {}; + eventColl = eventColl || {}; + if (!eventColl._list || eventColl._list[shortEventName] !== true) { + eventColl._list = eventColl._list || {}; + eventColl._list_length = eventColl._list_length || 0; + if (eventColl._list_length <= 500) { + eventColl._list[shortEventName] = true; + eventColl._list_length++; + rootUpdate.$addToSet = {list: shortEventName}; + } + else { + return; //do not record this event in aggregated data + } + } + eventColl._segments = eventColl._segments || {}; + var eventCollectionName = crypto.createHash('sha1').update(shortEventName + params.app_id).digest('hex'); + var updates = []; + + if (currEvent.s && common.isNumber(currEvent.s)) { + common.fillTimeObjectMonth(params, tmpEventObj, common.dbMap.sum, currEvent.s); + common.fillTimeObjectMonth(params, tmpTotalObj, shortEventName + '.' + common.dbMap.sum, currEvent.s); + } + else { + currEvent.s = 0; + } + + if (currEvent.dur && common.isNumber(currEvent.dur)) { + common.fillTimeObjectMonth(params, tmpEventObj, common.dbMap.dur, currEvent.dur); + common.fillTimeObjectMonth(params, tmpTotalObj, shortEventName + '.' + common.dbMap.dur, currEvent.dur); + } + else { + currEvent.dur = 0; + } + currEvent.c = currEvent.c || 1; + if (currEvent.c && common.isNumber(currEvent.c)) { + currEvent.count = parseInt(currEvent.c, 10); + } + + common.fillTimeObjectMonth(params, tmpEventObj, common.dbMap.count, currEvent.count); + common.fillTimeObjectMonth(params, tmpTotalObj, shortEventName + '.' + common.dbMap.count, currEvent.count); + + + for (var seg in currEvent.sg) { + if (forbiddenSegValues.indexOf(currEvent.sg[seg] + "") !== -1) { + continue; + } + if (eventColl._omitted_segments && eventColl._omitted_segments[shortEventName]) { + if (eventColl._omitted_segments[shortEventName][seg]) { + continue; + } + } + if (eventColl._whitelisted_segments && eventColl._whitelisted_segments[shortEventName]) { + if (!eventColl._whitelisted_segments[shortEventName][seg]) { + continue; + } + } + if (Array.isArray(currEvent.sg[seg])) { + continue; //Skipping arrays + } + + //Segment is not registred in meta. + if (!eventColl._segments[shortEventName] || !eventColl._segments[shortEventName]._list[seg]) { + eventColl._segments[shortEventName] = eventColl._segments[shortEventName] || {_list: {}, _list_length: 0}; + eventColl._segments[shortEventName]._list[seg] = true; + rootUpdate.$addToSet = rootUpdate.$addToSet || {}; + if (rootUpdate.$addToSet["segments." + shortEventName]) { + if (rootUpdate.$addToSet["segments." + shortEventName].$each) { + rootUpdate.$addToSet["segments." + shortEventName].$each.push(seg); + } + else { + rootUpdate.$addToSet["segments." + shortEventName] = {$each: [rootUpdate.$addToSet["segments." + shortEventName], seg]}; + } + } + else { + rootUpdate.$addToSet["segments." + shortEventName] = seg; + } + } + + //load meta for this segment in cacher. Add new value if needed + + var tmpSegVal = currEvent.sg[seg] + ""; + tmpSegVal = tmpSegVal.replace(/^\$+/, "").replace(/\./g, ":"); + tmpSegVal = common.encodeCharacters(tmpSegVal); + + if (forbiddenSegValues.indexOf(tmpSegVal) !== -1) { + tmpSegVal = "[CLY]" + tmpSegVal; + } + + var postfix_seg = common.crypto.createHash("md5").update(tmpSegVal).digest('base64')[0]; + var meta = await common.readBatcher.getOne("events_meta", {"_id": eventCollectionName + "no-segment_" + common.getDateIds(params).zero + "_" + postfix_seg}); + + if (pluginsGetConfig.event_segmentation_value_limit && meta.meta_v2 && + meta.meta_v2[seg] && + meta.meta_v2[seg].indexOf(tmpSegVal) === -1 && + meta.meta_v2[seg].length >= pluginsGetConfig.event_segmentation_value_limit) { + continue; + } + + if (!meta.meta_v2 || !meta.meta_v2[seg] || meta.meta_v2[seg].indexOf(tmpSegVal) === -1) { + meta.meta_v2 = meta.meta_v2 || {}; + meta.meta_v2[seg] = meta.meta_v2[seg] || []; + meta.meta_v2[seg].push(tmpSegVal); + updates.push({ + id: currEvent.a + "_" + eventCollectionName + "_no-segment_" + common.getDateIds(params).zero + "_" + postfix_seg, + update: {"$set": {["meta_v2." + seg + "." + tmpSegVal]: true, ["meta_v2.segments." + seg]: true, "s": "no-segment", "e": shortEventName, "m": common.getDateIds(params).zero, "a": params.app_id + ""}} + }); + } + //record data + var tmpObj = {}; + + if (currEvent.s) { + common.fillTimeObjectMonth(params, tmpObj, tmpSegVal + '.' + common.dbMap.sum, currEvent.s); + } + + if (currEvent.dur) { + common.fillTimeObjectMonth(params, tmpEventObj, tmpSegVal + '.' + common.dbMap.dur, currEvent.dur); + } + + common.fillTimeObjectMonth(params, tmpObj, tmpSegVal + '.' + common.dbMap.count, currEvent.c); + updates.push({ + id: currEvent.a + "_" + eventCollectionName + "_" + seg + "_" + common.getDateIds(params).month + "_" + postfix_seg, + update: {$inc: tmpObj, $set: {"s": seg, "e": shortEventName, m: common.getDateIds(params).month, a: params.app_id + ""}} + }); + } + + var dateIds = common.getDateIds(params); + var postfix2 = common.crypto.createHash("md5").update(shortEventName).digest('base64')[0]; + + tmpEventColl["no-segment" + "." + dateIds.month] = tmpEventObj; + + for (var z = 0; z < updates.length; z++) { + writeBatcher.add("events_data", updates[z].id, updates[z].update, "countly", {token: token}); + } + //ID is - appID_hash_no-segment_month + + var _id = currEvent.a + "_" + eventCollectionName + "_no-segment_" + dateIds.month; + //Current event + writeBatcher.add("events_data", _id, { + "$set": { + "m": dateIds.month, + "s": "no-segment", + "a": params.app_id + "", + "e": shortEventName + }, + "$inc": tmpEventObj + }, "countly", + {token: token}); + + //Total event + writeBatcher.add("events_data", currEvent.a + "_all_key_" + dateIds.month + "_" + postfix2, { + "$set": { + "m": dateIds.month, + "s": "key", + "a": params.app_id + "", + "e": "all" + }, + "$inc": tmpTotalObj + }, "countly", + {token: token}); + + //Meta document for all events: + writeBatcher.add("events_data", params.app_id + "_all_" + "no-segment_" + dateIds.zero + "_" + postfix2, { + $set: { + m: dateIds.zero, + s: "no-segment", + a: params.app_id + "", + e: "all", + ["meta_v2.key." + shortEventName]: true, + "meta_v2.segments.key": true + + } + }, "countly", + {token: token}); + //Total event meta data + + if (Object.keys(rootUpdate).length) { + common.db.collection("events").updateOne({_id: common.db.ObjectID(currEvent.a)}, rootUpdate, {upsert: true}); + } + + }); + } + }); +}; + + +usage.processSessionMetricsFromStream = function(currEvent, uniqueLevelsZero, uniqueLevelsMonth, params) { + /** + * + * @param {string} id - document id + * @param {function} callback - calback function + */ + function fetchMeta(id, callback) { + common.readBatcher.getOne(metaToFetch[id].coll, {'_id': metaToFetch[id].id}, {meta_v2: 1}, (err, metaDoc) => { + var retObj = metaDoc || {}; + retObj.coll = metaToFetch[id].coll; + callback(null, retObj); + }); + } + + var isNewUser = true; + var userProps = {}; + if (currEvent.sg && currEvent.sg.prev_session) { + isNewUser = false; + //Not a new user + + } + //We can't do metric changes unless we fetch previous session doc. + var predefinedMetrics = usage.getPredefinedMetrics(params, userProps); + + var dateIds = common.getDateIds(params); + var metaToFetch = {}; + + if ((plugins.getConfig("api", params.app && params.app.plugins, true).metric_limit || 1000) > 0) { + var postfix; + for (let i = 0; i < predefinedMetrics.length; i++) { + for (let j = 0; j < predefinedMetrics[i].metrics.length; j++) { + let tmpMetric = predefinedMetrics[i].metrics[j], + recvMetricValue = currEvent.up[tmpMetric.short_code]; + postfix = null; + + // We check if country data logging is on and user's country is the configured country of the app + if (tmpMetric.name === "country" && (plugins.getConfig("api", params.app && params.app.plugins, true).country_data === false)) { + continue; + } + // We check if city data logging is on and user's country is the configured country of the app + if (tmpMetric.name === "city" && (plugins.getConfig("api", params.app && params.app.plugins, true).city_data === false)) { + continue; + } + + if (recvMetricValue) { + recvMetricValue = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":"); + postfix = common.crypto.createHash("md5").update(recvMetricValue).digest('base64')[0]; + metaToFetch[predefinedMetrics[i].db + params.app_id + "_" + dateIds.zero + "_" + postfix] = { + coll: predefinedMetrics[i].db, + id: params.app_id + "_" + dateIds.zero + "_" + postfix + }; + } + } + } + + var metas = {}; + async.map(Object.keys(metaToFetch), fetchMeta, function(err, metaDocs) { + for (let i = 0; i < metaDocs.length; i++) { + if (metaDocs[i].coll && metaDocs[i].meta_v2) { + metas[metaDocs[i]._id] = metaDocs[i].meta_v2; + } + } + + for (let i = 0; i < predefinedMetrics.length; i++) { + for (let j = 0; j < predefinedMetrics[i].metrics.length; j++) { + let tmpTimeObjZero = {}, + tmpTimeObjMonth = {}, + tmpSet = {}, + needsUpdate = false, + zeroObjUpdate = [], + monthObjUpdate = [], + tmpMetric = predefinedMetrics[i].metrics[j], + recvMetricValue = "", + escapedMetricVal = ""; + + postfix = ""; + + recvMetricValue = currEvent.up[tmpMetric.short_code]; + + // We check if country data logging is on and user's country is the configured country of the app + if (tmpMetric.name === "country" && (plugins.getConfig("api", params.app && params.app.plugins, true).country_data === false)) { + continue; + } + // We check if city data logging is on and user's country is the configured country of the app + if (tmpMetric.name === "city" && (plugins.getConfig("api", params.app && params.app.plugins, true).city_data === false)) { + continue; + } + + if (recvMetricValue) { + escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":"); + postfix = common.crypto.createHash("md5").update(escapedMetricVal).digest('base64')[0]; + + var tmpZeroId = params.app_id + "_" + dateIds.zero + "_" + postfix; + var ignore = false; + if (metas[tmpZeroId] && + metas[tmpZeroId][tmpMetric.set] && + Object.keys(metas[tmpZeroId][tmpMetric.set]).length && + Object.keys(metas[tmpZeroId][tmpMetric.set]).length >= plugins.getConfig("api", params.app && params.app.plugins, true).metric_limit && + typeof metas[tmpZeroId][tmpMetric.set][escapedMetricVal] === "undefined") { + ignore = true; + } + + //should metric be ignored for reaching the limit + if (!ignore) { + //making sure metrics are strings + needsUpdate = true; + tmpSet["meta_v2." + tmpMetric.set + "." + escapedMetricVal] = true; + + monthObjUpdate.push(escapedMetricVal + '.' + common.dbMap.total); + + if (isNewUser) { + zeroObjUpdate.push(escapedMetricVal + '.' + common.dbMap.unique); + monthObjUpdate.push(escapedMetricVal + '.' + common.dbMap.new); + monthObjUpdate.push(escapedMetricVal + '.' + common.dbMap.unique); + } + else { + for (let k = 0; k < uniqueLevelsZero.length; k++) { + if (uniqueLevelsZero[k] === "Y") { + tmpTimeObjZero['d.' + escapedMetricVal + '.' + common.dbMap.unique] = 1; + } + else { + tmpTimeObjZero['d.' + uniqueLevelsZero[k] + '.' + escapedMetricVal + '.' + common.dbMap.unique] = 1; + } + } + + for (let l = 0; l < uniqueLevelsMonth.length; l++) { + tmpTimeObjMonth['d.' + uniqueLevelsMonth[l] + '.' + escapedMetricVal + '.' + common.dbMap.unique] = 1; + } + } + } + + common.fillTimeObjectZero(params, tmpTimeObjZero, zeroObjUpdate); + common.fillTimeObjectMonth(params, tmpTimeObjMonth, monthObjUpdate); + + if (needsUpdate) { + tmpSet.m = dateIds.zero; + tmpSet.a = params.app_id + ""; + var tmpMonthId = params.app_id + "_" + dateIds.month + "_" + postfix, + updateObjZero = {$set: tmpSet}; + + if (Object.keys(tmpTimeObjZero).length) { + updateObjZero.$inc = tmpTimeObjZero; + } + + if (Object.keys(tmpTimeObjZero).length || Object.keys(tmpSet).length) { + common.writeBatcher.add(predefinedMetrics[i].db, tmpZeroId, updateObjZero); + } + + common.writeBatcher.add(predefinedMetrics[i].db, tmpMonthId, { + $set: { + m: dateIds.month, + a: params.app_id + "" + }, + '$inc': tmpTimeObjMonth + }); + } + } + } + } + }); + } +}; + + +usage.getPredefinedMetrics = function(params, userProps) { + var predefinedMetrics = [ + { + db: "carriers", + metrics: [{ + name: "_carrier", + set: "carriers", + short_code: common.dbUserMap.carrier + }] + }, + { + db: "devices", + metrics: [ + { + name: "_device", + set: "devices", + short_code: common.dbUserMap.device + }, + { + name: "_manufacturer", + set: "manufacturers", + short_code: common.dbUserMap.manufacturer + } + ] + }, + { + db: "device_details", + metrics: [ + { + name: "_app_version", + set: "app_versions", + short_code: common.dbUserMap.app_version + }, + { + name: "_os", + set: "os", + short_code: common.dbUserMap.platform + }, + { + name: "_device_type", + set: "device_type", + short_code: common.dbUserMap.device_type + }, + { + name: "_os_version", + set: "os_versions", + short_code: common.dbUserMap.platform_version + }, + { + name: "_resolution", + set: "resolutions", + short_code: common.dbUserMap.resolution + }, + { + name: "_has_hinge", + set: "has_hinge", + short_code: common.dbUserMap.has_hinge + } + ] + }, + { + db: "cities", + metrics: [{ + is_user_prop: true, + name: "city", + set: "cities", + short_code: common.dbUserMap.city + }] + } + ]; + var isNewUser = (params.app_user && params.app_user[common.dbUserMap.first_seen]) ? false : true; + plugins.dispatch("/session/metrics", { + params: params, + predefinedMetrics: predefinedMetrics, + userProps: userProps, + user: params.app_user, + isNewUser: isNewUser + }); + + return predefinedMetrics; +}; + +module.exports = usage; \ No newline at end of file diff --git a/api/api.js b/api/api.js index df712833b8b..cb0f2f20a18 100644 --- a/api/api.js +++ b/api/api.js @@ -1,35 +1,24 @@ const http = require('http'); -const cluster = require('cluster'); const formidable = require('formidable'); -const os = require('os'); -const countlyConfig = require('./config', 'dont-enclose'); +const countlyConfig = require('./config'); const plugins = require('../plugins/pluginManager.js'); -const jobs = require('./parts/jobs'); const log = require('./utils/log.js')('core:api'); const common = require('./utils/common.js'); const {processRequest} = require('./utils/requestProcessor'); const frontendConfig = require('../frontend/express/config.js'); -const {CacheMaster, CacheWorker} = require('./parts/data/cache.js'); const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher.js'); const pack = require('../package.json'); const versionInfo = require('../frontend/express/version.info.js'); const moment = require("moment"); +var {MongoDbQueryRunner} = require('./utils/mongoDbQueryRunner.js'); + var t = ["countly:", "api"]; common.processRequest = processRequest; -if (cluster.isMaster) { - console.log("Starting Countly", "version", versionInfo.version, "package", pack.version); - if (!common.checkDatabaseConfigMatch(countlyConfig.mongodb, frontendConfig.mongodb)) { - log.w('API AND FRONTEND DATABASE CONFIGS ARE DIFFERENT'); - } - t.push("master"); - t.push("node"); - t.push(process.argv[1]); -} -else { - t.push("worker"); - t.push("node"); +console.log("Starting Countly", "version", versionInfo.version, "package", pack.version); +if (!common.checkDatabaseConfigMatch(countlyConfig.mongodb, frontendConfig.mongodb)) { + log.w('API AND FRONTEND DATABASE CONFIGS ARE DIFFERENT'); } // Finaly set the visible title @@ -39,12 +28,13 @@ plugins.connectToAllDatabases().then(function() { common.writeBatcher = new WriteBatcher(common.db); common.readBatcher = new ReadBatcher(common.db); common.insertBatcher = new InsertBatcher(common.db); + + if (common.drillDb) { common.drillReadBatcher = new ReadBatcher(common.drillDb); + common.drillQueryRunner = new MongoDbQueryRunner(common.drillDb); } - let workers = []; - /** * Set Max Sockets */ @@ -123,22 +113,15 @@ plugins.connectToAllDatabases().then(function() { /** * Set Plugins Logs Config */ - plugins.setConfigs('logs', { - debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '', - info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '', - warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '', - error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '', - default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn', - }, undefined, () => { - const cfg = plugins.getConfig('logs'), msg = { - cmd: 'log', - config: cfg - }; - if (process.send) { - process.send(msg); + plugins.setConfigs('logs', + { + debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '', + info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '', + warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '', + error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '', + default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn', } - require('./utils/log.js').ipcHandler(msg); - }); + ); /** * Initialize Plugins @@ -151,6 +134,19 @@ plugins.connectToAllDatabases().then(function() { */ async function storeBatchedData(code) { try { + + await new Promise((resolve) => { + server.close((err) => { + if (err) { + console.log("Error closing server:", err); + } + else { + console.log("Server closed successfully"); + resolve(); + } + }); + }); + await common.writeBatcher.flushAll(); await common.insertBatcher.flushAll(); console.log("Successfully stored batch state"); @@ -204,241 +200,113 @@ plugins.connectToAllDatabases().then(function() { console.trace(); }); - /** - * Pass To Master - * @param {cluster.Worker} worker - worker thatw as spawned by master - */ - const passToMaster = (worker) => { - worker.on('message', (msg) => { - if (msg.cmd === 'log') { - workers.forEach((w) => { - if (w !== worker) { - w.send({ - cmd: 'log', - config: msg.config - }); - } - }); - require('./utils/log.js').ipcHandler(msg); - } - else if (msg.cmd === "checkPlugins") { - plugins.checkPluginsMaster(); - } - else if (msg.cmd === "startPlugins") { - plugins.startSyncing(); - } - else if (msg.cmd === "endPlugins") { - plugins.stopSyncing(); - } - else if (msg.cmd === "batch_insert") { - const {collection, doc, db} = msg.data; - common.insertBatcher.insert(collection, doc, db); - } - else if (msg.cmd === "batch_write") { - const {collection, id, operation, db} = msg.data; - common.writeBatcher.add(collection, id, operation, db); - } - else if (msg.cmd === "batch_read") { - const {collection, query, projection, multi, msgId} = msg.data; - common.readBatcher.get(collection, query, projection, multi).then((data) => { - worker.send({ cmd: "batch_read", data: {msgId, data} }); - }) - .catch((err) => { - worker.send({ cmd: "batch_read", data: {msgId, err} }); - }); - } - else if (msg.cmd === "batch_invalidate") { - const {collection, query, projection, multi} = msg.data; - common.readBatcher.invalidate(collection, query, projection, multi); - } - else if (msg.cmd === "dispatchMaster" && msg.event) { - plugins.dispatch(msg.event, msg.data); - } - else if (msg.cmd === "dispatch" && msg.event) { - workers.forEach((w) => { - w.send(msg); - }); - } - }); - }; - - if (cluster.isMaster) { - plugins.installMissingPlugins(common.db); - common.runners = require('./parts/jobs/runner'); - common.cache = new CacheMaster(); - common.cache.start().then(() => { - setImmediate(() => { - plugins.dispatch('/cache/init', {}); - }); - }, e => { - console.log(e); - process.exit(1); - }); - - const workerCount = (countlyConfig.api.workers) - ? countlyConfig.api.workers - : os.cpus().length; - - for (let i = 0; i < workerCount; i++) { - // there's no way to define inspector port of a worker in the code. So if we don't - // pick a unique port for each worker, they conflict with each other. - let nodeOptions = {}; - if (countlyConfig?.symlinked !== true) { // countlyConfig.symlinked is passed when running in a symlinked setup - const inspectorPort = i + 1 + (common?.config?.masterInspectorPort || 9229); - nodeOptions = { NODE_OPTIONS: "--inspect-port=" + inspectorPort }; - } - const worker = cluster.fork(nodeOptions); - workers.push(worker); + var utcMoment = moment.utc(); + var incObj = {}; + incObj.r = 1; + incObj[`d.${utcMoment.format("D")}.${utcMoment.format("H")}.r`] = 1; + common.db.collection("diagnostic").updateOne({"_id": "no-segment_" + utcMoment.format("YYYY:M")}, {"$set": {"m": utcMoment.format("YYYY:M")}, "$inc": incObj}, {"upsert": true}, function(err) { + if (err) { + log.e(err); } + }); - workers.forEach(passToMaster); - cluster.on('exit', (worker) => { - workers = workers.filter((w) => { - return w !== worker; - }); - const newWorker = cluster.fork(); - workers.push(newWorker); - passToMaster(newWorker); - }); + plugins.installMissingPlugins(common.db); + const taskManager = require('./utils/taskmanager.js'); + //since process restarted mark running tasks as errored + taskManager.errorResults({db: common.db}); - plugins.dispatch("/master", {}); - - // Allow configs to load & scanner to find all jobs classes - setTimeout(() => { - jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); - jobs.job('api:ping').replace().schedule('every 1 day'); - jobs.job('api:clear').replace().schedule('every 1 day'); - jobs.job('api:clearTokens').replace().schedule('every 1 day'); - jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); - jobs.job('api:task').replace().schedule('every 5 minutes'); - jobs.job('api:userMerge').replace().schedule('every 10 minutes'); - jobs.job("api:ttlCleanup").replace().schedule("every 1 minute"); - //jobs.job('api:appExpire').replace().schedule('every 1 day'); - }, 10000); - - //Record as restarted - - var utcMoment = moment.utc(); - - var incObj = {}; - incObj.r = 1; - incObj[`d.${utcMoment.format("D")}.${utcMoment.format("H")}.r`] = 1; - common.db.collection("diagnostic").updateOne({"_id": "no-segment_" + utcMoment.format("YYYY:M")}, {"$set": {"m": utcMoment.format("YYYY:M")}, "$inc": incObj}, {"upsert": true}, function(err) { - if (err) { - log.e(err); - } - }); - } - else { - console.log("Starting worker", process.pid, "parent:", process.ppid); - const taskManager = require('./utils/taskmanager.js'); + plugins.dispatch("/master", {}); // init hook - common.cache = new CacheWorker(); - common.cache.start(); + const server = http.Server((req, res) => { + const params = { + qstring: {}, + res: res, + req: req + }; - //since process restarted mark running tasks as errored - taskManager.errorResults({db: common.db}); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('Keep-Alive', 'timeout=5, max=1000'); - process.on('message', common.log.ipcHandler); + if (req.method.toLowerCase() === 'post') { + const formidableOptions = {}; + if (countlyConfig.api.maxUploadFileSize) { + formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize; + } - process.on('message', (msg) => { - if (msg.cmd === 'log') { - common.log.ipcHandler(msg); + const form = new formidable.IncomingForm(formidableOptions); + if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) { + req.body = []; + req.on('data', (data) => { + req.body.push(data); + }); } - else if (msg.cmd === "dispatch" && msg.event) { - plugins.dispatch(msg.event, msg.data || {}); + else { + req.body = ''; + req.on('data', (data) => { + req.body += data; + }); } - }); - process.on('exit', () => { - console.log('Exiting due to master exited'); - }); - - plugins.dispatch("/worker", {common: common}); - - http.Server((req, res) => { - const params = { - qstring: {}, - res: res, - req: req - }; + let multiFormData = false; + // Check if we have 'multipart/form-data' + if (req.headers['content-type']?.startsWith('multipart/form-data')) { + multiFormData = true; + } - if (req.method.toLowerCase() === 'post') { - const formidableOptions = {}; - if (countlyConfig.api.maxUploadFileSize) { - formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize; + form.parse(req, (err, fields, files) => { + //handle bakcwards compatability with formiddble v1 + for (let i in files) { + if (files[i].filepath) { + files[i].path = files[i].filepath; + } + if (files[i].mimetype) { + files[i].type = files[i].mimetype; + } + if (files[i].originalFilename) { + files[i].name = files[i].originalFilename; + } } - - const form = new formidable.IncomingForm(formidableOptions); - if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) { - req.body = []; - req.on('data', (data) => { - req.body.push(data); - }); + params.files = files; + if (multiFormData) { + let formDataUrl = []; + for (const i in fields) { + params.qstring[i] = fields[i]; + formDataUrl.push(`${i}=${fields[i]}`); + } + params.formDataUrl = formDataUrl.join('&'); } else { - req.body = ''; - req.on('data', (data) => { - req.body += data; - }); + for (const i in fields) { + params.qstring[i] = fields[i]; + } } - - let multiFormData = false; - // Check if we have 'multipart/form-data' - if (req.headers['content-type']?.startsWith('multipart/form-data')) { - multiFormData = true; + if (!params.apiPath) { + processRequest(params); } + }); + } + else if (req.method.toLowerCase() === 'options') { + const headers = {}; + headers["Access-Control-Allow-Origin"] = "*"; + headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS"; + headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type"; + res.writeHead(200, headers); + res.end(); + } + //attempt process GET request + else if (req.method.toLowerCase() === 'get') { + processRequest(params); + } + else { + common.returnMessage(params, 405, "Method not allowed"); + } + }); + server.listen(common.config.api.port, common.config.api.host || ''); + server.timeout = common.config.api.timeout || 120000; + server.keepAliveTimeout = common.config.api.timeout || 120000; + server.headersTimeout = (common.config.api.timeout || 120000) + 1000; // Slightly higher - form.parse(req, (err, fields, files) => { - //handle bakcwards compatability with formiddble v1 - for (let i in files) { - if (files[i].filepath) { - files[i].path = files[i].filepath; - } - if (files[i].mimetype) { - files[i].type = files[i].mimetype; - } - if (files[i].originalFilename) { - files[i].name = files[i].originalFilename; - } - } - params.files = files; - if (multiFormData) { - let formDataUrl = []; - for (const i in fields) { - params.qstring[i] = fields[i]; - formDataUrl.push(`${i}=${fields[i]}`); - } - params.formDataUrl = formDataUrl.join('&'); - } - else { - for (const i in fields) { - params.qstring[i] = fields[i]; - } - } - if (!params.apiPath) { - processRequest(params); - } - }); - } - else if (req.method.toLowerCase() === 'options') { - const headers = {}; - headers["Access-Control-Allow-Origin"] = "*"; - headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS"; - headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type"; - res.writeHead(200, headers); - res.end(); - } - //attempt process GET request - else if (req.method.toLowerCase() === 'get') { - processRequest(params); - } - else { - common.returnMessage(params, 405, "Method not allowed"); - } - }).listen(common.config.api.port, common.config.api.host || '').timeout = common.config.api.timeout || 120000; - plugins.loadConfigs(common.db); - } + plugins.loadConfigs(common.db); }); diff --git a/api/config.sample.js b/api/config.sample.js index d3532c70d9f..0d7ef4e9d1a 100644 --- a/api/config.sample.js +++ b/api/config.sample.js @@ -5,6 +5,14 @@ /** @lends module:api/config */ var countlyConfig = { + /** + * Drill events database driver configuration + * @type {string} + * @property {string} [drill_events_driver=mongodb] - database driver to use for drill events storage + * Possible values are: "mongodb", "clickhouse" + */ + drill_events_driver: "clickhouse", + /** * MongoDB connection definition and options * @type {object} @@ -24,6 +32,7 @@ var countlyConfig = { db: "countly", port: 27017, max_pool_size: 500, + replicaName: "rs0", //username: test, //password: test, //mongos: false, @@ -50,9 +59,56 @@ var countlyConfig = { }, */ /* or define as a url - //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] - mongodb: "mongodb://localhost:27017/countly", + //mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] + mongodb: "mongodb://localhost:27017/countly", */ + /** + * ClickHouse connection definition and options + * @type {object|string} + * @property {string} [url=http://localhost:8123] - ClickHouse server URL + * @property {string} [username=default] - username for authenticating user + * @property {string} [password=] - password for authenticating user + * @property {string} [database=countly_drill] - ClickHouse database name + * @property {object} [compression] - compression settings + * @property {string} [application] - application name for connection + * @property {number} [request_timeout=1200000] - request timeout in milliseconds + * @property {object} [keep_alive] - keep alive settings + * @property {number} [max_open_connections=10] - maximum number of open connections + * @property {object} [clickhouse_settings] - ClickHouse specific settings + */ + clickhouse: { + url: "http://localhost:8123", + username: "default", + password: "", + database: "countly_drill", + compression: { + request: false, + response: false, + }, + application: "", + request_timeout: 1200_000, + keep_alive: { + enabled: true, + idle_socket_ttl: 10000, + }, + max_open_connections: 10, + clickhouse_settings: { + idle_connection_timeout: 11000 + '', + async_insert: 1, + wait_for_async_insert: 1, + wait_end_of_query: 1, + optimize_on_insert: 1, + allow_suspicious_types_in_group_by: 1, + allow_suspicious_types_in_order_by: 1, + optimize_move_to_prewhere: 1, + query_plan_optimize_lazy_materialization: 1 + } + }, + /* or define as a url + //http://[username:password@]host[:port][/database] + clickhouse: "http://localhost:8123/countly_drill", + */ + /** * Default API configuration * @type {object} @@ -71,6 +127,23 @@ var countlyConfig = { maxUploadFileSize: 200 * 1024 * 1024, // 200MB }, /** + * Default Ingestor configuration + * @type {object} + * @property {number} [port=3010] - api port number to use, default 3010 + * @property {string} [host=localhost] - host to which to bind connection + * @property {number} [max_sockets=1024] - maximal amount of sockets to open simultaneously + * @property {number} workers - amount of paralel countly processes to run, defaults to cpu/core amount + * @property {number} [timeout=120000] - nodejs server request timeout, need to also increase nginx timeout too for longer requests + * @property {number} maxUploadFileSize - limit the size of uploaded file + */ + ingestor: { + port: 3010, + host: "localhost", + max_sockets: 1024, + timeout: 120000, + maxUploadFileSize: 200 * 1024 * 1024, // 200MB + }, + /** * Path to use for countly directory, empty path if installed at root of website * @type {string} */ diff --git a/api/configextender.js b/api/configextender.js index 670c736c24d..d5175c31438 100644 --- a/api/configextender.js +++ b/api/configextender.js @@ -30,6 +30,13 @@ const OVERRIDES = { SERVEROPTIONS: 'serverOptions' }, + CLICKHOUSE: { + REQUEST_TIMEOUT: 'request_timeout', + MAX_OPEN_CONNECTIONS: 'max_open_connections', + CLICKHOUSE_SETTINGS: 'clickhouse_settings', + KEEP_ALIVE: 'keep_alive' + }, + API: { MAX_SOCKETS: 'max_sockets', MAX_UPLOAD_FILE_SIZE: 'maxUploadFileSize' diff --git a/api/ingestor.js b/api/ingestor.js new file mode 100644 index 00000000000..1c6e5fac78f --- /dev/null +++ b/api/ingestor.js @@ -0,0 +1,294 @@ +const http = require('http'); +const formidable = require('formidable'); +const countlyConfig = require('./config'); +const plugins = require('../plugins/pluginManager.js'); +const log = require('./utils/log.js')('ingestor-core:api'); +const {processRequest} = require('./ingestor/requestProcessor'); +const common = require('./utils/common.js'); +const {Cacher} = require('./parts/data/cacher.js'); + +var t = ["countly:", "ingestor"]; +t.push("node"); + +// Finaly set the visible title +process.title = t.join(' '); + +console.log("Connecting to databases"); + +//Overriding function +plugins.loadConfigs = plugins.loadConfigsIngestor; + +/** + * TODO + * temporarily change this false since it fails at + * Cannot create uid TypeError: common.db.ObjectID is not a function + * at usersApi.getUid (api/parts/mgmt/app_users.js:434:90) + */ +plugins.connectToAllDatabases(false).then(function() { + log.i("Db connections done"); + // common.writeBatcher = new WriteBatcher(common.db); + common.readBatcher = new Cacher(common.db); + //common.insertBatcher = new InsertBatcher(common.db); + if (common.drillDb) { + common.drillReadBatcher = new Cacher(common.drillDb); + } + /** + * Set Max Sockets + */ + http.globalAgent.maxSockets = countlyConfig.api.max_sockets || 1024; + /** + * Set Plugins APIs Config + */ + //Put in single file outside(all set configs) + plugins.setConfigs("api", { + domain: "", + safe: false, + session_duration_limit: 86400, + country_data: true, + city_data: true, + event_limit: 500, + event_segmentation_limit: 100, + event_segmentation_value_limit: 1000, + array_list_limit: 10, + metric_limit: 1000, + sync_plugins: false, + session_cooldown: 15, + request_threshold: 30, + total_users: true, + export_limit: 10000, + prevent_duplicate_requests: true, + metric_changes: true, + offline_mode: false, + reports_regenerate_interval: 3600, + send_test_email: "", + //data_retention_period: 0, + batch_processing: true, + //batch_on_master: false, + batch_period: 10, + batch_read_processing: true, + //batch_read_on_master: false, + batch_read_ttl: 600, + batch_read_period: 60, + user_merge_paralel: 1, + trim_trailing_ending_spaces: false + }); + + /** + * Set Plugins APPs Config + */ + plugins.setConfigs("apps", { + country: "TR", + timezone: "Europe/Istanbul", + category: "6" + }); + + /** + * Set Plugins Security Config + */ + plugins.setConfigs("security", { + login_tries: 3, + login_wait: 5 * 60, + password_min: 8, + password_char: true, + password_number: true, + password_symbol: true, + password_expiration: 0, + password_rotation: 3, + password_autocomplete: true, + robotstxt: "User-agent: *\nDisallow: /", + dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000 ; includeSubDomains\nX-Content-Type-Options: nosniff", + api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nAccess-Control-Allow-Origin:*", + dashboard_rate_limit_window: 60, + dashboard_rate_limit_requests: 500, + proxy_hostname: "", + proxy_port: "", + proxy_username: "", + proxy_password: "", + proxy_type: "https" + }); + + /** + * Set Plugins Logs Config + */ + plugins.setConfigs('logs', { + debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '', + info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '', + warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '', + error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '', + default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn', + }, undefined, () => { + const cfg = plugins.getConfig('logs'), msg = { + cmd: 'log', + config: cfg + }; + if (process.send) { + process.send(msg); + } + require('./utils/log.js').ipcHandler(msg); + }); + + /** + * Initialize Plugins + */ + + // plugins.init(); - should run new init ingestor + + /** + * Trying to gracefully handle the batch state + * @param {number} code - error code + */ + async function storeBatchedData(code) { + try { + //await common.writeBatcher.flushAll(); + //await common.insertBatcher.flushAll(); + console.log("Successfully stored batch state"); + } + catch (ex) { + console.log("Could not store batch state", ex); + } + process.exit(typeof code === "number" ? code : 1); + } + + /** + * Handle before exit for gracefull close + */ + process.on('beforeExit', (code) => { + console.log('Received exit, trying to save batch state: ', code); + storeBatchedData(code); + }); + + /** + * Handle exit events for gracefull close + */ + ['SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT', + 'SIGBUS', 'SIGFPE', 'SIGSEGV', 'SIGTERM', + ].forEach(function(sig) { + process.on(sig, async function() { + storeBatchedData(sig); + console.log('Got signal: ' + sig); + }); + }); + + /** + * Uncaught Exception Handler + */ + process.on('uncaughtException', (err) => { + console.log('Caught exception: %j', err, err.stack); + if (log && log.e) { + log.e('Logging caught exception'); + } + console.trace(); + storeBatchedData(1); + }); + + /** + * Unhandled Rejection Handler + */ + process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled rejection for %j with reason %j stack ', p, reason, reason ? reason.stack : undefined); + if (log && log.e) { + log.e('Logging unhandled rejection'); + } + console.trace(); + }); + console.log("Starting ingestor", process.pid); + //since process restarted mark running tasks as errored + plugins.dispatch("/ingestor", {common: common}); + plugins.init({"skipDependencies": true, "filename": "ingestor"}); + console.log("Loading configs"); + plugins.loadConfigs(common.db, function() { + console.log("Configs loaded. Opening server connection"); + console.log(JSON.stringify(common.config.ingestor || {})); + http.Server((req, res) => { + const params = { + qstring: {}, + res: res, + req: req + }; + + console.log("recieved some data"); + params.tt = Date.now().valueOf(); + if (req.method.toLowerCase() === 'post') { + const formidableOptions = {}; + if (countlyConfig.api.maxUploadFileSize) { + formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize; + } + + const form = new formidable.IncomingForm(formidableOptions); + if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) { + req.body = []; + req.on('data', (data) => { + req.body.push(data); + }); + } + else { + req.body = ''; + req.on('data', (data) => { + req.body += data; + }); + } + + let multiFormData = false; + // Check if we have 'multipart/form-data' + if (req.headers['content-type']?.startsWith('multipart/form-data')) { + multiFormData = true; + } + + form.parse(req, (err, fields, files) => { + //handle bakcwards compatability with formiddble v1 + for (let i in files) { + if (files[i].filepath) { + files[i].path = files[i].filepath; + } + if (files[i].mimetype) { + files[i].type = files[i].mimetype; + } + if (files[i].originalFilename) { + files[i].name = files[i].originalFilename; + } + } + params.files = files; + if (multiFormData) { + let formDataUrl = []; + for (const i in fields) { + params.qstring[i] = fields[i]; + formDataUrl.push(`${i}=${fields[i]}`); + } + params.formDataUrl = formDataUrl.join('&'); + } + else { + for (const i in fields) { + params.qstring[i] = fields[i]; + } + } + if (!params.apiPath) { + processRequest(params); + } + }); + } + else if (req.method.toLowerCase() === 'options') { + const headers = {}; + headers["Access-Control-Allow-Origin"] = "*"; + headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS"; + headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type"; + res.writeHead(200, headers); + res.end(); + } + //attempt process GET request + else if (req.method.toLowerCase() === 'get') { + processRequest(params); + } + else { + common.returnMessage(params, 405, "Method not allowed"); + } + }).listen(common.config?.ingestor?.port || 3010, common.config?.ingestor?.host || '').timeout = common.config?.ingestor?.timeout || 120000; + }); +}); + + +/** + * On incoming request + * 1)Get App data (Batcher) + * 2)Get overall configs + * + */ \ No newline at end of file diff --git a/api/ingestor/requestProcessor.js b/api/ingestor/requestProcessor.js new file mode 100644 index 00000000000..c40f3540e58 --- /dev/null +++ b/api/ingestor/requestProcessor.js @@ -0,0 +1,1134 @@ +const usage = require('./usage.js'); //special usage file for ingestor +const common = require('../utils/common.js'); +const url = require('url'); +const plugins = require("../../plugins/pluginManager.js"); +const log = require('../utils/log.js')('core:ingestor'); +const crypto = require('crypto'); +const { ignorePossibleDevices, checksumSaltVerification, validateRedirect} = require('../utils/requestProcessorCommon.js'); +const countlyApi = { + mgmt: { + appUsers: require('../parts/mgmt/app_users.js'), + } +}; + +const escapedViewSegments = { "name": true, "segment": true, "height": true, "width": true, "y": true, "x": true, "visit": true, "uvc": true, "start": true, "bounce": true, "exit": true, "type": true, "view": true, "domain": true, "dur": true, "_id": true, "_idv": true, "utm_source": true, "utm_medium": true, "utm_campaign": true, "utm_term": true, "utm_content": true, "referrer": true}; + + +//Do not restart. If fails to creating, ail request. +/** + * @param {object} params - request parameters + * @param {function} done - callback function + */ +function processUser(params, done) { + if (params && params.qstring && params.qstring.old_device_id && params.qstring.old_device_id !== params.qstring.device_id) { + const old_id = common.crypto.createHash('sha1') + .update(params.qstring.app_key + params.qstring.old_device_id + "") + .digest('hex'); + + countlyApi.mgmt.appUsers.merge(params.app_id, params.app_user, params.app_user_id, old_id, params.qstring.device_id, params.qstring.old_device_id, function(err0, userdoc) { + //remove old device ID and retry request + params.qstring.old_device_id = null; + if (err0) { + log.e(err0); + done('Cannot update user'); + } + else if (userdoc) { + if (!userdoc.uid) { + countlyApi.mgmt.appUsers.createUserDocument(params, function(err, userDoc2) { + if (err) { + log.e(err); + done('Cannot update user'); + } + else if (!userDoc2) { + done('Cannot update user'); + } + else { + params.app_user = userDoc2; + done(); + } + }); + } + else { + params.app_user = userdoc; + done(); + } + } + else { + done('User merged. Failed to record data.'); + } + }); + } + else if (params && params.app_user && !params.app_user.uid) { + countlyApi.mgmt.appUsers.createUserDocument(params, function(err, userDoc2) { + if (err) { + done(err); + } + else if (userDoc2) { + params.app_user = userDoc2; + done(); + } + else { + done("User creation failed"); + } + + }); + } + else { + done(); + } +} + +var preset = { + up: { + fs: { name: "first_seen", type: "d" }, + ls: { name: "last_seen", type: "d" }, + tsd: { name: "total_session_duration", type: "n" }, + sc: { name: "session_count", type: "n" }, + d: { name: "device", type: "l" }, + dt: { name: "device_type", type: "l" }, + mnf: { name: "manufacturer", type: "l" }, + ornt: { name: "ornt", type: "l" }, + cty: { name: "city", type: "l" }, + rgn: { name: "region", type: "l" }, + cc: { name: "country_code", type: "l" }, + p: { name: "platform", type: "l" }, + pv: { name: "platform_version", type: "l" }, + av: { name: "app_version", type: "l" }, + c: { name: "carrier", type: "l" }, + r: { name: "resolution", type: "l" }, + dnst: { name: "dnst", type: "l" }, + brw: { name: "brw", type: "l" }, + brwv: { name: "brwv", type: "l" }, + la: { name: "la", type: "l" }, + lo: { name: "lo", type: "l" }, + src: { name: "src", type: "l" }, + src_ch: { name: "src_ch", type: "l" }, + name: { name: "name", type: "s" }, + username: { name: "username", type: "s" }, + email: { name: "email", type: "s" }, + organization: { name: "organization", type: "s" }, + phone: { name: "phone", type: "s" }, + gender: { name: "gender", type: "l" }, + byear: { name: "byear", type: "n" }, + age: { name: "age", type: "n" }, + engagement_score: { name: "engagement_score", type: "n" }, + lp: { name: "lp", type: "d" }, + lpa: { name: "lpa", type: "n" }, + tp: { name: "tp", type: "n" }, + tpc: { name: "tpc", type: "n" }, + lv: { name: "lv", type: "l" }, + cadfs: { name: "cadfs", type: "n" }, + cawfs: { name: "cawfs", type: "n" }, + camfs: { name: "camfs", type: "n" }, + hour: { name: "hour", type: "l" }, + dow: { name: "dow", type: "l" }, + hh: { name: "hh", type: "l" }, + }, + sg: { + "[CLY]_view": { + start: { name: "start", type: "l" }, + exit: { name: "exit", type: "l" }, + bounce: { name: "bounce", type: "l" } + }, + "[CLY]_session": { + request_id: { name: "request_id", type: "s" }, + prev_session: { name: "prev_session", type: "s" }, + prev_start: { name: "prev_start", type: "d" }, + postfix: { name: "postfix", type: "s" }, + ended: {name: "ended", type: "l"} + }, + "[CLY]_action": { + x: { name: "x", type: "n" }, + y: { name: "y", type: "n" }, + width: { name: "width", type: "n" }, + height: { name: "height", type: "n" } + }, + "[CLY]_crash": { + name: { name: "name", type: "s" }, + manufacture: { name: "manufacture", type: "l" }, + cpu: { name: "cpu", type: "l" }, + opengl: { name: "opengl", type: "l" }, + view: { name: "view", type: "l" }, + browser: { name: "browser", type: "l" }, + os: { name: "os", type: "l" }, + orientation: { name: "orientation", type: "l" }, + nonfatal: { name: "nonfatal", type: "l" }, + root: { name: "root", type: "l" }, + online: { name: "online", type: "l" }, + signal: { name: "signal", type: "l" }, + muted: { name: "muted", type: "l" }, + background: { name: "background", type: "l" }, + app_version: { name: "app_version", type: "l" }, + ram_current: { name: "ram_current", type: "n" }, + ram_total: { name: "ram_total", type: "n" }, + disk_current: { name: "disk_current", type: "n" }, + disk_total: { name: "disk_total", type: "n" }, + bat_current: { name: "bat_current", type: "n" }, + bat_total: { name: "bat_total", type: "n" }, + bat: { name: "bat", type: "n" }, + run: { name: "run", type: "n" } + }, + "[CLY]_star_rating": { + email: { name: "email", type: "s" }, + comment: { name: "comment", type: "s" }, + widget_id: { name: "widget_id", type: "l" }, + contactMe: { name: "contactMe", type: "s" }, + rating: { name: "rating", type: "n" }, + platform_version_rate: { name: "platform_version_rate", type: "s" } + }, + "[CLY]_nps": { + comment: { name: "comment", type: "s" }, + widget_id: { name: "widget_id", type: "l" }, + rating: { name: "rating", type: "n" }, + shown: { name: "shown", type: "s" }, + answered: { name: "answered", type: "s" } + }, + "[CLY]_survey": { + widget_id: { name: "widget_id", type: "l" }, + shown: { name: "shown", type: "s" }, + answered: { name: "answered", type: "s" } + }, + "[CLY]_push_action": { + i: { name: "i", type: "s" } + }, + "[CLY]_push_sent": { + i: { name: "i", type: "s" } + } + } +}; + +/** + * Fills user properties from dbAppUser object + * @param {object} dbAppUser - app user object + * @param {object} meta_doc - meta document + * @returns {object} userProperties, userCustom, userCampaign + */ +function fillUserProperties(dbAppUser, meta_doc) { + var userProperties = {}, + userCustom = {}, + userCampaign = {}; + var setType = ""; + + if (!dbAppUser) { + return {up: userProperties, upCustom: userCustom, upCampaign: userCampaign }; + } + var countlyUP = preset.up; + for (let i in countlyUP) { + var shortRep = common.dbUserMap[countlyUP[i].name] || countlyUP[i].name; + + if (shortRep === "fs") { + dbAppUser.fs = (dbAppUser.fac) ? dbAppUser.fac : dbAppUser.fs; + } + else if (shortRep === "ls") { + dbAppUser.ls = (dbAppUser.lac) ? dbAppUser.lac : dbAppUser.ls; + } + + if (dbAppUser[shortRep]) { + setType = countlyUP[i].type || ""; + if (meta_doc && meta_doc.up && meta_doc.up[i]) { + setType = meta_doc.up[i]; + } + userProperties[i] = dbAppUser[shortRep]; + if (setType === 's') { + userProperties[i] = userProperties[i] + ""; + } + else if (setType === 'n' && common.isNumber(userProperties[i])) { + userProperties[i] = parseFloat(userProperties[i]); + } + } + } + if (dbAppUser.custom) { + let key; + for (let i in dbAppUser.custom) { + key = common.fixEventKey(i); + + if (!key) { + continue; + } + setType = ""; + if (meta_doc && meta_doc.custom && meta_doc.custom[key] && meta_doc.custom[key]) { + setType = meta_doc.custom[key]; + } + let tmpVal; + + if (Array.isArray(dbAppUser.custom[i])) { + for (var z = 0; z < dbAppUser.custom[i].length; z++) { + dbAppUser.custom[i][z] = dbAppUser.custom[i][z] + ""; + } + tmpVal = dbAppUser.custom[i]; + } + else if (setType === "s") { + tmpVal = dbAppUser.custom[i] + ""; + } + else if (setType === "n" && common.isNumber(dbAppUser.custom[i])) { + if (dbAppUser.custom[i].length && dbAppUser.custom[i].length <= 16) { + tmpVal = parseFloat(dbAppUser.custom[i]); + } + else { + tmpVal = dbAppUser.custom[i]; + } + } + else { + tmpVal = dbAppUser.custom[i]; + } + + userCustom[key] = tmpVal; + } + } + + //add referral campaign data if any + //legacy campaign data + if (dbAppUser.cmp) { + let key; + for (let i in dbAppUser.cmp) { + key = common.fixEventKey(i); + + if (!key || key === "_id" || key === "did" || key === "bv" || key === "ip" || key === "os" || key === "r" || key === "cty" || key === "last_click") { + continue; + } + + setType = ""; + if (meta_doc && meta_doc.cmp && meta_doc.cmp[key] && meta_doc.cmp[key]) { + setType = meta_doc.cmp[key]; + } + + let tmpVal; + if (setType && setType === 's') { + tmpVal = dbAppUser.cmp[i] + ""; + } + else if (common.isNumber(dbAppUser.cmp[i])) { + if (dbAppUser.cmp[i].length && dbAppUser.cmp[i].length <= 16) { + tmpVal = parseFloat(dbAppUser.cmp[i]); + } + else { + tmpVal = dbAppUser.cmp[i]; + } + } + else if (Array.isArray(dbAppUser.cmp[i])) { + tmpVal = dbAppUser.cmp[i]; + } + else { + tmpVal = dbAppUser.cmp[i]; + } + + userCampaign[key] = tmpVal; + } + } + else { + userCampaign.c = "Organic"; + } + + return { + up: userProperties, + upCustom: userCustom, + upCampaign: userCampaign + }; +} + +var processToDrill = async function(params, drill_updates, callback) { + var events = params.qstring.events || []; + if (!Array.isArray(events)) { + log.w("invalid events passed for recording" + JSON.stringify(events)); + callback(); + return; + } + var dbAppUser = params.app_user; + //Data we would expect to have + //app Timezone to store correct ts in database(we assume passed one is in app timezone) + //Configs with events to record(for each app) + //app_user object + //Event count and segment count limit if we want to enforce those + existng values + + var eventsToInsert = []; + var timestamps = {}; + var viewUpdate = {}; + if (events.length > 0) { + for (let i = 0; i < events.length; i++) { + var currEvent = events[i]; + if (!currEvent.key) { + continue; + } + if (!currEvent.key || (currEvent.key.indexOf('[CLY]_') === 0 && plugins.internalDrillEvents.indexOf(currEvent.key) === -1)) { + continue; + } + /* + if (currEvent.key === "[CLY]_session" && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_sessions) { + continue; + } + + if (currEvent.key === "[CLY]_view" && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_views) { + continue; + }*/ + + if (currEvent.key === "[CLY]_view" && !(currEvent.segmentation && currEvent.segmentation.visit)) { + continue; + } + + /* + if (currEvent.key === "[CLY]_action" && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_actions) { + continue; + } + + if ((currEvent.key === "[CLY]_push_action" || currEvent.key === "[CLY]_push_open") && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_pushes) { + continue; + } + + if ((currEvent.key === "[CLY]_push_sent") && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_pushes_sent) { + continue; + } + + if (currEvent.key === "[CLY]_crash" && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_crashes) { + continue; + } + + if ((currEvent.key === "[CLY]_star_rating") && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_star_rating) { + continue; + } + + if ((currEvent.key.indexOf('[CLY]_apm_') === 0) && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_apm) { + continue; + } + + if ((currEvent.key === "[CLY]_consent") && !plugins.getConfig("drill", params.app && params.app.plugins, true).record_consent) { + continue; + }*/ + + + var dbEventObject = { + "a": params.app_id + "", + "e": events[i].key, + "cd": new Date(), + "ts": events[i].timestamp || Date.now().valueOf(), + "uid": params.app_user.uid, + "_uid": params.app_user._id, + "did": params.app_user.did + //d, w,m,h + }; + if (currEvent.key.indexOf('[CLY]_') === 0) { + dbEventObject.n = events[i].key; + } + else { + dbEventObject.n = events[i].key; + dbEventObject.e = "[CLY]_custom"; + } + if (currEvent.name) { + dbEventObject.n = currEvent.name; + } + + if (dbAppUser && dbAppUser[common.dbUserMap.user_id]) { + dbEventObject[common.dbUserMap.user_id] = dbAppUser[common.dbUserMap.user_id]; + } + var upWithMeta = fillUserProperties(dbAppUser, params.app?.ovveridden_types?.prop); + dbEventObject[common.dbUserMap.device_id] = params.qstring.device_id; + dbEventObject.lsid = dbAppUser.lsid; + dbEventObject[common.dbEventMap.user_properties] = upWithMeta.up; + dbEventObject.custom = upWithMeta.upCustom; + dbEventObject.cmp = upWithMeta.upCampaign; + + var eventKey = events[i].key; + //Setting params depending in event + if (eventKey === "[CLY]_session") { + dbEventObject._id = params.request_id; + } + else { + dbEventObject._id = params.request_hash + "_" + params.app_user.uid + "_" + Date.now().valueOf() + "_" + i; + } + eventKey = currEvent.key; + + var time = params.time; + if (events[i].timestamp) { + time = common.initTimeObj(params.appTimezone, events[i].timestamp); + } + if (events[i].cvid) { + dbEventObject.cvid = events[i].cvid; + } + + if (events[i].pvid) { + dbEventObject.pvid = events[i].pvid; + } + + if (events[i].id) { + dbEventObject.id = events[i].id; + } + + if (events[i].peid) { + dbEventObject.peid = events[i].peid; + } + + if (eventKey === "[CLY]_view" && currEvent && currEvent.segmentation && currEvent.segmentation._idv) { + dbEventObject._id = params.app_id + "_" + dbAppUser.uid + "_" + currEvent.segmentation._idv; + if (!events[i].id) { + events[i].id = currEvent.segmentation._idv; + } + } + if (eventKey === "[CLY]_consent") { + dbEventObject.after = dbAppUser.consent; + } + + dbEventObject[common.dbEventMap.timestamp] = time.mstimestamp; + + while (timestamps[dbEventObject[common.dbEventMap.timestamp]]) { //if we have this timestamp there somewhere - make it slighty different + dbEventObject[common.dbEventMap.timestamp] += 1; + } + timestamps[dbEventObject[common.dbEventMap.timestamp]] = true; + + /*dbEventObject.d = momentDate.year() + ":" + (momentDate.month() + 1) + ":" + momentDate.format("D"); + dbEventObject.w = momentDate.isoWeekYear() + ":w" + momentDate.isoWeek(); + dbEventObject.m = momentDate.year() + ":m" + (momentDate.month() + 1); + dbEventObject.h = momentDate.year() + ":" + (momentDate.month() + 1) + ":" + momentDate.format("D") + ":h" + momentD*/ + + events[i].hour = (typeof events[i].hour !== "undefined") ? events[i].hour : params.qstring.hour; + if (typeof events[i].hour !== "undefined") { + events[i].hour = parseInt(events[i].hour); + if (events[i].hour === 24) { + events[i].hour = 0; + } + if (events[i].hour >= 0 && events[i].hour < 24) { + upWithMeta.up.hour = events[i].hour; + } + } + + events[i].dow = (typeof events[i].dow !== "undefined") ? events[i].dow : params.qstring.dow; + if (typeof events[i].dow !== "undefined") { + events[i].dow = parseInt(events[i].dow); + if (events[i].dow === 0) { + events[i].dow = 7; + } + if (events[i].dow > 0 && events[i].dow <= 7) { + upWithMeta.up.dow = events[i].dow; + } + } + + if (currEvent.segmentation) { + var tmpSegVal; + var meta_doc = params.app?.ovveridden_types?.events; + for (var segKey in currEvent.segmentation) { + var segKeyAsFieldName = segKey.replace(/^\$|\./g, ""); + + if (segKey === "" || segKeyAsFieldName === "" || currEvent.segmentation[segKey] === null || typeof currEvent.segmentation[segKey] === "undefined") { + continue; + } + var setType = ""; + if (meta_doc && meta_doc[eventKey] && meta_doc[eventKey][segKey]) { //some type is set via settings. + setType = meta_doc[eventKey][segKey]; + } + + if (Array.isArray(currEvent.segmentation[segKey])) { + var pluginsGetConfig = plugins.getConfig("api", params.app && params.app.plugins, true); + currEvent.segmentation[segKey] = currEvent.segmentation[segKey].splice(0, (pluginsGetConfig.array_list_limit || 10)); + for (var z = 0; z < currEvent.segmentation[segKey].length; z++) { + currEvent.segmentation[segKey][z] = currEvent.segmentation[segKey][z] + ""; + currEvent.segmentation[segKey][z] = common.encodeCharacters(currEvent.segmentation[segKey][z]); + } + } + //max number check + if (setType) { //if type is set as string - we use it as string + if (setType === "s" || setType === "l" || setType === "bl" || setType === "a") { + tmpSegVal = currEvent.segmentation[segKey] + ""; + tmpSegVal = common.encodeCharacters(tmpSegVal); + } + else if (setType === "n") { + if (common.isNumber(currEvent.segmentation[segKey])) { + tmpSegVal = parseFloat(currEvent.segmentation[segKey]); + } + else { + tmpSegVal = currEvent.segmentation[segKey]; + } + } + } + else { + tmpSegVal = currEvent.segmentation[segKey]; + } + dbEventObject[common.dbEventMap.segmentations] = dbEventObject[common.dbEventMap.segmentations] || {}; + dbEventObject[common.dbEventMap.segmentations][segKeyAsFieldName] = tmpSegVal; + } + } + if (currEvent.sum && common.isNumber(currEvent.sum)) { + currEvent.sum = parseFloat(parseFloat(currEvent.sum).toFixed(5)); + } + + if (currEvent.dur && common.isNumber(currEvent.dur)) { + currEvent.dur = parseFloat(currEvent.dur); + } + + if (currEvent.count && common.isNumber(currEvent.count)) { + currEvent.count = parseInt(currEvent.count, 10); + } + else { + currEvent.count = 1; + } + dbEventObject.s = currEvent.sum || 0; + dbEventObject.dur = currEvent.dur || 0; + dbEventObject.c = currEvent.count || 1; + eventsToInsert.push({"insertOne": {"document": dbEventObject}}); + if (eventKey === "[CLY]_view") { + var view_id = crypto.createHash('md5').update(currEvent.segmentation.name).digest('hex'); + viewUpdate[view_id] = {"lvid": dbEventObject._id, "ts": dbEventObject.ts, "a": params.app_id + ""}; + if (currEvent.segmentation) { + var sgm = {}; + var have_sgm = false; + for (var key in currEvent.segmentation) { + if (key === 'platform' || !escapedViewSegments[key]) { + sgm[key] = currEvent.segmentation[key]; + have_sgm = true; + } + } + if (have_sgm) { + viewUpdate[view_id].sg = sgm; + } + } + + } + + } + } + if (drill_updates && drill_updates.length > 0) { + for (var z4 = 0; z4 < drill_updates.length;z4++) { + eventsToInsert.push(drill_updates[z4]); + } + } + if (eventsToInsert.length > 0) { + try { + await common.drillDb.collection("drill_events").bulkWrite(eventsToInsert, {ordered: false}); + callback(null); + if (Object.keys(viewUpdate).length) { + //updates app_viewdata colelction.If delayed new incoming view updates will not have reference. (So can do in aggregator only if we can insure minimal delay) + try { + await common.db.collection("app_userviews").updateOne({_id: params.app_id + "_" + params.app_user.uid}, {$set: viewUpdate}, {upsert: true}); + } + catch (err) { + log.e(err); + } + } + + } + catch (errors) { + var realError; + if (errors && Array.isArray(errors)) { + log.e(JSON.stringify(errors)); + for (let i = 0; i < errors.length; i++) { + if ([11000, 10334, 17419].indexOf(errors[i].code) === -1) { + realError = true; + } + } + + if (realError) { + callback(realError); + } + else { + callback(null); + if (Object.keys(viewUpdate).length) { + //updates app_viewdata colelction.If delayed new incoming view updates will not have reference. (So can do in aggregator only if we can insure minimal delay) + try { + await common.db.collection("app_userviews").updateOne({_id: params.app_id + "_" + params.app_user.uid}, {$set: viewUpdate}, {upsert: true}); + } + catch (err) { + log.e(err); + } + } + + } + } + else { + console.log(errors); + callback(errors); + } + } + } + else { + callback(null); + } +}; + +plugins.register("/sdk/process_user", async function(ob) { + await usage.processUserProperties(ob); +}); + +/* +Needed only if we want to enforce event limit +const processEventsArray = async function(params) { + if (params && params.qstring && params.qstring.events) { + var event_limit = plugins.getConfig("api", params.app && params.app.plugins, true).event_limit || 500; + eventDoc = await common.readBatcher.getOne("events", {"_id": common.db.ObjectID(params.app_id)}, {"transformation": "event_object"}); + var currDateWithoutTimestamp = moment(); + var missingEvents = {}; + for (var z = 0; z < params.qstring.events.length; z++) { + params.qstring.events[z].key = common.fixEventKey(params.qstring.events[z].key); + if (params.qstring.events[z].key && params.qstring.events[z].key.indexOf("[CLY]_") !== 0) { + if (eventDoc && eventDoc._list) { + if (!eventDoc._list[params.qstring.events[z].key]) { + missingEvents[params.qstring.events[z].key] = true; + } + } + else { + missingEvents[params.qstring.events[z].key] = true; + } + if (!params.qstring.events[z].timestamp || parseInt(params.qstring.events[z].timestamp, 10) > currDateWithoutTimestamp.unix()) { + params.qstring.events[z].timestamp = params.time.mstimestamp; + } + } + else { + params.qstring.events[z]._system_event = true; + } + } + + if (Object.keys(missingEvents).length) { + if (!eventDoc || (eventDoc && eventDoc._list_length < event_limit)) { + var rr = await common.db.collection("events").bulkWrite([ + {updateOne: {filter: {"_id": common.db.ObjectID(params.app_id)}, upsert: true, update: {"$addToSet": {"list": {"$each": Object.keys(missingEvents)}}}}}, + {updateOne: {filter: {"_id": common.db.ObjectID(params.app_id)}, update: {"$push": {"list": {"$each": [], "$slice": event_limit}}}}} + ], {ordered: true}); + + //Refetch + eventDoc = await common.readBatcher.getOne("events", {"_id": common.db.ObjectID(params.app_id)}, {"transformation": "event_object", "refetch": true}); + } + } + + for (var z = 0; z < params.qstring.events.length; z++) { + if (!params.qstring.events[z]._system_event && !eventDoc._list[params.qstring.events[z].key]) { + params.qstring.events[z].key = null; + } + } + } +};*/ +/** + * + * @param {object} ob - request parameters + * @param {function} done - callback function + * + * 1)Process users props from request + * 2)Update App Users + * 3)Process and insert drill + */ +const processRequestData = (ob, done) => { + //preserve time for user's previous session + var update = {}; + //check if we already processed app users for this request + if (ob.params.app_user.last_req !== ob.params.request_hash && ob.updates.length) { + for (let i = 0; i < ob.updates.length; i++) { + update = common.mergeQuery(update, ob.updates[i]); + } + } + //var SaveAppUser = Date.now().valueOf(); + + common.updateAppUser(ob.params, update, false, function() { + + /*var AfterSaveAppUser = Date.now().valueOf(); + if (AfterSaveAppUser - SaveAppUser > treshold) { + console.log("SaveAppUser time: " + (AfterSaveAppUser - SaveAppUser)); + }*/ + processToDrill(ob.params, ob.drill_updates, function(error) { + if (error) { + common.returnMessage(ob.params, 400, 'Could not record events:' + error); + } + else { + common.returnMessage(ob.params, 200, 'Success'); + done(); + } + }); + + }); +}; + +plugins.register("/sdk/process_request", async function(ob) { + //Deals with duration. Determines if we keep session start if there is one passed. + //Adds some app_user updates if needed. + await usage.setLocation(ob.params); + usage.processSession(ob); +}); + +/** + * + * @param {*} params - request parameters + * @param {*} done - callback function + * + * + * 1)Get App collection settings + * -->update app with lu. Better if we do not need to invalidate. + * 2)Get App User + * ->Process User (merge or create+refetch if new) + * 3)Process request + */ +const validateAppForWriteAPI = (params, done) => { + if (ignorePossibleDevices(params)) { + common.returnMessage(params, 400, "Device ignored"); + done(); + return; + } + + common.readBatcher.getOne("apps", {'key': params.qstring.app_key + ""}, {}, (err, app) => { + if (err) { + log.e(err); + } + if (!app || !app._id) { + common.returnMessage(params, 400, 'App does not exist'); + params.cancelRequest = "App not found or no Database connection"; + done(); + return; + } + + if (app.paused) { + common.returnMessage(params, 400, 'App is currently not accepting data'); + params.cancelRequest = "App is currently not accepting data"; + plugins.dispatch("/sdk/cancel", {params: params}); + done(); + return; + } + + if ((params.populator || params.qstring.populator) && app.locked) { + common.returnMessage(params, 403, "App is locked"); + params.cancelRequest = "App is locked"; + plugins.dispatch("/sdk/cancel", {params: params}); + done(); + return; + } + if (!validateRedirect({params: params, app: app})) { + if (!params.res.finished && !params.waitForResponse) { + common.returnOutput(params, {result: 'Success', info: 'Request redirected: ' + params.cancelRequest}); + } + done(); + return; + } + + params.app_id = app._id + ""; + params.app_cc = app.country; + params.app_name = app.name; + params.appTimezone = app.timezone; + params.app = app; + params.time = common.initTimeObj(params.appTimezone, params.qstring.timestamp); + + var time = Date.now().valueOf(); + time = Math.round((time || 0) / 1000); + if (params.app && (!params.app.last_data || params.app.last_data < time - 60 * 60 * 24) && !params.populator && !params.qstring.populator) { //update if more than day passed + //set new value for cache + common.readBatcher.updateCacheOne("apps", {'key': params.qstring.app_key + ""}, {"last_data": time}); + //set new value in database + try { + common.db.collection("apps").findOneAndUpdate({"_id": common.db.ObjectID(params.app._id)}, {"$set": {"last_data": time}}); + params.app.last_data = time; + } + catch (err3) { + log.e(err3); + } + } + + if (!checksumSaltVerification(params)) { + done(); + return; + } + + plugins.dispatch("/sdk/validate_request", {params: params}, async function() { //validates request if there is no reason to block/cancel it + if (params.cancelRequest) { + if (!params.res.finished && !params.waitForResponse) { + common.returnOutput(params, {result: 'Success', info: 'Request ignored: ' + params.cancelRequest}); + //common.returnMessage(params, 200, 'Request ignored: ' + params.cancelRequest); + } + common.log("request").i('Request ignored: ' + params.cancelRequest, params.req.url, params.req.body); + done(); + return; + } + try { + var user = await common.db.collection('app_users' + params.app_id).findOne({'_id': params.app_user_id}); + } + catch (err2) { + common.returnMessage(params, 400, 'Cannot get app user'); + params.cancelRequest = "Cannot get app user or no Database connection"; + done(); + return; + } + + params.app_user = user || {}; + params.collectedMetrics = {}; + + let payload = params.href.substr(3) || ""; + if (params.req && params.req.method && params.req.method.toLowerCase() === 'post') { + payload += "&" + params.req.body; + } + //remove dynamic parameters + payload = payload.replace(new RegExp("[?&]?(rr=[^&\n]+)", "gm"), ""); + payload = payload.replace(new RegExp("[?&]?(checksum=[^&\n]+)", "gm"), ""); + payload = payload.replace(new RegExp("[?&]?(checksum256=[^&\n]+)", "gm"), ""); + params.request_hash = common.crypto.createHash('sha1').update(payload).digest('hex') + (params.qstring.timestamp || params.time.mstimestamp); + if (plugins.getConfig("api", params.app && params.app.plugins, true).prevent_duplicate_requests) { + //check unique millisecond timestamp, if it is the same as the last request had, + //then we are having duplicate request, due to sudden connection termination + if (params.app_user.last_req === params.request_hash) { + params.cancelRequest = "Duplicate request"; + } + } + + if (params.qstring.metrics && typeof params.qstring.metrics === "string") { + try { + params.qstring.metrics = JSON.parse(params.qstring.metrics); + } + catch (SyntaxError) { + console.log('Parse metrics JSON failed', params.qstring.metrics, params.req.url, params.req.body); + } + } + if (!params.cancelRequest) { + processUser(params, function(userErr) { + if (userErr) { + if (!params.res.finished) { + common.returnMessage(params, 400, userErr); + } + } + else { + var ob = {params: params, app: app, updates: [], drill_updates: []}; + ob.params.request_id = ob.params.request_hash + "_" + ob.params.app_user.uid + "_" + ob.params.time.mstimestamp; + plugins.dispatch("/sdk/process_request", ob, function() { //collects all metrics + plugins.dispatch("/sdk/validate_user", ob, function() { + if (params.cancelRequest) { + if (!params.res.finished && !params.waitForResponse) { + common.returnOutput(params, {result: 'Success', info: 'Request ignored: ' + params.cancelRequest}); + } + common.log("request").i('Request ignored: ' + params.cancelRequest, params.req.url, params.req.body); + done(); + return; + } + else { + ob.params.previous_session = ob.params.app_user.lsid; + ob.params.previous_session_start = ob.params.app_user.ls; + + if (ob.params.qstring.begin_session) { + params.qstring.events = params.qstring.events || []; + params.qstring.events.unshift({ + key: "[CLY]_session", + dur: params.qstring.session_duration || 0, + count: 1, + timestamp: params.time.mstimestamp, + segmentation: { + request_id: params.request_id, + prev_session: params.previous_session, + prev_start: params.previous_session_start, + postfix: crypto.createHash('md5').update(params.app_user.did + "").digest('base64')[0], + ended: "false" + } + }); + } + plugins.dispatch("/sdk/process_user", ob, function() { // + processRequestData(ob, done); + }); + } + }); + }); + } + }); + } + else { + if (!params.res.finished && !params.waitForResponse) { + common.returnOutput(params, {result: 'Success', info: 'Request ignored: ' + params.cancelRequest}); + //common.returnMessage(params, 200, 'Request ignored: ' + params.cancelRequest); + } + common.log("request").i('Request ignored: ' + params.cancelRequest, params.req.url, params.req.body); + done(); + return; + } + }); + }); +}; +/** + * + * @param {array} requests - array with requests + * @param {object} params - params object + */ +const processBulkRequest = async function(requests, params) { + const appKey = params.qstring.app_key; + var skippedRequests = []; + for (var i = 0; i < requests.length; i++) { + if (!requests[i] || (!requests[i].app_key && !appKey) || !requests[i].device_id) { + continue; + } + else { + requests[i].app_key = requests[i].app_key || appKey; + const tmpParams = { + 'app_id': '', + 'app_cc': '', + 'ip_address': requests[i].ip_address || common.getIpAddress(params.req), + 'user': { + 'country': requests[i].country_code || 'Unknown', + 'city': requests[i].city || 'Unknown' + }, + 'qstring': requests[i], + 'href': "/i", + 'res': params.res, + 'req': params.req, + 'promises': [], + 'bulk': true, + 'populator': params.qstring.populator, + 'blockResponses': true + }; + + tmpParams.qstring.device_id += ""; + tmpParams.app_user_id = common.crypto.createHash('sha1') + .update(tmpParams.qstring.app_key + tmpParams.qstring.device_id + "") + .digest('hex'); + + await new Promise((resolve) => { + validateAppForWriteAPI(tmpParams, () => { + //log request + if (tmpParams.cancelRequest) { + skippedRequests.push(tmpParams.qstring); + } + plugins.dispatch("/sdk/log", {params: tmpParams}); + resolve(); + }); + }); + + } + } + common.unblockResponses(params); + common.returnMessage(params, 200, 'Success'); +}; + + +/** + * Process request function + * @param {object} params - request parameters + * @returns {boolean} - returns false if request is cancelled + */ +const processRequest = (params) => { + if (!params.req || !params.req.url) { + return common.returnMessage(params, 400, "Please provide request data"); + } + + params.tt = Date.now().valueOf(); + const urlParts = url.parse(params.req.url, true), + queryString = urlParts.query, + paths = urlParts.pathname.split("/"); + + params.href = urlParts.href; + params.qstring = params.qstring || {}; + params.res = params.res || {}; + params.urlParts = urlParts; + params.paths = paths; + + params.req.headers = params.req.headers || {}; + params.req.socket = params.req.socket || {}; + params.req.connection = params.req.connection || {}; + + //copying query string data as qstring param + if (queryString) { + for (let i in queryString) { + params.qstring[i] = queryString[i]; + } + } + + //copying body as qstring param + if (params.req.body && typeof params.req.body === "object") { + for (let i in params.req.body) { + params.qstring[i] = params.req.body[i]; + } + } + + if (params.qstring.app_id && params.qstring.app_id.length !== 24) { + common.returnMessage(params, 400, 'Invalid parameter "app_id"'); + return false; + } + + if (params.qstring.user_id && params.qstring.user_id.length !== 24) { + common.returnMessage(params, 400, 'Invalid parameter "user_id"'); + return false; + } + + //remove countly path + if (common.config.path === "/" + paths[1]) { + paths.splice(1, 1); + } + + let apiPath = ''; + + for (let i = 1; i < paths.length; i++) { + if (i > 2) { + break; + } + + apiPath += "/" + paths[i]; + } + + params.apiPath = apiPath; + params.fullPath = paths.join("/"); + + switch (apiPath) { + case '/i': { + if ([true, "true"].includes(plugins.getConfig("api", params.app && params.app.plugins, true).trim_trailing_ending_spaces)) { + params.qstring = common.trimWhitespaceStartEnd(params.qstring); + } + params.ip_address = params.qstring.ip_address || common.getIpAddress(params.req); + params.user = {}; + + if (!params.qstring.app_key || !params.qstring.device_id) { + common.returnMessage(params, 400, 'Missing parameter "app_key" or "device_id"'); + return false; + } + else { + //make sure device_id is string + params.qstring.device_id += ""; + params.qstring.app_key += ""; + // Set app_user_id that is unique for each user of an application. + params.app_user_id = common.crypto.createHash('sha1') + .update(params.qstring.app_key + params.qstring.device_id + "") + .digest('hex'); + } + + if (params.qstring.events && typeof params.qstring.events === "string") { + try { + params.qstring.events = JSON.parse(params.qstring.events); + } + catch (SyntaxError) { + console.log('Parse events JSON failed', params.qstring.events, params.req.url, params.req.body); + params.qstring.events = []; + + } + } + + if (!params.qstring.events && !Array.isArray(params.qstring.events)) { + params.qstring.events = []; + } + validateAppForWriteAPI(params, () => { + //log request + plugins.dispatch("/sdk/log", {params: params}); + }); + break; + } + case '/i/bulk': { + let requests = params.qstring.requests; + if (requests && typeof requests === "string") { + try { + requests = JSON.parse(requests); + } + catch (SyntaxError) { + console.log('Parse bulk JSON failed', requests, params.req.url, params.req.body); + requests = null; + } + } + if (!requests) { + common.returnMessage(params, 400, 'Missing parameter "requests"'); + return false; + } + if (!Array.isArray(requests)) { + console.log("Passed invalid param for request. Expected Array, got " + typeof requests); + common.returnMessage(params, 400, 'Invalid parameter "requests"'); + return false; + } + common.blockResponses(params);//no response till finished processing + + processBulkRequest(requests, params); + break; + } + default: + if (!plugins.dispatch(apiPath, { + params: params, + paths: paths + })) { + if (!plugins.dispatch(params.fullPath, { + params: params, + paths: paths + })) { + common.returnMessage(params, 400, 'Invalid path'); + } + } + } + +}; + +module.exports = {processRequest: processRequest}; diff --git a/api/ingestor/usage.js b/api/ingestor/usage.js new file mode 100644 index 00000000000..ad154d474b0 --- /dev/null +++ b/api/ingestor/usage.js @@ -0,0 +1,643 @@ +var usage = {}, + common = require('./../utils/common.js'), + geoip = require('geoip-lite'), + geocoder = require('offline-geocoder')(), + log = require('./../utils/log.js')('ingestor:usage'), + plugins = require('./../../plugins/pluginManager.js'), + moment = require('moment-timezone'); + +/** +* Get location either from coordinate to populate country and city, or from country and city to get coordinates +* @param {params} params - params object +* @param {object} loc - location object +* @param {number} loc.lat - lattitude +* @param {number} loc.lon - longitude +* @param {string} loc.country - country +* @param {string} loc.city - city +* @param {string} loc.tz - timezone +* @returns {Promise} promise which resolves missing location parameters +**/ +function locFromGeocoder(params, loc) { + return new Promise(resolve => { + try { + let promise; + if (loc.lat !== undefined && loc.lon !== undefined) { + loc.gps = true; + promise = geocoder.reverse(loc.lat, loc.lon); + } + else if (loc.city && loc.country) { + loc.gps = false; + promise = geocoder.location(loc.city, loc.country); + } + else { + promise = Promise.resolve(); + } + promise.then(data => { + loc.country = loc.country || (data && data.country && data.country.id); + loc.city = loc.city || (data && data.name); + loc.lat = loc.lat === undefined ? data && data.coordinates && data.coordinates.latitude : loc.lat; + loc.lon = loc.lon === undefined ? data && data.coordinates && data.coordinates.longitude : loc.lon; + if (!loc.tz && data && data.tz) { + var zone = moment.tz.zone(data.tz); + if (zone) { + loc.tz = -zone.utcOffset(new Date(params.time.mstimestamp || Date.now())); + } + } + resolve(loc); + }, err => { + log.w('Error to reverse geocode: %j', err); + resolve(loc); + }); + } + catch (err) { + log.e('Error in geocoder: %j', err, err.stack); + resolve(loc); + } + }); +} + +/** +* Get location data from ip address +* @param {object} loc - location object +* @param {number} loc.lat - lattitude +* @param {number} loc.lon - longitude +* @param {string} loc.country - country +* @param {string} loc.city - city +* @param {string} loc.tz - timezone +* @param {string} ip_address - User's ip address +* @returns {Promise} promise which resolves missing location parameters +**/ +function locFromGeoip(loc, ip_address) { + return new Promise(resolve => { + try { + var data = geoip.lookup(ip_address); + if (data) { + loc.country = loc.country || (data && data.country); + loc.city = loc.city || (data && data.city); + loc.region = loc.region || (data && data.region); + loc.lat = loc.lat === undefined ? (data && data.ll && data.ll[0]) : loc.lat; + loc.lon = loc.lon === undefined ? (data && data.ll && data.ll[1]) : loc.lon; + resolve(loc); + } + else { + return resolve(loc); + } + } + catch (e) { + log.e('Error in geoip: %j', e); + resolve(loc); + } + }); +} + +/** + * Set Location information in params but donot update it in users document + * @param {params} params - params object + * @returns {Promise} promise which resolves upon completeing processing + */ +usage.setLocation = function(params) { + if ('tz' in params.qstring) { + params.user.tz = parseInt(params.qstring.tz); + if (isNaN(params.user.tz)) { + delete params.user.tz; + } + } + + return new Promise(resolve => { + var loc = { + country: params.qstring.country_code, + city: params.qstring.city, + tz: params.user.tz + }; + + if ('location' in params.qstring) { + if (params.qstring.location) { + var coords = params.qstring.location.split(','); + if (coords.length === 2) { + var lat = parseFloat(coords[0]), + lon = parseFloat(coords[1]); + + if (!isNaN(lat) && !isNaN(lon)) { + loc.lat = lat; + loc.lon = lon; + } + } + } + } + + if (loc.lat !== undefined || (loc.country && loc.city)) { + locFromGeocoder(params, loc).then(loc2 => { + if (loc2.city && loc2.country && loc2.lat !== undefined) { + usage.setUserLocation(params, loc2); + return resolve(); + } + else { + loc2.city = loc2.country === undefined ? undefined : loc2.city; + loc2.country = loc2.city === undefined ? undefined : loc2.country; + locFromGeoip(loc2, params.ip_address).then(loc3 => { + usage.setUserLocation(params, loc3); + return resolve(); + }); + } + }); + } + else { + locFromGeoip(loc, params.ip_address).then(loc2 => { + usage.setUserLocation(params, loc2); + return resolve(); + }); + } + }); +}; + +/** + * Set user location in params + * @param {params} params - params object + * @param {object} loc - location info + */ +usage.setUserLocation = function(params, loc) { + params.user.country = plugins.getConfig('api', params.app && params.app.plugins, true).country_data === false ? undefined : loc.country; + params.user.region = plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true ? loc.region : undefined; + params.user.city = (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === false || + plugins.getConfig('api', params.app && params.app.plugins, true).country_data === false) ? undefined : loc.city; +}; + +usage.processCoreMetrics = function(params) { + if (params && params.qstring && params.qstring.metrics) { + common.processCarrier(params.qstring.metrics); + + if (params.qstring.metrics._carrier) { + params.collectedMetrics[common.dbUserMap.carrier] = params.qstring.metrics._carrier; + } + if (params.qstring.metrics._os) { + params.qstring.metrics._os += ""; + if (params.qstring.metrics._os_version && !params.is_os_processed) { + params.qstring.metrics._os_version += ""; + + if (common.os_mapping[params.qstring.metrics._os.toLowerCase()] && !params.qstring.metrics._os_version.startsWith(common.os_mapping[params.qstring.metrics._os.toLowerCase()])) { + params.qstring.metrics._os_version = common.os_mapping[params.qstring.metrics._os.toLowerCase()] + params.qstring.metrics._os_version; + params.is_os_processed = true; + } + else { + params.qstring.metrics._os = params.qstring.metrics._os.replace(/\[|\]/g, ''); + params.qstring.metrics._os_version = "[" + params.qstring.metrics._os + "]" + params.qstring.metrics._os_version; + params.is_os_processed = true; + } + params.collectedMetrics[common.dbUserMap.platform_version] = params.qstring.metrics._os_version; + } + params.collectedMetrics[common.dbUserMap.platform] = params.qstring.metrics._os; + } + if (params.qstring.metrics._app_version) { + params.qstring.metrics._app_version += ""; + if (params.qstring.metrics._app_version.indexOf('.') === -1 && common.isNumber(params.qstring.metrics._app_version)) { + params.qstring.metrics._app_version += ".0"; + } + params.collectedMetrics[common.dbUserMap.app_version] = params.qstring.metrics._app_version; + } + if (!params.qstring.metrics._device_type && params.qstring.metrics._device) { + var device = (params.qstring.metrics._device + ""); + if (params.qstring.metrics._os === "iOS" && (device.startsWith("iPhone") || device.startsWith("iPod"))) { + params.qstring.metrics._device_type = "mobile"; + } + else if (params.qstring.metrics._os === "iOS" && device.startsWith("iPad")) { + params.qstring.metrics._device_type = "tablet"; + } + else if (params.qstring.metrics._os === "watchOS" && device.startsWith("Watch")) { + params.qstring.metrics._device_type = "wearable"; + } + else if (params.qstring.metrics._os === "tvOS" && device.startsWith("AppleTV")) { + params.qstring.metrics._device_type = "smarttv"; + } + else if (params.qstring.metrics._os === "macOS" && (device.startsWith("Mac") || device.startsWith("iMac"))) { + params.qstring.metrics._device_type = "desktop"; + } + } + if (params.qstring.metrics._device_type) { + params.collectedMetrics[common.dbUserMap.device_type] = params.qstring.metrics._device_type; + } + if (params.qstring.metrics._device) { + params.collectedMetrics[common.dbUserMap.device] = params.qstring.metrics._device; + } + if (!params.qstring.metrics._manufacturer && params.qstring.metrics._os) { + if (params.qstring.metrics._os === "iOS") { + params.qstring.metrics._manufacturer = "Apple"; + } + else if (params.qstring.metrics._os === "watchOS") { + params.qstring.metrics._manufacturer = "Apple"; + } + else if (params.qstring.metrics._os === "tvOS") { + params.qstring.metrics._manufacturer = "Apple"; + } + else if (params.qstring.metrics._os === "macOS") { + params.qstring.metrics._manufacturer = "Apple"; + } + } + if (params.qstring.metrics._manufacturer) { + params.collectedMetrics[common.dbUserMap.manufacturer] = params.qstring.metrics._manufacturer; + } + if (params.qstring.metrics._has_hinge) { + var hasHingeValue = params.qstring.metrics._has_hinge; + if (hasHingeValue === "true" || hasHingeValue === true || hasHingeValue === "hinged") { + params.qstring.metrics._has_hinge = "hinged"; + } + else { + params.qstring.metrics._has_hinge = "not_hinged"; + } + params.collectedMetrics[common.dbUserMap.has_hinge] = params.qstring.metrics._has_hinge; + } + if (params.qstring.metrics._resolution) { + params.collectedMetrics[common.dbUserMap.resolution] = params.qstring.metrics._resolution; + } + } +}; + +usage; + + +/** + * Process all metrics and return + * @param {params} params - params object + * @returns {object} params + */ +usage.returnRequestMetrics = function(params) { + usage.processCoreMetrics(params); + for (var key in params.collectedMetrics) { + // We check if country data logging is on and user's country is the configured country of the app + if (key === "cc" && (plugins.getConfig("api", params.app && params.app.plugins, true).country_data === false || params.app_cc !== params.user.country)) { + continue; + } + // We check if city data logging is on and user's country is the configured country of the app + if (key === "cty" && (plugins.getConfig("api", params.app && params.app.plugins, true).city_data === false || params.app_cc !== params.user.country)) { + continue; + } + + if (params.collectedMetrics[key]) { + var escapedMetricVal = (params.collectedMetrics[key] + "").replace(/^\$/, "").replace(/\./g, ":"); + params.collectedMetrics[key] = escapedMetricVal; + } + else { + delete params.collectedMetrics[key]; + } + } + return params.collectedMetrics; +}; + +usage.processSession = function(ob) { + var params = ob.params; + var userProps = {}; + var session_duration = 0; + var update = {}; + + if (params.qstring.session_duration) { + session_duration = parseInt(params.qstring.session_duration); + var session_duration_limit = parseInt(plugins.getConfig("api", params.app && params.app.plugins, true).session_duration_limit); + if (session_duration) { + if (session_duration_limit && session_duration > session_duration_limit) { + session_duration = session_duration_limit; + } + if (session_duration < 0) { + session_duration = 30; + } + } + } + + if (params.qstring.begin_session) { + var lastEndSession = params.app_user[common.dbUserMap.last_end_session_timestamp]; + if (!params.app_user[common.dbUserMap.has_ongoing_session]) { + userProps[common.dbUserMap.has_ongoing_session] = true; + } + + if (!params.qstring.ignore_cooldown && lastEndSession && (params.time.timestamp - lastEndSession) < plugins.getConfig("api", params.app && params.app.plugins, true).session_cooldown) { + console.log("Skipping because of cooldown"); + console.log(params.time.timestamp - lastEndSession); + console.log(plugins.getConfig("api", params.app && params.app.plugins, true).session_cooldown); + delete params.qstring.begin_session;//do not start a new session. + } + else { + + if (params.app_user[common.dbUserMap.has_ongoing_session]) { + params.qstring.end_session = {"lsid": ob.params.app_user.lsid, "ls": ob.params.app_user.ls, "sd": ob.params.app_user.sd}; + } + userProps[common.dbUserMap.last_begin_session_timestamp] = params.time.timestamp; + userProps.lsid = params.request_id; + + if (params.app_user[common.dbUserMap.has_ongoing_session]) { + var drill_updates = {}; + if (params.app_user.lsid) { + if (params.app_user.sd > 0) { + drill_updates.dur = params.app_user.sd; + } + if (params.app_user.custom && Object.keys(params.app_user.custom).length > 0) { + drill_updates.custom = JSON.parse(JSON.stringify(params.app_user.custom)); + } + drill_updates["sg.ended"] = "true"; + drill_updates.lu = new Date(); + //if (drill_updates.dur || drill_updates.custom) { + ob.drill_updates.push({"updateOne": {"filter": {"_id": params.app_user.lsid}, "update": {"$set": drill_updates}}}); + //} + } + userProps.sd = 0 + session_duration; + userProps.data = {}; + } + //new session + var isNewUser = (params.app_user && params.app_user[common.dbUserMap.first_seen]) ? false : true; + if (isNewUser) { + userProps[common.dbUserMap.first_seen] = params.time.timestamp; + userProps[common.dbUserMap.last_seen] = params.time.timestamp; + } + else { + if (parseInt(params.app_user[common.dbUserMap.last_seen], 10) < params.time.timestamp) { + userProps[common.dbUserMap.last_seen] = params.time.timestamp; + } + } + + if (!update.$inc) { + update.$inc = {}; + } + update.$inc.sc = 1; + } + } + else if (params.qstring.end_session) { + // check if request is too old, ignore it + if (!params.qstring.ignore_cooldown) { + userProps[common.dbUserMap.last_end_session_timestamp] = params.time.timestamp; + } + else { + var drill_updates2 = {}; + if (params.app_user.lsid) { + if (params.app_user.sd > 0 || session_duration > 0) { + drill_updates2.dur = params.app_user.sd + (session_duration || 0); + } + if (params.app_user.custom && Object.keys(params.app_user.custom).length > 0) { + drill_updates2.custom = JSON.parse(JSON.stringify(params.app_user.custom)); + } + drill_updates2["sg.ended"] = "true"; + drill_updates2.lu = new Date(); + //if (drill_updates2.dur || drill_updates2.custom) { + ob.drill_updates.push({"updateOne": {"filter": {"_id": params.app_user.lsid}, "update": {"$set": drill_updates2}}}); + //} + } + userProps.data = {}; + } + if (params.app_user[common.dbUserMap.has_ongoing_session]) { + if (!update.$unset) { + update.$unset = {}; + } + update.$unset[common.dbUserMap.has_ongoing_session] = ""; + } + } + + if (!params.qstring.begin_session) { + if (session_duration) { + if (!update.$inc) { + update.$inc = {}; + } + update.$inc.sd = session_duration; + update.$inc.tsd = session_duration; + params.session_duration = (params.app_user.sd || 0) + session_duration; + } + } + + for (var key in userProps) { + if (userProps[key] === params.app_user[key]) { + delete userProps[key]; + } + } + + if (Object.keys(userProps).length) { + update.$set = userProps; + } + + if (Object.keys(update).length) { + ob.updates.push(update); + } + usage.processCoreMetrics(params); //Collexts core metrics + +}; + +usage.processUserProperties = async function(ob) { + var params = ob.params; + var userProps = {}; + var update = {}; + params.user = {}; + var config = plugins.getConfig("api", params.app && params.app.plugins, true); + + if (params.qstring.tz) { + var tz = parseInt(params.qstring.tz); + if (isNaN(tz)) { + userProps.tz = tz; + } + } + + if (params.qstring.country_code) { + userProps.cc = params.qstring.country_code; + } + + if (params.qstring.region) { + userProps.rgn = params.qstring.region; + } + + if (params.qstring.city) { + userProps.cty = params.qstring.city; + } + var locationData; + if (params.qstring.location) { + var coords = (params.qstring.location + "").split(','); + if (coords.length === 2) { + var lat = parseFloat(coords[0]), + lon = parseFloat(coords[1]); + + if (!isNaN(lat) && !isNaN(lon)) { + userProps.loc = { + gps: true, + geo: { + type: 'Point', + coordinates: [lon, lat] + }, + date: params.time.mstimestamp + }; + locationData = await locFromGeocoder(params, { + country: userProps.cc, + city: userProps.cc, + tz: userProps.tz, + lat: userProps.loc && userProps.loc.geo.coordinates[1], + lon: userProps.loc && userProps.loc.geo.coordinates[0] + }); + + if (!userProps.cc && locationData.country) { + userProps.cc = locationData.country; + } + + if (!userProps.rgn && locationData.region) { + userProps.rgn = locationData.region; + } + + if (!userProps.cty && locationData.city) { + userProps.cty = locationData.city; + } + } + } + } + if (params.qstring.begin_session && params.qstring.location === "") { + //user opted out of location tracking + userProps.cc = userProps.rgn = userProps.cty = 'Unknown'; + if (userProps.loc) { + delete userProps.loc; + } + if (params.app_user.loc) { + if (!update.$unset) { + update.$unset = {}; + } + update.$unset = {loc: 1}; + } + } + else if (params.qstring.begin_session && params.qstring.location !== "") { + if (userProps.loc !== undefined || (userProps.cc && userProps.cty)) { + let data = locationData || await locFromGeocoder(params, { + country: userProps.cc, + city: userProps.cc, + tz: userProps.tz, + lat: userProps.loc && userProps.loc.geo.coordinates[1], + lon: userProps.loc && userProps.loc.geo.coordinates[0] + }); + if (data) { + if (!userProps.cc && data.country) { + userProps.cc = data.country; + } + + if (!userProps.rgn && data.region) { + userProps.rgn = data.region; + } + + if (!userProps.cty && data.city) { + userProps.cty = data.city; + } + + if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && typeof data.lat !== "undefined" && typeof data.lon !== "undefined") { + // only override lat/lon if no recent gps location exists in user document + if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) { + userProps.loc = { + gps: false, + geo: { + type: 'Point', + coordinates: [data.ll[1], data.ll[0]] + }, + date: params.time.mstimestamp + }; + } + } + } + } + else { + try { + let data = geoip.lookup(params.ip_address); + if (data) { + if (!userProps.cc && data.country) { + userProps.cc = data.country; + } + + if (!userProps.rgn && data.region) { + userProps.rgn = data.region; + } + + if (!userProps.cty && data.city) { + userProps.cty = data.city; + } + + if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && data.ll && typeof data.ll[0] !== "undefined" && typeof data.ll[1] !== "undefined") { + // only override lat/lon if no recent gps location exists in user document + if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) { + userProps.loc = { + gps: false, + geo: { + type: 'Point', + coordinates: [data.ll[1], data.ll[0]] + }, + date: params.time.mstimestamp + }; + } + } + } + } + catch (e) { + log.e('Error in geoip: %j', e); + } + } + if (!userProps.cc) { + userProps.cc = "Unknown"; + } + if (!userProps.cty) { + userProps.cty = "Unknown"; + } + if (!userProps.rgn) { + userProps.rgn = "Unknown"; + } + } + + if (config.country_data === false) { + userProps.cc = 'Unknown'; + userProps.cty = 'Unknown'; + } + + if (config.city_data === false) { + userProps.cty = 'Unknown'; + } + + params.user.country = userProps.cc || "Unknown"; + params.user.city = userProps.cty || "Unknown"; + + //if we have metrics, let's process metrics + if (params.qstring.metrics) { + //Collect all metrics + var up = usage.returnRequestMetrics(params); + if (Object.keys(up).length) { + for (let key in up) { + userProps[key] = up[key]; + } + } + } + + + if (params.qstring.events) { + var eventCount = 0; + for (let i = 0; i < params.qstring.events.length; i++) { + let currEvent = params.qstring.events[i]; + if (currEvent.key === "[CLY]_orientation") { + if (currEvent.segmentation && currEvent.segmentation.mode) { + userProps.ornt = currEvent.segmentation.mode; + } + } + if (!(currEvent.key + "").startsWith("[CLY]_")) { + eventCount++; + currEvent.ce = false; + } + else { + currEvent.ce = true; + } + } + if (eventCount > 0) { + if (!update.$inc) { + update.$inc = {}; + } + + update.$inc["data.events"] = eventCount; + } + } + + //do not write values that are already assignd to user + for (var key in userProps) { + if (userProps[key] === params.app_user[key]) { + delete userProps[key]; + } + } + + if (Object.keys(userProps).length) { + update.$set = userProps; + } + + if (Object.keys(update).length) { + ob.updates.push(update); + } +}; + +module.exports = usage; diff --git a/api/jobs/appExpire.js b/api/jobs/appExpire.js index 7fb968c19f7..e33b9bae3df 100644 --- a/api/jobs/appExpire.js +++ b/api/jobs/appExpire.js @@ -1,15 +1,34 @@ -'use strict'; - -const job = require('../parts/jobs/job.js'), - async = require('async'), - plugins = require('../../plugins/pluginManager.js'), - log = require('../utils/log.js')('job:appExpire'), - common = require('../utils/common.js'), - crypto = require('crypto'); +const async = require('async'); +const plugins = require('../../plugins/pluginManager.js'); +const log = require('../utils/log.js')('job:appExpire'); +const common = require('../utils/common.js'); +const crypto = require('crypto'); +const {Job} = require("../../jobServer"); /** Class for the user mergind job **/ -class AppExpireJob extends job.Job { +class AppExpireJob extends Job { + + /** + * Determines if the job should be enabled when created + * @public + * @returns {boolean} True if job should be enabled by default, false otherwise + */ + getEnabled() { + return false; + } + + /** + * Get schedule for the job + * @returns {GetScheduleConfig} Schedule configuration object + */ + getSchedule() { + return { + type: 'schedule', + value: '15 4 * * *' // every day at 4:15 AM + }; + } + /** * Run the job * @param {Db} database connection diff --git a/api/jobs/clear.js b/api/jobs/clear.js deleted file mode 100644 index 1150e4c757a..00000000000 --- a/api/jobs/clear.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict'; - -const job = require('../parts/jobs/job.js'), - log = require('../utils/log.js')('job:clear'); - -/** Class for job of clearing old jobs **/ -class ClearJob extends job.Job { - /** - * Run the job - * @param {Db} db connection - * @param {done} done callback - */ - run(db, done) { - log.d('Clearing jobs ...'); - var query = { - $and: [ - {status: {$in: [job.STATUS.DONE, job.STATUS.CANCELLED]}}, - { - $or: [ - {finished: {$exists: false}}, - {finished: {$lt: Date.now() - 60 * 60 * 24 * 1000}}, - {finished: null} - ] - } - ] - }; - - db.collection('jobs').deleteMany(query, (err, result) => { - if (err) { - log.e('Error while clearing jobs: ', err); - } - else { - log.d('Done clearing old jobs done before %j:', query.$and[1].$or[1].finished.$lt, result.deletedCount); - } - done(err); - }); - } -} - -module.exports = ClearJob; \ No newline at end of file diff --git a/api/jobs/clearAutoTasks.js b/api/jobs/clearAutoTasks.js index f2bf6e929ac..8174626a96b 100644 --- a/api/jobs/clearAutoTasks.js +++ b/api/jobs/clearAutoTasks.js @@ -1,8 +1,10 @@ "use strict"; -const job = require("../parts/jobs/job.js"); +// const job = require("../parts/jobs/job.js"); const log = require('../utils/log.js')('job:clearAutoTasks'); const taskManager = require('../utils/taskmanager'); +const Job = require("../../jobServer/Job"); + /** * clear task record in db with task id * @param {string} taskId - the id of task in db. @@ -20,7 +22,18 @@ const clearTaskRecord = (taskId) => { }; /** Class for job of clearing auto tasks created long time ago **/ -class ClearAutoTasks extends job.Job { +class ClearAutoTasks extends Job { + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "0 2 * * *" // Every day at 2:00 AM + }; + } + /** * Run the job * @param {Db} db connection diff --git a/api/jobs/clearTokens.js b/api/jobs/clearTokens.js index 2a3932c2313..9587479acc7 100644 --- a/api/jobs/clearTokens.js +++ b/api/jobs/clearTokens.js @@ -1,10 +1,23 @@ 'use strict'; -const job = require('../parts/jobs/job.js'), - authorize = require('../utils/authorizer.js'); +// const job = require('../parts/jobs/job.js'), +const authorize = require('../utils/authorizer.js'); +const Job = require("../../jobServer/Job"); /** Class for job of clearing tokens **/ -class CleanTokensJob extends job.Job { +class CleanTokensJob extends Job { + + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "30 2 * * *" // Every day at 2:30 AM + }; + } + /** * Run the job * @param {Db} db connection diff --git a/api/jobs/ipcTest.js b/api/jobs/ipcTest.js deleted file mode 100644 index e7e63d39bd6..00000000000 --- a/api/jobs/ipcTest.js +++ /dev/null @@ -1,128 +0,0 @@ -'use strict'; - -const job = require('../parts/jobs/job.js'), - res = require('../parts/jobs/resource.js'), - log = require('../utils/log.js')('job:ipcTest'); - -/** Class for testing resource handling for jobs **/ -class TestResource extends res.Resource { - /** - * Open resource - * @returns {Promise} promise - **/ - open() { - this.a = 0; - return new Promise((resolve) => { - log.d('open'); - setTimeout(() => { - this.opened(); - resolve(); - }, 2000); - }); - } - - /** - * Close resource - * @returns {Promise} promise - **/ - close() { - return new Promise((resolve) => { - log.d('close'); - setTimeout(() => { - this.closed(); - resolve(); - }, 2000); - }); - } - - /** - * Check if resource is used - * @returns {Promise} promise - **/ - checkActive() { - return new Promise((resolve) => { - log.d('checkActive'); - setTimeout(() => { - resolve(this.a++ > 0 ? false : true); - }, 2000); - }); - } - -} -/** Class for testing ipc jobs **/ -class Test extends job.IPCJob { - /** - * Create resource - * @returns {Resource} resourse - **/ - createResource() { - return new TestResource(); - } - - /** - * Create between processes - * @returns {Promise} promise - **/ - divide() { - return new Promise((resolve) => { - log.d('dividing ', this._json); - - setTimeout(() => { - resolve([{ - smth: 1, - size: 40 - }], [{ - smth: 1, - size: 50 - }]); - }, 500); - }); - } - - /** - * Run the job - * @param {Db} db connection - * @param {done} done callback - * @param {function} progress to report progress of the job - */ - run(db, done, progress) { - log.d('running ', this._json); - log.d('resource is ', typeof this.resource); - log.d('resource is open? ', this.resource.isOpen); - log.d('resource is active? ', this.resource.isAssigned); - - if (this.done < 10) { - setTimeout(() => { - progress(this._json.size, 10, 'ten'); - }, 1000); - } - - if (this.done < 20) { - setTimeout(() => { - progress(this._json.size, 20, 'twenty'); - // a = b; - }, 5000); - } - - if (this.done < 30) { - setTimeout(() => { - progress(this._json.size, 30, 'thirty'); - }, 6000); - } - - setTimeout(() => { - progress(100, 100, 'sixty'); - done(); - }, 60000); - - setTimeout(() => { - log.d('after done resource is ', typeof this.resource); - done('Big fat error'); - db.collection('jobs').findOne({_id: this._json._id}, (err, obj) => { - log.d('after done job findOne ', err, obj); - }); - }, 120000); - } -} - -module.exports = Test; \ No newline at end of file diff --git a/api/jobs/ping.js b/api/jobs/ping.js index 984e82c566c..f9c81a5780d 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -1,28 +1,45 @@ 'use strict'; -const job = require('../parts/jobs/job.js'), - log = require('../utils/log.js')('job:ping'), - countlyConfig = require("../../frontend/express/config.js"), - versionInfo = require('../../frontend/express/version.info'), - plugins = require('../../plugins/pluginManager.js'), - request = require('countly-request')(plugins.getConfig("security")); +// const job = require('../parts/jobs/job.js'); +const log = require('../utils/log.js')('job:ping'); +const countlyConfig = require("../../frontend/express/config.js"); +const versionInfo = require('../../frontend/express/version.info'); +const plugins = require('../../plugins/pluginManager.js'); + + +const Job = require("../../jobServer/Job"); /** Class for the job of pinging servers **/ -class PingJob extends job.Job { +class PingJob extends Job { + + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "0 1 * * *" // Every day at 1:00 AM + }; + } + /** * Run the ping job * @param {Db} db connection * @param {done} done callback */ run(db, done) { - request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {}); - var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig)); - var url = "https://count.ly/configurations/ce/tracking"; - if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { - url = "https://count.ly/configurations/ee/tracking"; - } + plugins.loadConfigs(db, function() { + const request = require('countly-request')(plugins.getConfig("security")); + request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {}); + var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig)); + var url = "https://count.ly/configurations/ce/tracking"; + if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { + url = "https://count.ly/configurations/ee/tracking"; + } + const offlineMode = plugins.getConfig("api").offline_mode; const { countly_tracking } = plugins.getConfig('frontend'); if (!offlineMode) { @@ -56,12 +73,12 @@ class PingJob extends job.Job { let domain = plugins.getConfig('api').domain; try { - // try to extract hostname from full domain url + // try to extract hostname from full domain url const urlObj = new URL(domain); domain = urlObj.hostname; } catch (_) { - // do nothing, domain from config will be used as is + // do nothing, domain from config will be used as is } request({ diff --git a/api/jobs/task.js b/api/jobs/task.js index 06b6893a7e7..adf4d2cc294 100644 --- a/api/jobs/task.js +++ b/api/jobs/task.js @@ -1,9 +1,10 @@ 'use strict'; -const job = require('../parts/jobs/job.js'), - log = require('../utils/log.js')('api:task'), - asyncjs = require("async"), - plugins = require('../../plugins/pluginManager.js'); +// const job = require('../parts/jobs/job.js'); +const Job = require("../../jobServer/Job"); +const log = require('../utils/log.js')('api:task'); +const asyncjs = require("async"); +const plugins = require('../../plugins/pluginManager.js'); const common = require('../utils/common.js'); const taskmanager = require('../utils/taskmanager.js'); @@ -14,7 +15,19 @@ common.processRequest = processRequest; /** * Task Monitor Job extend from Countly Job */ -class MonitorJob extends job.Job { +class MonitorJob extends Job { + + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "*/5 * * * *" // Every 5 minutes + }; + } + /** * Run the job * @param {Db} db connection diff --git a/api/jobs/test.js b/api/jobs/test.js deleted file mode 100644 index b25d3825e99..00000000000 --- a/api/jobs/test.js +++ /dev/null @@ -1,129 +0,0 @@ -'use strict'; - -/* jshint ignore:start */ - -const J = require('../parts/jobs/job.js'), - R = require('../parts/jobs/resource.js'), - RET = require('../parts/jobs/retry.js'); - -/** Class for testing resource handling for jobs **/ -class TestResource extends R.Resource { - /** - * Open resource - * @returns {Promise} promise - **/ - open() { - console.log('resource: opening in %d', process.pid); - this.opened(); - this.openedTime = Date.now(); - return Promise.resolve(); - } - - /** - * Close resource - * @returns {Promise} promise - **/ - close() { - console.log('resource: closing in %d', process.pid); - this.closed(); - return Promise.resolve(); - } - - /** - * Kill resource - * @returns {Promise} promise - **/ - kill() { - console.log('resource: killed in %d', process.pid); - return Promise.resolve(); - } - - /** - * Check if resource is used - * @returns {Promise} promise - **/ - checkActive() { - console.log('resource: checkActive in %d', process.pid); - return Promise.resolve(Date.now() - this.openedTime < 20000); - } - - /** Start using resource **/ - start() { - this.openedTime = Date.now(); - super.start.apply(this, arguments); - } -} -/** Class for testing ipc jobs **/ -class IPCTestJob extends J.IPCJob { - /** - * Prepare the job - * @param {object} manager - resource manager - * @param {Db} db - db connection - */ - async prepare(manager, db) { - console.log('preparing in %d', process.pid); - await new Promise((res, rej) => db.collection('jobs').updateOne({_id: this._id}, {$set: {'data.prepared': 1}}, err => err ? rej(err) : res())); - } - - /** - * Get resource name - * @returns {string} resource name - **/ - resourceName() { - return 'resource:test'; - } - - /** - * Create resource - * @param {string} _id - resource _id - * @param {string} name - resource name - * @param {Db} db - db connection - * @returns {Resource} resource - */ - createResource(_id, name, db) { - return new TestResource(_id, name, db); - } - - /** - * Get retry policy - * @returns {RetryPolicy} retry policy - **/ - retryPolicy() { - return new RET.NoRetryPolicy(); - } - - /** - * Get concurrency - * @returns {number} concurency - **/ - getConcurrency() { - return this.data && this.data.concurrency || 0; - } - - /** - * Run the job - * @param {Db} db connection - */ - async run(db) { - console.log('running in %d', process.pid); - if (!this.resource) { - throw new Error('Resource should exist'); - } - if (!(this.resource instanceof TestResource)) { - throw new Error('Resource should be TestResource'); - } - await new Promise((res, rej) => db.collection('jobs').updateOne({_id: this._id}, {$set: {'data.run': 1}}, err => err ? rej(err) : res())); - - if (this.data && this.data.fail) { - throw new Error(this.data.fail); - } - - if (this.data && this.data.concurrency) { - await new Promise(res => setTimeout(res, 3000)); - } - - console.log('done running in %d', process.pid); - } -} - -module.exports = IPCTestJob; \ No newline at end of file diff --git a/api/jobs/topEvents.js b/api/jobs/topEvents.js index d5043b2235e..de811288447 100644 --- a/api/jobs/topEvents.js +++ b/api/jobs/topEvents.js @@ -1,4 +1,5 @@ -const job = require("../parts/jobs/job.js"); +// const job = require("../parts/jobs/job.js"); +const Job = require("../../jobServer/Job"); const crypto = require("crypto"); const Promise = require("bluebird"); const countlyApi = { @@ -14,7 +15,7 @@ const common = require('../utils/common.js'); const log = require('../utils/log.js')('job:topEvents'); /** Class for job of top events widget **/ -class TopEventsJob extends job.Job { +class TopEventsJob extends Job { /** * TopEvents initialize function @@ -243,6 +244,17 @@ class TopEventsJob extends job.Job { done(error); } } + + /** + * Get schedule + * @returns {GetScheduleConfig} schedule + */ + getSchedule() { + return { + type: "schedule", + value: "1 0 * * *" // every day at 00:01 + }; + } } /** diff --git a/api/jobs/ttlCleanup.js b/api/jobs/ttlCleanup.js index 1c168d38b21..5de0df26dbd 100644 --- a/api/jobs/ttlCleanup.js +++ b/api/jobs/ttlCleanup.js @@ -1,12 +1,25 @@ const plugins = require("../../plugins/pluginManager.js"); const common = require('../utils/common'); -const job = require("../parts/jobs/job.js"); +// const job = require("../parts/jobs/job.js"); const log = require("../utils/log.js")("job:ttlCleanup"); +const Job = require("../../jobServer/Job"); /** * Class for job of cleaning expired records inside ttl collections */ -class TTLCleanup extends job.Job { +class TTLCleanup extends Job { + + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "* * * * *" // Every minute + }; + } + /** * Run the job */ diff --git a/api/jobs/userMerge.js b/api/jobs/userMerge.js index 17d3a8022b7..c8c4ffa5363 100644 --- a/api/jobs/userMerge.js +++ b/api/jobs/userMerge.js @@ -1,10 +1,10 @@ -'use strict'; -const job = require('../parts/jobs/job.js'), - plugins = require('../../plugins/pluginManager.js'), - log = require('../utils/log.js')('job:userMerge'); -var Promise = require("bluebird"); -var usersApi = require('../parts/mgmt/app_users.js'); +// const job = require('../parts/jobs/job.js'); +const Job = require("../../jobServer/Job"); +const plugins = require('../../plugins/pluginManager.js'); +const log = require('../utils/log.js')('job:userMerge'); +const Promise = require("bluebird"); +const usersApi = require('../parts/mgmt/app_users.js'); var getMergeDoc = function(data) { @@ -180,7 +180,13 @@ var handleMerges = function(db, callback) { } } else { - resolve(); + //delete invalid document + db.collection('app_user_merges').remove({"_id": user._id}, function(err5) { + if (err5) { + log.e(err5); + } + resolve(); + }); } } } @@ -197,12 +203,11 @@ var handleMerges = function(db, callback) { for (var z = 0; z < paralel_cn; z++) { promises.push(new Promise((resolve)=>{ processMerging(dataObj, resolve); - })); } Promise.all(promises).then(()=>{ - if (mergedocs.length === 100) { + if (mergedocs.length === limit) { setTimeout(()=>{ handleMerges(db, callback); }, 0); //To do not grow stack. @@ -223,7 +228,19 @@ var handleMerges = function(db, callback) { }); }; /** Class for the user mergind job **/ -class UserMergeJob extends job.Job { +class UserMergeJob extends Job { + + /** + * Get the schedule configuration for this job + * @returns {GetScheduleConfig} schedule configuration + */ + getSchedule() { + return { + type: "schedule", + value: "*/5 * * * *" // Every 5 minutes + }; + } + /** * Run the job * @param {Db} db connection diff --git a/api/parts/data/batcher.js b/api/parts/data/batcher.js index b6979740941..dcfd3143a22 100644 --- a/api/parts/data/batcher.js +++ b/api/parts/data/batcher.js @@ -190,12 +190,22 @@ class WriteBatcher { constructor(db) { this.dbs = {countly: db}; this.data = {countly: {}}; + this.flushCallbacks = {}; plugins.loadConfigs(db, () => { this.loadConfig(); this.schedule(); }); } + /** + * Function to call once flushed + * @param {string} name - name of collection + * @param {function} callback - callback function + */ + addFlushCallback(name, callback) { + this.flushCallbacks[name] = callback; + } + /** * Add another database to batch * @param {string} name - name of the database @@ -224,33 +234,36 @@ class WriteBatcher { async flush(db, collection) { var no_fallback_errors = [10334, 17419, 14, 56]; var notify_errors = [10334, 17419]; - if (Object.keys(this.data[db][collection]).length) { + if (this.data[db] && this.data[db][collection] && this.data[db][collection].data && Object.keys(this.data[db][collection].data).length) { + var token0; var queries = []; - for (let key in this.data[db][collection]) { - if (Object.keys(this.data[db][collection][key]).length) { + for (let key in this.data[db][collection].data) { + if (Object.keys(this.data[db][collection].data[key]).length) { + var upsert = true; + if (typeof this.data[db][collection].data[key].upsert !== "undefined") { + upsert = this.data[db][collection].data[key].upsert; + } queries.push({ updateOne: { - filter: {_id: this.data[db][collection][key].id}, - update: this.data[db][collection][key].value, - upsert: this.data[db][collection][key].upsert + filter: {_id: this.data[db][collection].data[key].id}, + update: this.data[db][collection].data[key].value, + upsert: upsert } }); } } - this.data[db][collection] = {}; + token0 = this.data[db][collection].t; + this.data[db][collection] = {"data": {}}; batcherStats.update_queued -= queries.length; batcherStats.update_processing += queries.length; + + var self = this; try { - await new Promise((resolve, reject) => { - this.dbs[db].collection(collection).bulkWrite(queries, {ordered: false, ignore_errors: [11000]}, function(err, res) { - if (err) { - reject(err); - return; - } - batcherStats.update_processing -= queries.length; - resolve(res); - }); - }); + await this.dbs[db].collection(collection).bulkWrite(queries, {ordered: false, ignore_errors: [11000]}); + if (self.flushCallbacks[collection] && token0) { + self.flushCallbacks[collection](token0); + } + batcherStats.update_processing -= queries.length; } catch (ex) { if (ex.code !== 11000) { @@ -289,6 +302,12 @@ class WriteBatcher { } } } + else { + //No data. Acknowledge token if there is any + if (this.flushCallbacks[collection] && this.data[db] && this.data[db][collection] && this.data[db][collection].t) { + this.flushCallbacks[collection](this.data[db][collection].t); + } + } } /** @@ -302,6 +321,9 @@ class WriteBatcher { promises.push(this.flush(db, collection)); } } + if (this.flushCallbacks["*"] && this._token) { + this.flushCallbacks["*"](this._token); + } return Promise.all(promises).finally(() => { this.schedule(); }); @@ -340,36 +362,48 @@ class WriteBatcher { * @param {string} id - id of the document * @param {object} operation - operation * @param {string} db - name of the database for which to write data - * @param {object=} options - options for operation ((upsert: false) - if you don't want to upsert document) + * @param {object} options - options for the operation */ add(collection, id, operation, db = "countly", options) { + options = options || {}; if (!this.shared || cluster.isMaster) { if (!this.data[db][collection]) { - this.data[db][collection] = {}; - } - if (!this.data[db][collection][id]) { - this.data[db][collection][id] = {id: id, value: operation, upsert: true}; - if (options && options.upsert === false) { - this.data[db][collection][id].upsert = false; - } - batcherStats.update_queued++; + this._token = options.token; + this.data[db][collection] = {data: {}, "t": options.token, "upsert": options.upsert}; } else { - this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation); - if (options && options.upsert === false) { - this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false; + if (options.token) { + this._token = options.token; + this.data[db][collection].t = options.token; } + } - if (!this.process) { - this.flush(db, collection); + + if (id) { + if (!this.data[db][collection].data[id]) { + this.data[db][collection].data[id] = {id: id, value: operation}; + batcherStats.update_queued++; + } + else { + this.data[db][collection].data[id].value = common.mergeQuery(this.data[db][collection].data[id].value, operation); + } + + if (typeof options.upsert !== "undefined" && this.data[db][collection].data[id]) { + this.data[db][collection].data[id].upsert = options.upsert; + } + if (!this.process) { + this.flush(db, collection); + } } } else { process.send({ cmd: "batch_write", data: {collection, id, operation, db} }); } } + } + /** * Class for caching read from database * @example @@ -425,38 +459,34 @@ class ReadBatcher { * @param {boolean} multi - true if multiple documents * @returns {Promise} promise */ - getData(collection, id, query, projection, multi) { - return new Promise((resolve, reject) => { - if (multi) { - this.db.collection(collection).find(query, projection).toArray((err, res) => { - if (!err) { - this.cache(collection, id, query, projection, res, true); - resolve(res); - } - else { - if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { - this.data[collection][id].promise = null; - } - reject(err); - } - }); + getData = async function(collection, id, query, projection, multi) { + if (multi) { + try { + var res = await this.db.collection(collection).find(query, projection).toArray(); + this.cache(collection, id, query, projection, res, true); + return res; } - else { - this.db.collection(collection).findOne(query, projection, (err, res) => { - if (!err) { - this.cache(collection, id, query, projection, res, false); - resolve(res); - } - else { - if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { - this.data[collection][id].promise = null; - } - reject(err); - } - }); + catch (err) { + if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { + this.data[collection][id].promise = null; + } + throw err; } - }); - } + } + else { + try { + var res2 = await this.db.collection(collection).findOne(query, projection); + this.cache(collection, id, query, projection, res2, false); + return res2; + } + catch (err) { + if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { + this.data[collection][id].promise = null; + } + throw err; + } + } + }; /** * Check all cache @@ -693,4 +723,4 @@ function getId() { return crypto.randomBytes(16).toString("hex"); } -module.exports = {WriteBatcher, ReadBatcher, InsertBatcher}; +module.exports = {WriteBatcher, ReadBatcher, InsertBatcher}; \ No newline at end of file diff --git a/api/parts/data/cache.js b/api/parts/data/cache.js index a2da942aaf1..cc4263b6e8f 100644 --- a/api/parts/data/cache.js +++ b/api/parts/data/cache.js @@ -1,13 +1,11 @@ 'use strict'; -const log = require('../../utils/log.js')('cache:' + process.pid), - // common = require('../../utils/common.js'), - { CentralWorker, CentralMaster } = require('../jobs/ipc.js'), +const log = require('../../utils/log.js')('cache'), { Jsonable } = require('../../utils/models'), LRU = require('lru-cache'), config = require('../../config.js'); -const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPDATE: 'u'}; +// const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPDATE: 'u'}; // new job: {o: 2, k: 'ObjectId', g: 'jobs', d: '{"name": "jobs:clean", "status": 0, ...}'} // job update: {o: 3, k: 'ObjectId', g: 'jobs', d: '{"status": 1}'} // job retreival: {o: 1, k: 'ObjectId', g: 'jobs'} @@ -16,10 +14,12 @@ const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPD /** * Get value in nested objects - * @param {object} obj - object to checl + * @param {object} obj - object to check * @param {string|array} is - keys for nested value * @param {any} value - if provided acts as setter setting this value in nested object * @return {varies} returns value in provided key in nested object + * @note + * TODO: change to iterative if profiling finds memory issues */ const dot = function(obj, is, value) { if (typeof is === 'string') { @@ -54,7 +54,13 @@ class DataStore { constructor(size, age, dispose, Cls) { this.size = size; this.age = age; - this.lru = new LRU({max: size || 10000, ttl: age || Number.MAX_SAFE_INTEGER, dispose: dispose, noDisposeOnSet: true, updateAgeOnGet: true}); + this.lru = new LRU({ + max: size || 10000, + ttl: age || Number.MAX_SAFE_INTEGER, + dispose: dispose, + noDisposeOnSet: true, + updateAgeOnGet: true + }); if (Cls) { this.Cls = Cls; this.Clas = require('../../../' + Cls[0])[Cls[1]]; @@ -151,415 +157,14 @@ class DataStore { * - notifies master about updates; * - loads data from master when local copy misses a particular key. */ -class CacheWorker { +class Cache { /** * Constructor - * * @param {Number} size max number of cache groups */ constructor(size = 100) { this.data = new DataStore(size); - this.started = false; - - this.ipc = new CentralWorker(CENTRAL, (m, reply) => { - let {o, k, g, d} = m || {}; - - if (!g) { - return; - } - log.d('handling %s: %j', reply ? 'reply' : 'broadcast', m); - - let store = this.data.read(g); - - if (o === OP.INIT) { - this.data.write(g, new DataStore(d.size, d.age, undefined, d.Cls)); - return; - } - else if (!store) { - log.d('Group store is not initialized'); - return; - } - - if (o === OP.PURGE) { - if (k) { - store.write(k, null); - } - else { // purgeAll - store.iterate(id => store.write(id, null)); - } - } - else if (o === OP.READ) { - store.write(k, d); - } - else if (o === OP.WRITE) { - store.write(k, d); - } - else if (o === OP.UPDATE) { - store.update(k, d); - } - else { - throw new Error(`Illegal cache operaion: ${o}, ${k}, ${g}, ${d}`); - } - - // store.iterate((k, v) => { - // log.d('have %s: %j', k, v); - // }); - }); - } - - /** - * Start listening to IPC messages - */ - async start() { - if (this.started === true) { - return; - } - - if (this.started === false) { - log.d('starting worker'); - this.started = new Promise((resolve, reject) => { - let timeout = setTimeout(() => { - reject(new Error('Failed to start CacheWorker on timeout')); - }, 10000); - this.ipc.attach(); - this.ipc.request({o: OP.INIT}).then(ret => { - log.d('got init response: %j', ret); - Object.keys(ret).forEach(g => { - if (!this.data.read(g)) { - this.data.write(g, new DataStore(ret[g].size, ret[g].age, undefined, ret[g].Cls)); - if (ret[g].data) { - log.d('got %d data objects in init response', Object.keys(ret[g].data).length); - for (let k in ret[g].data) { - this.data.read(g).write(k, ret[g].data[k]); - } - } - } - }); - this.started = true; - clearTimeout(timeout); - resolve(); - }); - }); - } - - await this.started; - } - - /** - * Stop worker - */ - async stop() { - this.ipc.detach(); - } - - /** - * Write data to cache: - * - send a write to the master; - * - wait for a response with write status; - * - write the data to local copy and return it in case of success, throw error otherwise. - * - * @param {String} group group key - * @param {String} id data key - * @param {Object} data data to store - * @return {Object} data if succeeded, null otherwise, throws in case of an error - */ - async write(group, id, data) { - await this.start(); - - if (!group || !id || !data || typeof id !== 'string') { - throw new Error('Where are my args?'); - } - else if (!this.data.read(group)) { - throw new Error('No such cache group'); - } - log.d(`writing ${group}:${id}`); - let rsp = await this.ipc.request({o: OP.WRITE, g: group, k: id, d: data instanceof Jsonable ? data.json : data}); - if (rsp) { - this.data.read(group).write(id, rsp); - } - - return this.has(group, id); - } - - /** - * Update data in the cache: - * - send an update to the master; - * - wait for a response with update status; - * - update the data in the local copy and return updated object in case of success, throw error otherwise. - * - * @param {String} group group key - * @param {String} id data key - * @param {Object} update data to store - * @return {Object} data if succeeded, null otherwise, throws in case of an error - */ - async update(group, id, update) { - await this.start(); - - if (!group || !id || !update || typeof id !== 'string') { - throw new Error('Where are my args?!'); - } - else if (!this.data.read(group)) { - throw new Error('No such cache group'); - } - log.d(`updating ${group}:${id}`); - let rsp = await this.ipc.request({o: OP.UPDATE, g: group, k: id, d: update}), - store = this.data.read(group); - if (rsp) { - store.update(id, rsp); - } - else { - store.remove(id); - - } - return this.has(group, id); - } - - /** - * Remove a record from cache. - * - * @param {String} group group key - * @param {String} id data key - * @return {Boolean} true if removed - */ - async remove(group, id) { - await this.start(); - - if (!group || !id || typeof id !== 'string') { - throw new Error('Where are my args?!'); - } - else if (!this.data.read(group)) { - throw new Error('No such cache group'); - } - log.d(`removing ${group}:${id}`); - await this.ipc.request({o: OP.WRITE, g: group, k: id, d: null}); - let store = this.data.read(group); - if (store) { - store.remove(id); - } - return this.has(group, id) === null; - } - - /** - * Remove a record from cache. - * - * @param {String} group group key - * @param {String} id data key - * @return {Boolean} true if removed - */ - async purge(group, id) { - await this.start(); - - if (!group || !id || typeof id !== 'string') { - throw new Error('Where are my args?!'); - } - else if (!this.data.read(group)) { - throw new Error('No such cache group'); - } - log.d(`purging ${group}:${id}`); - await this.ipc.request({o: OP.PURGE, g: group, k: id}); - let store = this.data.read(group); - if (store) { - store.remove(id); - } - return this.has(group, id) === null; - } - - /** - * Remove from cache all records for a given group. - * - * @param {String} group group key - */ - async purgeAll(group) { - await this.start(); - - if (!group) { - throw new Error('Where are my args?!'); - } - else if (!this.data.read(group)) { - throw new Error('No such cache group'); - } - log.d(`purging ${group}`); - await this.ipc.request({o: OP.PURGE, g: group}); - let store = this.data.read(group); - store.iterate(id => store.write(id, null)); - } - - /** - * Read a record from cache: - * - from local copy if exists; - * - send a read request to master otherwise. - * - * @param {String} group group key - * @param {String} id data key - * @return {Object} data if any, null otherwise - */ - async read(group, id) { - await this.start(); - - if (!group || !id) { - throw new Error('Where are my args?!'); - } - let data = this.has(group, id); - if (data) { - return data; - } - else { - let rsp = await this.ipc.request({o: OP.READ, g: group, k: id}); - if (rsp) { - let store = this.data.read(group); - if (!store) { - throw new Error(`No store for a group ${group}?!`); - // store = this.data.write(group, new DataStore(this.size)); - } - store.write(id, rsp); - } - return this.has(group, id); - } - } - - /** - * Check if local copy has data under the key. - * - * @param {String} group group key - * @param {String} id data key - * @return {Object} data if any, null otherwise - */ - has(group, id) { - if (!group) { - throw new Error('Where are my args?!'); - } - let store = this.data.read(group); - if (id) { - return store && store.read(id) || null; - } - else { - return store; - } - } - - /** - * Just a handy method which returns an object with partials with given group. - * - * @param {String} group group name - * @return {Object} object with all the {@code CacheWorker} methods without group - */ - cls(group) { - return { - read: this.read.bind(this, group), - write: this.write.bind(this, group), - update: this.update.bind(this, group), - remove: this.remove.bind(this, group), - purge: this.purge.bind(this, group), - purgeAll: this.purgeAll.bind(this, group), - has: this.has.bind(this, group), - iterate: f => { - let g = this.data.read(group); - if (g) { - g.iterate(f); - } - else { - log.e('no cache group %s to iterate on', group); - } - } - }; - } -} - -/** - * Cache instance for master process: - * - listen for requests from workers; - * - call group operators to read/write/udpate - */ -class CacheMaster { - /** - * Constructor - * - * @param {Number} size max number of cache groups - */ - constructor(size = 100) { - this.data = new DataStore(size, Number.MAX_SAFE_INTEGER); this.operators = {}; - this.initialized = {}; - this.delayed_messages = []; - this.ipc = new CentralMaster(CENTRAL, ({o, g, k, d}, reply, from) => { - log.d('handling %s: %j / %j / %j / %j', reply ? 'reply' : 'broadcast', o, g, k, d); - - if (o === OP.INIT) { - this.initialized[from] = true; - let ret = {}; - this.data.iterate((group, store) => { - let data = {}; - store.iterate((key, obj) => { - data[key] = obj instanceof Jsonable ? obj.json : obj; - }); - ret[group] = {size: store.size, age: store.age, Cls: store.Cls, data}; - }); - setImmediate(() => { - let remove = []; - this.delayed_messages.filter(arr => arr[0] === from).forEach(arr => { - remove.push(arr); - this.ipc.send(arr[0], arr[1]); - }); - if (remove.length) { - log.d('sent %d delayed messages after %d worker\'s init', remove.length, from); - remove.forEach(m => { - const i = this.delayed_messages.indexOf(m); - if (i !== -1) { - this.delayed_messages.splice(i, 1); - } - }); - } - }); - return ret; - } - - let store = this.data.read(g); - if (!store) { - log.d(`No store for group ${g}`); - throw new Error('No such store ' + g); - } - - if (o === OP.PURGE) { - if (k) { - return this.purge(g, k, from); - } - else { - return this.purgeAll(g, from); - } - } - else if (o === OP.READ) { - return this.read(g, k, from); - } - else if (o === OP.WRITE) { - return this.write(g, k, d, from); - } - else if (o === OP.UPDATE) { - return this.update(g, k, d, from); - } - else if (o === OP.REMOVE) { - return this.remove(g, k, from); - } - else { - throw new Error(`Illegal cache operaion: ${o}, ${k}, ${g}, ${d}`); - } - }); - } - - /** - * Attach to IPC - * - * @return {Promise} void - */ - async start() { - this.ipc.attach(); - log.d('started master'); - } - - /** - * Detaches IPC instance - */ - stop() { - this.ipc.detach(); } /** @@ -569,55 +174,44 @@ class CacheMaster { * @param {Function} options.init initializer - an "async () => [Object]" kind of function, preloads data to cache on startup * @param {string[]} options.Cls class - an optional array of ["require path", "export name"] which resolves to a Jsonable subclass to construct instances * @param {Function} options.read reader - an "async (key) => Object" kind of function, returns data to cache if any for the key supplied - * @param {Function} options.write writer - an "async (key, data) => Object" kind of function, persists the data cached if needed (must return the data persisted on success) + * @param {Function} options.write writer - an "async (key, data) => Object" kind of function, persists the data cached if needed * @param {Function} options.update updater - an "async (key, update) => Object" kind of function, updates persisted data if needed * @param {Function} options.remove remover - an "async (key) => Object" kind of function, removes persisted data if needed * @param {int} age how long in ms to keep records in memory for the group * @param {int} size how much records to keep in memory for the group */ - init(group, {init, Cls, read, write, update, remove}, size = null, age = null) { + async init(group, {init, Cls, read, write, update, remove}, size = null, age = null) { this.operators[group] = {init, Cls, read, write, update, remove}; if (!size && size !== 0) { - size = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].size !== undefined ? config.api.cache[group].size : 10000; + size = config.api?.cache?.[group]?.size ?? 10000; } if (!age && age !== 0) { - age = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].age !== undefined ? config.api.cache[group].age : Number.MAX_SAFE_INTEGER; + age = config.api?.cache?.[group]?.age ?? Number.MAX_SAFE_INTEGER; } - this.data.write(group, new DataStore(size, age, k => { - this.ipc.send(0, {o: OP.PURGE, g: group, k}); - }, Cls)); - - this.ipc.send(0, {o: OP.INIT, g: group, d: {size, age, Cls}}); + this.data.write(group, new DataStore(size, age, null, Cls)); - init().then(arr => { + try { + const arr = await init(); (arr || []).forEach(([k, d]) => { this.data.read(group).write(k, d); - const msg = {o: OP.READ, g: group, k, d: d && (d instanceof Jsonable) ? d.json : d}; - for (const pid in this.ipc.workers) { - if (this.initialized[pid]) { - this.ipc.send(parseInt(pid), msg); - } - else { - this.delayed_messages.push([parseInt(pid), msg]); - } - } }); - }, log.e.bind(log, 'Error during initialization of cache group %s', group)); + } + catch (err) { + log.e('Error during initialization of cache group %s', group, err); + } } /** * Write data to the cache - * * @param {String} group group key * @param {String} id data key * @param {Object} data data to store - * @param {int} from originating pid if any * @return {Object} data if succeeded, null otherwise, throws in case of an error */ - async write(group, id, data, from = 0) { + async write(group, id, data) { if (!group || !id || (data === undefined) || typeof id !== 'string') { throw new Error('Where are my args?!'); } @@ -627,38 +221,29 @@ class CacheMaster { log.d(`writing ${group}:${id}: %j`, data); if (group in this.operators) { - return this.operators[group][data === null ? 'remove' : 'write'](id, data).then(rc => { - if (rc) { - if (data === null) { - rc = null; - } - if (rc instanceof Jsonable) { - rc = rc.json; - } - this.data.read(group)[data === null ? 'remove' : 'write'](id, rc); - this.ipc.send(-from, {o: OP.WRITE, g: group, k: id, d: rc}); - return data === null ? true : rc; - } - else { - return null; + const rc = await this.operators[group][data === null ? 'remove' : 'write'](id, data); + if (rc) { + if (data === null) { + this.data.read(group).remove(id); + return true; } - }); - } - else { + const toStore = rc instanceof Jsonable ? rc.json : rc; + this.data.read(group).write(id, toStore); + return toStore; + } return null; } + return null; } /** * Update data in the cache - * * @param {String} group group key * @param {String} id data key * @param {Object} update data to store - * @param {int} from originating pid if any * @return {Object} data if succeeded, null otherwise, throws in case of an error */ - async update(group, id, update, from = 0) { + async update(group, id, update) { if (!group || !id || !update || typeof id !== 'string') { throw new Error('Where are my args?!'); } @@ -668,15 +253,11 @@ class CacheMaster { log.d(`updating ${group}:${id} with %j`, update); if (group in this.operators) { - return this.operators[group].update(id, update).then(() => { - this.data.read(group).update(id, update); - this.ipc.send(-from, {o: OP.UPDATE, g: group, k: id, d: update}); - return update; - }); - } - else { - return null; + await this.operators[group].update(id, update); + this.data.read(group).update(id, update); + return update; } + return null; } /** @@ -684,33 +265,18 @@ class CacheMaster { * * @param {String} group group key * @param {String} id data key - * @param {int} from originating pid if any * @return {Boolean} true if removed */ - async remove(group, id, from) { + async remove(group, id) { if (!group || !id || typeof id !== 'string') { throw new Error('Where are my args?!'); } else if (!this.data.read(group)) { throw new Error('No such cache group'); } - log.d(`removing ${group}:${id}`); - if (group in this.operators) { - return this.operators[group].remove(id).then(rc => { - if (rc) { - this.data.read(group).remove(id); - this.ipc.send(-from, {o: OP.WRITE, g: group, k: id, d: null}); - return true; - } - else { - return null; - } - }); - } - else { - return null; - } + this.data.read(group).remove(id); + return true; } /** @@ -718,10 +284,9 @@ class CacheMaster { * * @param {String} group group key * @param {String} id data key - * @param {int} from originating pid if any * @return {Boolean} true if removed */ - async purge(group, id, from = 0) { + async purge(group, id) { if (!group || !id || typeof id !== 'string') { throw new Error('Where are my args?!'); } @@ -729,42 +294,33 @@ class CacheMaster { throw new Error('No such cache group'); } log.d(`purging ${group}:${id}`); - this.data.read(group).write(id, null); - this.ipc.send(-from, {o: OP.PURGE, g: group, k: id}); return true; } /** - * Remove from cache all record for given group. + * Remove from cache all records for a given group. * * @param {String} group group key - * @param {int} from originating pid if any - * @return {Boolean} true if removed */ - async purgeAll(group, from = 0) { + async purgeAll(group) { if (!group) { throw new Error('Where are my args?!'); } + else if (!this.data.read(group)) { + throw new Error('No such cache group'); + } log.d(`purging ${group}`); - - let grp = this.data.read(group); - grp.iterate(k => grp.write(k, null)); - this.ipc.send(-from, {o: OP.PURGE, g: group}); - return true; + this.data.read(group).iterate(id => this.data.read(group).write(id, null)); } /** - * Read a record from cache: - * - from local copy if exists; - * - send a read request to master otherwise. - * + * Read data from the cache * @param {String} group group key * @param {String} id data key - * @param {int} from originating pid if any - * @return {Object} data if any, null otherwise + * @return {Object} data if succeeded, null otherwise, throws in case of an error */ - async read(group, id, from = 0) { + async read(group, id) { if (!group || !id || typeof id !== 'string') { throw new Error('Where are my args?!'); } @@ -772,75 +328,52 @@ class CacheMaster { throw new Error('No such cache group'); } - let store = this.data.read(group), - rc = store.read(id); - if (rc) { - return rc; - } - else if (group in this.operators) { - return this.operators[group].read(id).then(x => { - if (x) { - this.ipc.send(-from, {o: OP.READ, g: group, k: id, d: x instanceof Jsonable ? x.json : x}); - store.write(id, x); - return x; - } - else { - return null; - } - }); + let data = this.has(group, id); + if (data) { + return data; } - else { - return null; + + if (group in this.operators) { + const rc = await this.operators[group].read(id); + if (rc) { + const toStore = rc instanceof Jsonable ? rc.json : rc; + this.data.read(group).write(id, toStore); + return toStore; + } } + return null; } /** - * Check if local copy has data under the key. - * + * Check if data exists in cache * @param {String} group group key * @param {String} id data key - * @return {Object} data if any, undefined otherwise + * @return {Object|null} data if exists, null otherwise */ has(group, id) { - if (!group) { + if (!group || !id || typeof id !== 'string') { throw new Error('Where are my args?!'); } - let store = this.data.read(group); - if (id) { - return store && store.read(id) || null; - } - else { - return store; + else if (!this.data.read(group)) { + throw new Error('No such cache group'); } + + return this.data.read(group).read(id); } /** - * Just a handy method which returns an object with partials with given group. - * + * Get class interface for a group * @param {String} group group name - * @return {Object} object with all the {@code CacheWorker} methods without group + * @return {Object} object with read/write/update methods bound to the group */ cls(group) { return { read: this.read.bind(this, group), write: this.write.bind(this, group), update: this.update.bind(this, group), - remove: this.remove.bind(this, group), - purge: this.purge.bind(this, group), - purgeAll: this.purgeAll.bind(this, group), - has: this.has.bind(this, group), - iterate: f => { - let g = this.data.read(group); - if (g) { - g.iterate(f); - } - else { - log.e('no cache group %s to iterate on', group); - } - } + remove: this.remove.bind(this, group) }; } - } /** @@ -856,4 +389,4 @@ class TestDataClass extends Jsonable { } -module.exports = {CacheMaster, CacheWorker, TestDataClass}; +module.exports = {Cache, TestDataClass}; diff --git a/api/parts/data/cacher.js b/api/parts/data/cacher.js new file mode 100644 index 00000000000..13465681966 --- /dev/null +++ b/api/parts/data/cacher.js @@ -0,0 +1,306 @@ +const plugins = require('../../../plugins/pluginManager.js'); +//const log = require('../../utils/log.js')("cacher"); + +/** + * Use it to read data and keep cached in memory in transformed state. + */ +class Cacher { +/** + * Create batcher instance + * @param {Db} db - database object + * @param {object} options - options object + */ + constructor(db, options) { + this.db = db; + this.data = {}; + this.promises = {}; + this.options = options || {}; + + this.transformationFunctions = {}; + plugins.loadConfigs(db, () => { + this.loadConfig(options); + this.schedule(); + }); + } + + /** + * Reloads server configs + */ + loadConfig() { + let config = plugins.getConfig("api"); + this.period = config.batch_read_period * 1000; + this.ttl = config.batch_read_ttl * 1000; + this.process = config.batch_processing; + + if (this.options && this.options.ttl) { + this.ttl = this.options.ttl * 1000; + this.process = true; + } + if (this.options && this.options.period) { + this.period = this.options.period * 1000; + this.process = true; + } + } + + /** + * Check all cache + */ + checkAll() { + for (let collection in this.data) { + if (Object.keys(this.data[collection]).length) { + for (let id in this.data[collection]) { + if (this.data[collection][id].last_used < Date.now() - this.ttl) { + delete this.data[collection][id]; + } + } + } + } + this.schedule(); + } + + /** + * Schedule next flush + */ + schedule() { + setTimeout(() => { + this.loadConfig(); + this.checkAll(); + }, this.period); + } + + /** + * Get data from database + * @param {string} collection - name of the collection for which to write data + * @param {string} id - id of cache + * @param {string} query - query for the document + * @param {string} projection - which fields to return + * @param {string} transformation - transformation function name + * @param {bool} multi - true if multiple documents + * @returns {Promise} promise + */ + getData = async function(collection, id, query, projection, transformation, multi) { + var res; + if (multi) { + try { + res = await this.db.collection(collection).find(query, projection).toArray(); + if (transformation && this.transformationFunctions[transformation]) { + res = this.transformationFunctions[transformation](res); + } + this.cache(collection, id, query, projection, res, true); + return res; + } + catch (err) { + if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { + this.data[collection][id].promise = null; + } + throw err; + } + } + else { + try { + res = await this.db.collection(collection).findOne(query, projection); + res = res || {}; + if (transformation && this.transformationFunctions[transformation]) { + res = this.transformationFunctions[transformation](res); + } + this.cache(collection, id, query, projection, res, false); + return res; + } + catch (err) { + if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].promise) { + this.data[collection][id].promise = null; + } + throw err; + } + } + }; + + /** + * Get data from cache or from db and cache it + * @param {string} collection - name of the collection where to update data + * @param {object} query - query for the document + * @param {object} projection - which fields to return + * @param {object} transformation - transformation function name + * @param {bool} refetch - true if need to refetch + * @param {bool} multi - true if multiple documents + * @returns {Promise} promise + */ + get(collection, query, projection, transformation, refetch, multi) { + var id = JSON.stringify(query) + "_" + multi; + if (transformation) { + id += "_" + JSON.stringify(transformation); + } + if (!this.data[collection]) { + this.data[collection] = {}; + } + var good_projection = true; + var keysSaved = this.keysFromProjectionObject(this.data[collection][id] && this.data[collection][id].projection); + var keysNew = this.keysFromProjectionObject(projection); + + if (this.data[collection][id] && (keysSaved.have_projection || keysNew.have_projection)) { + if (keysSaved.have_projection) { + for (let p = 0; p < keysNew.keys.length; p++) { + if (keysSaved.keys.indexOf(keysNew.keys[p]) === -1) { + good_projection = false; + keysSaved.keys.push(keysNew.keys[p]); + } + } + } + if (!good_projection) { + projection = {}; + for (var p = 0; p < keysSaved.keys.length; p++) { + projection[keysSaved.keys[p]] = 1; + } + } + } + + if (refetch || !this.process || !good_projection || !this.data[collection][id] || (this.data[collection][id].last_updated < Date.now() - this.period)) { + if (this.process) { + this.data[collection][id] = { + query: query, + promise: this.getData(collection, id, query, projection, transformation, multi), + projection: projection, + last_used: Date.now(), + last_updated: Date.now(), + multi: multi + }; + return this.data[collection][id].promise; + } + else { + return this.getData(collection, id, query, projection, transformation, multi); + } + } + //we already have a read for this + else if (this.data[collection][id] && this.data[collection][id].promise) { + return this.data[collection][id].promise; + } + else { + this.data[collection][id].last_used = Date.now(); + + return new Promise((resolve) => { + resolve(this.data[collection][id].data); + }); + } + + } + + /** + * Get single document from cache or from db and cache it + * @param {string} collection - name of the collection where to update data + * @param {string} query - query for the document + * @param {object} options - options object + * @param {function=} callback - optional to get result, or else will return promise + * @returns {Promise} if callback not passed, returns promise + */ + getOne(collection, query, options, callback) { + if (typeof options === "function") { + callback = options; + options = {}; + } + if (!options) { + options = {}; + } + return promiseOrCallback(this.get(collection, query, options.projection, options.transformation, options.refetch, false), callback); + } + + /** + * + * @param {string} collection - collection name + * @param {object} query - query to match data + * @param {object} update - data object to update + */ + updateCacheOne(collection, query, update) { + var id = JSON.stringify(query) + "_" + false; + if (this.data && this.data[collection] && this.data[collection][id] && this.data[collection][id].data) { + for (var key in update) { + this.data[key] = update[key]; + } + } + } + + /** + * Gets list of keys from projection object which are included + * @param {object} projection - which fields to return + * @returns {object} {keys - list of keys, have_projection - true if projection not empty} + */ + keysFromProjectionObject(projection) { + var keysSaved = []; + var have_projection = false; + projection = projection || {}; + + if (projection.projection && typeof projection.projection === 'object') { + projection = projection.projection; + } + + if (projection.fields && typeof projection.fields === 'object') { + projection = projection.fields; + } + + for (var k in projection) { + have_projection = true; + if (projection[k] === 1) { + keysSaved.push(k); + } + } + return {"keys": keysSaved, "have_projection": have_projection}; + } + + /** + * Invalidate specific cache + * @param {string} collection - name of the collection where to update data + * @param {object} query - query for the document + * @param {object} projection - which fields to return + * @param {bool} multi - true if multiple documents + */ + invalidate(collection, query, projection, multi) { + var id = JSON.stringify(query) + "_" + multi; + if (!this.data[collection]) { + this.data[collection] = {}; + } + if (this.data[collection][id] && !this.data[collection][id].promise) { + delete this.data[collection][id]; + } + } + + /** + * Cache data read from database + * @param {string} collection - name of the collection where to update data + * @param {string} id - id of the cache + * @param {string} query - query for the document + * @param {string} projection - which fields to return + * @param {object} data - data from database + * @param {bool} multi - true if multiple documents + */ + cache(collection, id, query, projection, data, multi) { + if (this.process) { + this.data[collection][id] = { + query: query, + data: data, + projection: projection, + last_used: Date.now(), + last_updated: Date.now(), + multi: multi + }; + } + } +} + +/** + * Return promise or callback based on params + * @param {Promise} promise - promise for data + * @param {function=} callback - callback to call + * @returns {Promise} Returned promise + */ +function promiseOrCallback(promise, callback) { + if (typeof callback === "function") { + return promise.then(function(data) { + callback(null, data); + }) + .catch(function(err) { + callback(err, null); + }); + } + return promise; +} + +module.exports = {Cacher}; \ No newline at end of file diff --git a/api/parts/data/changeStreamReader.js b/api/parts/data/changeStreamReader.js new file mode 100644 index 00000000000..8144bbdfdd0 --- /dev/null +++ b/api/parts/data/changeStreamReader.js @@ -0,0 +1,300 @@ +const common = require("../../utils/common"); +const log = require('../../utils/log.js')("changeStreamReader"); +var Timestamp = require('mongodb').Timestamp; + +/** + * Class to ruse change streams to read from mongodb. + */ +class changeStreamReader { + /** + * @param {Object} db - Database object + * @param {Object} options - Options object + * @param {function} onData - Function to call when getting new data from stream + */ + constructor(db, options, onData,) { + this.db = db; + this.pipeline = options.pipeline || []; + this.lastToken = null; + this.name = options.name || ""; + this.collection = options.collection || "drill_events"; + this.options = options.options; + this.onClose = options.onClose; + this.firstDocAfterReset = null; + this.startupFailure = null; + this.onData = onData; + this.interval = options.interval || 10000; + this.intervalRunner = null; + this.keep_closed = false; + this.waitingForAcknowledgement = false; + this.fallback = options.fallback; + + if (this.fallback && !this.fallback.inteval) { + this.fallback.interval = 1000; + } + + //I give data + //Processor function processes. Sends last processed token from time to time. + //Update the last processed token to database + //On startup - read token, resume from that token. + this.setUp(onData, false); + + if (this.intervalRunner) { + clearInterval(this.intervalRunner); + } + this.intervalRunner = setInterval(this.checkState.bind(this), this.interval); + + } + + /** + * Check if stream is closed and restart if needed + */ + checkState() { + if ((!this.stream || this.stream.closed) && !this.keep_closed) { + console.log("Stream is closed. Setting up again"); + this.setUp(this.onData); + } + else if (this.waitingForAcknowledgement && Date.now() - this.waitingForAcknowledgement > 60000) { + console.log("Waiting for acknowledgement for more than 60 seconds. Closing stream and restarting"); + this.keep_closed = false; + this.stream.close(); + } + } + + /** + * Processes range of dates + * @param {date} cd - start time + */ + async processNextDateRange(cd) { + if (this.fallback) { + var cd2 = cd.valueOf() + 60000; + var now = Date.now().valueOf(); + cd2 = cd2 > now ? now : cd2; + + cd2 = new Date(cd2); + var pipeline = JSON.parse(JSON.stringify(this.fallback.pipeline)) || []; + var match = this.fallback.match || {}; + + if (this.fallback.timefield) { + match[this.fallback.timefield] = {$gte: new Date(cd), $lt: cd2}; + } + else { + match.cd = {$gte: new Date(cd), $lt: cd2}; + } + pipeline.unshift({"$match": match}); + //console.log(this.name + " Processing fallback pipeline for range: " + JSON.stringify(match)); + var cursor = this.db.collection(this.collection).aggregate(pipeline); + + while (await cursor.hasNext()) { + var doc = await cursor.next(); + this.onData({"token": "timed", "cd": doc.cd, "_id": doc._id}, doc); + } + setTimeout(() => { + this.processNextDateRange(cd2); + }, this.fallback.interval || 10000); + } + } + + /** + * Process bad range((when token can't continue)) + * @param {Object} options - Options object + * @param {Object} tokenInfo - Token info object + */ + async processBadRange(options, tokenInfo) { + console.log("Processing bad range"); + console.log(JSON.stringify({cd: {$gte: options.cd1, $lt: options.cd2}})); + var gotTokenDoc = false; + var doc; + var cursor = this.db.collection(this.collection).find({cd: {$gte: new Date(options.cd1), $lt: new Date(options.cd2)}}).sort({cd: 1}); + while (await cursor.hasNext() && !gotTokenDoc) { + doc = await cursor.next(); + if (JSON.stringify(doc._id) === JSON.stringify(tokenInfo._id) || doc.cd > tokenInfo.cd) { + gotTokenDoc = true; + } + console.log("SKIP:" + JSON.stringify(doc)); + } + if (doc && doc.cd > tokenInfo.cd) { + tokenInfo.cd = doc.cd; + tokenInfo._id = doc._id; + console.log(this.name + " Process:" + JSON.stringify(doc)); + this.onData(tokenInfo, doc); + } + + while (await cursor.hasNext()) { + doc = await cursor.next(); + console.log(this.name + " Process:" + JSON.stringify(doc)); + tokenInfo.cd = doc.cd; + tokenInfo._id = doc._id; + this.onData(tokenInfo, doc); + } + console.log("done"); + } + + /** + * Sets up stream to read data from mongodb + * @param {function} onData - function to call on new data + */ + async setUp(onData) { + var token; + try { + if (this.stream && !this.stream.closed) { + console.log("Stream is already open. returning"); + return; + } + var options = JSON.parse(JSON.stringify(this.options || {})); + var tokenFailed = false; + var res = await common.db.collection("plugins").findOne({"_id": "_changeStreams"}, {projection: {[this.name]: 1}}); + if (res && res[this.name] && res[this.name].token) { + token = res[this.name]; + options.startAfter = token.token; + } + if (this.failedToken && JSON.stringify(this.failedToken.token) === JSON.stringify(token.token)) { + console.log("Do not use failed token"); + tokenFailed = true; + delete options.startAfter; + var startTime = Date.now().valueOf() / 1000 - 60; + if (startTime) { + options.startAtOperationTime = new Timestamp({t: startTime, i: 1}); + } + } + console.log("Stream options: " + JSON.stringify(options)); + if (this.collection) { + this.stream = await this.db.collection(this.collection).watch(this.pipeline, options); + } + else { + this.stream = await this.db.watch(this.pipeline, options); + } + var self = this; + + if (tokenFailed) { + //fetch data while cd is less than failed token + console.log("Fetching data while cd is less or equal cd to failed token"); + var doc; + do { + doc = await this.stream.next(); + console.log(JSON.stringify(doc)); + } + while (doc && doc.cd && doc.cd <= token.cd); + this.keep_closed = true; + this.stream.close(); + var next_token = {"token": this.stream.resumeToken}; + next_token._id = doc.__id; + next_token.cd = doc.cd; + try { + this.processBadRange({name: this.name, cd1: token.cd, cd2: next_token.cd}, this.failedToken); + this.onData(next_token, doc); + this.waitingForAcknowledgement = Date.now(); + this.restartStream = true; + this.failedToken = null; + } + catch (err) { + log.e("Error on processing bad range", err); + if (this.onClose) { + this.onClose(function() { + this.keep_closed = false; + }); + } + } + } + else { + this.stream.on('change', (change) => { + var my_token = {token: self.stream.resumeToken}; + my_token._id = change.__id; + if (change.cd) { + my_token.cd = change.cd; + onData(my_token, change); + } + else { + onData(my_token, change); + } + }); + + this.stream.on('error', async(err) => { + if (err.code === 286 || err.code === 50811 || err.code === 9 || err.code === 14 || err.code === 280) { //Token is not valid + log.e("Set Failed token", token); + this.failedToken = token; + } + else if (err.code === 40573) { //change stream is not supported + console.log("Change stream is not supported. Keeping streams closed"); + this.keep_closed = true; + var newCD = Date.now(); + if (token && token.cd) { + await this.processBadRange({name: this.name, cd1: token.cd, cd2: newCD}, token); + } + + this.processNextDateRange(newCD); + } + else { + log.e("Error on change stream", err); + } + }); + //Turns out it is closing on exhausted. So we have to let aggregator know to flush aggregated data. + this.stream.on('close', () => { + //Trigger flushing data + if (this.onClose) { + this.onClose(function() {}); + } + log.e("Stream closed."); + }); + } + } + catch (err) { + if (err.code === 286 || err.code === 50811 || err.code === 9) { //failed because of bad token + console.log("Set Failed token", token); + this.failedToken = token; + } + //Failed because of db does not support change streams. Run in "query mode"; + else if (err.code === 40573) { //change stream is not supported + console.log("Change stream is not supported. Keeping streams closed"); + this.keep_closed = true; + var newCD = Date.now(); + if (token && token.cd) { + await this.processBadRange({name: this.name, cd1: token.cd, cd2: newCD}, token); + } + + this.processNextDateRange(newCD); + //Call process bad range if there is any info about last token. + //Switch to query mode + } + else { + log.e("Error on change stream", err); + } + } + } + + /** + * Acknowledges token as recorded + * @param {object} token - token info + */ + async acknowledgeToken(token) { + this.lastToken = token; + //Update last processed token to database + try { + await common.db.collection("plugins").updateOne({"_id": "_changeStreams"}, {$set: {[this.name]: token}}, {"upsert": true}); + if (this.restartStream) { + this.waitingForAcknowledgement = false; + this.keep_closed = false; + this.restartStream = false; + this.stream.close(); + } + } + catch (err) { + log.e("Error on acknowledging token", JSON.stringify(err)); + } + } + + /** + * Closes stream permanently + */ + close() { + console.log("Closing permanently"); + if (this.intervalRunner) { + clearInterval(this.intervalRunner); + } + this.keep_closed = true; + this.stream.close(true); + } + + +} + +module.exports = {changeStreamReader}; \ No newline at end of file diff --git a/api/parts/data/events.js b/api/parts/data/events.js index e1dd8e1e6ce..79382165fa6 100644 --- a/api/parts/data/events.js +++ b/api/parts/data/events.js @@ -11,6 +11,9 @@ var countlyEvents = {}, Promise = require("bluebird"), plugins = require('../../../plugins/pluginManager.js'); + + + /** * Process JSON decoded events data from request * @param {params} params - params object @@ -221,7 +224,6 @@ function processEvents(appEvents, appSegments, appSgValues, params, omitted_segm } for (let i = 0; i < params.qstring?.events.length; i++) { - var currEvent = params.qstring.events[i]; tmpEventObj = {}; tmpEventColl = {}; diff --git a/api/parts/data/fetch.js b/api/parts/data/fetch.js index 210c277571a..7c9d466c8f0 100644 --- a/api/parts/data/fetch.js +++ b/api/parts/data/fetch.js @@ -20,18 +20,68 @@ var fetch = {}, _ = require('underscore'), crypto = require('crypto'), usage = require('./usage.js'), - STATUS_MAP = require('../jobs/job').STATUS_MAP, plugins = require('../../../plugins/pluginManager.js'); + + +fetch.fetchFromGranuralData = async function(queryData, callback) { + var data; + if (queryData.queryName === "uniqueCount") { + data = await common.drillQueryRunner.getUniqueCount(queryData); + callback(null, data); + } + else if (queryData.queryName === "uniqueGraph") { + data = await common.drillQueryRunner.getUniqueGraph(queryData); + callback(null, data); + } + else if (queryData.queryName === "viewsTableData") { + data = await common.drillQueryRunner.getViewsTableData(queryData); + callback(null, data); + } + else if (queryData.queryName === "aggregatedSessionData") { + data = await common.drillQueryRunner.aggregatedSessionData(queryData); + callback(null, data); + } + else if (queryData.queryName === "segmentValuesForPeriod") { + data = await common.drillQueryRunner.segmentValuesForPeriod(queryData); + callback(null, data); + } + else if (common.drillQueryRunner[queryData.queryName] && typeof common.drillQueryRunner[queryData.queryName] === "function") { + data = await common.drillQueryRunner[queryData.queryName](queryData); + callback(null, data); + } + else { + + callback("Invalid query name - " + queryData.queryName, null); + } + /*else { + try { + data = await common.drillQueryRunner.getAggregatedData(queryData); + callback(null, data); + + } + catch (e) { + console.log(e); + return callback(e); + } + }*/ +}; /** -* Prefetch event data, either by provided key or first event in the list and output result to browser +* Prefetch event data, either by provided key or first event all events * @param {string} collection - event key * @param {params} params - params object **/ fetch.prefetchEventData = function(collection, params) { if (!params.qstring.event) { common.readBatcher.getOne("events", { '_id': params.app_id }, (err, result) => { + if (err) { + console.log(err); + } + var events = []; if (result && result.list) { + events = result.list.filter((event) => { + return event.indexOf("[CLY]") !== 0; + }); if (result.order && result.order.length) { for (let i = 0; i < result.order.length; i++) { if (result.order[i].indexOf("[CLY]") !== 0) { @@ -49,9 +99,10 @@ fetch.prefetchEventData = function(collection, params) { } } } - var collectionName = crypto.createHash('sha1').update(collection + params.app_id).digest('hex'); - fetch.fetchTimeObj("events_data", params, true, {'id_prefix': params.app_id + "_" + collectionName + '_'}); + fetch.getMergedEventData(params, events, {'id_prefix': params.app_id + "_" + collectionName + '_'}, function(result3) { + common.returnOutput(params, result3); + }); } else { common.returnOutput(params, {}); @@ -334,6 +385,7 @@ fetch.getMergedEventData = function(params, events, options, callback) { var collectionName = crypto.createHash('sha1').update(eventKey.e + eventKey.a).digest('hex'); var optionsCopy = JSON.parse(JSON.stringify(options)); optionsCopy.id_prefix = eventKey.a + "_" + collectionName + "_"; + params.qstring.event = eventKey.e; fetchTimeObj("events_data", params, true, optionsCopy, function(output) { done(null, output || {}); }); @@ -1069,6 +1121,39 @@ fetch.metricToCollection = function(metric) { } }; +fetch.metricToProperty = function(metric) { + switch (metric) { + case 'locations': + case 'countries': + return "up.cc"; + case 'cities': + return "up.cc"; + case 'sessions': + case 'users': + return "u"; + case 'app_versions': + return "up.av"; + case 'os': + case 'platforms': + return "up.os"; + case 'os_versions': + case 'platform_version': + return "up.os_version"; + case 'resolutions': + return "up.r"; + case 'device_type': + return "up.d"; + case 'device_details': + return "up.d"; + case 'devices': + return "up.d"; + case 'manufacturers': + return "up.m"; + default: + return metric; + } +}; + /** * Get metric data for metric api and output to browser * @param {params} params - params object @@ -1088,7 +1173,6 @@ fetch.fetchMetric = function(params) { else { common.returnOutput(params, []); } - } }; @@ -1594,6 +1678,62 @@ fetch.formatTotalUsersObj = function(obj, forMetric, prev) { return tmpObj; }; +/** + * Caluclates model data from granural data + * @param {string} collection - collection name + * @param {object} params - request parameters + * @param {object} options - options of query + * @param {funtyion} callback - callback function with result + */ +async function fetchFromGranural(collection, params, options, callback) { + if (params.qstring.segmentation) { + if (params.qstring.segmentation === "key") { + params.qstring.segmentation = "n"; + } + else { + params.qstring.segmentation = "sg." + params.qstring.segmentation; + } + } + + if (params.qstring.refresh) { + params.qstring.period = "hour"; + } + + var queryObject = { + "appID": params.app_id, + "timezone": params.appTimezone, + "period": params.qstring.period, + "segmentation": params.qstring.segmentation, + "graphData": true + }; + if (params.qstring.event !== "[CLY]_star_rating") { + queryObject.event = "[CLY]_custom"; + queryObject.name = params.qstring.event; + } + if (collection === "users") { + queryObject.event = "[CLY]_session"; + } + + var data = await common.drillQueryRunner.getAggregatedData(queryObject); + var modelData = common.convertArrayToModel(data, params.qstring.segmentation); + modelData.lu = Date.now(); + var pp = options.id_prefix.split("_"); + try { + var meta = await common.drillReadBatcher.getOne("drill_meta", {"_id": pp[0] + "_meta_" + pp[1]}); + if (meta && meta.sg) { + modelData.meta = modelData.meta || {}; + modelData.meta.segments = []; + for (var val in meta.sg) { + modelData.meta.segments.push(val); + } + } + } + catch (e) { + console.log(e); + } + callback(modelData); +} + /** * Fetch db data in standard format * @param {string} collection - from which collection to fetch @@ -1607,8 +1747,10 @@ fetch.formatTotalUsersObj = function(obj, forMetric, prev) { * @param {array=} options.levels.daily - which metrics to expect on daily level, default ["t", "n", "c", "s", "dur"] * @param {array=} options.levels.monthly - which metrics to expect on monthly level, default ["t", "n", "d", "e", "c", "s", "dur"] * @param {function} callback - to call when fetch done + * @returns {void} void **/ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { + if (typeof options === "function") { callback = options; options = {}; @@ -1622,6 +1764,14 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { options.db = common.db; } + if (!params || !params.app_id || !params.qstring) { + return callback({}); + } + if (params.qstring.fetchFromGranural) { + fetchFromGranural(collection, params, options, callback); + return; + } + if (typeof options.unique === "undefined") { options.unique = common.dbUniqueMap[collection] || common.dbUniqueMap["*"]; } @@ -1647,7 +1797,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { } if (params.qstring.fullRange) { - options.db.collection(collection).find({ '_id': { $regex: "^" + (options.id_prefix || "") + options.id + ".*" } }).toArray(function(err1, data) { + options.db.collection(collection).find({ '_id': { $regex: "^" + (options.id_prefix || "") + options.id + ".*" + (options.id_postfix || "") } }).toArray(function(err1, data) { callback(getMergedObj(data, true, options.levels, params.truncateEventValuesList)); }); } @@ -1706,11 +1856,10 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { var monthDocs = [monthIdToFetch]; if (!options.dontBreak) { for (let i = 0; i < common.base64.length; i++) { - zeroDocs.push(zeroIdToFetch + "_" + common.base64[i]); - monthDocs.push(monthIdToFetch + "_" + common.base64[i]); + zeroDocs.push(zeroIdToFetch + "_" + common.base64[i] + (options.id_postfix || "")); + monthDocs.push(monthIdToFetch + "_" + common.base64[i] + (options.id_postfix || "")); } } - options.db.collection(collection).find({ '_id': { $in: zeroDocs } }, fetchFromZero).toArray(function(err1, zeroObject) { options.db.collection(collection).find({ '_id': { $in: monthDocs } }, fetchFromMonth).toArray(function(err2, monthObject) { zeroObject = zeroObject || []; @@ -1746,7 +1895,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { } else { for (let i = 0; i < periodObj.reqZeroDbDateIds.length; i++) { - documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqZeroDbDateIds[i]); + documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqZeroDbDateIds[i] + (options.id_postfix || "")); if (!(options && options.dontBreak)) { for (let m = 0; m < common.base64.length; m++) { documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqZeroDbDateIds[i] + "_" + common.base64[m]); @@ -1755,7 +1904,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { } for (let i = 0; i < periodObj.reqMonthDbDateIds.length; i++) { - documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqMonthDbDateIds[i]); + documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqMonthDbDateIds[i] + (options.id_postfix || "")); if (!(options && options.dontBreak)) { for (let m = 0; m < common.base64.length; m++) { documents.push((options.id_prefix || "") + options.id + "_" + periodObj.reqMonthDbDateIds[i] + "_" + common.base64[m]); @@ -1763,7 +1912,6 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) { } } } - options.db.collection(collection).find({ '_id': { $in: documents } }, {}).toArray(function(err, dataObjects) { if (err) { console.log(err); @@ -2007,130 +2155,6 @@ function union(x, y) { return res; } -/** -* Get data for jobs listing for jobs api -* @param {string} metric - name of the collection where to get data from -* @param {params} params - params object with app_id and date -*/ -fetch.fetchJobs = async function(metric, params) { - try { - if (params.qstring.name) { - await fetch.jobDetails(metric, params); - } - else { - await fetch.alljobs(metric, params); - } - } - catch (e) { - console.log(e); - common.returnOutput(params, 500, "Fetching jobs failed"); - } -}; - -/** -* Get all jobs grouped by job name for jobs api -* @param {string} metric - name of the collection where to get data from -* @param {params} params - params object with app_id and date -*/ -fetch.alljobs = async function(metric, params) { - const columns = ["name", "schedule", "next", "finished", "status", "total"]; - let sort = {}; - let total = await common.db.collection('jobs').aggregate([ - { - $group: { _id: "$name" } - }, - { - $count: 'total' - } - ]).toArray(); - total = total.length > 0 ? total[0].total : 0; - const pipeline = [ - { - $addFields: { - sortKey: { - $cond: { - if: { $eq: ["$status", 0] }, - then: 0, - else: { - $cond: { - if: { $eq: ["$status", 7] }, - then: 1, - else: 2 - } - } - } - } - } - }, - { - $sort: { - sortKey: 1, - finished: -1 - } - }, - { - $group: { - _id: "$name", - name: { $first: "$name" }, - status: { $first: "$status" }, - schedule: { $first: "$schedule" }, - next: { $first: "$next" }, - finished: { $first: "$finished" }, - total: { $sum: 1 }, - rowId: { $first: "$_id" } - } - } - ]; - if (params.qstring.sSearch) { - var rr; - try { - rr = new RegExp(params.qstring.sSearch, "i"); - pipeline.unshift({ - $match: { name: { $regex: rr } } - }); - } - catch (e) { - console.log('Could not use as regex:' + params.qstring.sSearch); - } - } - const cursor = common.db.collection('jobs').aggregate(pipeline, { allowDiskUse: true }); - sort[columns[params.qstring.iSortCol_0 || 0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1; - cursor.sort(sort); - cursor.skip(Number(params.qstring.iDisplayStart || 0)); - cursor.limit(Number(params.qstring.iDisplayLength || 10)); - let items = await cursor.toArray(); - items = items.map((job) => { - job.status = STATUS_MAP[job.status]; - return job; - }); - cursor.close(); - common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: total, aaData: items || [] }); -}; - -/** -* Get all documents for a given job name -* @param {string} metric - name of the collection where to get data from -* @param {params} params - params object with app_id and date -*/ -fetch.jobDetails = async function(metric, params) { - const columns = ["schedule", "next", "finished", "status", "data", "duration"]; - let sort = {}; - const total = await common.db.collection('jobs').count({ name: params.qstring.name }); - const cursor = common.db.collection('jobs').find({ name: params.qstring.name }); - sort[columns[params.qstring.iSortCol_0 || 0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1; - cursor.sort(sort); - cursor.skip(Number(params.qstring.iDisplayStart || 0)); - cursor.limit(Number(params.qstring.iDisplayLength || 10)); - let items = await cursor.toArray(); - items = items.map((job) => { - job.status = STATUS_MAP[job.status]; - return job; - }); - cursor.close(); - common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: total, aaData: items || [] }); -}; - - /** * Fetch data for tops * @param {params} params - params object diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js index 67ce22fc244..568e2037d85 100644 --- a/api/parts/data/usage.js +++ b/api/parts/data/usage.js @@ -1278,4 +1278,4 @@ plugins.register("/sdk/user_properties", async function(ob) { } }); -module.exports = usage; +module.exports = usage; \ No newline at end of file diff --git a/api/parts/jobs/index.js b/api/parts/jobs/index.js index 231d30e2d89..d7316ba9200 100644 --- a/api/parts/jobs/index.js +++ b/api/parts/jobs/index.js @@ -1,6 +1,6 @@ 'use strict'; -const countlyConfig = require('./../../config', 'dont-enclose'); +const countlyConfig = require('./../../config'); if (require('cluster').isMaster && process.argv[1].endsWith('api/api.js') && !(countlyConfig && countlyConfig.preventJobs)) { module.exports = require('./manager.js'); diff --git a/api/parts/jobs/scanner.js b/api/parts/jobs/scanner.js index dad13d46845..2050a0eb1c7 100644 --- a/api/parts/jobs/scanner.js +++ b/api/parts/jobs/scanner.js @@ -2,7 +2,30 @@ const log = require('../../utils/log.js')('jobs:scanner'), manager = require('../../../plugins/pluginManager.js'), - fs = require('fs'); + fs = require('fs'), + {Job, IPCJob, IPCFaçadeJob, TransientJob} = require('./job.js'); + +/** + * Validates if a job class has the required methods + * @param {Function} JobClass - The job class to validate + * @returns {boolean} - True if valid, throws error if invalid + */ +const validateJobClass = (JobClass) => { + // Check if it's a class/constructor + if (typeof JobClass !== 'function') { + throw new Error('Job must be a class constructor'); + } + + // Check if it inherits from one of the valid base classes + if (!(JobClass.prototype instanceof Job || + JobClass.prototype instanceof IPCJob || + JobClass.prototype instanceof IPCFaçadeJob || + JobClass.prototype instanceof TransientJob)) { + throw new Error('Job class must extend Job, IPCJob, IPCFaçadeJob, or TransientJob'); + } + + return true; +}; module.exports = (db, filesObj, classesObj) => { return new Promise((resolve, reject) => { @@ -52,12 +75,20 @@ module.exports = (db, filesObj, classesObj) => { (arr || []).forEach(job => { try { let name = job.category + ':' + job.name; - filesObj[name] = job.file; - classesObj[name] = require(job.file); - log.d('Found job %j at %j', name, job.file); + const JobClass = require(job.file); + if (validateJobClass(JobClass)) { + filesObj[name] = job.file; + classesObj[name] = JobClass; + log.d('Found valid job %j at %j', name, job.file); + } } catch (e) { - log.e('Error when loading job %s: %j ', job.file, e, e.stack); + if (e.message === "Job class must extend Job, IPCJob, IPCFaçadeJob, or TransientJob") { + // do nothing + } + else { + log.e('Error when loading job %s: %j ', job.file, e, e.stack); + } } }); }); diff --git a/api/parts/mgmt/app_users.js b/api/parts/mgmt/app_users.js index 3ee62d4e005..150ffd57740 100644 --- a/api/parts/mgmt/app_users.js +++ b/api/parts/mgmt/app_users.js @@ -108,6 +108,77 @@ usersApi.create = function(app_id, doc, params, callback) { }); }; +usersApi.createUserDocument = function(params, done) { + usersApi.getUid(params.app_id, async function(err, uid) { + if (uid) { + params.app_user.uid = uid; + if (!params.app_user._id) { + var doc = { + uid: uid, + did: params.qstring.device_id + }; + if (params && params.href) { + doc.first_req_get = (params.href + "") || ""; + } + else { + doc.first_req_get = ""; + } + if (params && params.req && params.req.body) { + doc.first_req_post = (params.req.body + "") || ""; + } + else { + doc.first_req_post = ""; + } + doc._id = params.app_user_id; + try { + var appUserDoc = await common.db.collection('app_users' + params.app_id).findOneAndUpdate({"_id": params.app_user_id}, {"$setOnInsert": doc}, {upsert: true, returnDocument: "after"}); + if (appUserDoc) { + if (appUserDoc.value && appUserDoc.value.uid) { + appUserDoc = appUserDoc.value; + } + done(null, appUserDoc); + } + else { + done("Failed user processing."); + } + } + catch (error) { + done(error); + } + } + else { + //document was created, but has no uid + //here we add uid only if it does not exist in db + //so if paralel request inserted it, we will not overwrite it + //and retrieve that uid on retry + try { + var userdoc = await common.db.collection('app_users' + params.app_id).findOneAndUpdate({ + _id: params.app_user_id, + uid: {$exists: false} + }, {$set: {uid: uid}}, {upsert: true, ignore_errors: [11000]}); + if (userdoc) { + if (userdoc.value && userdoc.value.uid) { + userdoc = userdoc.value; + } + done(null, userdoc); + } + else { + done('User merged. Failed to record data.'); + } + } + catch (ee) { + done(ee); + } + } + } + else { + //cannot create uid, so cannot process request now + console.log("Cannot create uid", err, uid); + done("User creation failed"); + } + }); +}; + /** * Update existing app_users document. Cannot replace document, must have modifiers like $set, $unset, etc * @param {string} app_id - _id of the app @@ -216,51 +287,61 @@ usersApi.delete = function(app_id, query, params, callback) { return; } - common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) { - if (res[0].exported) { - //delete exports if exist - for (let i = 0;i < res[0].exported.length; i++) { - let id = res[0].exported[i].split("/"); - id = id[id.length - 1]; //last one is filename - id = id.substr(id.length - 7); - - deleteMyExport(id).then( - function() {}, - function(err1) { - console.log(err1); - } - ); - } + //remove from drill_events + + common.drillDb.collection('drill_events').remove({"a": (app_id + ""), uid: {$in: res[0].uid}}, function(err1) { + if (err1) { + log.e("Failed to delete data from drill_events collection", err1); + common.returnMessage(params, 500, { errorMessage: "User deletion failed. Failed to delete some data related to this user." }); + return; + } - //deleting userimages(if they exist); - if (res[0].picture) { - for (let i = 0;i < res[0].picture.length; i++) { - //remove /userimages/ - let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12); - var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id); - countlyFs.deleteFile("userimages", pp, {id: id}, function(err1) { - if (err1) { - console.log(err1); - } - }); + common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) { + if (res[0].exported) { + //delete exports if exist + for (let i = 0;i < res[0].exported.length; i++) { + let id = res[0].exported[i].split("/"); + id = id[id.length - 1]; //last one is filename + id = id.substr(id.length - 7); + + deleteMyExport(id).then( + function() {}, + function(err5) { + console.log(err5); + } + ); + } } - } - try { - fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8"); - } - catch (err2) { - console.log(err2); - } - plugins.dispatch("/systemlogs", { - params: params, - action: "app_user_deleted", - data: { - app_id: app_id, - query: JSON.stringify(query), - uids: res[0].uid, + //deleting userimages(if they exist); + if (res[0].picture) { + for (let i = 0;i < res[0].picture.length; i++) { + //remove /userimages/ + let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12); + var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id); + countlyFs.deleteFile("userimages", pp, {id: id}, function(err6) { + if (err6) { + console.log(err6); + } + }); + } } + try { + fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8"); + } + catch (err2) { + console.log(err2); + } + plugins.dispatch("/systemlogs", { + params: params, + action: "app_user_deleted", + data: { + app_id: app_id, + query: JSON.stringify(query), + uids: res[0].uid, + } + }); + callback(err, res[0].uid); }); - callback(err, res[0].uid); }); }); }); @@ -364,24 +445,33 @@ usersApi.count = function(app_id, query, callback) { * @param {string} app_id - _id of the app * @param {function} callback - called when finished providing error (if any) as first param and new uid as second */ -usersApi.getUid = function(app_id, callback) { - common.db.collection('apps').findAndModify({_id: common.db.ObjectID(app_id)}, {}, {$inc: {seq: 1}}, { - new: true, - upsert: true - }, function(err, result) { - result = result && result.ok ? result.value : null; +usersApi.getUid = async function(app_id, callback) { + try { + var result = await common.db.collection('apps').findOneAndUpdate({_id: common.db.ObjectID(app_id + "")}, {$inc: {seq: 1}}, { + returnDocument: 'after', + upsert: false + }); + //When connected through our wrapper it returns value in value property not as root doc. + if (result && result.value && result.value.seq) { + result = result.value; + } if (result && result.seq) { if (callback) { - callback(err, common.parseSequence(result.seq)); + callback(null, common.parseSequence(result.seq)); } } else if (callback) { - callback(err); + callback("Document not returned for app:" + app_id); } - }); + } + catch (ee) { + callback(ee); + } }; + + usersApi.mergeOtherPlugins = function(options, callback) { var db = options.db; var app_id = options.app_id; @@ -436,18 +526,22 @@ usersApi.mergeOtherPlugins = function(options, callback) { log.e(err9); } var retry = false; + var retry_error = ""; if (result && result.length) { for (let index = 0; index < result.length; index++) { if (result[index].status === "rejected") { - log.e(result[index]); + console.log(JSON.stringify(result[index])); + retry_error += result[index].reason.message + "\n"; retry = true; break; } } + retry_error = retry_error.substring(0, 1000); + } if (retry) { //Unmark cc to let it be retried later in job. - common.db.collection('app_user_merges').update({"_id": iid}, {'$unset': {"cc": ""}, "$set": {"lu": Math.round(new Date().getTime() / 1000)}}, {upsert: false}, function(err4) { + common.db.collection('app_user_merges').update({"_id": iid}, {'$unset': {"cc": ""}, "$set": {"retry_error": retry_error, "lu": Math.round(new Date().getTime() / 1000)}}, {upsert: false}, function(err4) { if (err4) { log.e(err4); } @@ -457,13 +551,29 @@ usersApi.mergeOtherPlugins = function(options, callback) { }); } else { - //data merged. Delete record from merges collection - common.db.collection('app_user_merges').remove({"_id": iid}, function(err5) { - if (err5) { - log.e("Failed to remove merge document", err5); + //Merge data in drill_events collection + common.drillDb.collection('drill_events').updateMany({"a": app_id, "uid": oldAppUser.uid}, {'$set': {"uid": newAppUser.uid}}, function(err1) { + if (err1) { + log.e("Failed to update drill_events collection", err1); + common.db.collection('app_user_merges').update({"_id": iid}, {'$unset': {"cc": ""}, "$set": {"retry_error": "Failure while merging drill_events data", "lu": Math.round(new Date().getTime() / 1000)}}, {upsert: false}, function(err4) { + if (err4) { + log.e(err4); + } + if (callback && typeof callback === 'function') { + callback(err4); + } + }); } - if (callback && typeof callback === 'function') { - callback(err5); + else { + //data merged. Delete record from merges collection + common.db.collection('app_user_merges').remove({"_id": iid}, function(err5) { + if (err5) { + log.e("Failed to remove merge document", err5); + } + if (callback && typeof callback === 'function') { + callback(err5); + } + }); } }); } @@ -619,7 +729,7 @@ async function updateStatesInTransaction(common, app_id, newAppUserP, oldAppUser * @param {string} old_device_id - old user's device_id * @param {function} callback - called when finished providing error (if any) as first param and resulting merged document as second */ -usersApi.merge = function(app_id, newAppUser, new_id, old_id, new_device_id, old_device_id, callback) { +usersApi.merge = async function(app_id, newAppUser, new_id, old_id, new_device_id, old_device_id, callback) { /** * Inner function to merge user data * @param {object} newAppUserP - new user data @@ -631,100 +741,50 @@ usersApi.merge = function(app_id, newAppUser, new_id, old_id, new_device_id, old app_id: app_id, newAppUser: newAppUserP, oldAppUser: oldAppUser - }, function() { + }, async function() { //merge user data - usersApi.mergeUserProperties(newAppUserP, oldAppUser); - //update states in transaction to ensure integrity - - //we could use transactions, which makes it more stable, but we can't for now on all servers. - //keeping for future reference - /* - updateStatesInTransaction(common, app_id, newAppUserP, oldAppUser, function(err) { - if (err) { - log.e("Failed to update states in transaction", err); - } - if (callback && typeof callback === 'function') { - callback(err, newAppUserP); - } - if (!err) { - common.db.collection("metric_changes" + app_id).update({uid: oldAppUser.uid}, {'$set': {uid: newAppUserP.uid}}, {multi: true}, function(err7) { - if (err7) { - log.e("Failed metric changes update in app_users merge", err7); - } - else { - usersApi.mergeOtherPlugins(common.db, app_id, newAppUserP, oldAppUser, {"cc": true, "mc": true}, function() { + try { + await common.db.collection('app_users' + app_id).updateOne({_id: newAppUserP._id}, {'$set': newAppUserP}); + usersApi.mergeUserProperties(newAppUserP, oldAppUser); + await common.db.collection('app_users' + app_id).deleteOne({_id: oldAppUser._id}); - }); - } - }); - } - }); - */ - common.db.collection('app_users' + app_id).update({_id: newAppUserP._id}, {'$set': newAppUserP}, function(err) { - if (err) { - if (callback && typeof callback === 'function') { - callback(err, newAppUserP); //Filed. Old and new exists. SDK will re - } - } - else { - common.db.collection('app_users' + app_id).remove({_id: oldAppUser._id}, function(errRemoving) { - if (errRemoving) { - log.e("Failed to remove merged user from database", errRemoving); //Failed. Old and new exists. SDK will retry - } - if (callback && typeof callback === 'function') { - callback(errRemoving, newAppUserP); - } - //Dispatch to other plugins only after callback. - if (!errRemoving) { - //If it fails now - job will retry. - //update metric changes document - var iid = app_id + "_" + newAppUser.uid + "_" + oldAppUser.uid; - common.db.collection('app_user_merges').update({"_id": iid, "cc": {"$ne": true}}, {'$set': {"u": true}}, {upsert: false}, function(err1) { - if (err1) { - log.e(err1); - } - else { - common.db.collection("metric_changes" + app_id).update({uid: oldAppUser.uid}, {'$set': {uid: newAppUserP.uid}}, {multi: true}, function(err7) { - if (err7) { - log.e("Failed metric changes update in app_users merge", err7); - } - usersApi.mergeOtherPlugins({db: common.db, app_id: app_id, newAppUser: newAppUserP, oldAppUser: oldAppUser, updateFields: {"cc": true, "u": true, "mc": true}}, function() {}); - }); - } - }); - } - }); + } + catch (err) { + if (callback && typeof callback === 'function') { + callback(err, newAppUserP); } - }); + return; + } + callback(null, newAppUserP); + var iid = app_id + "_" + newAppUser.uid + "_" + oldAppUser.uid; + try { + await common.db.collection('app_user_merges').updateOne({"_id": iid, "cc": {"$ne": true}}, {'$set': {"u": true}}, {upsert: false}); + await common.db.collection("metric_changes" + app_id).updateMany({uid: oldAppUser.uid}, {'$set': {uid: newAppUserP.uid}}); + } + catch (e) { + log.e("Failed metric changes update in app_users merge", e); + } }); } - common.db.collection('app_users' + app_id).findOne({'_id': old_id }, function(err, oldAppUser) { - if (err) { - //problem getting old user data - return callback(err, oldAppUser); - } + try { + var oldAppUser = await common.db.collection('app_users' + app_id).findOne({'_id': old_id }); if (!oldAppUser) { - //there is no old user, process request return callback(null, newAppUser); } if (!newAppUser || !Object.keys(newAppUser).length) { - //new user does not exist yet - //simply copy user document with old uid - //no harm is done oldAppUser.did = new_device_id + ""; oldAppUser._id = new_id; - common.db.collection('app_users' + app_id).insert(oldAppUser, function(err2) { - if (err) { - callback(err2, oldAppUser); - } - else { - common.db.collection('app_users' + app_id).remove({_id: old_id}, function(err3) { - callback(err3, oldAppUser); - }); - } - }); + try { + await common.db.collection('app_users' + app_id).insertOne(oldAppUser); + await common.db.collection('app_users' + app_id).deleteOne({_id: old_id}); + } + catch (e) { + log.e("Failed to update in database. This does not prevent processing."); + log.e(e); + } + callback(null, oldAppUser); } else { //we have to merge user data @@ -747,19 +807,21 @@ usersApi.merge = function(app_id, newAppUser, new_id, old_id, new_device_id, old oldAppUser = newAppUser; newAppUser = tempDoc; } - common.db.collection('app_user_merges').insert({ + await common.db.collection("app_user_merges").insertOne({ //If we want to ensure order later then for each A->B we should check if there is B->C in progress and wait for it to finish first. So we could recheck using $regex _id: app_id + "_" + newAppUser.uid + "_" + oldAppUser.uid, merged_to: newAppUser.uid, ts: Math.round(new Date().getTime() / 1000), lu: Math.round(new Date().getTime() / 1000), t: 0 //tries - }, {ignore_errors: [11000]}, function() { - //If there is any merge inserted New->somethingElse, do not merge data yet. skip till that finishes. - mergeUserData(newAppUser, oldAppUser); - }); + }, {ignore_errors: [11000]}); + mergeUserData(newAppUser, oldAppUser); } - }); + + } + catch (e) { + callback(e, null); + } }; var deleteMyExport = function(exportID) { //tries to delete packed file, exported folder and saved export in gridfs diff --git a/api/parts/mgmt/apps.js b/api/parts/mgmt/apps.js index bd7f2c82412..f79922b1b64 100644 --- a/api/parts/mgmt/apps.js +++ b/api/parts/mgmt/apps.js @@ -836,6 +836,8 @@ function deleteAllAppData(appId, fromAppDelete, params, app) { common.db.collection('cities').remove({'_id': {$regex: "^" + appId + ".*"}}, function() {}); common.db.collection('top_events').remove({'app_id': common.db.ObjectID(appId)}, function() {}); common.db.collection('app_user_merges').remove({'_id': {$regex: "^" + appId + "_.*"}}, function() {}); + + common.drillDb.collection("drill_events").remove({"a": appId}, function() {}); deleteAppLongTasks(appId); /** * Deletes all app's events @@ -857,12 +859,8 @@ function deleteAllAppData(appId, fromAppDelete, params, app) { common.db.collection('metric_changes' + appId).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {}); common.db.collection('metric_changes' + appId).ensureIndex({uid: 1}, { background: true }, function() {}); }); - common.db.collection('app_user_merges' + appId).drop(function() { - common.db.collection('app_user_merges' + appId).ensureIndex({cd: 1}, { - expireAfterSeconds: 60 * 60 * 3, - background: true - }, function() {}); - }); + //Removes old app_user_merges collection + common.db.collection('app_user_merges' + appId).drop(function() {}); if (params.qstring.args.period === "reset") { plugins.dispatch("/i/apps/reset", { params: params, @@ -949,6 +947,7 @@ function deletePeriodAppData(appId, fromAppDelete, params, app) { common.db.collection('device_details').remove({$and: [{'_id': {$regex: appId + ".*"}}, {'_id': {$nin: skip}}]}, function() {}); common.db.collection('cities').remove({$and: [{'_id': {$regex: appId + ".*"}}, {'_id': {$nin: skip}}]}, function() {}); + common.drillDb.collection("drill_events").remove({"a": appId, "ts": {$lt: (oldestTimestampWanted * 1000)}}, function() {}); common.db.collection('events').findOne({'_id': common.db.ObjectID(appId)}, function(err, events) { if (!err && events && events.list) { common.arrayAddUniq(events.list, plugins.internalEvents); diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index d878ab7188f..835bc951c35 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -13,7 +13,6 @@ var tracker = {}, countlyConfig = require("../../../frontend/express/config.js"), versionInfo = require('../../../frontend/express/version.info'), ip = require('./ip.js'), - cluster = require('cluster'), os = require('os'), fs = require('fs'), asyncjs = require('async'), @@ -107,20 +106,19 @@ tracker.enable = function() { if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { Countly.track_errors(); } - if (cluster.isMaster) { - setTimeout(function() { - if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { - Countly.begin_session(true); - setTimeout(function() { - collectServerStats(); - collectServerData(); - }, 20000); - } - }, 1000); - //report app start trace - if (Countly.report_app_start) { - Countly.report_app_start(); + + setTimeout(function() { + if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { + Countly.begin_session(true); + setTimeout(function() { + collectServerStats(); + collectServerData(); + }, 20000); } + }, 1000); + //report app start trace + if (Countly.report_app_start) { + Countly.report_app_start(); } }; diff --git a/api/tcp_example.js b/api/tcp_example.js index 387ff224850..5d53fa48d37 100644 --- a/api/tcp_example.js +++ b/api/tcp_example.js @@ -1,5 +1,5 @@ const net = require('net'); -const countlyConfig = require('./config', 'dont-enclose'); +const countlyConfig = require('./config'); const plugins = require('../plugins/pluginManager.js'); const log = require('./utils/log.js')('core:tcp'); const common = require('./utils/common.js'); diff --git a/api/utils/calculatedDataManager.js b/api/utils/calculatedDataManager.js new file mode 100644 index 00000000000..f44ada6f036 --- /dev/null +++ b/api/utils/calculatedDataManager.js @@ -0,0 +1,137 @@ +/** +* Module for handling possibly long running tasks +* @module api/utils/taskmanager +*/ + +/** @lends module:api/utils/taskmanager */ +var calculatedDataManager = {}; +var common = require("./common.js"); +var crypto = require("crypto"); +var fetch = require("../parts/data/fetch.js"); +var plugins = require("../../plugins/pluginManager.js"); + +var collection = "drill_data_cache"; +const log = require('./log.js')('core:calculatedDataManager'); + + +//1. check if there is cached value based on params and it is not old- then return it. +//2. if there is none - try creating calculating document (_id base don hash+calculating) +//3. if it is already calculating - return id for cache that is beeing calucalted. + +/** + * Looks if there is any cached data for given query, if there is not marks it as calculating. returns result if can be calculated in time + * @param {object} options - options object + * @param {object} options.query_data - query data + * @param {object} options.db - db connection + * @param {function} options.outputData - function to output data + * @param {number} options.threshold - threshold in seconds + */ +calculatedDataManager.longtask = async function(options) { + options.id = calculatedDataManager.getId(options.query_data); + options.db = options.db || common.db; + var timeout; + var keep = parseInt(plugins.getConfig("drill").drill_snapshots_cache_time, 10) || 60 * 60 * 24; + keep = keep * 1000; + + /** + * Return message in case it takes too long + * @param {object} options5 - options + */ + function notifyClient(options5) { + if (!options5.retuned) { + options5.returned = true; + options5.outputData(null, {"_id": options5.id, "running": true}); + } + } + /** switching to long task + * @param {object} my_options - options + * @param {object} my_options.query_data - query data + * @param {object} my_options.db - db connection + * @param {function} my_options.outputData - function to output data + * @param {number} my_options.threshold - threshold in seconds + */ + async function switchToLongTask(my_options) { + timeout = setTimeout(notifyClient, my_options.threshold * 1000); + try { + await common.db.collection(collection).insertOne({_id: my_options.id, status: "calculating", "lu": new Date()}); + } + catch (e) { + my_options.outputData(e, {"_id": my_options.id, "running": false, "error": true}); + clearTimeout(timeout); + return; + } + fetch.fetchFromGranuralData(my_options.query_data, function(err, res) { + if (err) { + my_options.errored = true; + my_options.errormsg = err; + } + calculatedDataManager.saveResult(my_options, res); + clearTimeout(timeout); + if (!my_options.returned) { + my_options.outputData(err, {"_id": my_options.id, "data": res, "lu": new Date()}); + } + }); + } + var data = await common.db.collection(collection).findOne({_id: options.id}); + + if (data) { + if (data.status === "done") { + //check if it is not too old + if (data.lu && (new Date().getTime() - data.lu.getTime()) < keep) { + options.outputData(null, {"data": data.data, "lu": data.lu, "_id": options.id}); + return; + } + else { + common.db.collection(collection).deleteOne({_id: options.id}, function(ee) { + if (ee) { + log.e("Error while deleting calculated data", ee); + } + switchToLongTask(options); + }); + } + } + else if (data.status === "calculating") { + if (data.start && (new Date().getTime() - data.start) > 1000 * 60 * 60) { + options.outputData(null, {"_id": options.id, "running": true}); + return; + } + else { + common.db.collection(collection).deleteOne({_id: options.id}, function(ee) { + if (ee) { + log.e("Error while deleting calculated data", ee); + } + switchToLongTask(options); + }); + + } + } + } + else { + switchToLongTask(options); + return; + } +}; + +calculatedDataManager.saveResult = function(options, data) { + options.db.collection(collection).updateOne({_id: options.id}, {$set: {status: "done", data: data, lu: new Date()}}, {upsert: true}, function(err) { + if (err) { + log.e("Error while saving calculated data", err); + } + }); +}; +calculatedDataManager.getId = function(data) { + //Period should be given as 2 date + var keys = ["appID", "event", "name", "queryName", "query", "period", "periodOffset", "bucket", "segmentation"]; + var dataString = ""; + for (var i = 0; i < keys.length; i++) { + if (data[keys[i]]) { + dataString += data[keys[i]]; + } + } + console.log(dataString); + console.log(crypto.createHash('sha1').update(dataString).digest('hex')); + return crypto.createHash('sha1').update(dataString).digest('hex'); +}; + + +module.exports = calculatedDataManager; \ No newline at end of file diff --git a/api/utils/common.js b/api/utils/common.js index a6f2e3558b5..aade9f4626f 100644 --- a/api/utils/common.js +++ b/api/utils/common.js @@ -19,7 +19,7 @@ const crypto = require('crypto'); const logger = require('./log.js'); const mcc_mnc_list = require('mcc-mnc-list'); const plugins = require('../../plugins/pluginManager.js'); -const countlyConfig = require('./../config', 'dont-enclose'); +const countlyConfig = require('./../config'); const argon2 = require('argon2'); const mongodb = require('mongodb'); const getRandomValues = require('get-random-values'); @@ -95,6 +95,15 @@ common.encodeCharacters = function(str) { } }; +common.dbEncode = function(str) { + return str.replace(/^\$/g, "$").replace(/\./g, '.'); +}; + +/** +* Decode escaped html +* @param {string} string - The string to decode +* @returns {string} escaped string +**/ common.decode_html = function(string) { string = string.replace(/'/g, "'"); string = string.replace(/"/g, '"'); @@ -1883,6 +1892,8 @@ function recordMetric(params, metric, props, tmpSet, updateUsersZero, updateUser common.fillTimeObjectMonth(params, updateUsersMonth, monthObjUpdate, props.value); } +common.collectMetric = recordMetric; + /** * Record specific metric segment * @param {Params} params - params object @@ -2342,7 +2353,15 @@ common.clearClashingQueryOperations = function(query) { }; -common.updateAppUser = function(params, update, no_meta, callback) { +/** +* Single method to update app_users document for specific user for SDK requests +* @param {params} params - params object +* @param {object} update - update query for mongodb, should contain operators on highest level, as $set or $unset +* @param {boolean} no_meta - if true, won't update some auto meta data, like first api call, last api call, etc. +* @param {function} callback - function to run when update is done or failes, passing error and result as arguments +*/ +common.updateAppUser = async function(params, update, no_meta, callback) { + //backwards compatability if (typeof no_meta === "function") { callback = no_meta; @@ -2353,7 +2372,7 @@ common.updateAppUser = function(params, update, no_meta, callback) { if (i.indexOf("$") !== 0) { let err = "Unkown modifier " + i + " in " + update + " for " + params.href; console.log(err); - if (callback) { + if (callback && typeof callback === "function") { callback(err); } return; @@ -2485,24 +2504,36 @@ common.updateAppUser = function(params, update, no_meta, callback) { } if (callback) { - common.db.collection('app_users' + params.app_id).findAndModify({'_id': params.app_user_id}, {}, common.clearClashingQueryOperations(update), { - new: true, - upsert: true, - skipDataMasking: true - }, function(err, res) { - if (!err && res && res.value) { - params.app_user = res.value; - } - callback(err, res); - }); + try { + var res = await common.db.collection('app_users' + params.app_id).findOneAndUpdate({'_id': params.app_user_id}, common.clearClashingQueryOperations(update), { + returnDocument: 'after', + upsert: true, + }); + if (res) { + params.app_user = res; + } + if (callback && typeof callback === "function") { + callback(null, res); + } + } + catch (err) { + if (callback && typeof callback === "function") { + callback(err); + } + } } else { // using updateOne costs less than findAndModify, so we should use this // when acknowledging writes and updated information is not relevant (aka callback is not passed) - common.db.collection('app_users' + params.app_id).updateOne({'_id': params.app_user_id}, common.clearClashingQueryOperations(update), {upsert: true}, function() {}); + try { + await common.db.collection('app_users' + params.app_id).findOneAndUpdate({'_id': params.app_user_id}, common.clearClashingQueryOperations(update), {upsert: true, skipDataMasking: true, returnDocument: 'after'}); + } + catch (err) { + console.log(err); + } } } - else if (callback) { + else if (callback && typeof callback === "function") { callback(); } }; @@ -3379,6 +3410,247 @@ class DataTable { common.DataTable = DataTable; +common.applyUniqueOnModel = function(model, uniqueData, prop) { + for (var z = 0; z < uniqueData.length; z++) { + var value = uniqueData[z][prop]; + var iid = uniqueData[z]._id.split(":"); + if (iid.length > 1) { + if (!model[iid[0]]) { + model[iid[0]] = {}; + } + if (!model[iid[0]][iid[1]]) { + model[iid[0]][iid[1]] = {}; + } + if (iid.length > 2) { + if (!model[iid[0]][iid[1]][iid[2]]) { + model[iid[0]][iid[1]][iid[2]] = {}; + } + if (iid.length > 3) { + if (!model[iid[0]][iid[1]][iid[2]][iid[3]]) { + model[iid[0]][iid[1]][iid[2]][iid[3]] = {}; + } + model[iid[0]][iid[1]][iid[2]][iid[3]][prop] = value; + } + else { + model[iid[0]][iid[1]][iid[2]][prop] = value; + } + + } + else { + model[iid[0]][iid[1]][prop] = value; + + } + } + } +}; +/** + * Shifts hourly data (To be in different timezone) + * @param {*} data array of data + * @param {*} offset (integer) + * @returns {Array} shifted data + */ +common.shiftHourlyData = function(data, offset) { + if (typeof offset === "number") { + for (var z = 0; z < data.length; z++) { + var iid = data[z]._id.replace("h", "").split(":"); + var dd = Date.UTC(parseInt(iid[0], 10), parseInt(iid[1]), parseInt(iid[2]), parseInt(iid[3]), 0, 0); + dd = new Date(dd.valueOf() + offset * 60 * 60 * 1000); + iid = dd.getFullYear() + ":" + dd.getMonth() + ":" + dd.getDate() + ":" + dd.getHours(); + data[z]._id = iid; + } + } + return data; +}; + +/** + * Function converts usual Countly model data to array. (Not useful for unique values). Normally used to shift data and turn back to model. + * @param {object} model - countly model data + * @param {boolean} segmented - true if segmented + * @returns {Array} model data as array + */ +common.convertModelToArray = function(model, segmented) { + var data = []; + for (var year in model) { + if (common.isNumber(year)) { + for (var month in model[year]) { + if (common.isNumber(month)) { + for (var day in model[year][month]) { + if (common.isNumber(day)) { + for (var hour = 0; hour < 24; hour++) { + if (model[year][month][day][hour + ""]) { + var id = year + ":" + month + ":" + day + ":" + hour; + if (segmented) { + for (var segment in model[year][month][day][hour]) { + var obj = {_id: id, "sg": segment}; + for (var prop in model[year][month][day][hour][segment]) { + obj[prop] = model[year][month][day][hour][segment][prop]; + } + data.push(obj); + } + } + else { + var obj3 = {_id: id}; + for (var prop3 in model[year][month][day][hour]) { + obj3[prop3] = model[year][month][day][hour][prop3]; + } + data.push(obj3); + } + } + } + } + } + } + } + } + } + return data; + +}; + +/** + * Converts array of data to typical Countly model format. (Querying from granural data will give array. Use this to transform to model expected in frontend) + * @param {Array} arr data in format {"_id":"2014:1:1:1","u":1,"t":1,"n":1,"d":1,"m":1,"c":1,"b":1,"e":1,"s":1,"dur":1, "sg": "segment"} + * @param {boolean} segmented - if it is segmented. If not true - will ignore sg field + * @param {object} props - all expected props + * @returns {object} model data + */ +common.convertArrayToModel = function(arr, segmented, props) { + props = props || {"c": true, "s": true, "dur": true}; + /** + * Creates empty object with all property values set to 0 + * @param {object} my_props - all properies + * @returns {object} - object with 0 values for each from + */ + function createEmptyObj(my_props) { + var obj = {}; + for (var pp2 in my_props) { + obj[pp2] = 0; + } + return obj; + } + var model = createEmptyObj(props); + var iid; + var z; + if (segmented) { + segmented = segmented.replace("sg.", ""); + model.meta = {}; + var values = {}; + for (z = 0;z < arr.length;z++) { + iid = arr[z]._id.split(":"); + values[arr[z].sg] = true; + for (var p in props) { + if (arr[z][p]) { + model[arr[z].sg] = model[arr[z].sg] || createEmptyObj(props); + model[arr[z].sg][p] += arr[z][p]; + } + } + if (iid.length > 0) { + if (!model[iid[0]]) { + model[iid[0]] = {}; + } + if (!model[iid[0]][arr[z].sg]) { + model[iid[0]][arr[z].sg] = createEmptyObj(props); + } + for (var p0 in props) { + if (arr[z][p0]) { + model[iid[0]][arr[z].sg][p0] += arr[z][p0]; + } + } + + if (iid.length > 1) { + + if (!model[iid[0]][iid[1]]) { + model[iid[0]][iid[1]] = {}; + } + + if (!model[iid[0]][iid[1]][arr[z].sg]) { + model[iid[0]][iid[1]][arr[z].sg] = createEmptyObj(props); + } + for (var p1 in props) { + if (arr[z][p1]) { + model[iid[0]][iid[1]][arr[z].sg][p1] += arr[z][p1]; + } + } + if (iid.length > 2) { + + if (!model[iid[0]][iid[1]][iid[2]]) { + model[iid[0]][iid[1]][iid[2]] = {}; + } + + if (!model[iid[0]][iid[1]][iid[2]][arr[z].sg]) { + model[iid[0]][iid[1]][iid[2]][arr[z].sg] = createEmptyObj(props); + } + for (var p2 in props) { + if (arr[z][p2]) { + model[iid[0]][iid[1]][iid[2]][arr[z].sg][p2] += arr[z][p2]; + } + } + } + } + } + } + model.meta[segmented] = Object.keys(values); + } + else { + for (z = 0;z < arr.length;z++) { + iid = arr[z]._id.split(":"); + for (var pp6 in props) { + if (arr[z][pp6]) { + model[pp6] += arr[z][pp6]; + } + } + if (iid.length > 0) { + if (!model[iid[0]]) { + model[iid[0]] = createEmptyObj(props); + } + for (var p00 in props) { + if (arr[z][p00]) { + model[iid[0]][p00] += arr[z][p00]; + } + } + + if (iid.length > 1) { + if (!model[iid[0]][iid[1]]) { + model[iid[0]][iid[1]] = createEmptyObj(props); + } + for (var p10 in props) { + if (arr[z][p10]) { + model[iid[0]][iid[1]][p10] += arr[z][p10]; + } + } + if (iid.length > 2) { + if (!model[iid[0]][iid[1]][iid[2]]) { + model[iid[0]][iid[1]][iid[2]] = createEmptyObj(props); + } + for (var p20 in props) { + if (arr[z][p20]) { + model[iid[0]][iid[1]][iid[2]][p20] += arr[z][p20]; + } + } + if (iid.length > 3) { + if (!model[iid[0]][iid[1]][iid[2]][iid[3]]) { + model[iid[0]][iid[1]][iid[2]][iid[3]] = createEmptyObj(props); + } + for (var p3 in props) { + if (arr[z][p3]) { + model[iid[0]][iid[1]][iid[2]][iid[3]][p3] += arr[z][p3]; + } + } + } + } + } + } + } + } + return model; +}; + +/** + * Sync license check results to request (and session if present) + * + * @param {object} req request + * @param {object|undefined} check check results + */ common.licenseAssign = function(req, check) { if (check && check.error) { req.licenseError = check.error; diff --git a/api/utils/log.js b/api/utils/log.js index 40bdc3e6a3d..bbd6ca9f033 100644 --- a/api/utils/log.js +++ b/api/utils/log.js @@ -1,68 +1,47 @@ -'use strict'; +const pino = require('pino'); + +// Optional OpenTelemetry imports +let trace; +let context; +let metrics; +let semanticConventions; +try { + trace = require('@opentelemetry/api').trace; + context = require('@opentelemetry/api').context; + metrics = require('@opentelemetry/api').metrics; + semanticConventions = require('@opentelemetry/semantic-conventions'); + // eslint-disable-next-line no-empty +} +catch (e) { + // do nothing +} /** - * Log provides a wrapper over debug or console functions with log level filtering, module filtering and ability to store log in database. - * Uses configuration require('../config.js').logging: - * { - * 'info': ['app', 'auth', 'static'], // log info and higher level for modules 'app*', 'auth*', 'static*' - * 'debug': ['api.users'], // log debug and higher (in fact everything) for modules 'api.users*' - * 'default': 'warn', // log warn and higher for all other modules - * } - * Note that log levels supported are ['debug', 'info', 'warn', 'error'] - * - * Usage is quite simple: - * var log = require('common.js').log('module[:submodule[:subsubmodule]]'); - * log.i('something happened: %s, %j', 'string', {obj: 'ect'}); - * log.e('something really bad happened: %j', new Error('Oops')); - * - * Whenever DEBUG is in process.env, log outputs all filtered messages with debug module instead of console so you could have pretty colors in console. - * In other cases only log.d is logged using debug module. - * - * To control log level at runtime, call require('common.js').log.setLevel('events', 'debug'). From now on 'events' logger will log everything. - * - * There is also a handy method for generating standard node.js callbacks which log error. Only applicable if no actions in case of error needed: - * collection.find().toArray(log.callback(function(arg1, arg2){ // all good })); - * - if error didn't happen, function is called - * - if error happened, it will be logged, but function won't be called - * - if error happened, arg1 is a first argument AFTER error, it's not an error - * @module api/utils/log + * Fallback for performance.now() if not available + * @returns {number} Current timestamp in milliseconds */ - -var prefs = require('../config.js', 'dont-enclose').logging || {}; -prefs.default = prefs.default || "warn"; -var colors = require('colors'); -var deflt = (prefs && prefs.default) ? prefs.default : 'error'; - -for (let level in prefs) { - if (prefs[level].sort) { - prefs[level].sort(); +const getNow = () => { + if (typeof performance !== 'undefined' && performance.now) { + return performance.now(); } -} + return Date.now(); +}; -var styles = { - moduleColors: { - // 'push:*api': 0 // green - '[last]': -1 - }, - colors: ['green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'gray', 'red'], - stylers: { - warn: function(args) { - for (var i = 0; i < args.length; i++) { - if (typeof args[i] === 'string') { - args[i] = colors.bgYellow.black(args[i].black); - } - } - }, - error: function(args) { - for (var i = 0; i < args.length; i++) { - if (typeof args[i] === 'string') { - args[i] = colors.bgRed.white(args[i].white); - } - } - } - } +/** + * Mapping of short level codes to full level names + * @type {Object.} + */ +const LEVELS = { + d: 'debug', + i: 'info', + w: 'warn', + e: 'error' }; +/** + * Mapping of log levels to acceptable log levels + * @type {Object.} + */ const ACCEPTABLE = { d: ['debug'], i: ['debug', 'info'], @@ -70,61 +49,97 @@ const ACCEPTABLE = { e: ['debug', 'info', 'warn', 'error'], }; -const NAMES = { - d: 'DEBUG', - i: 'INFO', - w: 'WARN', - e: 'ERROR' -}; + +// Initialize configuration with defaults +let prefs = require('../config.js').logging || {}; +prefs.default = prefs.default || "warn"; +let deflt = (prefs && prefs.default) ? prefs.default : 'error'; /** - * Returns logger function for given preferences - * @param {string} level - log level - * @param {string} prefix - add prefix to message - * @param {boolean} enabled - whether function should log anything - * @param {object} outer - this for @out - * @param {function} out - output function (console or debug) - * @param {function} styler - function to apply styles - * @returns {function} logger function + * Current levels for all modules + * @type {Object.} */ -var log = function(level, prefix, enabled, outer, out, styler) { - return function() { - // console.log(level, prefix, enabled(), arguments); - if (enabled()) { - var args = Array.prototype.slice.call(arguments, 0); - var color = styles.moduleColors[prefix]; - if (color === undefined) { - color = (++styles.moduleColors['[last]']) % styles.colors.length; - styles.moduleColors[prefix] = color; - } - color = styles.colors[color]; - if (styler) { - args[0] = new Date().toISOString() + ': ' + level + '\t' + '[' + (prefix || '') + ']\t' + args[0]; - styler(args); - } - else { - args[0] = (new Date().toISOString() + ': ' + level + '\t').gray + colors[color]('[' + (prefix || '') + ']\t') + args[0]; - } - // args[0] = (new Date().toISOString() + ': ' + (prefix || '')).gray + args[0]; - // console.log('Logging %j', args); - if (typeof out === 'function') { - out.apply(outer, args); - } - else { - for (var k in out) { - out[k].apply(outer, args); - } - } - } +const levels = {}; + +// Metrics setup if OpenTelemetry is available +let logCounter; +let logDurationHistogram; + +if (metrics) { + const meter = metrics.getMeter('logger'); + logCounter = meter.createCounter('log_entries_total', { + description: 'Number of log entries by level and module', + }); + logDurationHistogram = meter.createHistogram('log_duration_seconds', { + description: 'Duration of logging operations', + }); +} + +/** + * Gets the current trace context if OpenTelemetry is available + * @returns {Object|null} Trace context object or null if unavailable + */ +function getTraceContext() { + if (!trace) { + return null; + } + + const currentSpan = trace.getSpan(context.active()); + if (!currentSpan) { + return null; + } + + const spanContext = currentSpan.spanContext(); + return { + 'trace.id': spanContext.traceId, + 'span.id': spanContext.spanId, + 'trace.flags': spanContext.traceFlags.toString(16) }; -}; +} + +/** + * Creates a logging span if OpenTelemetry is available + * @param {string} name - The module name + * @param {string} level - The log level + * @param {string} message - The log message + * @returns {Span|null} The created span or null if unavailable + */ +function createLoggingSpan(name, level, message) { + if (!trace) { + return null; + } + + const tracer = trace.getTracer('logger'); + return tracer.startSpan(`log.${level}`, { + attributes: { + [semanticConventions.SemanticAttributes.CODE_FUNCTION]: name, + [semanticConventions.SemanticAttributes.CODE_NAMESPACE]: 'logger', + 'logging.level': level, + 'logging.message': message + } + }); +} + +/** + * Records metrics for logging operations + * @param {string} name - The module name + * @param {string} level - The log level + */ +function recordMetrics(name, level) { + if (logCounter) { + logCounter.add(1, { + module: name, + level: level + }); + } +} /** * Looks for logging level in config for a particular module - * @param {string} name - module name - * @returns {string} log level + * @param {string} name - The module name + * @returns {string} The configured log level */ -var logLevel = function(name) { +const logLevel = function(name) { if (typeof prefs === 'undefined') { return 'error'; } @@ -132,20 +147,17 @@ var logLevel = function(name) { return prefs; } else { - for (var level in prefs) { + for (let level in prefs) { if (typeof prefs[level] === 'string' && name.indexOf(prefs[level]) === 0) { return level; } if (typeof prefs[level] === 'object' && prefs[level].length) { - for (var i = prefs[level].length - 1; i >= 0; i--) { - var opt = prefs[level][i]; + for (let i = prefs[level].length - 1; i >= 0; i--) { + let opt = prefs[level][i]; if (opt === name || name.indexOf(opt) === 0) { return level; } } - // for (var m in prefs[level]) { - // if (name.indexOf(prefs[level][m]) === 0) { return level; } - // } } } return deflt; @@ -153,250 +165,242 @@ var logLevel = function(name) { }; /** - * Current levels for all modules + * Build a transport config: pretty in dev, JSON in prod. + * @returns {Object} Transport config */ -var levels = { - // mongo: 'info' -}; -/** -* Sets current logging level -* @static -* @param {string} module - name of the module for logging -* @param {string} level - level of logging, possible values are: debug, info, warn, error -**/ -var setLevel = function(module, level) { - levels[module] = level; -}; -/** -* Sets default logging level for all modules, that do not have specific level set -* @static -* @param {string} level - level of logging, possible values are: debug, info, warn, error -**/ -var setDefault = function(level) { - deflt = level; -}; +// function getTransport() { +// if (process.env.NODE_ENV === 'development') { +// return { +// target: 'pino-pretty', +// options: { +// colorize: true, +// translateTime: 'yyyy-mm-dd HH:MM:ss.l', +// ignore: 'pid,hostname' +// } +// }; +// } +// return undefined; +// } + + /** -* Get currently set logging level for module -* @static -* @param {string} module - name of the module for logging -* @returns {string} level of logging, possible values are: debug, info, warn, error -**/ -var getLevel = function(module) { - return levels[module] || deflt; + * Creates a Pino logger instance with the appropriate configuration + * @param {string} name - The module name + * @param {string} [level] - The log level + * @returns {Logger} Configured Pino logger instance + */ +const createLogger = (name, level) => { + return pino({ + name, + level: level || deflt, + timestamp: pino.stdTimeFunctions.isoTime, + formatters: { + level: (label) => { + return { level: label.toUpperCase() }; + }, + log: (obj) => { + const traceContext = getTraceContext(); + return traceContext ? { ...obj, ...traceContext } : obj; + } + }, + sync: false, + browser: false, + // transport: getTransport() + }); }; -var getEnabledWithLevel = function(acceptable, module) { - return function() { - // if (acceptable.indexOf(levels[module]) === -1) { - // console.log('Won\'t log %j because %j doesn\'t have %j (%j)', module, acceptable, levels[module], levels); - // } - return acceptable.indexOf(levels[module] || deflt) !== -1; - }; -}; /** -* Handle messages from ipc -* @static -* @param {string} msg - message received from other processes -**/ -var ipcHandler = function(msg) { - var m, l, modules, i; - - if (!msg || msg.cmd !== 'log' || !msg.config) { - return; - } - - // console.log('%d: Setting logging config to %j (was %j)', process.pid, msg.config, levels); - - if (msg.config.default) { - deflt = msg.config.default; - } - - for (m in levels) { - var found = null; - for (l in msg.config) { - modules = msg.config[l].split(',').map(function(v) { - return v.trim(); - }); - - for (i = 0; i < modules.length; i++) { - if (modules[i] === m) { - found = l; + * Creates a logging function for a specific level + * @param {Logger} logger - The Pino logger instance + * @param {string} name - The module name + * @param {string} level - The log level code (d, i, w, e) + * @returns {Function} The logging function + */ +const createLogFunction = (logger, name, level) => { + return function(...args) { + const currentLevel = levels[name] || deflt; + if (ACCEPTABLE[level].indexOf(currentLevel) !== -1) { + const startTime = getNow(); + const message = args[0]; + + // Create span for this logging operation + const span = createLoggingSpan(name, LEVELS[level], message); + + try { + if (args.length === 1) { + logger[LEVELS[level]](args[0]); + } + else { + const msg = args.shift(); + logger[LEVELS[level]](msg, ...args); } - } - } - if (found === null) { - for (l in msg.config) { - modules = msg.config[l].split(',').map(function(v) { - return v.trim(); - }); + // Record metrics + recordMetrics(name, LEVELS[level]); - for (i = 0; i < modules.length; i++) { - if (modules[i].indexOf('*') === -1 && modules[i] === m.split(':')[0]) { - found = l; - } - else if (modules[i].indexOf('*') !== -1 && modules[i].split(':')[1] === '*' && modules[i].split(':')[0] === m.split(':')[0]) { - found = l; - } + // Record duration + if (logDurationHistogram) { + const duration = (getNow() - startTime) / 1000; // Convert to seconds + logDurationHistogram.record(duration, { + module: name, + level: LEVELS[level] + }); } } - } - - if (found !== null) { - levels[m] = found; - } - else { - levels[m] = deflt; - } - } - - for (l in msg.config) { - if (msg.config[l] && l !== 'default') { - modules = msg.config[l].split(',').map(function(v) { - return v.trim(); - }); - prefs[l] = modules; - - for (i in modules) { - m = modules[i]; - if (!(m in levels)) { - levels[m] = l; + finally { + if (span) { + span.end(); } } } - else { - prefs[l] = []; - } - } + }; +}; - prefs.default = msg.config.default; +/** + * Sets current logging level for a module + * @param {string} module - The module name + * @param {string} level - The log level + */ +const setLevel = function(module, level) { + levels[module] = level; +}; - // console.log('%d: Set logging config to %j (now %j)', process.pid, msg.config, levels); +/** + * Sets default logging level + * @param {string} level - The log level + */ +const setDefault = function(level) { + deflt = level; }; /** - * @typedef {Object} Logger - * @property {function(): string} id - Get the logger id - * @example - * const loggerId = logger.id(); - * console.log(`Current logger ID: ${loggerId}`); - * - * @property {function(...*): void} d - Log debug level messages - * @example - * logger.d('Debug message: %s', 'Some debug info'); - * - * @property {function(...*): void} i - Log information level messages - * @example - * logger.i('Info message: User %s logged in', username); - * - * @property {function(...*): void} w - Log warning level messages - * @example - * logger.w('Warning: %d attempts failed', attempts); - * - * @property {function(...*): void} e - Log error level messages - * @example - * logger.e('Error occurred: %o', errorObject); - * - * @property {function(string, function, string, ...*): boolean} f - Log variable level messages - * @example - * logger.f('d', (log) => { - * const expensiveOperation = performExpensiveCalculation(); - * log('Debug: Expensive operation result: %j', expensiveOperation); - * }, 'i', 'Skipped expensive debug logging'); - * - * @property {function(function=): function} callback - Create a callback function for logging - * @example - * const logCallback = logger.callback((result) => { - * console.log('Operation completed with result:', result); - * }); - * someAsyncOperation(logCallback); - * - * @property {function(string, function=, function=): function} logdb - Create a callback function for logging database operations - * @example - * const dbCallback = logger.logdb('insert user', - * (result) => { console.log('User inserted:', result); }, - * (error) => { console.error('Failed to insert user:', error); } - * ); - * database.insertUser(userData, dbCallback); - * - * @property {function(string): Logger} sub - Create a sub-logger - * @example - * const subLogger = logger.sub('database'); - * subLogger.i('Connected to database'); + * Gets current logging level for a module + * @param {string} module - The module name + * @returns {string} The current log level */ +const getLevel = function(module) { + return levels[module] || deflt; +}; /** - * Creates a new logger object for the provided module - * @module api/utils/log - * @param {string} name - Name of the module - * @returns {Logger} Logger object - * @example - * const logger = require('./log.js')('myModule'); - * logger.i('MyModule initialized'); + * Creates a new logger instance + * @param {string} name - The module name + * @returns {Object} Logger instance with various methods */ module.exports = function(name) { setLevel(name, logLevel(name)); - // console.log('Got level for ' + name + ': ' + levels[name] + ' ( prefs ', prefs); + const logger = createLogger(name, levels[name]); + /** - * @type Logger - **/ + * Creates a sub-logger with the parent's name as prefix + * @param {string} subname - The sub-logger name + * @returns {Object} Sub-logger instance + */ + const createSubLogger = (subname) => { + const full = name + ':' + subname; + setLevel(full, logLevel(full)); + const subLogger = createLogger(full, levels[full]); + + return { + /** + * Returns the full identifier of this sub-logger + * @returns {string} Full logger identifier + */ + id: () => full, + + /** + * Logs a debug message + * @param {...*} args - Message and optional parameters + */ + d: createLogFunction(subLogger, full, 'd'), + + /** + * Logs an info message + * @param {...*} args - Message and optional parameters + */ + i: createLogFunction(subLogger, full, 'i'), + + /** + * Logs a warning message + * @param {...*} args - Message and optional parameters + */ + w: createLogFunction(subLogger, full, 'w'), + + /** + * Logs an error message + * @param {...*} args - Message and optional parameters + */ + e: createLogFunction(subLogger, full, 'e'), + + /** + * Conditionally executes a function based on current log level + * @param {string} l - Log level code + * @param {Function} fn - Function to execute if level is enabled + * @param {string} [fl] - Fallback log level + * @param {...*} fargs - Arguments for fallback + * @returns {boolean} True if the function was executed + */ + f: function(l, fn, fl, ...fargs) { + if (ACCEPTABLE[l].indexOf(levels[full] || deflt) !== -1) { + fn(createLogFunction(subLogger, full, l)); + return true; + } + else if (fl) { + this[fl].apply(this, fargs); + } + }, + + /** + * Creates a nested sub-logger + * @param {string} subname - The nested sub-logger name + * @returns {Object} Nested sub-logger instance + */ + sub: createSubLogger + }; + }; + return { /** - * Get logger id - * @returns {string} id of this logger - * @example - * const loggerId = logger.id(); - * console.log(`Current logger ID: ${loggerId}`); - */ + * Returns the identifier of this logger + * @returns {string} Logger identifier + */ id: () => name, + /** - * Log debug level messages - * @param {...*} var_args - string and values to format string with - * @example - * logger.d('Debug message: %s', 'Some debug info'); - */ - d: log(NAMES.d, name, getEnabledWithLevel(ACCEPTABLE.d, name), this, console.log), + * Logs a debug message + * @param {...*} args - Message and optional parameters + */ + d: createLogFunction(logger, name, 'd'), /** - * Log information level messages - * @param {...*} var_args - string and values to format string with - * @example - * logger.i('Info message: User %s logged in', username); + * Logs an info message + * @param {...*} args - Message and optional parameters */ - i: log(NAMES.i, name, getEnabledWithLevel(ACCEPTABLE.i, name), this, console.info), + i: createLogFunction(logger, name, 'i'), /** - * Log warning level messages - * @param {...*} var_args - string and values to format string with - * @example - * logger.w('Warning: %d attempts failed', attempts); + * Logs a warning message + * @param {...*} args - Message and optional parameters */ - w: log(NAMES.w, name, getEnabledWithLevel(ACCEPTABLE.w, name), this, console.warn, styles.stylers.warn), + w: createLogFunction(logger, name, 'w'), /** - * Log error level messages - * @param {...*} var_args - string and values to format string with - * @example - * logger.e('Error occurred: %o', errorObject); + * Logs an error message + * @param {...*} args - Message and optional parameters */ - e: log(NAMES.e, name, getEnabledWithLevel(ACCEPTABLE.e, name), this, console.error, styles.stylers.error), + e: createLogFunction(logger, name, 'e'), /** - * Log variable level messages (for cases when logging parameters calculation are expensive enough and shouldn't be done unless the level is enabled) - * @param {string} l - log level (d, i, w, e) - * @param {function} fn - function to call with single argument - logging function - * @param {string} fl - fallback level if l is disabled - * @param {...*} fargs - fallback level arguments - * @returns {boolean} true if f() has been called - * @example - * logger.f('d', (log) => { - * const expensiveOperation = performExpensiveCalculation(); - * log('Debug: Expensive operation result: %j', expensiveOperation); - * }, 'i', 'Skipped expensive debug logging'); + * Conditionally executes a function based on current log level + * @param {string} l - Log level code + * @param {Function} fn - Function to execute if level is enabled + * @param {string} [fl] - Fallback log level + * @param {...*} fargs - Arguments for fallback + * @returns {boolean} True if the function was executed */ f: function(l, fn, fl, ...fargs) { if (ACCEPTABLE[l].indexOf(levels[name] || deflt) !== -1) { - fn(log(NAMES[l], name, getEnabledWithLevel(ACCEPTABLE[l], name), this, l === 'e' ? console.error : l === 'w' ? console.warn : console.log, l === 'w' ? styles.stylers.warn : l === 'e' ? styles.stylers.error : undefined)); + fn(createLogFunction(logger, name, l)); return true; } else if (fl) { @@ -405,42 +409,32 @@ module.exports = function(name) { }, /** - * Logging inside callbacks - * @param {function=} next - next function to call, after callback executed - * @returns {function} function to pass as callback - * @example - * const logCallback = logger.callback((result) => { - * console.log('Operation completed with result:', result); - * }); - * someAsyncOperation(logCallback); + * Creates a callback function that logs errors + * @param {Function} [next] - Function to call on success + * @returns {Function} Callback function */ callback: function(next) { - var self = this; + const self = this; return function(err) { if (err) { self.e(err); } else if (next) { - var args = Array.prototype.slice.call(arguments, 1); + const args = Array.prototype.slice.call(arguments, 1); next.apply(this, args); } }; }, + /** - * Logging database callbacks - * @param {string} opname - name of the performed operation - * @param {function=} next - next function to call, after callback executed - * @param {function=} nextError - function to pass error to - * @returns {function} function to pass as callback - * @example - * const dbCallback = logger.logdb('insert user', - * (result) => { console.log('User inserted:', result); }, - * (error) => { console.error('Failed to insert user:', error); } - * ); - * database.insertUser(userData, dbCallback); + * Creates a database operation callback that logs results + * @param {string} opname - Operation name + * @param {Function} [next] - Function to call on success + * @param {Function} [nextError] - Function to call on error + * @returns {Function} Database callback function */ logdb: function(opname, next, nextError) { - var self = this; + const self = this; return function(err) { if (err) { self.e('Error while %j: %j', opname, err); @@ -456,99 +450,39 @@ module.exports = function(name) { } }; }, + /** - * Add one more level to the logging output while leaving loglevel the same - * @param {string} subname - sublogger name - * @returns {Logger} new logger - * @example - * const subLogger = logger.sub('database'); - * subLogger.i('Connected to database'); + * Creates a sub-logger with the current logger's name as prefix + * @param {string} subname - The sub-logger name + * @returns {Object} Sub-logger instance */ - - sub: function(subname) { - let full = name + ':' + subname, - self = this; - - setLevel(full, logLevel(full)); - - return { - /** - * Get logger id - * @returns {string} id of this logger - */ - id: () => full, - /** - * Log debug level messages - * @memberof module:api/utils/log~Logger - * @param {...*} var_args - string and values to format string with - **/ - d: log(NAMES.d, full, getEnabledWithLevel(ACCEPTABLE.d, full), this, console.log), - - /** - * Log information level messages - * @memberof module:api/utils/log~Logger - * @param {...*} var_args - string and values to format string with - **/ - i: log(NAMES.i, full, getEnabledWithLevel(ACCEPTABLE.i, full), this, console.info), - - /** - * Log warning level messages - * @memberof module:api/utils/log~Logger - * @param {...*} var_args - string and values to format string with - **/ - w: log(NAMES.w, full, getEnabledWithLevel(ACCEPTABLE.w, full), this, console.warn, styles.stylers.warn), - - /** - * Log error level messages - * @memberof module:api/utils/log~Logger - * @param {...*} var_args - string and values to format string with - **/ - e: log(NAMES.e, full, getEnabledWithLevel(ACCEPTABLE.e, full), this, console.error, styles.stylers.error), - - /** - * Log variable level messages (for cases when logging parameters calculation are expensive enough and shouldn't be done unless the level is enabled) - * @param {String} l log level (d, i, w, e) - * @param {function} fn function to call with single argument - logging function - * @param {String} fl fallback level if l is disabled - * @param {any[]} fargs fallback level arguments - * @returns {boolean} true if f() has been called - */ - f: function(l, fn, fl, ...fargs) { - if (ACCEPTABLE[l].indexOf(levels[name] || deflt) !== -1) { - fn(log(NAMES[l], full, getEnabledWithLevel(ACCEPTABLE[l], full), this, l === 'e' ? console.error : l === 'w' ? console.warn : console.log, l === 'w' ? styles.stylers.warn : l === 'e' ? styles.stylers.error : undefined)); - return true; - } - else if (fl) { - this[fl].apply(this, fargs); - } - }, - - /** - * Pass sub one level up - */ - sub: self.sub.bind(self) - }; - } + sub: createSubLogger }; - // return { - // d: log('DEBUG\t', getEnabledWithLevel(ACCEPTABLE.d, name), this, debug(name)), - // i: log('INFO\t', getEnabledWithLevel(ACCEPTABLE.i, name), this, debug(name)), - // w: log('WARN\t', getEnabledWithLevel(ACCEPTABLE.w, name), this, debug(name)), - // e: log('ERROR\t', getEnabledWithLevel(ACCEPTABLE.e, name), this, debug(name)), - // callback: function(next){ - // var self = this; - // return function(err) { - // if (err) { self.e(err); } - // else if (next) { - // var args = Array.prototype.slice.call(arguments, 1); - // next.apply(this, args); - // } - // }; - // }, - // }; }; +// Export static methods +/** + * Sets logging level for a specific module + * @param {string} module - The module name + * @param {string} level - The log level + */ module.exports.setLevel = setLevel; + +/** + * Sets default logging level for all modules without explicit configuration + * @param {string} level - The log level + */ module.exports.setDefault = setDefault; + +/** + * Gets current logging level for a module + * @param {string} module - The module name + * @returns {string} The current log level + */ module.exports.getLevel = getLevel; -module.exports.ipcHandler = ipcHandler; \ No newline at end of file + +/** + * Indicates if OpenTelemetry integration is available + * @type {boolean} + */ +module.exports.hasOpenTelemetry = Boolean(trace && metrics); \ No newline at end of file diff --git a/api/utils/mongoDbQueryRunner.js b/api/utils/mongoDbQueryRunner.js new file mode 100644 index 00000000000..6e257d94b51 --- /dev/null +++ b/api/utils/mongoDbQueryRunner.js @@ -0,0 +1,655 @@ +const moment = require('moment-timezone'); + +/** + * Class for running Mongodb drill queries + */ +class MongoDbQueryRunner { + /** + * Constructor + * @param {object} mongoDb - mongodb connection + */ + constructor(mongoDb) { + this.db = mongoDb; + } + + + /** +* Returns number for timestamp making sure it is 13 digits +* @param {integer}ts - number we need to se as timestamp +* @returns {integer} timestamp in ms +**/ + fixTimestampToMilliseconds(ts) { + if ((ts + "").length > 13) { + ts = (ts + '').substring(0, 13); + ts = parseInt(ts, 10); + } + return ts; + } + + /** + * Full query runner + * dbFilter - what we can set as query also in normal drill query + * event - single or array + * name - single or array + * period + * timezoneOffset + * props to calculate (object); + * byVal + * is_graph - boolean, if true, check bucket + */ + + /** + * + * @param {object} period - common period in Countly + * @param {string} timezone - app timezone + * @param {number} offset - offset in minutes + * @returns {object} - describes period range + */ + getPeriodRange(period, timezone, offset) { + //Gets start and end points of period for querying in drill + var startTimestamp = 0; + var endTimestamp = 0; + var _currMoment = moment(); + + if (typeof period === 'string' && period.indexOf(",") !== -1) { + try { + period = JSON.parse(period); + } + catch (SyntaxError) { + console.log("period JSON parse failed"); + period = "30days"; + } + } + + var excludeCurrentDay = period.exclude_current_day || false; + if (period.period) { + period = period.period; + } + + if (period.since) { + period = [period.since, endTimestamp.clone().valueOf()]; + } + + + endTimestamp = excludeCurrentDay ? _currMoment.clone().subtract(1, 'days').endOf('day') : _currMoment.clone().endOf('day'); + + if (Array.isArray(period)) { + if ((period[0] + "").length === 10) { + period[0] *= 1000; + } + if ((period[1] + "").length === 10) { + period[1] *= 1000; + } + var fromDate, toDate; + + if (Number.isInteger(period[0]) && Number.isInteger(period[1])) { + period[0] = this.fixTimestampToMilliseconds(period[0]); + period[1] = this.fixTimestampToMilliseconds(period[1]); + fromDate = moment(period[0]); + toDate = moment(period[1]); + } + else { + fromDate = moment(period[0], ["DD-MM-YYYY HH:mm:ss", "DD-MM-YYYY"]); + toDate = moment(period[1], ["DD-MM-YYYY HH:mm:ss", "DD-MM-YYYY"]); + } + + startTimestamp = fromDate.clone().startOf("day"); + endTimestamp = toDate.clone().endOf("day"); + + fromDate.tz(timezone); + toDate.tz(timezone); + + if (fromDate.valueOf() > toDate.valueOf()) { + //incorrect range - reset to 30 days + let nDays = 30; + + startTimestamp = _currMoment.clone().startOf("day").subtract(nDays - 1, "days"); + endTimestamp = _currMoment.clone().endOf("day"); + } + } + else if (period === "month") { + startTimestamp = _currMoment.clone().startOf("year"); + } + else if (period === "day") { + startTimestamp = _currMoment.clone().startOf("month"); + } + else if (period === "prevMonth") { + startTimestamp = _currMoment.clone().subtract(1, "month").startOf("month"); + endTimestamp = _currMoment.clone().subtract(1, "month").endOf("month"); + } + else if (period === "hour") { + startTimestamp = _currMoment.clone().startOf("day"); + } + else if (period === "yesterday") { + let yesterday = _currMoment.clone().subtract(1, "day"); + startTimestamp = yesterday.clone().startOf("day"); + endTimestamp = yesterday.clone().endOf("day"); + } + else if (/([1-9][0-9]*)minutes/.test(period)) { + let nMinutes = parseInt(/([1-9][0-9]*)minutes/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("minute").subtract(nMinutes - 1, "minutes"); + } + else if (/([1-9][0-9]*)hours/.test(period)) { + let nHours = parseInt(/([1-9][0-9]*)hours/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("hour").subtract(nHours - 1, "hours"); + } + else if (/([1-9][0-9]*)days/.test(period)) { + let nDays = parseInt(/([1-9][0-9]*)days/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("day").subtract(nDays - 1, "days"); + } + else if (/([1-9][0-9]*)weeks/.test(period)) { + const nWeeks = parseInt(/([1-9][0-9]*)weeks/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("week").subtract((nWeeks - 1), "weeks"); + } + else if (/([1-9][0-9]*)months/.test(period)) { + const nMonths = parseInt(/([1-9][0-9]*)months/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("month").subtract((nMonths - 1), "months"); + } + else if (/([1-9][0-9]*)years/.test(period)) { + const nYears = parseInt(/([1-9][0-9]*)years/.exec(period)[1]); + startTimestamp = _currMoment.clone().startOf("year").subtract((nYears - 1), "years"); + } + //incorrect period, defaulting to 30 days + else { + let nDays = 30; + startTimestamp = _currMoment.clone().startOf("day").subtract(nDays - 1, "days"); + } + if (!offset) { + offset = startTimestamp.utcOffset(); + offset = offset * -1; + } + + return {"$gt": startTimestamp.valueOf() + offset * 60000, "$lt": endTimestamp.valueOf() + offset * 60000}; + } + + /** + * Gets projection base don timezone and bucket + * @param {string} bucket - d, h, m, w + * @param {string} timezone - timezone for display + * @returns {object} - projection + */ + getDateStringProjection(bucket, timezone) { + if (!(bucket === "h" || bucket === "d" || bucket === "m" || bucket === "w")) { + bucket = "d"; + } + var dstr = {"$toDate": "$ts"}; + if (bucket === "h") { + dstr = {"$dateToString": {"date": dstr, "format": "%Y:%m:%d:%H", "timezone": (timezone || "UTC")}}; + } + if (bucket === "m") { + dstr = {"$dateToString": {"date": dstr, "format": "%Y:%m", "timezone": (timezone || "UTC")}}; + } + else if (bucket === "w") { + dstr = {"$dateToString": {"date": dstr, "format": "%Y:%U", "timezone": (timezone || "UTC")}}; + } + else if (bucket === "d") { + dstr = {"$dateToString": {"date": dstr, "format": "%Y:%m:%d", "timezone": (timezone || "UTC")}}; + } + return dstr; + } + + /** + * Calculates timezone for mongo based on offset + * @param {number} offset - offset in minutes + * @returns {string} timezone + */ + calculateTimezoneFromOffset(offset) { + var hours = Math.abs(Math.floor(offset / 60)); + var minutes = Math.abs(offset % 60); + return (offset < 0 ? "+" : "-") + (hours < 10 ? "0" : "") + hours + ":" + (minutes < 10 ? "0" : "") + minutes; + } + + /** + * Fetches list of most popular segment values for a given period + * @param {object} options - query options + * @returns {object} - fetched data + */ + async segmentValuesForPeriod(options) { + var match = options.dbFilter || {}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.event) { + match.e = options.event; + } + if (options.name) { + match.n = options.name; + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + + var pipeline = []; + pipeline.push({"$match": match}); + pipeline.push({"$group": {"_id": "$" + options.field, "c": {"$sum": 1}}}); + pipeline.push({"$sort": {"c": -1}}); + pipeline.push({"$limit": options.limit || 1000}); + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + return data; + } + + /** + * Gets aggregated data chosen timezone.If not set - returns in UTC timezone. + * @param {object} options - options + * options.appID - application id + * options.event - string as event key or array with event keys + * options.period - Normal Countly period + * + * @returns {object} - aggregated data + */ + async getAggregatedData(options) { + var pipeline = options.pipeline || []; + var match = options.match || {}; + + if (options.appID) { + match.a = options.appID + ""; + } + if (options.event) { + if (Array.isArray(options.event)) { + match.e = {"$in": options.event}; + } + else { + match.e = options.event; + } + } + if (options.name) { + if (Array.isArray(options.event)) { + match.n = {"$in": options.name}; + } + else { + match.n = options.name; + } + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, options.timezone, options.periodOffset); + } + if (options.bucket !== "h" && options.bucket !== "d" && options.bucket !== "m" && options.bucket !== "w") { + options.bucket = "h"; + } + pipeline.push({"$match": match}); + + pipeline.push({"$addFields": {"d": this.getDateStringProjection(options.bucket, options.timezone)}}); + if (options.segmentation) { + /*if (Array.isArray(options.segmentation)) { + + } + else {*/ + pipeline.push({"$unwind": ("$" + options.segmentation)}); + pipeline.push({"$group": {"_id": {"d": "$d", "sg": "$" + options.segmentation}, "c": {"$sum": "$c"}, "dur": {"$sum": "$dur"}, "s": {"$sum": "$s"}}}); + pipeline.push({"$project": { "_id": "$_id.d", "sg": "$_id.sg", "c": 1, "dur": 1, "s": 1}}); + //} + } + else { + pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "dur": {"$sum": "$dur"}, "s": {"$sum": "$s"}}}); + } + try { + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + }/* */ + catch (e) { + console.log(e); + } + + for (var z = 0; z < data.length; z++) { + data[z]._id = data[z]._id.replaceAll(/\:0/gi, ":"); + if (data[z].sg) { + data[z].sg = data[z].sg.replaceAll(/\./gi, ":"); + } + } + + return data; + //Group by hour + } + + /** + * Gets session table data + * @param {options} options - options + * @returns {object} - aggregated data + */ + async aggregatedSessionData(options) { + var pipeline = options.pipeline || []; + var match = {"e": "[CLY]_session"}; + + if (options.appID) { + match.a = options.appID + ""; + } + + if (options.period) { + match.ts = this.getPeriodRange(options.period, options.timezone, options.periodOffset); + } + + pipeline.push({"$match": match}); + pipeline.push({"$addFields": {"n": {"$cond": [{"$eq": ["$up.sc", 1]}, 1, 0]}}}); + if (options.segmentation) { + pipeline.push({"$group": {"_id": {"u": "$uid", "sg": "$" + options.segmentation}, "d": {"$sum": "$dur"}, "t": {"$sum": 1}, "n": {"$sum": "$n"}}}); + pipeline.push({"$group": {"_id": "$_id.sg", "t": {"$sum": "$t"}, "d": {"$sum": "$d"}, "n": {"$sum": "$n"}, "u": {"$sum": 1}}}); + } + else { + pipeline.push({"$group": {"_id": "$uid", "t": {"$sum": 1}, "d": {"$sum": "$dur"}, "n": {"$sum": "$n"}}}); + pipeline.push({"$group": {"_id": null, "u": {"$sum": 1}, "d": {"$sum": "$d"}, "t": {"$sum": "$t"}, "n": {"$sum": "$n"}}}); + + } + console.log(JSON.stringify(pipeline)); + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + for (var z = 0; z < data.length; z++) { + // data[z]._id = data[z]._id.replaceAll(/\:0/gi, ":"); + if (data[z].sg) { + data[z].sg = data[z].sg.replaceAll(/\./gi, ":"); + } + } + return data; + } + + /* + async runDrillDataQuery(options) { + var match = options.dbFilter || {}; + if (options.app_id) { + if (Array.isArray(options.app_id)) { + match.a = {"$in": options.app_id}; + } + else { + match.a = options.app_id + ""; + } + } + + if (options.event) { + if (Array.isArray(options.event)) { + match.e = {"$in": options.event}; + } + else { + match.e = options.event; + } + } + if (options.periodOffset) { + options.timezone = this.calculateTimezoneFromOffset(options.periodOffset); + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + + + }*/ + + /** + * Get grph data based on unique + * @param {object} options - options + * @returns {object} model data as array + */ + async getUniqueGraph(options) { + var match = options.dbFilter || {}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.event) { + match.e = options.event; + } + if (options.name) { + match.n = options.name; + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + var field = options.field || "uid"; + + if (options.bucket !== "h" && options.bucket !== "d" && options.bucket !== "m" && options.bucket !== "w") { + options.bucket = "d"; + } + + if (options.periodOffset) { + options.timezone = this.calculateTimezoneFromOffset(options.periodOffset); + } + var pipeline = []; + pipeline.push({"$match": match}); + pipeline.push({"$addFields": {"d": this.getDateStringProjection(options.bucket, options.timezone)}}); + pipeline.push({"$group": {"_id": {"d": "$d", "id": "$" + field}}}); + pipeline.push({"$group": {"_id": "$_id.d", "u": {"$sum": 1}}}); + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + for (var z = 0; z < data.length; z++) { + data[z]._id = data[z]._id.replaceAll(/:0/gi, ":"); + if (data[z].sg) { + data[z].sg = data[z].sg.replaceAll(/\./gi, ":"); + } + } + return data; + } + + /** + * Gets unique count(single number) + * @param {options} options - options + * @returns {number} number + */ + async getUniqueCount(options) { + var match = options.dbFilter || {}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.event) { + match.e = options.event; + } + if (options.name) { + match.n = options.name; + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + var field = options.field || "uid"; + var pipeline = [ + {"$match": match}, + {"$group": {"_id": "$" + field}}, + {"$group": {"_id": null, "c": {"$sum": 1}}} + ]; + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + data = data[0] || {"c": 0}; + return data.c; + } + + /** + * Gets views table data + * @param {object} options -options + * @returns {object} table data + */ + async getViewsTableData(options) { + var match = options.dbFilter || {}; + if (options.appID) { + match.a = options.appID + ""; + } + match.e = "[CLY]_view"; + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + + var pipeline = []; + pipeline.push({"$match": match}); + pipeline.push({"$group": {"_id": {"u": "$uid", "sg": "$n" }, "t": {"$sum": 1}, "d": {"$sum": "$dur"}, "s": {"$sum": "$sg.visit"}, "e": {"$sum": "$sg.exit"}, "b": {"$sum": "$sg.bounce"}}}); + pipeline.push({"$addFields": {"u": 1}}); + //Union with cly action + pipeline.push({ + "$unionWith": { + "coll": "drill_events", + "pipeline": [ + {"$match": {"e": "[CLY]_action", "sg.type": "scroll", "ts": match.ts, "a": match.a}}, + {"$group": {"_id": {"u": "$uid", "sg": "$n"}, "scr": {"$sum": "$sg.scr"}}} + ] + } + }); + pipeline.push({"$group": {"_id": "$_id.sg", "u": {"$sum": "$u"}, "t": {"$sum": "$t"}, "d": {"$sum": "$d"}, "s": {"$sum": "$s"}, "e": {"$sum": "$e"}, "b": {"$sum": "$b"}, "scr": {"$sum": "$scr"}}}); + pipeline.push({ + "$addFields": { + "scr-calc": { $cond: [ { $or: [{$eq: ["$t", 0]}, {$eq: ['$scr', 0]}]}, 0, {'$divide': ['$scr', "$t"]}] }, + "d-calc": { $cond: [ { $or: [{$eq: ["$t", 0]}, {$eq: ['$d', 0]}]}, 0, {'$divide': ['$d', "$t"]}] }, + "br": { $cond: [ { $or: [{$eq: ["$s", 0]}, {$eq: ['$b', 0]}]}, 0, {'$divide': [{"$min": ['$b', "$s"]}, "$s"]}] }, + "b": {"$min": [{"$ifNull": ["$b", 0]}, {"$ifNull": ["$s", 0]}]}, + "view": "$_id" + } + }); + console.log(JSON.stringify(pipeline)); + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + for (var z = 0; z < data.length; z++) { + data[z]._id = data[z]._id.replaceAll(/\:0/gi, ":"); + if (data[z].sg) { + data[z].sg = data[z].sg.replaceAll(/\./gi, ":"); + } + } + return data; + } + + /** + * Gets timeline table data for user profile + * @param {object} options - options + * @returns {object} table data + */ + async getTimelineSessionTable(options) { + var match = {"e": "[CLY]_session"}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.uid) { + match.uid = options.uid; + } + if (options.event) { + match.e = options.event; + } + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + + var projection = {"dur": 1, "ts": 1, "_id": 1}; + var cursor = await this.db.collection("drill_events").find(match, projection); + var total = await cursor.count(); + if (options.skip) { + cursor = cursor.skip(options.skip); + } + if (options.limit) { + cursor = cursor.limit(options.limit); + } + var data = await cursor.toArray(); + return {data: data || [], total: total || 0}; + } + + /** + * Gets timeline graph data for user profile + * @param {object} options - options + * @returns {object} table data + */ + async getTimelineGraph(options) { + var match = {}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.uid) { + match.uid = options.uid; + } + + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + if (options.periodOffset) { + options.timezone = this.calculateTimezoneFromOffset(options.periodOffset); + } + + var pipeline = [ + {"$match": match}, + { + "$project": { + "d": this.getDateStringProjection(options.bucket || "d", options.timezone), + "e": {"$cond": [{"$eq": ["$e", "[CLY]_session"]}, 0, 1]}, + "s": {"$cond": [{"$eq": ["$e", "[CLY]_session"]}, 1, 0]} + } + }, + {"$group": {"_id": "$d", "e": {"$sum": "$e"}, "s": {"$sum": "$s"}}}, + {"$project": {"label": "$_id", "_id": 0, "e": 1, "s": 1}} + ]; + + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + var totals = {"e": 0, "s": 0}; + var mapped = {}; + for (var i = 0; i < data.length; i++) { + data[i].label = data[i].label + ""; + data[i].label = data[i].label.replaceAll(/\:0/gi, ":"); + mapped[data[i].label] = {"e": data[i].e, "s": data[i].s}; + totals.e += data[i].e; + totals.s += data[i].s; + } + + return {"totals": totals, "graph": mapped}; + } + + /** + * Gets timeline table data for user profile + * @param {object} options - options + * @returns {object} table data + */ + async getTimelineTable(options) { + var match = options.match || {}; + if (options.appID) { + match.a = options.appID + ""; + } + if (options.uid) { + match.uid = options.uid; + } + if (options.session) { + match.lsid = options.session; + } + if (options.event) { + match.e = options.event; + } + else { + match.e = {"$ne": "[CLY]_session"}; + } + + if (options.period) { + match.ts = this.getPeriodRange(options.period, "UTC", options.periodOffset); + } + var projection = { + "key": "$e", + "ts": 1, + "c": 1, + "sum": 1, + "dur": 1, + "sg": 1 + }; + if (options.projection) { + projection = options.projection; + } + + var cursor = await this.db.collection("drill_events").find(match, projection); + var total = await cursor.count(); + if (options.sort) { + cursor = cursor.sort(options.sort); + } + if (options.skip) { + cursor = cursor.skip(options.skip); + } + if (options.limit) { + cursor = cursor.limit(options.limit); + } + + var data = await cursor.toArray(); + return {data: data || [], total: total || 0}; + } + + /** + * Fetches data for times of day plugin from granural + * @param {object} options options +* @returns {object} table data + */ + async timesOfDay(options) { + var match = options.match || {}; + if (options.appID) { + match.a = options.appID + ""; + } + + if (options.event) { + match.e = options.event; + } + + var pipeline = [ + {"$match": match}, + {"$group": {"_id": {"d": "$up.dow", "h": "$up.hour"}, "c": {"$sum": 1}}} + ]; + + var data = await this.db.collection("drill_events").aggregate(pipeline).toArray(); + return data || []; + } +} + +module.exports = {MongoDbQueryRunner}; \ No newline at end of file diff --git a/api/utils/render.js b/api/utils/render.js index ee01d946878..c8f538428e2 100644 --- a/api/utils/render.js +++ b/api/utils/render.js @@ -24,7 +24,7 @@ var alternateChrome = true; var chromePath = ""; var countlyFs = require('./countlyFs'); var log = require('./log.js')('core:render'); -var countlyConfig = require('./../config', 'dont-enclose'); +var countlyConfig = require('./../config'); /** diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js index ae202254b33..68710869f71 100644 --- a/api/utils/requestProcessor.js +++ b/api/utils/requestProcessor.js @@ -15,6 +15,7 @@ const countlyCommon = require('../lib/countly.common.js'); const { validateAppAdmin, validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, validateUpdate, validateDelete, validateCreate, getBaseAppFilter } = require('./rights.js'); const authorize = require('./authorizer.js'); const taskmanager = require('./taskmanager.js'); +const calculatedDataManager = require('./calculatedDataManager.js'); const plugins = require('../../plugins/pluginManager.js'); const versionInfo = require('../../frontend/express/version.info'); const packageJson = require('./../../package.json'); @@ -28,7 +29,14 @@ const validateUserForDataWriteAPI = validateUserForWrite; const validateUserForGlobalAdmin = validateGlobalAdmin; const validateUserForMgmtReadAPI = validateUser; const request = require('countly-request')(plugins.getConfig("security")); -const Handle = require('../../api/parts/jobs/index.js'); + +try { + require('../../jobServer/api'); + log.i('Job api loaded'); +} +catch (ex) { + log.e('Job api not available'); +} var loaded_configs_time = 0; @@ -1929,6 +1937,43 @@ const processRequest = (params) => { common.returnOutput(params, plugins.getPlugins()); }, params); break; + case 'aggregator': + validateUserForMgmtReadAPI(() => { + //fetch current aggregator status + common.db.collection("plugins").findOne({_id: "_changeStreams"}, function(err, pluginsData) { + if (err) { + common.returnMessage(params, 400, 'Error fetching aggregator status'); + } + else { + //find biggest cd value in drill database + common.drillDb.collection("drill_events").find({}, {cd: 1}).sort({cd: -1}).limit(1).toArray(function(err2, drillData) { + var data = []; + var now = Date.now().valueOf(); + var nowDrill = now; + if (drillData && drillData.length) { + nowDrill = new Date(drillData[0].cd).valueOf(); + } + + for (var key in pluginsData) { + if (key !== "_id") { + var lastAccepted = new Date(pluginsData[key].cd).valueOf(); + data.push({ + name: key, + last_cd: pluginsData[key].cd, + drill: drillData && drillData[0] && drillData[0].cd, + last_id: pluginsData[key]._id, + diff: (now - lastAccepted) / 1000, + diffDrill: (nowDrill - lastAccepted) / 1000 + }); + } + } + common.returnOutput(params, data); + }); + } + + }); + }, params); + break; default: if (!plugins.dispatch(apiPath, { params: params, @@ -2534,104 +2579,6 @@ const processRequest = (params) => { } switch (params.qstring.method) { - case 'jobs': - /** - * @api {get} /o?method=jobs Get Jobs Table Information - * @apiName GetJobsTableInfo - * @apiGroup Jobs - * - * @apiDescription Get jobs information in the jobs table - * @apiQuery {String} method which kind jobs requested, it should be 'jobs' - * - * @apiSuccess {Number} iTotalRecords Total number of jobs - * @apiSuccess {Number} iTotalDisplayRecords Total number of jobs by filtering - * @apiSuccess {Objects[]} aaData Job details - * @apiSuccess {Number} sEcho DataTable's internal counter - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "sEcho": "0", - * "iTotalRecords": 14, - * "iTotalDisplayRecords": 14, - * "aaData": [{ - * "_id": "server-stats:stats", - * "name": "server-stats:stats", - * "status": "SCHEDULED", - * "schedule": "every 1 day", - * "next": 1650326400000, - * "finished": 1650240007917, - * "total": 1 - * }] - * } - */ - - /** - * @api {get} /o?method=jobs/name Get Job Details Table Information - * @apiName GetJobDetailsTableInfo - * @apiGroup Jobs - * - * @apiDescription Get the information of the filtered job in the table - * @apiQuery {String} method Which kind jobs requested, it should be 'jobs' - * @apiQuery {String} name The job name is required to redirect to the selected job - * - * @apiSuccess {Number} iTotalRecords Total number of jobs - * @apiSuccess {Number} iTotalDisplayRecords Total number of jobs by filtering - * @apiSuccess {Objects[]} aaData Job details - * @apiSuccess {Number} sEcho DataTable's internal counter - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "sEcho": "0", - * "iTotalRecords": 1, - * "iTotalDisplayRecords": 1, - * "aaData": [{ - * "_id": "62596cd41307dc89c269b5a8", - * "name": "api:ping", - * "created": 1650027732240, - * "status": "SCHEDULED", - * "started": 1650240000865, - * "finished": 1650240000891, - * "duration": 30, - * "data": {}, - * "schedule": "every 1 day", - * "next": 1650326400000, - * "modified": 1650240000895, - * "error": null - * }] - * } - */ - - validateUserForGlobalAdmin(params, countlyApi.data.fetch.fetchJobs, 'jobs'); - break; - case 'suspend_job': { - /** - * @api {get} /o?method=suspend_job Suspend Job - * @apiName SuspendJob - * @apiGroup Jobs - * - * @apiDescription Suspend the selected job - * * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result": true, - * "message": "Job suspended successfully" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Updating job status failed" - * } - * - */ - validateUserForGlobalAdmin(params, async() => { - await Handle.suspendJob(params); - }); - break; - } case 'total_users': validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTotalUsersObj, params.qstring.metric || 'users'); break; @@ -2787,6 +2734,69 @@ const processRequest = (params) => { break; } + case '/o/aggregate': { + validateUser(params, () => { + //Long task to run specific drill query. Give back task_id if running, result if done. + if (params.qstring.query) { + + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ee) { + log.e(ee); + common.returnMessage(params, 400, 'Invalid query parameter'); + return; + } + + if (params.qstring.query.appID) { + if (Array.isArray(params.qstring.query.appID)) { + //make sure member has access to all apps in this list + for (var i = 0; i < params.qstring.query.appID.length; i++) { + if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID[i]) === -1) { + common.returnMessage(params, 401, 'User does not have access right for this app'); + return; + } + } + } + else { + if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID) === -1) { + common.returnMessage(params, 401, 'User does not have access right for this app'); + return; + } + } + } + else { + params.qstring.query.appID = params.qstring.app_id; + } + if (params.qstring.period) { + params.qstring.query.period = params.qstring.query.period || params.qstring.period || "30days"; + } + if (params.qstring.periodOffset) { + params.qstring.query.periodOffset = params.qstrig.query.periodOffset || params.qstring.periodOffset || 0; + } + + calculatedDataManager.longtask({ + db: common.db, + threshold: plugins.getConfig("api").request_threshold, + app_id: params.qstring.query.app_id, + query_data: params.qstring.query, + outputData: function(err, data) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, data); + } + } + }); + } + else { + common.returnMessage(params, 400, 'Missing parameter "query"'); + } + + }); + break; + } case '/o/countly_version': { validateUser(params, () => { //load previos version info if exist @@ -3727,4 +3737,4 @@ function loadDbVersionMarks(callback) { } /** @lends module:api/utils/requestProcessor */ -module.exports = {processRequest: processRequest, processUserFunction: processUser}; +module.exports = {processRequest: processRequest, processUserFunction: processUser}; \ No newline at end of file diff --git a/api/utils/requestProcessorCommon.js b/api/utils/requestProcessorCommon.js new file mode 100644 index 00000000000..c1009d38129 --- /dev/null +++ b/api/utils/requestProcessorCommon.js @@ -0,0 +1,146 @@ +const common = require('./common.js'); +var plugins = require('../../plugins/pluginManager.js'); +const request = require('countly-request')(plugins.getConfig("security")); +var log = require('./log.js')("core:api"); + +/** + * Ignore possible devices + * @param {object} params - params object + * @returns {boolean} - true if device_id is 00000000-0000-0000-0000-000000000000 + */ +const ignorePossibleDevices = (params) => { + //ignore possible opted out users for ios 10 + if (params.qstring.device_id === "00000000-0000-0000-0000-000000000000") { + common.returnMessage(params, 200, 'Ignoring device_id'); + common.log("request").i('Request ignored: Ignoring zero IDFA device_id', params.req.url, params.req.body); + params.cancelRequest = "Ignoring zero IDFA device_id"; + plugins.dispatch("/sdk/cancel", {params: params}); + return true; + } +}; + +const validateRedirect = function(ob) { + var params = ob.params, + app = ob.app; + if (!params.cancelRequest && app.redirect_url && app.redirect_url !== '') { + var newPath = params.urlParts.path; + + //check if we have query part + if (newPath.indexOf('?') === -1) { + newPath += "?"; + } + + var opts = { + uri: app.redirect_url + newPath + '&ip_address=' + params.ip_address, + method: 'GET' + }; + + //should we send post request + if (params.req.method.toLowerCase() === 'post') { + opts.method = "POST"; + //check if we have body from post method + if (params.req.body) { + opts.json = true; + opts.body = params.req.body; + } + } + request(opts, function(error, response, body) { + var code = 400; + var message = "Redirect error. Tried to redirect to:" + app.redirect_url; + + if (response && response.statusCode) { + code = response.statusCode; + } + + + if (response && response.body) { + try { + var resp = JSON.parse(response.body); + message = resp.result || resp; + } + catch (e) { + if (response.result) { + message = response.result; + } + else { + message = response.body; + } + } + } + if (error) { //error + log.e("Redirect error", error, body, opts, app, params); + } + + if (plugins.getConfig("api", params.app && params.app.plugins, true).safe || params.qstring?.safe_api_response) { + common.returnMessage(params, code, message); + } + }); + params.cancelRequest = "Redirected: " + app.redirect_url; + params.waitForResponse = false; + if (plugins.getConfig("api", params.app && params.app.plugins, true).safe || params.qstring?.safe_api_response) { + params.waitForResponse = true; + } + return false; + } + else { + return true; + } +}; + +/** + * @param {object} params - params object + * @param {String} type - source type + * @param {Function} done - done callback + * @returns {Function} - done or boolean value + */ +const checksumSaltVerification = (params) => { + params.app.checksum_salt = params.app.salt || params.app.checksum_salt;//checksum_salt - old UI, .salt - new UI. + if (params.app.checksum_salt && params.app.checksum_salt.length && !params.no_checksum) { + const payloads = []; + payloads.push(params.href.substr(params.fullPath.length + 1)); + + if (params.req.method.toLowerCase() === 'post') { + // Check if we have 'multipart/form-data' + if (params.formDataUrl) { + payloads.push(params.formDataUrl); + } + else { + payloads.push(params.req.body); + } + } + if (typeof params.qstring.checksum !== "undefined") { + for (let i = 0; i < payloads.length; i++) { + payloads[i] = (payloads[i] + "").replace("&checksum=" + params.qstring.checksum, "").replace("checksum=" + params.qstring.checksum, ""); + payloads[i] = common.crypto.createHash('sha1').update(payloads[i] + params.app.checksum_salt).digest('hex').toUpperCase(); + } + if (payloads.indexOf((params.qstring.checksum + "").toUpperCase()) === -1) { + common.returnMessage(params, 200, 'Request does not match checksum'); + console.log("Checksum did not match", params.href, params.req.body, payloads); + params.cancelRequest = 'Request does not match checksum sha1'; + return false; + } + } + else if (typeof params.qstring.checksum256 !== "undefined") { + for (let i = 0; i < payloads.length; i++) { + payloads[i] = (payloads[i] + "").replace("&checksum256=" + params.qstring.checksum256, "").replace("checksum256=" + params.qstring.checksum256, ""); + payloads[i] = common.crypto.createHash('sha256').update(payloads[i] + params.app.checksum_salt).digest('hex').toUpperCase(); + } + if (payloads.indexOf((params.qstring.checksum256 + "").toUpperCase()) === -1) { + common.returnMessage(params, 200, 'Request does not match checksum'); + console.log("Checksum did not match", params.href, params.req.body, payloads); + params.cancelRequest = 'Request does not match checksum sha256'; + return false; + } + } + else { + common.returnMessage(params, 200, 'Request does not have checksum'); + console.log("Request does not have checksum", params.href, params.req.body); + params.cancelRequest = "Request does not have checksum"; + return false; + } + } + + return true; +}; + +module.exports = {ignorePossibleDevices: ignorePossibleDevices, checksumSaltVerification: checksumSaltVerification, validateRedirect: validateRedirect}; \ No newline at end of file diff --git a/api/utils/utils.js b/api/utils/utils.js index b1c7925c81c..82e8ebdea8f 100644 --- a/api/utils/utils.js +++ b/api/utils/utils.js @@ -3,7 +3,7 @@ * @module api/utils/utils */ var crypto = require('crypto'), - countlyConfig = require('./../config', 'dont-enclose'); + countlyConfig = require('./../config'); if (!countlyConfig.encryption) { countlyConfig.encryption = {}; diff --git a/bin/commands/docker/countly-aggregator.sh b/bin/commands/docker/countly-aggregator.sh new file mode 100755 index 00000000000..4d4eb06c40f --- /dev/null +++ b/bin/commands/docker/countly-aggregator.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [ "$COUNTLY_MONGO_INSIDE" == "1" ] +then + until mongo localhost --eval "db.stats()" | grep "collections" + do + echo + echo "[api] waiting for MongoDB to allocate files ..." + sleep 1 + done + sleep 3 + echo "[api] MongoDB started" +fi + +exec /sbin/setuser countly /usr/bin/nodejs /opt/countly/api/aggregator.js diff --git a/bin/commands/docker/countly-ingestor.sh b/bin/commands/docker/countly-ingestor.sh new file mode 100755 index 00000000000..e4e472a70f6 --- /dev/null +++ b/bin/commands/docker/countly-ingestor.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [ "$COUNTLY_MONGO_INSIDE" == "1" ] +then + until mongo localhost --eval "db.stats()" | grep "collections" + do + echo + echo "[api] waiting for MongoDB to allocate files ..." + sleep 1 + done + sleep 3 + echo "[api] MongoDB started" +fi + +exec /sbin/setuser countly /usr/bin/nodejs /opt/countly/api/ingestor.js diff --git a/bin/config/nginx.server.block.conf b/bin/config/nginx.server.block.conf index 71e8cab1245..5bc8d8d705d 100644 --- a/bin/config/nginx.server.block.conf +++ b/bin/config/nginx.server.block.conf @@ -12,11 +12,19 @@ server { if ($http_content_type = "text/ping") { return 404; } + proxy_pass http://localhost:3010; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } - proxy_pass http://localhost:3001; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - } + location = /i/bulk { + if ($http_content_type = "text/ping") { + return 404; + } + proxy_pass http://localhost:3010; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } # Unblocked paths on API service # @@ -30,7 +38,7 @@ server { # /o/surveys/nps/widget # /i/campaign/click/* - location ~ (/i/bulk|/o/sdk|/o/actions|/o/feedback/multiple-widgets-by-id|/o/feedback/widget|/o/surveys/survey/widget|/o/surveys/nps/widget|/i/campaign/click/*) { + location ~ (/o/sdk|/o/actions|/o/feedback/multiple-widgets-by-id|/o/feedback/widget|/o/surveys/survey/widget|/o/surveys/nps/widget|/i/campaign/click/*) { if ($http_content_type = "text/ping") { return 404; } diff --git a/bin/config/nginx.server.conf b/bin/config/nginx.server.conf index 5927ce50abc..412b1e37d0d 100644 --- a/bin/config/nginx.server.conf +++ b/bin/config/nginx.server.conf @@ -9,7 +9,25 @@ server { if ($http_content_type = "text/ping") { return 404; } - proxy_pass http://localhost:3001; + proxy_pass http://localhost:3010; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } + + location = /i/bulk { + if ($http_content_type = "text/ping") { + return 404; + } + proxy_pass http://localhost:3010; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } + + location = /i/feedback/input { + if ($http_content_type = "text/ping") { + return 404; + } + proxy_pass http://localhost:3010; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Real-IP $remote_addr; } diff --git a/bin/config/nginx.server.ssl.conf b/bin/config/nginx.server.ssl.conf index 16089456631..d4a2cdfa9aa 100644 --- a/bin/config/nginx.server.ssl.conf +++ b/bin/config/nginx.server.ssl.conf @@ -64,7 +64,16 @@ server { if ($http_content_type = "text/ping") { return 404; } - proxy_pass http://localhost:3001; + proxy_pass http://localhost:3010; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } + + location = /i/bulk { + if ($http_content_type = "text/ping") { + return 404; + } + proxy_pass http://localhost:3010; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Real-IP $remote_addr; } diff --git a/bin/config/supervisord.example.conf b/bin/config/supervisord.example.conf index a0b80b382fd..51a5a9e4ce5 100644 --- a/bin/config/supervisord.example.conf +++ b/bin/config/supervisord.example.conf @@ -17,7 +17,7 @@ childlogdir=%(here)s/../../log/supervisord/ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [group:countly] -programs=countly-dashboard, countly-api +programs=countly-dashboard, countly-api, countly-ingestor, countly-aggregator [program:countly-dashboard] environment=NODE_ENV=production @@ -45,4 +45,32 @@ stdout_logfile_maxbytes=50MB stdout_logfile_backups=50 stdout_capture_maxbytes=1MB stdout_events_enabled=false +loglevel=warn + +[program:countly-ingestor] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/ingestor.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-ingestor.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false +loglevel=warn + +[program:countly-aggregator] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/aggregator.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-aggregator.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false loglevel=warn \ No newline at end of file diff --git a/bin/config/supervisord.ldaps.conf b/bin/config/supervisord.ldaps.conf index f58973e5df1..9624c23106a 100644 --- a/bin/config/supervisord.ldaps.conf +++ b/bin/config/supervisord.ldaps.conf @@ -17,7 +17,7 @@ childlogdir=%(here)s/../../log/supervisord/ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [group:countly] -programs=countly-dashboard, countly-api +programs=countly-dashboard, countly-api, countly-ingestor, countly-aggregator [program:countly-dashboard] environment=NODE_ENV=production,NODE_EXTRA_CA_CERTS= @@ -45,4 +45,32 @@ stdout_logfile_maxbytes=50MB stdout_logfile_backups=50 stdout_capture_maxbytes=1MB stdout_events_enabled=false +loglevel=warn + +[program:countly-ingestor] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/ingestor.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-ingestor.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false +loglevel=warn + +[program:countly-aggregator] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/aggregator.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-aggregator.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false loglevel=warn \ No newline at end of file diff --git a/bin/config/supervisort.wuser.conf b/bin/config/supervisort.wuser.conf index f9653d5dac3..c6316c6e960 100644 --- a/bin/config/supervisort.wuser.conf +++ b/bin/config/supervisort.wuser.conf @@ -17,7 +17,7 @@ childlogdir=/var/log/ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [group:countly] -programs=countly-dashboard, countly-api +programs=countly-dashboard, countly-api, countly-ingestor, countly-aggregator [program:countly-dashboard] environment=NODE_ENV=production @@ -45,4 +45,32 @@ stdout_logfile_maxbytes=50MB stdout_logfile_backups=50 stdout_capture_maxbytes=1MB stdout_events_enabled=false +loglevel=warn + +[program:countly-ingestor] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/ingestor.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-ingestor.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false +loglevel=warn + +[program:countly-aggregator] +environment=NODE_ENV=production +command=node --max-old-space-size=2048 %(here)s/../../api/aggregator.js +;command=nyc --nycrc-path nyc.config node %(here)s/../../api/api.js ;replace previous line with this in case you want to run nyc +directory=%(here)s +autorestart=true +redirect_stderr=true +stdout_logfile=%(here)s/../../log/countly-aggregator.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=50 +stdout_capture_maxbytes=1MB +stdout_events_enabled=false loglevel=warn \ No newline at end of file diff --git a/bin/scripts/data-cleanup/remove_old_views_collections.js b/bin/scripts/data-cleanup/remove_old_views_collections.js new file mode 100644 index 00000000000..a1fc8fbb099 --- /dev/null +++ b/bin/scripts/data-cleanup/remove_old_views_collections.js @@ -0,0 +1,72 @@ +/* +Script to delete Views collections after migrating to single collection +!! DO NOT RUN IT BEFORE MAKING SURE DATA IS MIGRATED TO SINGLE COLLECTION !! + + Path: $(countly dir)/bin/scripts/data-cleanup +To run this script use the following command +node /remove_old_views_collections.js +*/ +var pluginManager = require('../../../plugins/pluginManager.js'); + +var force = false; //if set to false will skip event collections which do not show up as migrated +if (force === false) { + console.log("Script will skip all collections without migration flag. If there are any collections which were skipped in migration because they could not be associated with any app - those will not be deleted. Safe approach is to run merging again and compare skipped collection list with collections not deleted by this scrip."); +} +console.log("Removing old Views collections"); +var failedCn = 0; +pluginManager.dbConnection().then(async(countlyDb) => { + //get list with all collections + try { + let allCollections = await countlyDb.listCollections().toArray(); + let collectionNames = allCollections.map(o => o.name); + + for (var z = 0; z < collectionNames.length; z++) { + if ((collectionNames[z].indexOf("app_viewdata") === 0 && collectionNames[z].length > 13) || + (collectionNames[z].indexOf("app_userviews") === 0 && collectionNames[z].length > 14) || + (collectionNames[z].indexOf("app_viewsmeta") === 0 && collectionNames[z].length > 13)) { + if (force) { + console.log("Dropping collection: " + collectionNames[z]); + await countlyDb.collection(collectionNames[z]).drop(); + } + else { + await new Promise((resolve) => { + countlyDb.collection(collectionNames[z]).findOne({"merged": {"$ne": true}}, {"_id": 1}, function(err, res) { + if (err) { + console.log(err); + failedCn++; + resolve(); + } + else { + if (res) { + console.log("Collection not migrated: " + collectionNames[z]); + failedCn++; + resolve(); + } + else { + countlyDb.collection(collectionNames[z]).drop(function(err) { + if (err) { + console.log(err); + failedCn++; + } + resolve(); + }); + } + } + }); + }); + } + } + } + } + catch (error) { + console.log(`Error removing old events collections: ${error}`); + countlyDb.close(); + } + finally { + if (failedCn > 0) { + console.log("Failed to remove collections: ", failedCn); + } + console.log("Done"); + countlyDb.close(); + } +}); \ No newline at end of file diff --git a/bin/scripts/expire-data/expire_index_on_timeline.js b/bin/scripts/expire-data/expire_index_on_timeline.js index adfbc1c087f..aaa03b14f90 100644 --- a/bin/scripts/expire-data/expire_index_on_timeline.js +++ b/bin/scripts/expire-data/expire_index_on_timeline.js @@ -9,7 +9,6 @@ var EXPIRE_AFTER = 60 * 60 * 24 * 365; //1 year in seconds var INDEX_NAME = "cd_1"; var async = require('async'), - Promise = require("bluebird"), plugins = require('../../../plugins/pluginManager.js'); //var db = plugins.dbConnection("countly"); @@ -17,7 +16,7 @@ var async = require('async'), var apps = [];//leave empty to set to all apps or put in some apps -Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).spread(function(db, db_drill) { +Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).then(function([db, db_drill]) { db.collections(function(err, collections) { if (err) { console.log(err); diff --git a/bin/scripts/modify-data/csv_tag/delete_user_from_csv.js b/bin/scripts/modify-data/csv_tag/delete_user_from_csv.js index 2366de9c727..fc2af44de29 100644 --- a/bin/scripts/modify-data/csv_tag/delete_user_from_csv.js +++ b/bin/scripts/modify-data/csv_tag/delete_user_from_csv.js @@ -14,7 +14,6 @@ var batchLimit = 1000; var asyncjs = require("async"); -var Promise = require("bluebird"); var common = require('../../../../api/utils/common.js'); var app_users = require('../../../../api/parts/mgmt/app_users.js'); var pluginManager = require("../../../../plugins/pluginManager"); @@ -43,7 +42,7 @@ csv() if (batch.length > 0) { batches.push(batch); } - Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).spread(function(db, drill) { + Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(function([db, drill]) { common.db = db; common.drillDb = drill; asyncjs.eachOfSeries(batches, function(batch, num, done) { diff --git a/bin/scripts/modify-data/csv_tag/update_property_from_csv.js b/bin/scripts/modify-data/csv_tag/update_property_from_csv.js index abf3b81f7c0..79fcecea796 100644 --- a/bin/scripts/modify-data/csv_tag/update_property_from_csv.js +++ b/bin/scripts/modify-data/csv_tag/update_property_from_csv.js @@ -56,7 +56,7 @@ csv() } }) .on('done', ()=>{ - Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).spread(function(db, dbDrill) { + Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(function([db, dbDrill]) { if (batch.length > 0) { batches.push(batch); } diff --git a/bin/scripts/modify-data/csv_tag/update_property_static_value.js b/bin/scripts/modify-data/csv_tag/update_property_static_value.js index f8ba36ed909..d2c4aee7e7a 100644 --- a/bin/scripts/modify-data/csv_tag/update_property_static_value.js +++ b/bin/scripts/modify-data/csv_tag/update_property_static_value.js @@ -41,7 +41,7 @@ csv() } }) .on('done', ()=>{ - Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).spread(function(db, dbDrill) { + Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(function([db, dbDrill]) { if (batch.length > 0) { batches.push(batch); } diff --git a/bin/scripts/testEventMergingScript.js b/bin/scripts/testEventMergingScript.js index e4f6b65209b..5b69f03f9f1 100644 --- a/bin/scripts/testEventMergingScript.js +++ b/bin/scripts/testEventMergingScript.js @@ -4,7 +4,6 @@ Script will delete documents matching selected event. Do not run on real event data, as it will delete it. */ const pluginManager = require('../../plugins/pluginManager.js'); -var Promise = require("bluebird"); //Change these values to correct ones. var collectionName = 'fe55f384d642312c76f6164a4f8ea40ffcc80795'; @@ -16,7 +15,7 @@ Promise.all( [ pluginManager.dbConnection("countly") ]) - .spread(async function(countlyDB) { + .then(async function([countlyDB]) { //insert some documents in collection await countlyDB.collection("events" + collectionName).drop();//cleanup in case await countlyDB.collection("events_data").deleteMany({'_id': {"$regex": "^" + app_ID + "_" + collectionName + "_.*"}}); @@ -226,10 +225,8 @@ Promise.all( if (!compareObjects(coppied, mapped[coppied._id], "", true)) { console.log("Document not same as original: " + coppied._id); } - } } - //delete all records from selected databases //await countlyDB.collection("events"+collectionName).drop(); // await countlyDB.collection("events_data").deleteMany({'_id':{"$regex":"^"+app_ID+"_"+collectionName+"_.*"}}); diff --git a/bin/upgrade/dev/scripts/merge_views_collections.js b/bin/upgrade/dev/scripts/merge_views_collections.js new file mode 100644 index 00000000000..b4b70c2663a --- /dev/null +++ b/bin/upgrade/dev/scripts/merge_views_collections.js @@ -0,0 +1,251 @@ +/*Script to merge all drill meta */ +const pluginManager = require('../../../../plugins/pluginManager.js'); +var Promise = require("bluebird"); +var crypto = require('crypto'); +const common = require('../../../../api/utils/common.js'); + +console.log("Merging views collections"); + +var maxActionCount = 10000;//limiting action count to avoid 16MB update operation limit +var maxRetries = 100; +var retryInterval = 10000; +var reports = { + "listed": 0, + "skipped": 0, + "merged": [], + "failed": [], +}; + +async function migrateApp(app, countlyDB, callback) { + var appId = app._id; + //load view data for this app + var base_data = await countlyDB.collection("views").findOne({"_id": countlyDB.ObjectID(appId)}); + if (!base_data) { + console.log("No view data found for app " + appId); + callback(); + return; + } + base_data.segments = base_data.segments || {}; + + //Migrate App_viewsmeta collection; + var batchSize = 1000; + var batch = []; + var cursor = await countlyDB.collection("app_viewsmeta" + appId).find({}); + var doc; + //Rewrite and get mapping old ids to new ids. + var mapped = {}; + doc = await cursor.next(); + while (doc) { + var newid = crypto.createHash('md5').update(doc.view).digest('hex'); + mapped[doc._id] = newid; + doc._id = appId + "_" + newid; + doc.a = appId + ""; + if (!doc.merged) { + batch.push({"insertOne": {"document": doc}}); + if (batch.length >= batchSize) { + try { + await countlyDB.collection("app_viewsmeta").bulkWrite(batch); + } + catch (e) { + if (e.code !== 11000) { + callback("Unable to update meta collecion"); + } + } + batch = []; + } + } + doc = await cursor.next(); + } + if (batch.length) { + try { + await countlyDB.collection("app_viewsmeta").bulkWrite(batch); + } + catch (e) { + if (e.code !== 11000) { //ignore duplicate error + callback("Unable to update meta collecion"); + } + } + } + await countlyDB.collection("app_viewsmeta" + appId).updateMany({}, {"$set": {"merged": true}}); + console.log("Migrated app_viewsmeta for app " + appId); + + //Migrate app_viewdata collections + var segments = ["no-segment"]; + for (var segment in base_data.segments) { + segments.push(segment); + } + + for (var z = 0; z < segments.length; z++) { + var collName = "app_viewdata" + crypto.createHash('sha1').update(segments[z] + appId).digest('hex'); + if (segments[z] === "no-segment") { + collName = "app_viewdata" + crypto.createHash('sha1').update(appId + "").digest('hex'); + } + await merge_data_from_collection(countlyDB, collName, {"a": appId + "", "s": segments[z]}, mapped); + } + console.log("Migrated app_viewdata for app " + appId); + console.log("Migrating app_userviews data"); + callback(); +} + +Promise.all( + [ + pluginManager.dbConnection("countly") + ]) + .spread(function(countlyDB) { + + //Process each app + countlyDB.collection('apps').find({}).toArray(function(err, apps) { + if (err) { + console.log("err"); + console.log("Failed to load app list. exiting"); + countlyDB.close(); + } + else { + Promise.each(apps, function(app) { + return new Promise(function(resolve, reject) { + migrateApp(app, countlyDB, function(err) { + if (err) { + reject(); + } + else { + resolve(); + } + }); + }); + }).then(function() { + console.log("All apps processed"); + } + ).catch(function(err) { + console.log(err); + console.log("All apps processed"); + console.log('Script failed. Exiting. PLEASE RERUN SCRIPT TO MIGRATE ALL DATA.'); + countlyDB.close(); + }); + } + }); + }) + .catch(function(err) { + console.log("Error wile getting connecion. Exiting."); + console.log(err); + }); + + +async function migrateAppUserviews(countlyDB, appId, mapped) { + var collection = "app_userviews" + appId; + var cursor = await countlyDB.collection(collection).find({"merged": {"$ne": true}}); + var doc; + doc = await cursor.next(); + while (doc) { + doc = await cursor.next(); + } +} + +async function merge_data_from_collection(countlyDB, collection, options, mapped) { + var app_id = options.a; + var prefix = app_id + "_" + options.s + "_"; + console.log(" checking collection: " + collection); + var cursor = await countlyDB.collection(collection).find(/*{"merged": {"$ne": true}}*/{}); + var doc; + doc = await cursor.next(); + while (doc) { + var splitted = doc._id.split("_"); + if (splitted.length == 2) { + var vw = doc.vw + ""; + if (mapped[vw]) { + var actionCounter = 0; + var original_id = doc._id; + doc._id = prefix + doc.m + "_" + mapped[vw]; + doc.vw = mapped[vw]; + var update = {"$set": {"vw": app_id + "_" + doc.vw, "m": doc.m, "a": doc.a, "s": doc.s || "no-segment"}}; + if (doc.meta_v2) { + for (var key0 in doc.meta_v2) { + for (var value in doc.meta_v2[key0]) { + update["$set"]["meta_v2." + key0 + "." + value] = doc.meta_v2[key0][value]; + actionCounter++; + if (actionCounter > maxActionCount) { + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + update = {"$set": {"vw": doc.vw, "m": doc.m, "a": doc.a, "e": doc.e, "s": doc.s || "no-segment"}}; + actionCounter = 0; + } + } + } + } + if (doc.d) { + for (var day in doc.d) { + for (var key in doc.d[day]) { //1.level + //if is Object + if (typeof doc.d[day][key] === 'object') { + for (var key2 in doc.d[day][key]) { //level2 + if (typeof doc.d[day][key][key2] === 'object') { + for (var key3 in doc.d[day][key][key2]) { //level3 + if (typeof doc.d[day][key][key2][key3] === 'object') { + for (var key4 in doc.d[day][key][key2][key3]) { //level4 + if (common.isNumber(doc.d[day][key][key2][key3][key4])) { + update["$inc"] = update["$inc"] || {}; + update["$inc"]["d." + day + "." + key + "." + key2 + "." + key3 + "." + key4] = doc.d[day][key][key2][key3][key4]; + + actionCounter++; + if (actionCounter > maxActionCount) { + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + update = {"$set": {"vw": doc.vw, "m": doc.m, "a": doc.a, "e": doc.e, "s": doc.s || "no-segment"}}; + actionCounter = 0; + } + } + } + } + else { + if (common.isNumber(doc.d[day][key][key2][key3])) { + update["$inc"] = update["$inc"] || {}; + update["$inc"]["d." + day + "." + key + "." + key2 + "." + key3] = doc.d[day][key][key2][key3]; + + actionCounter++; + if (actionCounter > maxActionCount) { + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + update = {"$set": {"vw": doc.vw, "m": doc.m, "a": doc.a, "e": doc.e, "s": doc.s || "no-segment"}}; + actionCounter = 0; + } + } + } + } + } + else { + if (common.isNumber(doc.d[day][key][key2])) { + update["$inc"] = update["$inc"] || {}; + update["$inc"]["d." + day + "." + key + "." + key2] = doc.d[day][key][key2]; + + actionCounter++; + if (actionCounter > maxActionCount) { + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + update = {"$set": {"vw": doc.vw, "m": doc.m, "a": doc.a, "e": doc.e, "s": doc.s || "no-segment"}}; + actionCounter = 0; + } + } + } + } + } + else { + if (common.isNumber(doc.d[day][key])) { + update["$inc"] = update["$inc"] || {}; + update["$inc"]["d." + day + "." + key] = doc.d[day][key]; + + actionCounter++; + if (actionCounter > maxActionCount) { + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + update = {"$set": {"vw": doc.vw, "m": doc.m, "a": doc.a, "e": doc.e, "s": doc.s || "no-segment"}}; + actionCounter = 0; + } + } + } + } + } + } + if (actionCounter > 0) { //we are splitting updates to make sure update operation do not reach 16MB + await countlyDB.collection("app_viewdata").updateOne({"_id": doc._id}, update, {upsert: true}); + } + await countlyDB.collection(collection).updateOne({"_id": original_id}, {"$set": {"merged": true}}); + } + } + doc = await cursor.next(); + } + console.log("done"); +} \ No newline at end of file diff --git a/bin/upgrade/dev/scripts/update_drill_collections.js b/bin/upgrade/dev/scripts/update_drill_collections.js new file mode 100644 index 00000000000..1d9018a1fdb --- /dev/null +++ b/bin/upgrade/dev/scripts/update_drill_collections.js @@ -0,0 +1,25 @@ +const { first } = require("underscore"); +const common = require("../../../../api/utils/common"); +var pluginManager = require("../../../pluginManager.js"); +Promise.all( + [ + pluginManager.dbConnection("countly"), + pluginManager.dbConnection("countly_drill") + ]) + .spread(async function(countlyDB, drill_db) { + console.log("Fixing viws events"); + + await common.drill_db.updateMany({"e": "[CLY]_view", "n": {"$exists": false}}, [{"$set": {"n": "$sg.name"}}]); + await common.drill_db.updateMany({"e": "[CLY]_action", "n": {"$exists": false}}, {"$set": {"n": "$sg.view"}}); + + //Fixing custom events + //get list of custom events. Run update query for each + var events = await countlyDB.collection("events").find({}, {_id: 1, list: 1}).toArray(); + for (var z = 0; z < events.list.length; z++) { + var aa = await common.drill_db.updateMany({"a": events._id + "", "e": events.list[z], "n": {"$exists": false}}, {"$set": {"n": event.name, "e": "[CLY]_custom"}}); + } + + }).catch(function(err) { + console.log(err); + } + ); \ No newline at end of file diff --git a/frontend/express/app.js b/frontend/express/app.js index 9b9b1bd9ece..26c0667dcc5 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -61,7 +61,7 @@ var versionInfo = require('./version.info'), preventBruteforce = require('./libs/preventBruteforce.js'), plugins = require('../../plugins/pluginManager.js'), request = require('countly-request')(plugins.getConfig("security")), - countlyConfig = require('./config', 'dont-enclose'), + countlyConfig = require('./config'), log = require('../../api/utils/log.js')('core:app'), url = require('url'), authorize = require('../../api/utils/authorizer.js'), //for token validations diff --git a/frontend/express/libs/members.js b/frontend/express/libs/members.js index a9c42ad2f83..e47b33c4cfd 100755 --- a/frontend/express/libs/members.js +++ b/frontend/express/libs/members.js @@ -13,7 +13,7 @@ var authorize = require('./../../../api/utils/authorizer.js'); //for token valid var common = require('./../../../api/utils/common.js'); var plugins = require('./../../../plugins/pluginManager.js'); var { getUserApps } = require('./../../../api/utils/rights.js'); -var configs = require('./../config', 'dont-enclose'); +var configs = require('./../config'); var countlyMail = require('./../../../api/parts/mgmt/mail.js'); var countlyStats = require('./../../../api/parts/data/stats.js'); var request = require('countly-request')(plugins.getConfig("security")); diff --git a/frontend/express/public/core/aggregator-status/javascripts/countly.models.js b/frontend/express/public/core/aggregator-status/javascripts/countly.models.js new file mode 100644 index 00000000000..bc468dcafe7 --- /dev/null +++ b/frontend/express/public/core/aggregator-status/javascripts/countly.models.js @@ -0,0 +1,63 @@ +/*global countlyVue, $, countlyCommon, CV, CountlyHelpers*/ +(function(countlyAggregationManager) { + countlyAggregationManager.fetchData = function() { + return $.ajax({ + type: "GET", + url: countlyCommon.API_PARTS.data.r + '/system/aggregator', + dataType: "json", + data: { + app_id: countlyCommon.ACTIVE_APP_ID, + "preventRequestAbort": true + }, + error: function(/*xhr, status, error*/) { + // TODO: handle error + } + }); + }; + + countlyAggregationManager.getVuexModule = function() { + var getInitialState = function() { + return { }; + }; + + var fetchData = function() { + return $.ajax({ + type: "GET", + url: countlyCommon.API_PARTS.data.r + '/system/aggregator', + dataType: "json", + data: { + app_id: countlyCommon.ACTIVE_APP_ID, + "preventRequestAbort": true + }, + error: function(/*xhr, status, error*/) { + // TODO: handle error + } + }); + }; + + var actions = { + fetchData: function() { + return fetchData.then(function() { + return true; + }).catch(function() { + CountlyHelpers.notify({ + message: CV.i18n('management.preset.created.error'), + type: "error" + }); + return false; + }); + }, + }; + + var mutations = { + + }; + return countlyVue.vuex.Module("countlyAppManagement", { + state: getInitialState, + actions: actions, + mutations: mutations, + }); + }; +}(window.countlyAggregationManager = window.countlyAggregationManager || {})); + + diff --git a/frontend/express/public/core/aggregator-status/javascripts/countly.views.js b/frontend/express/public/core/aggregator-status/javascripts/countly.views.js new file mode 100644 index 00000000000..c041a3ccc5a --- /dev/null +++ b/frontend/express/public/core/aggregator-status/javascripts/countly.views.js @@ -0,0 +1,41 @@ +/* global countlyVue, app, countlyAggregationManager, CV*/ + + +var AggregatorStatusView = countlyVue.views.create({ + template: CV.T('/core/aggregator-status/templates/aggregator-status.html'), + data: function() { + return { + tableData: [] + }; + }, + mounted: function() { + var self = this; + countlyAggregationManager.fetchData().then(function(data) { + self.tableData = data; + }); + }, + methods: { + getTable: function() { + return this.tableData; + }, + refresh: function() { + var self = this; + countlyAggregationManager.fetchData().then(function(data) { + self.tableData = data; + }); + } + }, + computed: { + aggregatorRows: function() { + return this.getTable(); + } + + } +}); + +app.route("/aggregator", 'aggregators', function() { + this.renderWhenReady(new CV.views.BackboneWrapper({ + component: AggregatorStatusView, + vuex: [{clyModel: countlyAggregationManager}] + })); +}); diff --git a/frontend/express/public/core/aggregator-status/stylesheets/_main.scss b/frontend/express/public/core/aggregator-status/stylesheets/_main.scss new file mode 100644 index 00000000000..df188ba040e --- /dev/null +++ b/frontend/express/public/core/aggregator-status/stylesheets/_main.scss @@ -0,0 +1,5 @@ +.header{ + display: flex; + flex-direction: column; /* Stack divs vertically */ + gap: 20px; /* Add space between divs */ + } \ No newline at end of file diff --git a/frontend/express/public/core/aggregator-status/templates/aggregator-status.html b/frontend/express/public/core/aggregator-status/templates/aggregator-status.html new file mode 100644 index 00000000000..bfc432e04c4 --- /dev/null +++ b/frontend/express/public/core/aggregator-status/templates/aggregator-status.html @@ -0,0 +1,23 @@ +
+ + + + + + + + + + +
+ diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js index 1bfc8c5ad48..e80e152976f 100644 --- a/frontend/express/public/core/jobs/javascripts/countly.views.js +++ b/frontend/express/public/core/jobs/javascripts/countly.views.js @@ -1,201 +1,514 @@ -/*global countlyAuth, countlyCommon, app, countlyVue, CV, countlyGlobal, CountlyHelpers, $ */ +/*global countlyAuth, countlyCommon, app, countlyVue, CV, countlyGlobal, CountlyHelpers, $, moment */ (function() { + /** + * Helper function to map the job status to a color tag + * @param {Object} row The job row object + * @returns {string} Color code for the status + */ var getColor = function(row) { - if (row.status === "SCHEDULED") { - return "yellow"; + // row is the merged job object + // Use _originalStatus if available, otherwise fall back to status + var status = row._originalStatus || row.status; + if (status) { + status = status.toUpperCase(); // Convert to uppercase for consistent comparison } - else if (row.status === "SUSPENDED") { + + // Use _originalLastRunStatus if available, otherwise fall back to lastRunStatus + var lastRunStatus = row._originalLastRunStatus || row.lastRunStatus; + if (lastRunStatus) { + lastRunStatus = lastRunStatus.toUpperCase(); // Convert to uppercase for consistent comparison + } + + // Check if job is disabled via config.enabled + if (!row.config || !row.config.enabled) { return "gray"; } - else if (row.status === "CANCELLED") { - return "red"; + + // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js) + // But also "success", "failed", "pending" (see getRunStatus in api.js) + switch (status) { + case "RUNNING": return "green"; + case "COMPLETED": return "green"; + case "SUCCESS": return "green"; + case "SCHEDULED": + case "PENDING": return "yellow"; + case "SUSPENDED": return "gray"; + case "CANCELLED": return "red"; } - else if (row.status === "RUNNING") { - return "green"; + + // If status doesn't match, check lastRunStatus + if (lastRunStatus === "FAILED") { + return "red"; } + + return "gray"; }; + + /** + * Helper to update row display fields (like nextRunDate, nextRunTime, lastRun) + * @param {Object} row The job row object to update + * @returns {void} + */ var updateScheduleRow = function(row) { - var index; - row.nextRunDate = countlyCommon.getDate(row.next); - row.nextRunTime = countlyCommon.getTime(row.next); - row.lastRun = countlyCommon.formatTimeAgo(row.finished); - row.scheduleLabel = row.schedule || ""; - index = row.scheduleLabel.indexOf("starting on"); - if (index > (-1)) { - row.scheduleLabel = row.schedule.substring(0, index).trim(); - row.scheduleDetail = row.schedule.substring(index).trim(); - } - if (row.schedule && row.schedule.startsWith("at")) { - index = row.schedule.indexOf("every"); - row.scheduleDetail = row.schedule.substring(0, index).trim(); - row.scheduleLabel = row.schedule.substring(index).trim(); + // Store original values for sorting + if (row.name !== undefined) { + row._sortName = String(row.name); + } + + if (row.status !== undefined) { + // Store the original status value for color determination and sorting + row._originalStatus = row.status; + row._sortStatus = String(row.status); + } + + // Add sortBy properties for date fields to ensure correct sorting + if (row.nextRunAt) { + var nextRunMoment = moment(row.nextRunAt); + row.nextRunDate = nextRunMoment.format('YYYY-MM-DD'); + row.nextRunTime = nextRunMoment.format('HH:mm:ss'); + + // Store original value for sorting + row._sortNextRunAt = new Date(row.nextRunAt).getTime(); + } + else { + row.nextRunDate = ''; + row.nextRunTime = ''; + } + + if (row.lastFinishedAt) { + var lastFinishedMoment = moment(row.lastFinishedAt); + row.lastRun = lastFinishedMoment.fromNow(); + + // Store original value for sorting + row._sortLastFinishedAt = new Date(row.lastFinishedAt).getTime(); + } + else { + row.lastRun = ''; + } + + if (row.scheduleLabel !== undefined) { + row._sortScheduleLabel = String(row.scheduleLabel); + } + + if (row.lastRunStatus !== undefined) { + // Store the original lastRunStatus value for color determination and sorting + row._originalLastRunStatus = row.lastRunStatus; + row._sortLastRunStatus = String(row.lastRunStatus); + } + + if (row.total !== undefined) { + row._sortTotal = Number(row.total); + } + + // If the row has .config.defaultConfig.schedule.value, use it as its "default schedule" + if (row.config && row.config.defaultConfig && row.config.defaultConfig.schedule) { + row.schedule = row.config.defaultConfig.schedule.value; + row.configuredSchedule = row.config.schedule; + row.scheduleOverridden = row.configuredSchedule && (row.configuredSchedule !== row.schedule); } }; + + /** + * Main view for listing jobs + */ var JobsView = countlyVue.views.create({ - template: CV.T('/core/jobs/templates/jobs.html'), + template: CV.T('/core/jobs/templates/jobs.html'), // your HTML template path data: function() { var self = this; - var tableStore = countlyVue.vuex.getLocalStore(countlyVue.vuex.ServerDataTable("jobsTable", { - columns: ['name', "schedule", "next", "finished", "status", "total"], - onRequest: function() { - self.loaded = false; - return { - type: "GET", - url: countlyCommon.API_URL + "/o", - data: { - app_id: countlyCommon.ACTIVE_APP_ID, - method: 'jobs' + + // Create a local vuex store for the server data table + var tableStore = countlyVue.vuex.getLocalStore( + countlyVue.vuex.ServerDataTable("jobsTable", { + // columns: ['name', "schedule", "next", "finished", "status", "total"], + columns: ["name", "status", "scheduleLabel", "nextRunAt", "lastFinishedAt", "lastRunStatus", "total"], + + onRequest: function() { + // Called before making the request + self.loaded = false; + return { + type: "GET", + url: countlyCommon.API_URL + "/o/jobs", // no ?name= param => list mode + data: { + app_id: countlyCommon.ACTIVE_APP_ID, + iDisplayStart: 0, + iDisplayLength: 50 + } + }; + }, + onReady: function(context, rows) { + // Called when request completes successfully + self.loaded = true; + + // rows.aaData is an array: [ { job: {...}, config: {...} }, ... ] + // We merge job + config into a single row object + var processedRows = []; + for (var i = 0; i < rows.length; i++) { + var mergedJob = rows[i].job; // from "job" + var config = rows[i].config; // from "config" + + mergedJob.enabled = config.enabled; + mergedJob.config = config; + + // Do any schedule display updates, etc. + updateScheduleRow(mergedJob); + + processedRows.push(mergedJob); } - }; - }, - onReady: function(context, rows) { - self.loaded = true; - var row; - for (var i = 0; i < rows.length; i++) { - row = rows[i]; - updateScheduleRow(row); + return processedRows; } - return rows; - } - })); + }) + ); + return { loaded: true, + saving: false, + scheduleDialogVisible: false, + selectedJobConfig: { + name: '', + schedule: '', + defaultSchedule: '', + scheduleLabel: '', + enabled: true + }, tableStore: tableStore, - remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable") + remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable"), + jobsTablePersistKey: "cly-jobs-table" }; }, computed: { + /** + * Whether the current user can enable/disable jobs + * @returns {boolean} True if user has admin rights + */ canSuspendJob: function() { return countlyGlobal.member.global_admin || countlyGlobal.admin_apps[countlyCommon.ACTIVE_APP_ID]; }, }, methods: { + formatDateTime: function(date) { + return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-'; + }, + getStatusColor: function(details) { + // Not strictly used in the listing, but you can keep it for reference + if (!details.config.enabled) { + return 'gray'; + } + if (details.currentState.status === 'RUNNING') { + return 'green'; + } + if (details.currentState.status === 'FAILED') { + return 'red'; + } + if (details.currentState.status === 'COMPLETED') { + return 'green'; + } + if (details.currentState.status === 'PENDING') { + return 'yellow'; + } + return 'yellow'; + }, + getRunStatusColor(status) { + // For run status + // Use _originalLastRunStatus if available + var statusValue = status && status._originalLastRunStatus ? status._originalLastRunStatus : status; + + // Convert to uppercase for consistent comparison with backend values + if (typeof statusValue === 'string') { + statusValue = statusValue.toUpperCase(); + } + + // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js) + // But also "success", "failed", "pending" (see getRunStatus in api.js) + switch (statusValue) { + case "SUCCESS": + case "COMPLETED": return 'green'; + case "RUNNING": return 'green'; + case "FAILED": return 'red'; + case "PENDING": + case "SCHEDULED": return 'yellow'; + default: return 'gray'; + } + }, refresh: function(force) { if (this.loaded || force) { this.loaded = false; this.tableStore.dispatch("fetchJobsTable"); } }, + /** + * Navigates to job details page + * @param {Object} row The job row to navigate to + * @returns {void} + */ goTo: function(row) { app.navigate("#/manage/jobs/" + row.name, true); }, getColor: getColor, + /** + * Called from the row's more options, e.g. "enable", "disable", "schedule", "runNow" + * @param {string} command The command to execute + * @param {Object} row The job row + * @returns {void} + */ handleCommand: function(command, row) { - if (row.rowId) { + if (row.name) { var self = this; - if (command === "change-job-status") { - const suspend = row.status !== "SUSPENDED" ? true : false; - var notifyType = "ok"; - $.ajax({ - type: "GET", - url: countlyCommon.API_URL + "/o", - data: { - app_id: countlyCommon.ACTIVE_APP_ID, - method: 'suspend_job', - id: row.rowId, - suspend: suspend - }, - contentType: "application/json", - success: function(res) { - if (res.result) { - self.refresh(true); - } - else { - notifyType = "error"; - } + if (command === 'schedule') { + // Show the schedule dialog + this.selectedJobConfig = { + name: row.name, + schedule: row.configuredSchedule || row.schedule, + defaultSchedule: row.schedule, + enabled: row.enabled + }; + this.scheduleDialogVisible = true; + return; + } + + // For enable, disable, runNow, etc. => /i/jobs + var data = { + app_id: countlyCommon.ACTIVE_APP_ID, + jobName: row.name, + action: command + }; + + $.ajax({ + type: "GET", // or POST if your server expects that + url: countlyCommon.API_URL + "/i/jobs", + data: data, + success: function(res) { + if (res.result === "Success") { + self.refresh(true); CountlyHelpers.notify({ - type: notifyType, - message: res.message + type: "ok", + message: CV.i18n("jobs." + command + "-success") }); - }, - error: function(err) { + } + else { CountlyHelpers.notify({ type: "error", - message: err.responseJSON.error + message: res.result }); } + }, + error: function(err) { + CountlyHelpers.notify({ + type: "error", + message: err.responseJSON?.result || "Error" + }); + } + }); + } + }, + /** + * Called when user clicks "Save" on the schedule dialog + */ + saveSchedule: function() { + var self = this; + self.saving = true; + + $.ajax({ + type: "GET", + url: countlyCommon.API_URL + "/i/jobs", + data: { + app_id: countlyCommon.ACTIVE_APP_ID, + jobName: this.selectedJobConfig.name, + action: 'updateSchedule', + schedule: this.selectedJobConfig.schedule + }, + success: function() { + self.saving = false; + self.scheduleDialogVisible = false; + self.refresh(true); + CountlyHelpers.notify({ + type: "ok", + message: CV.i18n("jobs.schedule-updated") + }); + }, + error: function(err) { + self.saving = false; + CountlyHelpers.notify({ + type: "error", + message: err.responseJSON?.result || "Error" }); } - } + }); }, } }); - var JobDetailView = countlyVue.views.create({ - template: CV.T('/core/jobs/templates/jobs-details.html'), + /** + * Detailed view for a single job + */ + var JobDetailsView = countlyVue.views.BaseView.extend({ + template: "#jobs-details-template", data: function() { - var self = this; - var tableStore = countlyVue.vuex.getLocalStore(countlyVue.vuex.ServerDataTable("jobsTable", { - columns: ['name', "schedule", "next", "finished", "status", "total"], - onRequest: function() { - self.loaded = false; - return { - type: "GET", - url: countlyCommon.API_URL + "/o", - data: { - app_id: countlyCommon.ACTIVE_APP_ID, - method: 'jobs', - name: self.job_name - } - }; - }, - onReady: function(context, rows) { - self.loaded = true; - var row; - for (var i = 0; i < rows.length; i++) { - row = rows[i]; - row.dataAsString = JSON.stringify(row.data, null, 2); - row.durationInSeconds = (row.duration / 1000) + 's'; - updateScheduleRow(row); - } - return rows; - } - })); return { - job_name: this.$route.params.job_name, - loaded: true, - tableStore: tableStore, - remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable") + job_name: this.$route.params.jobName, + jobDetails: null, + jobRuns: [], + isLoading: false, + // columns for the run history table + jobRunColumns: [ + { prop: "lastRunAt", label: CV.i18n('jobs.run-time'), sortable: true }, + { prop: "status", label: CV.i18n('jobs.status'), sortable: true }, + { prop: "duration", label: CV.i18n('jobs.duration'), sortable: true }, + { prop: "result", label: CV.i18n('jobs.result') } + ] }; }, + computed: { + /** + * Check if there are any scheduleOverride or retryOverride in the config + * @returns {boolean} True if overrides exist + */ + hasOverrides: function() { + return this.jobDetails && + (this.jobDetails.config?.scheduleOverride || + this.jobDetails.config?.retryOverride); + } + }, methods: { - refresh: function(force) { - if (this.loaded || force) { - this.loaded = false; - this.tableStore.dispatch("fetchJobsTable"); + /** + * Fetches jobDetails + normal docs from /o/jobs?name= + */ + fetchJobDetails: function() { + var self = this; + self.isLoading = true; + + CV.$.ajax({ + type: "GET", + url: countlyCommon.API_PARTS.data.r + "/jobs", + data: { + "app_id": countlyCommon.ACTIVE_APP_ID, + "name": self.job_name, + "iDisplayStart": 0, + "iDisplayLength": 50 + }, + dataType: "json", + success: function(response) { + // jobDetails => the main scheduled doc + overrides + self.jobDetails = response.jobDetails; + + // aaData => the array of normal run docs + self.jobRuns = (response.aaData || []).map(function(run) { + return { + lastRunAt: run.lastRunAt, + status: run.status, + duration: run.duration, + result: run.result, + failReason: run.failReason, + dataAsString: run.dataAsString + }; + }); + + self.isLoading = false; + }, + error: function() { + self.isLoading = false; + CountlyHelpers.notify({ + title: CV.i18n("common.error"), + message: CV.i18n("jobs.details-fetch-error"), + type: "error" + }); + } + }); + }, + formatDateTime: function(date) { + return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-'; + }, + calculateDuration: function(run) { + // (optional) if you want dynamic calculations + if (!run.lastRunAt || !run.lastFinishedAt) { + return '-'; } + return ((new Date(run.lastFinishedAt) - new Date(run.lastRunAt)) / 1000).toFixed(2); }, - navigate: function(id) { - app.navigate("#/manage/jobs/" + id); + /** + * Map jobDetails.currentState.status to a color + * @param {Object} jobDetails The job details object + * @returns {string} Color code for the status + */ + getStatusColor: function(jobDetails) { + if (!jobDetails.config?.enabled) { + return "gray"; + } + switch (jobDetails.currentState.status) { + case "RUNNING": return "green"; + case "FAILED": return "red"; + case "COMPLETED": return "green"; + case "PENDING": return "yellow"; + default: return "yellow"; + } }, - getColor: getColor + /** + * Map each run's status to a color + * @param {Object} run The job run object + * @returns {string} Color code for the status + */ + getRunStatusColor: function(run) { + // Handle both string and object status + // Use _originalLastRunStatus if available + var status = run._originalLastRunStatus || run.status; + + // Convert to uppercase for consistent comparison with backend values + if (typeof status === 'string') { + status = status.toUpperCase(); + } + + // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js) + // But also "success", "failed", "pending" (see getRunStatus in api.js) + switch (status) { + case "SUCCESS": + case "COMPLETED": return "green"; + case "RUNNING": return "green"; + case "FAILED": return "red"; + case "PENDING": + case "SCHEDULED": return "yellow"; + default: return "gray"; + } + } + }, + mounted: function() { + // On load, fetch data + this.fetchJobDetails(); } }); + /** + * Wrap the JobsView as a Countly Backbone view + * @returns {Object} Backbone wrapper view + */ var getMainView = function() { return new countlyVue.views.BackboneWrapper({ component: JobsView, - vuex: [] //empty array if none - }); - }; - - var getDetailedView = function() { - return new countlyVue.views.BackboneWrapper({ - component: JobDetailView, - vuex: [] //empty array if none + vuex: [] // empty array if none }); }; + /** + * Define routes for #/manage/jobs and #/manage/jobs/:jobName + */ if (countlyAuth.validateGlobalAdmin()) { app.route("/manage/jobs", "manageJobs", function() { this.renderWhenReady(getMainView()); }); - app.route("/manage/jobs/:name", "manageJobName", function(name) { - var view = getDetailedView(); - view.params = {job_name: name}; - this.renderWhenReady(view); + app.route("/manage/jobs/:jobName", 'jobs-details', function(jobName) { + var jobDetailsView = new countlyVue.views.BackboneWrapper({ + component: JobDetailsView, + templates: [ + { + namespace: "jobs", + mapping: { + "details-template": "/core/jobs/templates/jobs-details.html" + } + } + ] + }); + jobDetailsView.params = { jobName: jobName }; + this.renderWhenReady(jobDetailsView); }); } -})(); \ No newline at end of file +})(); diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html index 4062a534513..e67a5184964 100644 --- a/frontend/express/public/core/jobs/templates/jobs-details.html +++ b/frontend/express/public/core/jobs/templates/jobs-details.html @@ -1,49 +1,241 @@ +
- + + - - -