2473 lines
78 KiB
JavaScript
2473 lines
78 KiB
JavaScript
var crossCtl = require("../../src/crossCtl");
|
|
var utils = require("../../src/utils");
|
|
var mysql = require("mysql");
|
|
|
|
var config = {};
|
|
var stype = "";
|
|
var sid = "";
|
|
|
|
function getTag() {
|
|
return stype + ":" + sid;
|
|
}
|
|
exports.getTag = getTag;
|
|
|
|
const FastScanner = require("fastscan");
|
|
// let toxicWords = require('./filter.json');
|
|
let toxicWords = [];
|
|
exports.toxicWords = toxicWords;
|
|
let scanner = new FastScanner(toxicWords);
|
|
// const options = { quick: false, longest: true };
|
|
|
|
let scannerPool = {}
|
|
|
|
let hitCount = 0;
|
|
|
|
let elapsedSum = 0;
|
|
|
|
let elapsedAvg = 0;
|
|
|
|
function getElapsed(startTime) {
|
|
let stopTime = process.hrtime(startTime);
|
|
|
|
let precision = 3; // 3 decimal places
|
|
let elapsedRaw = stopTime[1] / 1000000; // divide by a million to get nano to milli
|
|
let elapsedString =
|
|
stopTime[0] + " s, " + elapsedRaw.toFixed(precision) + " ms";
|
|
|
|
let elapsedTime = (stopTime[0] * 1e9 + stopTime[1]) / 1e9;
|
|
|
|
return { text: elapsedString, value: elapsedTime };
|
|
}
|
|
|
|
function handleApiFinalResponse(req, res, cb_res) {
|
|
|
|
let reqDump = {
|
|
key: req.header("x-api-key"),
|
|
body: req.body,
|
|
keyInfo: req.keyInfo ? req.keyInfo : {},
|
|
};
|
|
if (res.Status.Code == 2000) {
|
|
let elapsed = getElapsed(req.startTime);
|
|
res.Elapsed = elapsed.text;
|
|
makeStat(elapsed.value);
|
|
}
|
|
req.workTag.res().status(200).send(res);
|
|
/*
|
|
crossCtl.boom.log("rawEvent", {
|
|
ts: utils.getNow(),
|
|
trId: res.trackingId,
|
|
req: JSON.stringify(reqDump),
|
|
res: JSON.stringify(res),
|
|
cb_res: JSON.stringify(cb_res),
|
|
// foot print
|
|
url: req.protocol + "://" + req.get("host") + req.originalUrl,
|
|
headers: req.headers,
|
|
referrer: utils.getReferrerFromReq(req),
|
|
ip: utils.getIPFromReq(req),
|
|
});
|
|
*/
|
|
|
|
if (res.Status.Code == 2000) {
|
|
crossCtl.boom.log("success", {
|
|
ts: utils.getNow(),
|
|
trackingId: res.TrackingId,
|
|
uid: req.keyInfo.uid,
|
|
key: req.header("x-api-key"),
|
|
hit: res.Detected.length,
|
|
words: res.Detected,
|
|
size: new TextEncoder().encode(req.body.text).length,
|
|
referrer: utils.getReferrerFromReq(req),
|
|
ip: utils.getIPFromReq(req),
|
|
});
|
|
} else {
|
|
var uid = req.isAuthenticated() ? req.user.uid : utils.uuid("uid_" + req.sessionID);
|
|
crossCtl.boom.log("error", {
|
|
ts: utils.getNow(),
|
|
uid: uid,
|
|
key: req.header("x-api-key"),
|
|
//
|
|
req: JSON.stringify(reqDump),
|
|
//
|
|
ip: utils.getIPFromReq(req),
|
|
referrer: utils.getReferrerFromReq(req),
|
|
result: res.Status
|
|
});
|
|
|
|
}
|
|
}
|
|
exports.handleApiFinalResponse = handleApiFinalResponse;
|
|
|
|
|
|
var wordCounted = 0;
|
|
|
|
function getDashboardData() {
|
|
return {
|
|
wordCount: wordCounted,
|
|
hitCount: hitCount,
|
|
elapsedAvg: elapsedAvg,
|
|
};
|
|
}
|
|
exports.getDashboardData = getDashboardData;
|
|
|
|
function splice(source, start, length, replacement) {
|
|
return source.substr(0, start) + replacement + source.substr(start + length);
|
|
}
|
|
exports.splice = splice;
|
|
|
|
function getAPIKeyInfo(apiKey, cb) {
|
|
crossCtl.redis.doEasyGetWithExpireAt(
|
|
"apiKeyInfoPool::" + apiKey,
|
|
60,
|
|
function (err, replies) {
|
|
if (err) {
|
|
cb(err, null);
|
|
} else {
|
|
if (replies == null) {
|
|
var qry =
|
|
"SELECT * FROM " +
|
|
db.options.database +
|
|
".tbl_key_list WHERE api_key = " +
|
|
mysql.escape(apiKey) +
|
|
" AND status = 0";
|
|
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
cb(err, null);
|
|
} else {
|
|
if (results.length == 0) {
|
|
cb(null, null);
|
|
} else if (results.length != 1) {
|
|
cb(
|
|
new Error("Too many target. count = " + results.length),
|
|
null
|
|
);
|
|
} else {
|
|
let apiKeyInfoString = JSON.stringify(results[0]);
|
|
crossCtl.redis.doEasySetWithExpireAt(
|
|
"apiKeyInfoPool::" + apiKey,
|
|
apiKeyInfoString,
|
|
60,
|
|
function (err, replies) {
|
|
cb(err, utils.safeJSON(apiKeyInfoString));
|
|
}
|
|
);
|
|
}
|
|
}
|
|
});
|
|
} else {
|
|
cb(null, utils.safeJSON(replies));
|
|
}
|
|
}
|
|
}
|
|
);
|
|
}
|
|
exports.getAPIKeyInfo = getAPIKeyInfo;
|
|
|
|
|
|
function updateAPIKeyInfo(serial, cb) {
|
|
var qry =
|
|
"SELECT * FROM " +
|
|
db.options.database +
|
|
".tbl_key_list WHERE serial = " +
|
|
mysql.escape(serial);
|
|
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
cb(err, null);
|
|
} else {
|
|
if (results.length == 0) {
|
|
cb(null, null);
|
|
} else if (results.length != 1) {
|
|
cb(
|
|
new Error("Too many target. count = " + results.length),
|
|
null
|
|
);
|
|
} else {
|
|
let apiKeyInfoString = JSON.stringify(results[0]);
|
|
let apiKey = results[0]['api_key']
|
|
let status = results[0]['status']
|
|
|
|
if (status == 4) {
|
|
crossCtl.redis.doEasyDel(
|
|
"apiKeyInfoPool::" + apiKey,
|
|
function (err, replies) {
|
|
cb(err, null);
|
|
}
|
|
);
|
|
} else {
|
|
crossCtl.redis.doEasySetWithExpireAt(
|
|
"apiKeyInfoPool::" + apiKey,
|
|
apiKeyInfoString,
|
|
60,
|
|
function (err, replies) {
|
|
cb(err, utils.safeJSON(apiKeyInfoString));
|
|
}
|
|
);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
exports.updateAPIKeyInfo = updateAPIKeyInfo;
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function makeStat(elapsed) {
|
|
hitCount++;
|
|
elapsedSum += elapsed;
|
|
elapsedAvg = elapsedSum / hitCount;
|
|
|
|
crossCtl.redis.doEasySet(
|
|
crossCtl.sConfig.boom.basePath + ":dashboardData",
|
|
JSON.stringify({
|
|
hitCount: hitCount,
|
|
elapsedSum: elapsedSum,
|
|
}),
|
|
function (error, replies) {}
|
|
);
|
|
}
|
|
exports.makeStat = makeStat;
|
|
|
|
async function doSearch(filterSource, options, cb) {
|
|
let offWords = scanner.search(filterSource, options);
|
|
|
|
cb(null, offWords);
|
|
}
|
|
exports.doSearch = doSearch;
|
|
|
|
|
|
async function doSearchWithLevel(levelTag, filterSource, options, cb) {
|
|
|
|
let tmpScanner = scannerPool[levelTag];
|
|
let offWords = tmpScanner.search(filterSource, options);
|
|
|
|
cb(null, offWords);
|
|
}
|
|
exports.doSearchWithLevel = doSearchWithLevel;
|
|
|
|
|
|
var db = require("../../src/newDbHandler")();
|
|
exports.db = db;
|
|
|
|
var nuxtHandler = null;
|
|
exports.nuxtHandler = nuxtHandler;
|
|
|
|
var routerMain = null;
|
|
exports.routerMain = routerMain;
|
|
|
|
function setRouterMain(_router) {
|
|
_router.use((req, res) => {
|
|
nuxtHandler.handler(req, res);
|
|
});
|
|
}
|
|
exports.setRouterMain = setRouterMain;
|
|
|
|
const { exec } = require("child_process");
|
|
|
|
function syncClientDist(from, to, cb) {
|
|
crossCtl.s3.downloadFile(from, to, function (error, path) {
|
|
if (error) {
|
|
cb(error);
|
|
} else {
|
|
exec(
|
|
"tar -zxvf " + path + " -C " + path.replace("client.tgz", ""),
|
|
(error, stdout, stderr) => {
|
|
if (error) {
|
|
console.log(`error: ${error.message}`);
|
|
}
|
|
if (stderr) {
|
|
console.log(`stderr: ${stderr}`);
|
|
}
|
|
// console.log(`stdout: ${stdout}`);
|
|
cb(error | stderr, null);
|
|
}
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
var async = require("async");
|
|
|
|
function syncFilterWithDb(level, cb) {
|
|
var qry =
|
|
"SELECT raw FROM " +
|
|
db.options.database +
|
|
"." + crossCtl.MAIN_WORD_TBL_NAME + " WHERE tag = 'SWEARWORD' AND status = 0 AND level >= " +
|
|
mysql.escape(level);
|
|
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
let newTWAry = [];
|
|
for (var i = 0; i < results.length; i++) {
|
|
newTWAry.push(results[i]["raw"]);
|
|
}
|
|
console.log("read done, count =", newTWAry.length);
|
|
if (level == 10) {
|
|
wordCounted = newTWAry.length;
|
|
}
|
|
toxicWords = newTWAry;
|
|
exports.toxicWords = toxicWords;
|
|
|
|
scanner = new FastScanner(toxicWords);
|
|
let scannerLevelTag = '';
|
|
switch (level) {
|
|
case 10:
|
|
scannerLevelTag = 'high';
|
|
break;
|
|
case 50:
|
|
scannerLevelTag = 'mid';
|
|
break;
|
|
case 100:
|
|
scannerLevelTag = 'low';
|
|
break;
|
|
|
|
}
|
|
scannerPool[scannerLevelTag] = scanner
|
|
cb(error, newTWAry.length);
|
|
});
|
|
}
|
|
exports.syncFilterWithDb = syncFilterWithDb;
|
|
|
|
|
|
function syncFilterWithDbAll(cb) {
|
|
syncFilterWithDb(10, function(error) {
|
|
syncFilterWithDb(50, function(error) {
|
|
syncFilterWithDb(100, function(error) {
|
|
cb(error)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
exports.syncFilterWithDbAll = syncFilterWithDbAll;
|
|
|
|
|
|
var targetTagList = ['SWEARWORD', 'SUICIDE', 'AD', 'GAMBLE', 'ADULT', 'WHITE'];
|
|
|
|
// var targetTagList = ['SUICIDE'];
|
|
|
|
function doFilterUpdate(req, res, cb) {
|
|
|
|
|
|
var finalWordsArray = [];
|
|
|
|
for (var i = 0; i < targetTagList.length; i++) {
|
|
let rawdata = fs.readFileSync(path.join(__dirname, './data/' + targetTagList[i] + '.txt'));
|
|
|
|
let tmpAry = rawdata.toString('utf-8').split('\n');
|
|
|
|
for (var j = 0; j < tmpAry.length; j++) {
|
|
finalWordsArray.push({tag: targetTagList[i], word: tmpAry[j]})
|
|
}
|
|
|
|
}
|
|
|
|
|
|
let chunkArrays = [];
|
|
const chunkSize = 100000;
|
|
for (let i = 0; i < finalWordsArray.length; i += chunkSize) {
|
|
const chunk = finalWordsArray.slice(i, i + chunkSize);
|
|
// do whatever
|
|
chunkArrays.push(chunk);
|
|
}
|
|
|
|
console.log("total chunk count is ", chunkArrays.length);
|
|
|
|
var maxWordlength = 0;
|
|
var maxWord = "";
|
|
|
|
async.mapSeries(
|
|
chunkArrays,
|
|
function (chunk, subCallback) {
|
|
// console.log('chunk=', chunk);
|
|
|
|
for (let i = 0; i < chunk.length; i++) {
|
|
if (chunk[i].length > maxWordlength) {
|
|
maxWordlength = chunk[i].word.length;
|
|
maxWord = chunk[i].word;
|
|
console.log("max changed ", maxWordlength);
|
|
}
|
|
}
|
|
|
|
var qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
"." + crossCtl.MAIN_WORD_TBL_NAME + " (tag, word, raw, level, memo) VALUES ? ";
|
|
|
|
var tmpPost = [];
|
|
|
|
for (var i = 0; i < chunk.length; i++) {
|
|
tmpPost[tmpPost.length] = [chunk[i].tag, chunk[i].word, chunk[i].word, 10, "Bulk Inserted"];
|
|
}
|
|
|
|
var post = [tmpPost];
|
|
|
|
// console.log('post=', post);
|
|
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
console.log("error=", error);
|
|
console.log("results=", results);
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on doFilterUpdate(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, finalWordsArray.length);
|
|
} else {
|
|
console.log("final max ", maxWordlength);
|
|
console.log("final max word ", maxWord);
|
|
cb(null, finalWordsArray.length);
|
|
}
|
|
}
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
exports.doFilterUpdate = doFilterUpdate;
|
|
|
|
function doFilterUpdateBackup(req, res, cb) {
|
|
// toxicWords
|
|
|
|
// let tempToxicWords = toxicWords.slice(0, 100000);
|
|
|
|
let sourcePath = path.join(__dirname, "./filter.json");
|
|
// console.log('sourcePath=', sourcePath);
|
|
|
|
let rawdata = fs.readFileSync(sourcePath);
|
|
// console.log('rawdata=', rawdata);
|
|
// let parsedWordsJson = JSON.parse(rawdata);
|
|
let tempToxicWords = JSON.parse(rawdata);
|
|
|
|
// let tempToxicWords = parsedWordsJson.slice(0, 100);
|
|
|
|
// console.log('tempToxicWords=', tempToxicWords);
|
|
|
|
let dupFilter = {};
|
|
let noDupArray = [];
|
|
|
|
for (let i = 0; i < tempToxicWords.length; i++) {
|
|
if (dupFilter[tempToxicWords[i]] == undefined) {
|
|
dupFilter[tempToxicWords[i]] = tempToxicWords[i];
|
|
noDupArray.push(tempToxicWords[i]);
|
|
} else {
|
|
console.log("dup item found ", tempToxicWords[i]);
|
|
}
|
|
}
|
|
|
|
tempToxicWords = noDupArray;
|
|
|
|
let chunkArrays = [];
|
|
const chunkSize = 100000;
|
|
for (let i = 0; i < tempToxicWords.length; i += chunkSize) {
|
|
const chunk = tempToxicWords.slice(i, i + chunkSize);
|
|
// do whatever
|
|
chunkArrays.push(chunk);
|
|
}
|
|
|
|
console.log("total chunk count is ", chunkArrays.length);
|
|
|
|
var maxWordlength = 0;
|
|
var maxWord = "";
|
|
|
|
async.mapSeries(
|
|
chunkArrays,
|
|
function (chunk, subCallback) {
|
|
// console.log('chunk=', chunk);
|
|
|
|
for (let i = 0; i < chunk.length; i++) {
|
|
if (chunk[i].length > maxWordlength) {
|
|
maxWordlength = chunk[i].length;
|
|
maxWord = chunk[i];
|
|
console.log("max changed ", maxWordlength);
|
|
}
|
|
}
|
|
|
|
var qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
"." + crossCtl.MAIN_WORD_TBL_NAME + " (word, raw, level, memo) VALUES ? ";
|
|
|
|
var tmpPost = [];
|
|
|
|
for (var i = 0; i < chunk.length; i++) {
|
|
tmpPost[tmpPost.length] = [chunk[i], chunk[i], 10, "Bulk Inserted"];
|
|
}
|
|
|
|
var post = [tmpPost];
|
|
|
|
// console.log('post=', post);
|
|
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
console.log("error=", error);
|
|
console.log("results=", results);
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on doFilterUpdate(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, tempToxicWords.length);
|
|
} else {
|
|
console.log("final max ", maxWordlength);
|
|
console.log("final max word ", maxWord);
|
|
cb(null, tempToxicWords.length);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
function init(id, cfg, cb) {
|
|
// utils.log('debug', 'local init(), ', id, cfg)
|
|
config = cfg;
|
|
stype = config.type;
|
|
sid = id;
|
|
|
|
crossCtl.redis.doEasyGet(
|
|
crossCtl.sConfig.boom.basePath + ":dashboardData",
|
|
function (error, replies) {
|
|
if (replies != null) {
|
|
let tmpDashboardData = utils.safeJSON(replies);
|
|
hitCount = Number(tmpDashboardData.hitCount);
|
|
elapsedSum = Number(tmpDashboardData.elapsedSum);
|
|
elapsedAvg = elapsedSum / hitCount;
|
|
}
|
|
}
|
|
);
|
|
|
|
if (crossCtl.sConfig.clientDistInfo != undefined) {
|
|
let clientPath = path.join(__dirname, ".", "client") + "/client.tgz";
|
|
let clientIndexPath = path.join(
|
|
__dirname,
|
|
".",
|
|
"client",
|
|
// crossCtl.sConfig.clientDistInfo.fromPath,
|
|
".output",
|
|
"server",
|
|
"index.mjs"
|
|
);
|
|
|
|
// console.log('clientIndexPath = ', clientIndexPath);
|
|
|
|
syncClientDist(
|
|
crossCtl.sConfig.clientDistInfo.fromPath,
|
|
clientPath,
|
|
function (error, result) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"s3.syncDist error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
console.log("final dist file list = ", result);
|
|
|
|
fs.access(clientIndexPath, fs.F_OK, (error) => {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"s3.syncDist error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
import(clientIndexPath).then((_nuxtHandler) => {
|
|
console.log("nuxt handler imported...", _nuxtHandler); // "Hello world!"
|
|
nuxtHandler = _nuxtHandler;
|
|
|
|
db.init(config.db, function (error) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"db init error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
syncFilterWithDb(10, function (error, count) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"word sync error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
|
|
syncFilterWithDb(50, function (error, count) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"word sync error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
syncFilterWithDb(100, function (error, count) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"word sync error : " + error + " " + utils.__where()
|
|
);
|
|
cb(error);
|
|
} else {
|
|
initializeStatisticsWork(function(error) {
|
|
cb(null);
|
|
});
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
});
|
|
});
|
|
}
|
|
});
|
|
}
|
|
}
|
|
);
|
|
} else {
|
|
cb(null);
|
|
}
|
|
}
|
|
exports.init = init;
|
|
|
|
/*
|
|
passEventToLocalHandler(
|
|
'cron',
|
|
{ cmd: 'cron', interval: 'day', tag: nowTag },
|
|
function (error) {
|
|
cb(error);
|
|
}
|
|
);
|
|
*/
|
|
|
|
var moment = require("moment");
|
|
// const tsFormat = () => moment().format('YYYY-MM-DD HH:mm:ss.SSS');
|
|
|
|
function makeStatisticsFromScrach(targetTag, dateTagBase, cb) {
|
|
|
|
let resultData = {boom: 0, event: 0};
|
|
|
|
initWordStatistics(targetTag, dateTagBase);
|
|
|
|
let statisticsResultPool = {
|
|
key: {},
|
|
uid: {},
|
|
};
|
|
|
|
crossCtl.s3.listObjects(
|
|
crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
|
|
function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrach(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
|
|
resultData.boom = data.Contents.length;
|
|
if (data.Contents.length == 0) {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
} else {
|
|
async.map(
|
|
data.Contents,
|
|
function (key, subCallback) {
|
|
// console.log('key = ', key);
|
|
let parts = key.Key.split("/");
|
|
let nameOnly =
|
|
crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
|
|
// console.log('nameOnly=', nameOnly);
|
|
crossCtl.s3.readTextFile(nameOnly, function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrach(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
subCallback(error, key);
|
|
} else {
|
|
// console.log('data = ', data);
|
|
let tmpAry = data.split("\n");
|
|
// console.log('tmpAry=', tmpAry);
|
|
resultData.event += tmpAry.length;
|
|
for (var i = 0; i < tmpAry.length; i++) {
|
|
if (tmpAry[i].trim() != "") {
|
|
let tmpLogObj = utils.safeJSON(tmpAry[i]);
|
|
if (
|
|
statisticsResultPool["key"][tmpLogObj.key] == undefined
|
|
) {
|
|
statisticsResultPool["key"][tmpLogObj.key] = [];
|
|
}
|
|
if (
|
|
statisticsResultPool["uid"][tmpLogObj.uid] == undefined
|
|
) {
|
|
statisticsResultPool["uid"][tmpLogObj.uid] = [];
|
|
}
|
|
statisticsResultPool["key"][tmpLogObj.key].push(tmpLogObj);
|
|
statisticsResultPool["uid"][tmpLogObj.uid].push(tmpLogObj);
|
|
}
|
|
}
|
|
subCallback(error, key);
|
|
}
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrach(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
} else {
|
|
let keys = [];
|
|
Object.keys(statisticsResultPool["key"]).forEach(function (key) {
|
|
keys.push(statisticsResultPool["key"][key]);
|
|
});
|
|
// console.log('keys = ', keys);
|
|
let uids = [];
|
|
Object.keys(statisticsResultPool["uid"]).forEach(function (uid) {
|
|
uids.push(statisticsResultPool["uid"][uid]);
|
|
});
|
|
// console.log('uids = ', uids);
|
|
|
|
async.mapSeries(
|
|
keys,
|
|
function (records, subCallback) {
|
|
let targetKey = records[0].key;
|
|
let targetUid = records[0].uid;
|
|
let total = records.length;
|
|
let hit = records.length;
|
|
let miss = 0;
|
|
let size = 0;
|
|
let ipPool = {};
|
|
let referrerPool = {};
|
|
|
|
for (var i = 0; i < records.length; i++) {
|
|
if (records[i].hit == 0) {
|
|
hit--;
|
|
miss++;
|
|
}
|
|
size += records[i].size;
|
|
if (ipPool[records[i].ip] == undefined) {
|
|
ipPool[records[i].ip] = records[i].ip;
|
|
}
|
|
if (referrerPool[records[i].referrer] == undefined) {
|
|
referrerPool[records[i].referrer] = records[i].referrer;
|
|
}
|
|
|
|
if (records[i].words != undefined) {
|
|
makeWordStatistics(targetTag, dateTagBase, records[i].uid, records[i].key, records[i].words);
|
|
}
|
|
}
|
|
let uniqIp = Object.keys(ipPool).length;
|
|
let uniqReferrer = Object.keys(referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(total) +
|
|
", hit = " +
|
|
mysql.escape(hit) +
|
|
", miss = " +
|
|
mysql.escape(miss) +
|
|
", size = " +
|
|
mysql.escape(size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
api_key: targetKey,
|
|
uid: targetUid,
|
|
total: total,
|
|
hit: hit,
|
|
miss: miss,
|
|
size: size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrach(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
async.mapSeries(
|
|
uids,
|
|
function (records, subCallback) {
|
|
let targetUid = records[0].uid;
|
|
let total = records.length;
|
|
let hit = records.length;
|
|
let miss = 0;
|
|
let size = 0;
|
|
let ipPool = {};
|
|
let referrerPool = {};
|
|
|
|
for (var i = 0; i < records.length; i++) {
|
|
if (records[i].hit == 0) {
|
|
hit--;
|
|
miss++;
|
|
}
|
|
size += records[i].size;
|
|
if (ipPool[records[i].ip] == undefined) {
|
|
ipPool[records[i].ip] = records[i].ip;
|
|
}
|
|
if (referrerPool[records[i].referrer] == undefined) {
|
|
referrerPool[records[i].referrer] =
|
|
records[i].referrer;
|
|
}
|
|
}
|
|
let uniqIp = Object.keys(ipPool).length;
|
|
let uniqReferrer = Object.keys(referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(total) +
|
|
", hit = " +
|
|
mysql.escape(hit) +
|
|
", miss = " +
|
|
mysql.escape(miss) +
|
|
", size = " +
|
|
mysql.escape(size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
uid: targetUid,
|
|
total: total,
|
|
hit: hit,
|
|
miss: miss,
|
|
size: size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (
|
|
error,
|
|
results
|
|
) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrach(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
}
|
|
}
|
|
);
|
|
}
|
|
exports.makeStatisticsFromScrach = makeStatisticsFromScrach;
|
|
|
|
|
|
var tagJarPool = {};
|
|
|
|
function initTagValHashs() {
|
|
tagJarPool = {};
|
|
}
|
|
|
|
function getHashFromTagVal(tag, val) {
|
|
let tmpJar = tagJarPool[tag]
|
|
|
|
if (tmpJar == undefined) {
|
|
tmpJar = tagJarPool[tag] = {
|
|
valPool: {},
|
|
vals: []
|
|
}
|
|
}
|
|
|
|
if (tmpJar.valPool[val] == undefined) {
|
|
tmpJar.vals.push(val);
|
|
tmpJar.valPool[val] = tmpJar.vals.length - 1;
|
|
}
|
|
|
|
return tmpJar.valPool[val];
|
|
}
|
|
|
|
function getValFromTagHash(tag, hash) {
|
|
let tmpJar = tagJarPool[tag]
|
|
|
|
if (tmpJar == undefined) {
|
|
return undefined;
|
|
}
|
|
|
|
if (hash < 0 || tmpJar.vals.length -1 < hash) {
|
|
return undefined;
|
|
}
|
|
|
|
return tmpJar.vals[hash];
|
|
}
|
|
|
|
|
|
|
|
|
|
var statisticsJar = {};
|
|
|
|
function initStatisticsJar() {
|
|
statisticsJar = {
|
|
boomCount: 0,
|
|
eventCount: 0,
|
|
uidPool: {},
|
|
keyPool: {}
|
|
}
|
|
}
|
|
|
|
function doCalcOnJar(tag, dateBase, jar, log) {
|
|
|
|
jar.total++;
|
|
|
|
if (log.hit == 0) {
|
|
jar.miss++;
|
|
} else {
|
|
jar.hit++;
|
|
}
|
|
|
|
jar.size += log.size;
|
|
|
|
if (jar.ipPool[log.ip] == undefined) {
|
|
jar.ipPool[log.ip] = 1;
|
|
} else {
|
|
jar.ipPool[log.ip]++;
|
|
}
|
|
if (jar.referrerPool[log.rferrer] == undefined) {
|
|
jar.referrerPool[log.rferrer] = 1;
|
|
} else {
|
|
jar.referrerPool[log.rferrer]++;
|
|
}
|
|
|
|
if (log.words != undefined) {
|
|
makeWordStatistics(
|
|
tag, dateBase,
|
|
log.uid,
|
|
log.key,
|
|
log.words
|
|
);
|
|
}
|
|
}
|
|
|
|
function doWorkOnStatisticsJar(tag, dateBase, data) {
|
|
|
|
statisticsJar.boomCount++;
|
|
|
|
let tmpAry = data.split("\n");
|
|
|
|
statisticsJar.eventCount += tmpAry.length;
|
|
|
|
|
|
if (statisticsJar.boomCount % 1000 == 0) {
|
|
console.log('statisticsJar.boomCount = ', statisticsJar.boomCount, ', statisticsJar.eventCount = ', statisticsJar.eventCount);
|
|
}
|
|
|
|
for (var i = 0; i < tmpAry.length; i++) {
|
|
if (tmpAry[i].trim() != "") {
|
|
let tmpJSONLogObj = utils.safeJSON(tmpAry[i]);
|
|
/*
|
|
tmpJSONLogObj.uid,
|
|
tmpJSONLogObj.key,
|
|
tmpJSONLogObj.hit,
|
|
tmpJSONLogObj.words,
|
|
tmpJSONLogObj.size,
|
|
tmpJSONLogObj.rferrer,
|
|
tmpJSONLogObj.ip,
|
|
*/
|
|
|
|
if (statisticsJar.uidPool[tmpJSONLogObj.uid] == undefined) {
|
|
statisticsJar.uidPool[tmpJSONLogObj.uid] = {
|
|
uid: tmpJSONLogObj.uid,
|
|
key: tmpJSONLogObj.key,
|
|
total: 0,
|
|
hit: 0,
|
|
miss: 0,
|
|
size: 0,
|
|
ipPool: {},
|
|
referrerPool: {},
|
|
}
|
|
}
|
|
if (statisticsJar.keyPool[tmpJSONLogObj.key] == undefined) {
|
|
statisticsJar.keyPool[tmpJSONLogObj.key] = {
|
|
uid: tmpJSONLogObj.uid,
|
|
key: tmpJSONLogObj.key,
|
|
total: 0,
|
|
hit: 0,
|
|
miss: 0,
|
|
size: 0,
|
|
ipPool: {},
|
|
referrerPool: {},
|
|
}
|
|
}
|
|
|
|
doCalcOnJar(tag, dateBase, statisticsJar.uidPool[tmpJSONLogObj.uid], tmpJSONLogObj)
|
|
doCalcOnJar(tag, dateBase, statisticsJar.keyPool[tmpJSONLogObj.key], tmpJSONLogObj)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
function makeStatisticsFromScrachDebug(targetTag, dateTagBase, cb) {
|
|
|
|
let resultData = {boom: 0, event: 0};
|
|
|
|
initStatisticsJar();
|
|
|
|
initWordStatistics(targetTag, dateTagBase);
|
|
|
|
let statisticsResultPool = {
|
|
key: {},
|
|
uid: {},
|
|
};
|
|
|
|
crossCtl.s3.listObjectsDebug(
|
|
crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
|
|
function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
|
|
resultData.boom = data.Contents.length;
|
|
if (data.Contents.length == 0) {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
} else {
|
|
console.log('start to fetch logs...');
|
|
var tmpBoomCount = 0;
|
|
var tmpEventCount = 0;
|
|
async.mapSeries(
|
|
data.Contents,
|
|
function (key, subCallback) {
|
|
// console.log('key = ', key);
|
|
// console.log('data.Prefix + key.Key = ', data.Prefix + key.Key);
|
|
let parts = (data.Prefix + key.Key).split("/");
|
|
let nameOnly =
|
|
crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
|
|
// console.log('nameOnly=', nameOnly);
|
|
crossCtl.s3.readTextFile(nameOnly, function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
} else {
|
|
doWorkOnStatisticsJar(targetTag, dateTagBase, data);
|
|
}
|
|
subCallback(error, key);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
} else {
|
|
|
|
resultData.event = statisticsJar.eventCount;
|
|
|
|
let keys = [];
|
|
Object.keys(statisticsJar.keyPool).forEach(function (key) {
|
|
keys.push(statisticsJar.keyPool[key]);
|
|
});
|
|
// console.log('keys = ', keys);
|
|
let uids = [];
|
|
Object.keys(statisticsJar.uidPool).forEach(function (uid) {
|
|
uids.push(statisticsJar.uidPool[uid]);
|
|
});
|
|
// console.log('uids = ', uids);
|
|
|
|
async.mapSeries(
|
|
keys,
|
|
function (record, subCallback) {
|
|
|
|
let uniqIp = Object.keys(record.ipPool).length;
|
|
let uniqReferrer = Object.keys(record.referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(record.total) +
|
|
", hit = " +
|
|
mysql.escape(record.hit) +
|
|
", miss = " +
|
|
mysql.escape(record.miss) +
|
|
", size = " +
|
|
mysql.escape(record.size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
api_key: record.key,
|
|
uid: record.uid,
|
|
total: record.total,
|
|
hit: record.hit,
|
|
miss: record.miss,
|
|
size: record.size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
async.mapSeries(
|
|
uids,
|
|
function (record, subCallback) {
|
|
|
|
let uniqIp = Object.keys(record.ipPool).length;
|
|
let uniqReferrer = Object.keys(record.referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(record.total) +
|
|
", hit = " +
|
|
mysql.escape(record.hit) +
|
|
", miss = " +
|
|
mysql.escape(record.miss) +
|
|
", size = " +
|
|
mysql.escape(record.size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
uid: record.uid,
|
|
total: record.total,
|
|
hit: record.hit,
|
|
miss: record.miss,
|
|
size: record.size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (
|
|
error,
|
|
results
|
|
) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
}
|
|
}
|
|
);
|
|
}
|
|
exports.makeStatisticsFromScrachDebug = makeStatisticsFromScrachDebug;
|
|
|
|
|
|
|
|
|
|
|
|
function makeStatisticsFromScrachDebugBakup(targetTag, dateTagBase, cb) {
|
|
|
|
let resultData = {boom: 0, event: 0};
|
|
|
|
initTagValHashs();
|
|
|
|
initWordStatistics(targetTag, dateTagBase);
|
|
|
|
let statisticsResultPool = {
|
|
key: {},
|
|
uid: {},
|
|
};
|
|
|
|
crossCtl.s3.listObjectsDebug(
|
|
crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
|
|
function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
|
|
resultData.boom = data.Contents.length;
|
|
if (data.Contents.length == 0) {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
} else {
|
|
console.log('start to fetch logs...');
|
|
var tmpBoomCount = 0;
|
|
var tmpEventCount = 0;
|
|
async.mapSeries(
|
|
data.Contents,
|
|
function (key, subCallback) {
|
|
// console.log('key = ', key);
|
|
// console.log('data.Prefix + key.Key = ', data.Prefix + key.Key);
|
|
let parts = (data.Prefix + key.Key).split("/");
|
|
let nameOnly =
|
|
crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
|
|
// console.log('nameOnly=', nameOnly);
|
|
crossCtl.s3.readTextFile(nameOnly, function (error, data) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
subCallback(error, key);
|
|
} else {
|
|
// console.log('data = ', data);
|
|
let tmpAry = data.split("\n");
|
|
// console.log('tmpAry=', tmpAry);
|
|
resultData.event += tmpAry.length;
|
|
|
|
tmpEventCount = resultData.event;
|
|
|
|
tmpBoomCount+=1;
|
|
|
|
if (tmpBoomCount % 1000 == 0) {
|
|
console.log('tmpBoomCount = ', tmpBoomCount, ', tmpEventCount = ', tmpEventCount);
|
|
} else {
|
|
// console.log('skip boom display..., condition is ', tmpBoomCount % 10)
|
|
// console.log('skip boom display..., tmpBoomCount is ', tmpBoomCount)
|
|
}
|
|
|
|
// console.log('resultData.event=', resultData.event);
|
|
for (var i = 0; i < tmpAry.length; i++) {
|
|
if (tmpAry[i].trim() != "") {
|
|
let tmpJSONLogObj = utils.safeJSON(tmpAry[i]);
|
|
let tmpLogObj = [
|
|
getHashFromTagVal('uid', tmpJSONLogObj.uid),
|
|
getHashFromTagVal('key', tmpJSONLogObj.key),
|
|
tmpJSONLogObj.hit,
|
|
tmpJSONLogObj.words,
|
|
tmpJSONLogObj.size,
|
|
getHashFromTagVal('rferrer', tmpJSONLogObj.rferrer),
|
|
getHashFromTagVal('ip', tmpJSONLogObj.ip),
|
|
|
|
/*
|
|
tmpJSONLogObj.uid,
|
|
tmpJSONLogObj.key,
|
|
tmpJSONLogObj.hit,
|
|
tmpJSONLogObj.words,
|
|
tmpJSONLogObj.size,
|
|
tmpJSONLogObj.rferrer,
|
|
tmpJSONLogObj.ip,
|
|
*/
|
|
];
|
|
if (
|
|
statisticsResultPool["key"][tmpJSONLogObj.key] == undefined
|
|
) {
|
|
statisticsResultPool["key"][tmpJSONLogObj.key] = [];
|
|
}
|
|
if (
|
|
statisticsResultPool["uid"][tmpJSONLogObj.uid] == undefined
|
|
) {
|
|
statisticsResultPool["uid"][tmpJSONLogObj.uid] = [];
|
|
}
|
|
statisticsResultPool["key"][tmpJSONLogObj.key].push(tmpLogObj);
|
|
statisticsResultPool["uid"][tmpJSONLogObj.uid].push(tmpLogObj);
|
|
|
|
delete tmpJSONLogObj;
|
|
}
|
|
}
|
|
subCallback(error, key);
|
|
}
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
} else {
|
|
|
|
// let make real statistics and save to db
|
|
// statisticsResultPool['key'][tmpLogObj.key].push(tmpLogObj)
|
|
// statisticsResultPool['uid'][tmpLogObj.uid].push(tmpLogObj)
|
|
let keys = [];
|
|
Object.keys(statisticsResultPool["key"]).forEach(function (key) {
|
|
keys.push(statisticsResultPool["key"][key]);
|
|
});
|
|
// console.log('keys = ', keys);
|
|
let uids = [];
|
|
Object.keys(statisticsResultPool["uid"]).forEach(function (uid) {
|
|
uids.push(statisticsResultPool["uid"][uid]);
|
|
});
|
|
// console.log('uids = ', uids);
|
|
|
|
async.mapSeries(
|
|
keys,
|
|
function (records, subCallback) {
|
|
// let targetKey = records[0].key;
|
|
// let targetKey = records[0][1];
|
|
let targetKey = getValFromTagHash('key', records[0][1]);
|
|
// let targetUid = records[0].uid;
|
|
// let targetUid = records[0][0];
|
|
let targetUid = getValFromTagHash('uid', records[0][0]);
|
|
let total = records.length;
|
|
let hit = records.length;
|
|
let miss = 0;
|
|
let size = 0;
|
|
let ipPool = {};
|
|
let referrerPool = {};
|
|
|
|
for (var i = 0; i < records.length; i++) {
|
|
// if (records[i].hit == 0) {
|
|
if (records[i][2] == 0) {
|
|
hit--;
|
|
miss++;
|
|
}
|
|
// size += records[i].size;
|
|
size += records[i][4];
|
|
// if (ipPool[records[i].ip] == undefined) {
|
|
if (ipPool[records[i][6]] == undefined) {
|
|
// ipPool[records[i].ip] = records[i].ip;
|
|
ipPool[records[i][6]] = records[i][6];
|
|
}
|
|
// if (referrerPool[records[i].referrer] == undefined) {
|
|
if (referrerPool[records[i][7]] == undefined) {
|
|
// referrerPool[records[i].referrer] = records[i].referrer;
|
|
referrerPool[records[i][7]] = records[i][7];
|
|
}
|
|
|
|
// if (records[i].words != undefined) {
|
|
if (records[i][3] != undefined) {
|
|
// makeWordStatistics(targetTag, dateTagBase, records[i].uid, records[i].key, records[i].words);
|
|
// makeWordStatistics(targetTag, dateTagBase, records[i][0], records[i][1], records[i][3]);
|
|
makeWordStatistics(
|
|
targetTag, dateTagBase,
|
|
getValFromTagHash('uid', records[i][0]),
|
|
getValFromTagHash('key', records[i][1]),
|
|
records[i][3]
|
|
);
|
|
}
|
|
}
|
|
let uniqIp = Object.keys(ipPool).length;
|
|
let uniqReferrer = Object.keys(referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(total) +
|
|
", hit = " +
|
|
mysql.escape(hit) +
|
|
", miss = " +
|
|
mysql.escape(miss) +
|
|
", size = " +
|
|
mysql.escape(size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
api_key: targetKey,
|
|
uid: targetUid,
|
|
total: total,
|
|
hit: hit,
|
|
miss: miss,
|
|
size: size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
async.mapSeries(
|
|
uids,
|
|
function (records, subCallback) {
|
|
// let targetUid = records[0].uid;
|
|
// let targetUid = records[0][0];
|
|
let targetUid = getHashFromTagVal('uid', records[0][0]);
|
|
let total = records.length;
|
|
let hit = records.length;
|
|
let miss = 0;
|
|
let size = 0;
|
|
let ipPool = {};
|
|
let referrerPool = {};
|
|
|
|
for (var i = 0; i < records.length; i++) {
|
|
// if (records[i].hit == 0) {
|
|
if (records[i][2] == 0) {
|
|
hit--;
|
|
miss++;
|
|
}
|
|
// size += records[i].size;
|
|
size += records[i][4];
|
|
// if (ipPool[records[i].ip] == undefined) {
|
|
if (ipPool[records[i][6]] == undefined) {
|
|
// ipPool[records[i].ip] = records[i].ip;
|
|
ipPool[records[i][6]] = records[i][6];
|
|
}
|
|
// if (referrerPool[records[i].referrer] == undefined) {
|
|
if (referrerPool[records[i][5]] == undefined) {
|
|
// referrerPool[records[i].referrer] = records[i].referrer;
|
|
referrerPool[records[i][5]] = records[i][5];
|
|
}
|
|
}
|
|
let uniqIp = Object.keys(ipPool).length;
|
|
let uniqReferrer = Object.keys(referrerPool).length;
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
|
|
mysql.escape(total) +
|
|
", hit = " +
|
|
mysql.escape(hit) +
|
|
", miss = " +
|
|
mysql.escape(miss) +
|
|
", size = " +
|
|
mysql.escape(size) +
|
|
", uniq_ip = " +
|
|
mysql.escape(uniqIp) +
|
|
", uniq_referrer = " +
|
|
mysql.escape(uniqReferrer);
|
|
let post = {
|
|
date_tag: dateTagBase,
|
|
uid: targetUid,
|
|
total: total,
|
|
hit: hit,
|
|
miss: miss,
|
|
size: size,
|
|
uniq_ip: uniqIp,
|
|
uniq_referrer: uniqReferrer,
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (
|
|
error,
|
|
results
|
|
) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on makeStatisticsFromScrachDebug(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
cb(error, resultData);
|
|
} else {
|
|
flushWordStatistics(targetTag, dateTagBase);
|
|
cb(error, resultData);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
}
|
|
}
|
|
);
|
|
}
|
|
exports.makeStatisticsFromScrachDebugBakup = makeStatisticsFromScrachDebugBakup;
|
|
|
|
|
|
|
|
|
|
function makeStatisticsFromDbInChain(termTag, cb) {
|
|
|
|
makeStatisticsFromDb(termTag, 'uid', function(error, results) {
|
|
|
|
makeStatisticsFromDb(termTag, 'key', function(error, results) {
|
|
|
|
makeStatisticsFromDb(termTag, 'word', function(error, results) {
|
|
|
|
cb(error, results);
|
|
|
|
})
|
|
|
|
})
|
|
|
|
})
|
|
}
|
|
|
|
|
|
function makeStatisticsFromDb(termTag, targetTag, cb) {
|
|
|
|
let qry = '';
|
|
|
|
switch (targetTag) {
|
|
case 'uid':
|
|
qry = "INSERT INTO " + db.options.database + ".tbl_uid_statistics (date_tag, uid, total, hit, miss, `error`, size) SELECT date_tag, uid, total, hit, miss, `error`, size FROM (SELECT '" + termTag + "' AS date_tag, uid, sum(total) as total, sum(hit) as hit, sum(miss) as miss, sum(`error`) as `error`, sum(size) as size FROM " + db.options.database + ".tbl_uid_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY uid) as t ON DUPLICATE KEY UPDATE total = t.total, hit = t.hit, miss = t.miss, `error` = t.`error`, size = t.size";
|
|
break;
|
|
case 'key':
|
|
qry = "INSERT INTO " + db.options.database + ".tbl_key_statistics (date_tag, api_key, uid, total, hit, miss, `error`, size) SELECT date_tag, api_key, uid, total, hit, miss, `error`, size FROM (SELECT '" + termTag + "' AS date_tag, api_key, uid, sum(total) as total, sum(hit) as hit, sum(miss) as miss, sum(`error`) as `error`, sum(size) as size FROM " + db.options.database + ".tbl_key_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY api_key) as t ON DUPLICATE KEY UPDATE total = t.total, hit = t.hit, miss = t.miss, `error` = t.`error`, size = t.size";
|
|
break;
|
|
case 'word':
|
|
qry = "INSERT INTO " + db.options.database + ".tbl_word_statistics (date_tag, uid, api_key, word, count) SELECT date_tag, uid, api_key, word, count FROM (SELECT '" + termTag + "' AS date_tag, uid, api_key, word, sum(count) as count FROM " + db.options.database + ".tbl_word_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY api_key, word) as t ON DUPLICATE KEY UPDATE count = t.count";
|
|
break;
|
|
}
|
|
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"makeStatisticsFromDb(), error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
cb(error, {boom: 0, event: results.affectedRows});
|
|
});
|
|
}
|
|
exports.makeStatisticsFromDb = makeStatisticsFromDb;
|
|
|
|
|
|
function enqueueStatisticsWork(termTag, dateTag, cb) {
|
|
|
|
// let currentDateTagByTerm = moment().subtract(1, 'hours').format("YYYYMMDDHH");
|
|
/*
|
|
let currentDateTagByTerm = utils.getDateTimeTag('M');
|
|
|
|
switch (termTag) {
|
|
case 'minute':
|
|
currentDateTagByTerm = utils.getDateTimeTag('M');
|
|
break;
|
|
case 'hour':
|
|
currentDateTagByTerm = utils.getDateTimeTag('h');
|
|
break;
|
|
case 'day':
|
|
currentDateTagByTerm = utils.getDateTimeTag('d');
|
|
break;
|
|
case 'month':
|
|
currentDateTagByTerm = utils.getDateTimeTag('m');
|
|
break;
|
|
case 'year':
|
|
currentDateTagByTerm = utils.getDateTimeTag('y');
|
|
break;
|
|
}
|
|
*/
|
|
|
|
let qry = "INSERT INTO " + config.db.database + ".tbl_statistics_queue SET ? ";
|
|
let post = {
|
|
term_tag: termTag,
|
|
date_tag: dateTag,
|
|
}
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
cb(error);
|
|
});
|
|
|
|
}
|
|
|
|
|
|
function initializeStatisticsWork(cb) {
|
|
if (crossCtl.sConfig.master == true) {
|
|
let qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET status = 0, retry_count = retry_count + 1 WHERE status = 1";
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "initializeStatisticsWork(), error=", error);
|
|
} else {
|
|
if (results.affectedRows != 0) {
|
|
utils.log("info", "initializeStatisticsWork(), retry = ", results.affectedRows);
|
|
}
|
|
}
|
|
cb(error);
|
|
});
|
|
} else {
|
|
cb(null);
|
|
}
|
|
}
|
|
|
|
function handleStatisticsWork (termTag, dateTag, cb) {
|
|
|
|
switch (termTag) {
|
|
|
|
case 'all':
|
|
// makeStatisticsFromScrach('hour', dateTag.substring(0, 10), cb);
|
|
makeStatisticsFromScrachDebug(termTag, dateTag, cb);
|
|
break;
|
|
|
|
|
|
case 'minute':
|
|
// makeStatisticsFromScrach('hour', dateTag.substring(0, 10), cb);
|
|
makeStatisticsFromScrach(termTag, dateTag, cb);
|
|
break;
|
|
|
|
case 'hour':
|
|
makeStatisticsFromDbInChain(dateTag, cb);
|
|
break;
|
|
|
|
case 'day':
|
|
makeStatisticsFromDbInChain(dateTag, cb);
|
|
// makeStatisticsFromScrach(termTag, dateTag, cb);
|
|
break;
|
|
|
|
case 'month':
|
|
makeStatisticsFromDbInChain(dateTag, cb);
|
|
// makeStatisticsFromScrach(termTag, dateTag, cb);
|
|
break;
|
|
|
|
case 'year':
|
|
makeStatisticsFromDbInChain(dateTag, cb);
|
|
|
|
break;
|
|
|
|
default:
|
|
cb(new Error('bad termTag = ' + termTag), {boom: 0, event: 0})
|
|
|
|
/*
|
|
case 'day':
|
|
makeStatisticsFromDbInChain(
|
|
date_tag,
|
|
function(error) {
|
|
makeStatisticsFromDbInChain(
|
|
date_tag.substring(0, 6),
|
|
function(error) {
|
|
makeStatisticsFromDbInChain(
|
|
date_tag.substring(0, 4),
|
|
function(error) {
|
|
cb(error)
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
break;
|
|
|
|
case 'month':
|
|
makeStatisticsFromDbInChain(
|
|
date_tag,
|
|
function(error) {
|
|
makeStatisticsFromDbInChain(
|
|
date_tag.substring(0, 6),
|
|
function(error) {
|
|
makeStatisticsFromDbInChain(
|
|
date_tag.substring(0, 4),
|
|
function(error) {
|
|
cb(error)
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
break;
|
|
|
|
case 'year':
|
|
makeStatisticsFromDbInChain(
|
|
date_tag,
|
|
cb
|
|
);
|
|
break;
|
|
|
|
*/
|
|
}
|
|
}
|
|
|
|
var statisticsInProgressFlag = false;
|
|
|
|
function dequeueStatisticsWork() {
|
|
|
|
if (statisticsInProgressFlag == true) {
|
|
console.log('dequeueStatisticsWork(), skip....')
|
|
} else {
|
|
statisticsInProgressFlag = true;
|
|
// console.log('dequeueStatisticsWork(), ok.')
|
|
let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE status = 0 ORDER BY serial ASC LIMIT 1";
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
statisticsInProgressFlag = false;
|
|
} else {
|
|
if (results.length == 0) {
|
|
console.log('dequeueStatisticsWork(), no target');
|
|
|
|
handlePostQueueWork(function (error) {
|
|
setImmediate((arg) => {
|
|
// console.log(`executing immediate: ${arg}`);
|
|
statisticsInProgressFlag = false;
|
|
|
|
}, 'dummyParam');
|
|
})
|
|
|
|
|
|
} else {
|
|
let {serial, term_tag, date_tag} = results[0];
|
|
console.log('dequeueStatisticsWork(), got one. ', serial, term_tag, date_tag)
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET started = now(), status = 1 WHERE serial = " + mysql.escape(serial);
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
setImmediate((arg) => {
|
|
// console.log(`executing immediate: ${arg}`);
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
}, 'dummyParam');
|
|
} else {
|
|
if (results.affectedRows != 1) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error= bad affted count. affectedRows = ", results.affectedRows);
|
|
setImmediate((arg) => {
|
|
// console.log(`executing immediate: ${arg}`);
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
}, 'dummyParam');
|
|
} else {
|
|
handleStatisticsWork(term_tag, date_tag, function(error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), status = 3 WHERE serial = " + mysql.escape(serial);
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
}
|
|
setImmediate((arg) => {
|
|
// console.log(`executing immediate: ${arg}`);
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
}, 'dummyParam');
|
|
});
|
|
} else {
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), ? WHERE serial = " + mysql.escape(serial);
|
|
let post = {
|
|
status: 2,
|
|
boom_count: results.boom,
|
|
event_count: results.event
|
|
}
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
}
|
|
handlePostQueueWork(function (error) {
|
|
setImmediate((arg) => {
|
|
// console.log(`executing immediate: ${arg}`);
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
}, 'dummyParam');
|
|
})
|
|
});
|
|
}
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
function handlePostQueueWork(cb) {
|
|
|
|
let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE term_tag = 'minute' ORDER BY serial DESC LIMIT 1";
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "handlePostQueueWork(), error=", error);
|
|
cb(error)
|
|
} else {
|
|
if (results.length == 0) {
|
|
enqueueStatisticsWork('minute', '202306190800', function(error) {
|
|
cb(error);
|
|
});
|
|
} else {
|
|
if (results[0].status != 2) {
|
|
// just wait...
|
|
cb(null);
|
|
} else {
|
|
let currentDateTag = results[0].date_tag;
|
|
let nextDateTag = moment(currentDateTag, 'YYYYMMDDHHmm').add(1, 'minutes').format("YYYYMMDDHHmm");
|
|
console.log('currentDateTag=', currentDateTag)
|
|
console.log('nextDateTag=', nextDateTag)
|
|
if (moment(currentDateTag, 'YYYYMMDDHHmm').isBefore(moment().subtract(5, 'minutes'))) {
|
|
// if hour diff
|
|
if (currentDateTag.substring(0, 10) != nextDateTag.substring(0, 10)) {
|
|
enqueueStatisticsWork('hour', currentDateTag.substring(0, 10), function(error) {
|
|
enqueueStatisticsWork('day', currentDateTag.substring(0, 8), function(error) {
|
|
enqueueStatisticsWork('month', currentDateTag.substring(0, 6), function(error) {
|
|
enqueueStatisticsWork('year', currentDateTag.substring(0, 4), function(error) {
|
|
enqueueStatisticsWork('minute', nextDateTag, function(error) {
|
|
cb(error);
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
} else {
|
|
if (moment(currentDateTag, 'YYYYMMDDHHmm').isBefore(moment().subtract(5, 'minutes'))) {
|
|
enqueueStatisticsWork('minute', nextDateTag, function(error) {
|
|
cb(error);
|
|
});
|
|
} else {
|
|
cb(error);
|
|
}
|
|
}
|
|
} else {
|
|
cb(error);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
|
|
function dequeueStatisticsWorkBack() {
|
|
|
|
if (statisticsInProgressFlag == true) {
|
|
console.log('dequeueStatisticsWork(), skip....')
|
|
} else {
|
|
statisticsInProgressFlag = true;
|
|
// console.log('dequeueStatisticsWork(), ok.')
|
|
let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE status = 0 ORDER BY serial ASC LIMIT 1";
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
statisticsInProgressFlag = false;
|
|
}
|
|
else {
|
|
if (results.length == 0) {
|
|
console.log('dequeueStatisticsWork(), no target')
|
|
statisticsInProgressFlag = false;
|
|
} else {
|
|
|
|
let {serial, term_tag, date_tag} = results[0];
|
|
|
|
console.log('dequeueStatisticsWork(), got one. ', serial, term_tag, date_tag)
|
|
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET started = now(), status = 1 WHERE serial = " + mysql.escape(serial);
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
statisticsInProgressFlag = false;
|
|
} else {
|
|
if (results.affectedRows != 1) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error= bad affted count. affectedRows = ", results.affectedRows);
|
|
statisticsInProgressFlag = false;
|
|
} else {
|
|
handleStatisticsWork(term_tag, date_tag, function(error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), status = 3 WHERE serial = " + mysql.escape(serial);
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
}
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
});
|
|
} else {
|
|
qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), ? WHERE serial = " + mysql.escape(serial);
|
|
let post = {
|
|
status: 2,
|
|
boom_count: results.boom,
|
|
event_count: results.event
|
|
}
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "dequeueStatisticsWork(), error=", error);
|
|
}
|
|
statisticsInProgressFlag = false;
|
|
dequeueStatisticsWork();
|
|
});
|
|
}
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
function handleEvent(eventTag, params, cb) {
|
|
// utils.log('debug', 'local handleEvent(), ', eventTag, params)
|
|
|
|
switch (eventTag) {
|
|
|
|
case "safekiso:filter:sync":
|
|
syncFilterWithDbAll(function (error) {
|
|
cb(error)
|
|
});
|
|
|
|
break;
|
|
|
|
|
|
case "safekiso:key:update":
|
|
|
|
updateAPIKeyInfo(params.target, function (error, result) {
|
|
cb(error)
|
|
})
|
|
break;
|
|
|
|
|
|
case "cron":
|
|
// console.log('event cron, params=', params)
|
|
// let statisticsTag = "";
|
|
|
|
dequeueStatisticsWork();
|
|
cb(null);
|
|
/*
|
|
switch (params.interval) {
|
|
|
|
case "minute":
|
|
enqueueStatisticsWork('minute', moment().subtract(0, 'hours').format("YYYYMMDDHHmm"), function(error) {
|
|
cb(error);
|
|
dequeueStatisticsWork();
|
|
});
|
|
break;
|
|
|
|
case "hour":
|
|
enqueueStatisticsWork('hour', moment().subtract(1, 'hours').format("YYYYMMDDHH"), function(error) {
|
|
enqueueStatisticsWork('day', moment().subtract(1, 'hours').format("YYYYMMDD"), function(error) {
|
|
cb(error);
|
|
dequeueStatisticsWork();
|
|
});
|
|
});
|
|
break;
|
|
|
|
|
|
case "day":
|
|
enqueueStatisticsWork('day', moment().subtract(1, 'days').format("YYYYMMDD"), function(error) {
|
|
enqueueStatisticsWork('month', moment().subtract(1, 'days').format("YYYYMM"), function(error) {
|
|
enqueueStatisticsWork('year', moment().subtract(1, 'days').format("YYYY"), function(error) {
|
|
cb(error);
|
|
dequeueStatisticsWork();
|
|
});
|
|
});
|
|
});
|
|
break;
|
|
|
|
|
|
default:
|
|
cb(null);
|
|
}
|
|
*/
|
|
break;
|
|
|
|
case "addUser":
|
|
/*
|
|
var userNick = "";
|
|
utils.log("debug", "user=", params);
|
|
if (params.user_info.provider === "facebook") {
|
|
userNick = "f " + params.user_info.displayName;
|
|
} else if (params.user_info.provider === "google") {
|
|
userNick = "g+ " + params.account_info.name;
|
|
} else if (params.user_info.provider === "kakao") {
|
|
userNick = "k " + params.user_info.displayName;
|
|
} else {
|
|
userNick = params.user_info.displayName;
|
|
}
|
|
|
|
var post = {
|
|
uid: params.uid,
|
|
nick: userNick,
|
|
};
|
|
|
|
var qry = "INSERT INTO " + config.db.database + ".tbl_user_list SET ? ";
|
|
//utils.log('debug', 'qry=', qry)
|
|
//ut//ils.log('debug', 'post=', post)
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
//utils.log('debug', 'error=', error)
|
|
cb(error);
|
|
});
|
|
*/
|
|
cb(null);
|
|
break;
|
|
|
|
case "withdrawal":
|
|
|
|
handleWithdrawal(params, cb);
|
|
break;
|
|
|
|
|
|
default:
|
|
cb(new Error("unhandled event: " + eventTag));
|
|
}
|
|
}
|
|
exports.handleEvent = handleEvent;
|
|
|
|
|
|
function handleWithdrawal(params, cb) {
|
|
let {req, uid} = params;
|
|
|
|
let qry = "SELECT * FROM " + config.db.database + ".tbl_key_list WHERE uid = " + mysql.escape(uid) + " AND status = 0";
|
|
db.doEasyQuery(qry, function (error, results) {
|
|
if (error) {
|
|
cb(error);
|
|
} else {
|
|
if (results.length == 0) {
|
|
cb(null);
|
|
} else {
|
|
async.mapSeries(
|
|
results,
|
|
function (record, subCallback) {
|
|
|
|
logUserAction(
|
|
uid,
|
|
'system',
|
|
"key:delete",
|
|
record.serial,
|
|
"system",
|
|
"API Key serial " +
|
|
record.serial +
|
|
"을 계정탈퇴로 인해 삭제",
|
|
req,
|
|
record
|
|
);
|
|
|
|
|
|
let qry =
|
|
"UPDATE " +
|
|
db.options.database +
|
|
".tbl_key_list SET status = 4 WHERE serial = " + record.serial;
|
|
db.doEasyQuery(qry, function (
|
|
error,
|
|
results
|
|
) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"update, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, record);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on handleWithdrawal(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
|
|
}
|
|
// console.log('flush word Statistics done!');
|
|
cb(error);
|
|
|
|
}
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
});
|
|
}
|
|
|
|
function logUserAction(uid, name, tag, targetKey, by, description, req, infos) {
|
|
let dumpedReq = {
|
|
url: req.protocol + "://" + req.get("host") + req.originalUrl,
|
|
headers: req.headers,
|
|
query: req.query,
|
|
body: req.body,
|
|
referrer: utils.getReferrerFromReq(req),
|
|
ip: utils.getIPFromReq(req),
|
|
};
|
|
|
|
var post = {
|
|
uid: uid,
|
|
name: name,
|
|
tag: tag,
|
|
target_key: targetKey,
|
|
by: by,
|
|
description: description,
|
|
req: JSON.stringify(dumpedReq),
|
|
infos: JSON.stringify(infos),
|
|
};
|
|
|
|
var qry = "INSERT INTO " + config.db.database + ".tbl_user_log SET ? ";
|
|
//utils.log('debug', 'qry=', qry)
|
|
//ut//ils.log('debug', 'post=', post)
|
|
db.doEasyQueryPost(qry, post, function (error, results) {
|
|
if (error) {
|
|
utils.log("debug", "logUserAction(), error=", error);
|
|
}
|
|
});
|
|
}
|
|
exports.logUserAction = logUserAction;
|
|
|
|
function getPagedList(
|
|
selectClause,
|
|
tableName,
|
|
joinClause,
|
|
whereClause,
|
|
groupByClause,
|
|
orderByClause,
|
|
targetPageSize,
|
|
targetPageNumber,
|
|
cb
|
|
) {
|
|
var pageResults = {};
|
|
var countAllQuery =
|
|
"SELECT COUNT(*) as totalCount FROM " +
|
|
tableName +
|
|
(whereClause === "" ? "" : " WHERE " + whereClause);
|
|
// console.log('countAllQuery = ', countAllQuery)
|
|
// console.log('targetPageNumber = ', targetPageNumber)
|
|
db.doEasyQuery(countAllQuery, function (error, results) {
|
|
if (error) {
|
|
cb(error, null);
|
|
} else {
|
|
var adjustedTargetPageSize =
|
|
parseInt(targetPageSize) === -1
|
|
? results[0].totalCount
|
|
: targetPageSize;
|
|
// console.log('adjustedTargetPageSize = ', adjustedTargetPageSize)
|
|
pageResults.pageSize = adjustedTargetPageSize;
|
|
pageResults.totalCount = results[0].totalCount;
|
|
pageResults.totalPageCount =
|
|
Math.ceil(pageResults.totalCount / adjustedTargetPageSize) < 1
|
|
? 1
|
|
: Math.ceil(pageResults.totalCount / adjustedTargetPageSize);
|
|
|
|
if (pageResults.totalPageCount < targetPageNumber) {
|
|
pageResults.currentPageNumber = pageResults.totalPageCount;
|
|
} else {
|
|
pageResults.currentPageNumber = targetPageNumber;
|
|
}
|
|
|
|
// console.log('pageResults', pageResults)
|
|
|
|
// get page contents
|
|
var contentQuery =
|
|
"SELECT " +
|
|
selectClause +
|
|
" FROM " +
|
|
tableName +
|
|
(joinClause === "" ? "" : " " + joinClause) +
|
|
(whereClause === "" ? "" : " WHERE " + whereClause) +
|
|
" ORDER BY " +
|
|
orderByClause +
|
|
" LIMIT " +
|
|
(pageResults.currentPageNumber - 1) * adjustedTargetPageSize +
|
|
", " +
|
|
adjustedTargetPageSize;
|
|
console.log("contentQuery = ", contentQuery);
|
|
db.doEasyQuery(contentQuery, function (error, results) {
|
|
if (error) {
|
|
cb(error, null);
|
|
} else {
|
|
pageResults.fileList = results;
|
|
cb(null, pageResults);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
exports.getPagedList = getPagedList;
|
|
|
|
|
|
var wordStatisticsPool = {};
|
|
|
|
function initWordStatistics(termTag, dateTag) {
|
|
/*
|
|
console.log('init word Statistics!');
|
|
console.log('termTag = ', termTag);
|
|
console.log('dateTag = ', dateTag);
|
|
*/
|
|
let tmpBucketKey = termTag + "::" + dateTag;
|
|
wordStatisticsPool[tmpBucketKey] = {}
|
|
|
|
}
|
|
|
|
|
|
function makeWordStatistics(termTag, dateTag, uid, key, words) {
|
|
|
|
/*
|
|
console.log('make word Statistics!');
|
|
console.log('termTag = ', termTag);
|
|
console.log('dateTag = ', dateTag);
|
|
console.log('uid = ', uid);
|
|
console.log('key = ', key);
|
|
console.log('words = ', words);
|
|
*/
|
|
let tmpBucketKey = termTag + "::" + dateTag;
|
|
|
|
for (var i = 0; i < words.length; i++) {
|
|
let tmpPoolKey = dateTag + "::" + uid + "::" + key + "::" + words[i][1]
|
|
// console.log('hit : ', tmpPoolKey)
|
|
let tmpAry = wordStatisticsPool[tmpBucketKey]
|
|
let tmpHit = tmpAry[tmpPoolKey]
|
|
if (tmpHit == undefined) {
|
|
tmpAry[tmpPoolKey] = {
|
|
uid: uid,
|
|
key: key,
|
|
word: words[i][1],
|
|
hit: 1
|
|
}
|
|
tmpHit = tmpAry[tmpPoolKey]
|
|
} else {
|
|
tmpAry[tmpPoolKey].hit++;
|
|
}
|
|
}
|
|
}
|
|
|
|
function flushWordStatistics(termTag, dateTag) {
|
|
/*
|
|
console.log('flush word Statistics!');
|
|
console.log('termTag = ', termTag);
|
|
console.log('dateTag = ', dateTag);
|
|
*/
|
|
let tmpBucketKey = termTag + "::" + dateTag;
|
|
// console.log('flush', wordStatisticsPool[tmpBucketKey])
|
|
|
|
|
|
let uids = []
|
|
|
|
Object.keys(wordStatisticsPool[tmpBucketKey]).forEach(function(key) {
|
|
var val = wordStatisticsPool[tmpBucketKey][key];
|
|
uids.push(val);
|
|
});
|
|
|
|
// console.log('uids=', uids);
|
|
|
|
|
|
|
|
async.mapSeries(
|
|
uids,
|
|
function (record, subCallback) {
|
|
|
|
let qry =
|
|
"INSERT INTO " +
|
|
db.options.database +
|
|
".tbl_word_statistics SET ? ON DUPLICATE KEY UPDATE count = " +
|
|
mysql.escape(record.hit);
|
|
let post = {
|
|
date_tag: dateTag,
|
|
uid: record.uid,
|
|
api_key: record.key,
|
|
word: record.word,
|
|
count: record.hit
|
|
};
|
|
|
|
db.doEasyQueryPost(qry, post, function (
|
|
error,
|
|
results
|
|
) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"insert, error=",
|
|
error,
|
|
utils.__where()
|
|
);
|
|
}
|
|
subCallback(error, results);
|
|
});
|
|
},
|
|
function (error, subResults) {
|
|
if (error) {
|
|
utils.log(
|
|
"error",
|
|
"error on flushWordStatistics(), error : " +
|
|
JSON.stringify(error) +
|
|
" " +
|
|
utils.__where()
|
|
);
|
|
|
|
}
|
|
// console.log('flush word Statistics done!');
|
|
|
|
}
|
|
);
|
|
|
|
|
|
}
|