var crossCtl = require("../../src/crossCtl");
var utils = require("../../src/utils");
var mysql = require("mysql");

var config = {};
var stype = "";
var sid = "";

function getTag() {
  return stype + ":" + sid;
}
exports.getTag = getTag;

const FastScanner = require("fastscan");
// let toxicWords = require('./filter.json');
let toxicWords = [];
exports.toxicWords = toxicWords;
let scanner = new FastScanner(toxicWords);
// const options = { quick: false, longest: true };

let scannerPool = {}

let hitCount = 0;

let elapsedSum = 0;

let elapsedAvg = 0;

function getElapsed(startTime) {
  let stopTime = process.hrtime(startTime);

  let precision = 3; // 3 decimal places
  let elapsedRaw = stopTime[1] / 1000000; // divide by a million to get nano to milli
  let elapsedString =
    stopTime[0] + " s, " + elapsedRaw.toFixed(precision) + " ms";

  let elapsedTime = (stopTime[0] * 1e9 + stopTime[1]) / 1e9;

  return { text: elapsedString, value: elapsedTime };
}

function handleApiFinalResponse(req, res, cb_res) {
  
  let reqDump = {
    key: req.header("x-api-key"),
    body: req.body,
    keyInfo: req.keyInfo ? req.keyInfo : {},
  };
  if (res.Status.Code == 2000) {
    let elapsed = getElapsed(req.startTime);
    res.Elapsed = elapsed.text;
    makeStat(elapsed.value);
  }
  req.workTag.res().status(200).send(res);
  /*
  crossCtl.boom.log("rawEvent", {
    ts: utils.getNow(),
    trId: res.trackingId,
    req: JSON.stringify(reqDump),
    res: JSON.stringify(res),
    cb_res: JSON.stringify(cb_res),
    // foot print
    url: req.protocol + "://" + req.get("host") + req.originalUrl,
    headers: req.headers,
    referrer: utils.getReferrerFromReq(req),
    ip: utils.getIPFromReq(req),
  });
  */

  if (res.Status.Code == 2000) {
    crossCtl.boom.log("success", {
      ts: utils.getNow(),
      trackingId: res.TrackingId,
      uid: req.keyInfo.uid,
      key: req.header("x-api-key"),
      hit: res.Detected.length,
      words: res.Detected,
      size: new TextEncoder().encode(req.body.text).length,
      referrer: utils.getReferrerFromReq(req),
      ip: utils.getIPFromReq(req),
    });
  } else {
    var uid = req.isAuthenticated() ? req.user.uid : utils.uuid("uid_" + req.sessionID);
    crossCtl.boom.log("error", {
      ts: utils.getNow(),      
      uid: uid,
      key: req.header("x-api-key"),
      //
      req: JSON.stringify(reqDump),
      //
      ip: utils.getIPFromReq(req),
      referrer: utils.getReferrerFromReq(req),
      result: res.Status
    });

  }
}
exports.handleApiFinalResponse = handleApiFinalResponse;


var wordCounted = 0;

function getDashboardData() {
  return {
    wordCount: wordCounted,
    hitCount: hitCount,
    elapsedAvg: elapsedAvg,
  };
}
exports.getDashboardData = getDashboardData;

function splice(source, start, length, replacement) {
  return source.substr(0, start) + replacement + source.substr(start + length);
}
exports.splice = splice;

function getAPIKeyInfo(apiKey, cb) {
  crossCtl.redis.doEasyGetWithExpireAt(
    "apiKeyInfoPool::" + apiKey,
    60,
    function (err, replies) {
      if (err) {
        cb(err, null);
      } else {
        if (replies == null) {
          var qry =
            "SELECT * FROM " +
            db.options.database +
            ".tbl_key_list WHERE api_key = " +
            mysql.escape(apiKey) +
            " AND status = 0";

          db.doEasyQuery(qry, function (error, results) {
            if (error) {
              cb(err, null);
            } else {
              if (results.length == 0) {
                cb(null, null);
              } else if (results.length != 1) {
                cb(
                  new Error("Too many target. count = " + results.length),
                  null
                );
              } else {
                let apiKeyInfoString = JSON.stringify(results[0]);
                crossCtl.redis.doEasySetWithExpireAt(
                  "apiKeyInfoPool::" + apiKey,
                  apiKeyInfoString,
                  60,
                  function (err, replies) {
                    cb(err, utils.safeJSON(apiKeyInfoString));
                  }
                );
              }
            }
          });
        } else {
          cb(null, utils.safeJSON(replies));
        }
      }
    }
  );
}
exports.getAPIKeyInfo = getAPIKeyInfo;


function updateAPIKeyInfo(serial, cb) {
  var qry =
    "SELECT * FROM " +
    db.options.database +
    ".tbl_key_list WHERE serial = " +
    mysql.escape(serial);

  db.doEasyQuery(qry, function (error, results) {
    if (error) {
      cb(err, null);
    } else {
      if (results.length == 0) {
        cb(null, null);
      } else if (results.length != 1) {
        cb(
          new Error("Too many target. count = " + results.length),
          null
        );
      } else {
        let apiKeyInfoString = JSON.stringify(results[0]);
        let apiKey = results[0]['api_key']
        let status = results[0]['status']

        if (status == 4) {
          crossCtl.redis.doEasyDel(
            "apiKeyInfoPool::" + apiKey,
            function (err, replies) {
              cb(err, null);
            }
          );
        } else {
          crossCtl.redis.doEasySetWithExpireAt(
            "apiKeyInfoPool::" + apiKey,
            apiKeyInfoString,
            60,
            function (err, replies) {
              cb(err, utils.safeJSON(apiKeyInfoString));
            }
          );
        }
      }
    }
  });
}
exports.updateAPIKeyInfo = updateAPIKeyInfo;

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function makeStat(elapsed) {
  hitCount++;
  elapsedSum += elapsed;
  elapsedAvg = elapsedSum / hitCount;

  crossCtl.redis.doEasySet(
    crossCtl.sConfig.boom.basePath + ":dashboardData",
    JSON.stringify({
      hitCount: hitCount,
      elapsedSum: elapsedSum,
    }),
    function (error, replies) {}
  );
}
exports.makeStat = makeStat;

async function doSearch(filterSource, options, cb) {
  let offWords = scanner.search(filterSource, options);

  cb(null, offWords);
}
exports.doSearch = doSearch;


async function doSearchWithLevel(levelTag, filterSource, options, cb) {

  let tmpScanner = scannerPool[levelTag];
  let offWords = tmpScanner.search(filterSource, options);

  cb(null, offWords);
}
exports.doSearchWithLevel = doSearchWithLevel;


var db = require("../../src/newDbHandler")();
exports.db = db;

var nuxtHandler = null;
exports.nuxtHandler = nuxtHandler;

var routerMain = null;
exports.routerMain = routerMain;

function setRouterMain(_router) {
  _router.use((req, res) => {
    nuxtHandler.handler(req, res);
  });
}
exports.setRouterMain = setRouterMain;

const { exec } = require("child_process");

function syncClientDist(from, to, cb) {
  crossCtl.s3.downloadFile(from, to, function (error, path) {
    if (error) {
      cb(error);
    } else {
      exec(
        "tar -zxvf " + path + " -C " + path.replace("client.tgz", ""),
        (error, stdout, stderr) => {
          if (error) {
            console.log(`error: ${error.message}`);
          }
          if (stderr) {
            console.log(`stderr: ${stderr}`);
          }
          // console.log(`stdout: ${stdout}`);
          cb(error | stderr, null);
        }
      );
    }
  });
}

var async = require("async");

function syncFilterWithDb(level, cb) {
  var qry =
    "SELECT raw FROM " +
    db.options.database +
    "." + crossCtl.MAIN_WORD_TBL_NAME + " WHERE tag = 'SWEARWORD' AND status = 0 AND level >= " +
    mysql.escape(level);

  db.doEasyQuery(qry, function (error, results) {
    let newTWAry = [];
    for (var i = 0; i < results.length; i++) {
      newTWAry.push(results[i]["raw"]);
    }
    console.log("read done, count =", newTWAry.length);
    if (level == 10) {
      wordCounted = newTWAry.length;
    }
    toxicWords = newTWAry;
    exports.toxicWords = toxicWords;

    scanner = new FastScanner(toxicWords);
    let scannerLevelTag = '';
    switch (level) {
      case 10:
        scannerLevelTag = 'high';
        break;
      case 50:
        scannerLevelTag = 'mid';
        break;
      case 100:
        scannerLevelTag = 'low';
        break;
      
    }
    scannerPool[scannerLevelTag] = scanner
    cb(error, newTWAry.length);
  });
}
exports.syncFilterWithDb = syncFilterWithDb;


function syncFilterWithDbAll(cb) {
  syncFilterWithDb(10, function(error) {
    syncFilterWithDb(50, function(error) {
      syncFilterWithDb(100, function(error) {
        cb(error)
    
      })
        
    })
  
  })

}
exports.syncFilterWithDbAll = syncFilterWithDbAll;


var targetTagList = ['SWEARWORD', 'SUICIDE', 'AD', 'GAMBLE', 'ADULT', 'WHITE'];

// var targetTagList = ['SUICIDE'];

function doFilterUpdate(req, res, cb) {


  var finalWordsArray = [];

  for (var i = 0; i < targetTagList.length; i++) {
    let rawdata = fs.readFileSync(path.join(__dirname, './data/' + targetTagList[i] + '.txt'));

    let tmpAry = rawdata.toString('utf-8').split('\n');

    for (var j = 0; j < tmpAry.length; j++) {
      finalWordsArray.push({tag: targetTagList[i], word: tmpAry[j]})
    }

  }


  let chunkArrays = [];
  const chunkSize = 100000;
  for (let i = 0; i < finalWordsArray.length; i += chunkSize) {
    const chunk = finalWordsArray.slice(i, i + chunkSize);
    // do whatever
    chunkArrays.push(chunk);
  }

  console.log("total chunk count is ", chunkArrays.length);

  var maxWordlength = 0;
  var maxWord = "";

  async.mapSeries(
    chunkArrays,
    function (chunk, subCallback) {
      // console.log('chunk=', chunk);

      for (let i = 0; i < chunk.length; i++) {
        if (chunk[i].length > maxWordlength) {
          maxWordlength = chunk[i].word.length;
          maxWord = chunk[i].word;
          console.log("max changed ", maxWordlength);
        }
      }

      var qry =
        "INSERT INTO " +
        db.options.database +
        "." + crossCtl.MAIN_WORD_TBL_NAME + " (tag, word, raw, level, memo) VALUES ? ";

      var tmpPost = [];

      for (var i = 0; i < chunk.length; i++) {
        tmpPost[tmpPost.length] = [chunk[i].tag, chunk[i].word, chunk[i].word, 10, "Bulk Inserted"];
      }

      var post = [tmpPost];

      // console.log('post=', post);

      db.doEasyQueryPost(qry, post, function (error, results) {
        console.log("error=", error);
        console.log("results=", results);
        subCallback(error, results);
      });
    },
    function (error, subResults) {
      if (error) {
        utils.log(
          "error",
          "error on doFilterUpdate(), error : " +
            JSON.stringify(error) +
            " " +
            utils.__where()
        );
        cb(error, finalWordsArray.length);
      } else {
        console.log("final max ", maxWordlength);
        console.log("final max word ", maxWord);
        cb(null, finalWordsArray.length);
      }
    }
  );










}
exports.doFilterUpdate = doFilterUpdate;

function doFilterUpdateBackup(req, res, cb) {
  // toxicWords

  // let tempToxicWords = toxicWords.slice(0, 100000);

  let sourcePath = path.join(__dirname, "./filter.json");
  // console.log('sourcePath=', sourcePath);

  let rawdata = fs.readFileSync(sourcePath);
  // console.log('rawdata=', rawdata);
  // let parsedWordsJson = JSON.parse(rawdata);
  let tempToxicWords = JSON.parse(rawdata);

  // let tempToxicWords = parsedWordsJson.slice(0, 100);

  // console.log('tempToxicWords=', tempToxicWords);

  let dupFilter = {};
  let noDupArray = [];

  for (let i = 0; i < tempToxicWords.length; i++) {
    if (dupFilter[tempToxicWords[i]] == undefined) {
      dupFilter[tempToxicWords[i]] = tempToxicWords[i];
      noDupArray.push(tempToxicWords[i]);
    } else {
      console.log("dup item found ", tempToxicWords[i]);
    }
  }

  tempToxicWords = noDupArray;

  let chunkArrays = [];
  const chunkSize = 100000;
  for (let i = 0; i < tempToxicWords.length; i += chunkSize) {
    const chunk = tempToxicWords.slice(i, i + chunkSize);
    // do whatever
    chunkArrays.push(chunk);
  }

  console.log("total chunk count is ", chunkArrays.length);

  var maxWordlength = 0;
  var maxWord = "";

  async.mapSeries(
    chunkArrays,
    function (chunk, subCallback) {
      // console.log('chunk=', chunk);

      for (let i = 0; i < chunk.length; i++) {
        if (chunk[i].length > maxWordlength) {
          maxWordlength = chunk[i].length;
          maxWord = chunk[i];
          console.log("max changed ", maxWordlength);
        }
      }

      var qry =
        "INSERT INTO " +
        db.options.database +
        "." + crossCtl.MAIN_WORD_TBL_NAME + " (word, raw, level, memo) VALUES ? ";

      var tmpPost = [];

      for (var i = 0; i < chunk.length; i++) {
        tmpPost[tmpPost.length] = [chunk[i], chunk[i], 10, "Bulk Inserted"];
      }

      var post = [tmpPost];

      // console.log('post=', post);

      db.doEasyQueryPost(qry, post, function (error, results) {
        console.log("error=", error);
        console.log("results=", results);
        subCallback(error, results);
      });
    },
    function (error, subResults) {
      if (error) {
        utils.log(
          "error",
          "error on doFilterUpdate(), error : " +
            JSON.stringify(error) +
            " " +
            utils.__where()
        );
        cb(error, tempToxicWords.length);
      } else {
        console.log("final max ", maxWordlength);
        console.log("final max word ", maxWord);
        cb(null, tempToxicWords.length);
      }
    }
  );
}


const fs = require("fs");
const path = require("path");
function init(id, cfg, cb) {
  // utils.log('debug', 'local init(), ', id, cfg)
  config = cfg;
  stype = config.type;
  sid = id;

  crossCtl.redis.doEasyGet(
    crossCtl.sConfig.boom.basePath + ":dashboardData",
    function (error, replies) {
      if (replies != null) {
        let tmpDashboardData = utils.safeJSON(replies);
        hitCount = Number(tmpDashboardData.hitCount);
        elapsedSum = Number(tmpDashboardData.elapsedSum);
        elapsedAvg = elapsedSum / hitCount;
      }
    }
  );

  if (crossCtl.sConfig.clientDistInfo != undefined) {
    let clientPath = path.join(__dirname, ".", "client") + "/client.tgz";
    let clientIndexPath = path.join(
      __dirname,
      ".",
      "client",
      // crossCtl.sConfig.clientDistInfo.fromPath,
      ".output",
      "server",
      "index.mjs"
    );

    // console.log('clientIndexPath = ', clientIndexPath);

    syncClientDist(
      crossCtl.sConfig.clientDistInfo.fromPath,
      clientPath,
      function (error, result) {
        if (error) {
          utils.log(
            "error",
            "s3.syncDist error : " + error + " " + utils.__where()
          );
          cb(error);
        } else {
          console.log("final dist file list = ", result);

          fs.access(clientIndexPath, fs.F_OK, (error) => {
            if (error) {
              utils.log(
                "error",
                "s3.syncDist error : " + error + " " + utils.__where()
              );
              cb(error);
            } else {
              import(clientIndexPath).then((_nuxtHandler) => {
                console.log("nuxt handler imported...", _nuxtHandler); //  "Hello world!"
                nuxtHandler = _nuxtHandler;

                db.init(config.db, function (error) {
                  if (error) {
                    utils.log(
                      "error",
                      "db init error : " + error + " " + utils.__where()
                    );
                    cb(error);
                  } else {
                    syncFilterWithDb(10, function (error, count) {
                      if (error) {
                        utils.log(
                          "error",
                          "word sync error : " + error + " " + utils.__where()
                        );
                        cb(error);
                      } else {

                        syncFilterWithDb(50, function (error, count) {
                          if (error) {
                            utils.log(
                              "error",
                              "word sync error : " + error + " " + utils.__where()
                            );
                            cb(error);
                          } else {
                            syncFilterWithDb(100, function (error, count) {
                              if (error) {
                                utils.log(
                                  "error",
                                  "word sync error : " + error + " " + utils.__where()
                                );
                                cb(error);
                              } else {
                                initializeStatisticsWork(function(error) {
                                  cb(null);
                                });
                              }
                            });
                          }
                        });
                      }
                    });
                  }
                });
              });
            }
          });
        }
      }
    );
  } else {
    cb(null);
  }
}
exports.init = init;

/*
passEventToLocalHandler(
    'cron',
    { cmd: 'cron', interval: 'day', tag: nowTag },
    function (error) {
        cb(error);
    }
);
*/

var moment = require("moment");
// const tsFormat = () => moment().format('YYYY-MM-DD HH:mm:ss.SSS');

function makeStatisticsFromScrach(targetTag, dateTagBase, cb) {

  let resultData = {boom: 0, event: 0};

  initWordStatistics(targetTag, dateTagBase);

  let statisticsResultPool = {
    key: {},
    uid: {},
  };
  
  crossCtl.s3.listObjects(
    crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
    function (error, data) {
      if (error) {
        utils.log(
          "error",
          "error on makeStatisticsFromScrach(), error : " +
            JSON.stringify(error) +
            " " +
            utils.__where()
        );
        cb(error, resultData);
      } else {
        
        resultData.boom = data.Contents.length;
        if (data.Contents.length == 0) {
          flushWordStatistics(targetTag, dateTagBase);
          cb(error, resultData);
        } else {
          async.map(
            data.Contents,
            function (key, subCallback) {
              // console.log('key = ', key);
              let parts = key.Key.split("/");
              let nameOnly =
                crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
              // console.log('nameOnly=', nameOnly);
              crossCtl.s3.readTextFile(nameOnly, function (error, data) {
                if (error) {
                  utils.log(
                    "error",
                    "error on makeStatisticsFromScrach(), error : " +
                      JSON.stringify(error) +
                      " " +
                      utils.__where()
                  );
                  subCallback(error, key);
                } else {
                  // console.log('data = ', data);
                  let tmpAry = data.split("\n");
                  // console.log('tmpAry=', tmpAry);
                  resultData.event += tmpAry.length;
                  for (var i = 0; i < tmpAry.length; i++) {
                    if (tmpAry[i].trim() != "") {
                      let tmpLogObj = utils.safeJSON(tmpAry[i]);
                      if (
                        statisticsResultPool["key"][tmpLogObj.key] == undefined
                      ) {
                        statisticsResultPool["key"][tmpLogObj.key] = [];
                      }
                      if (
                        statisticsResultPool["uid"][tmpLogObj.uid] == undefined
                      ) {
                        statisticsResultPool["uid"][tmpLogObj.uid] = [];
                      }
                      statisticsResultPool["key"][tmpLogObj.key].push(tmpLogObj);
                      statisticsResultPool["uid"][tmpLogObj.uid].push(tmpLogObj);
                    }
                  }
                  subCallback(error, key);
                }
              });
            },
            function (error, subResults) {
              if (error) {
                utils.log(
                  "error",
                  "error on makeStatisticsFromScrach(), error : " +
                    JSON.stringify(error) +
                    " " +
                    utils.__where()
                );
              } else {
                let keys = [];
                Object.keys(statisticsResultPool["key"]).forEach(function (key) {
                  keys.push(statisticsResultPool["key"][key]);
                });
                // console.log('keys = ', keys);
                let uids = [];
                Object.keys(statisticsResultPool["uid"]).forEach(function (uid) {
                  uids.push(statisticsResultPool["uid"][uid]);
                });
                // console.log('uids = ', uids);
  
                async.mapSeries(
                  keys,
                  function (records, subCallback) {
                    let targetKey = records[0].key;
                    let targetUid = records[0].uid;
                    let total = records.length;
                    let hit = records.length;
                    let miss = 0;
                    let size = 0;
                    let ipPool = {};
                    let referrerPool = {};
  
                    for (var i = 0; i < records.length; i++) {
                      if (records[i].hit == 0) {
                        hit--;
                        miss++;
                      }
                      size += records[i].size;
                      if (ipPool[records[i].ip] == undefined) {
                        ipPool[records[i].ip] = records[i].ip;
                      }
                      if (referrerPool[records[i].referrer] == undefined) {
                        referrerPool[records[i].referrer] = records[i].referrer;
                      }
  
                      if (records[i].words != undefined) {
                        makeWordStatistics(targetTag, dateTagBase, records[i].uid, records[i].key, records[i].words);
                      }
                    }
                    let uniqIp = Object.keys(ipPool).length;
                    let uniqReferrer = Object.keys(referrerPool).length;
  
                    let qry =
                      "INSERT INTO " +
                      db.options.database +
                      ".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                      mysql.escape(total) +
                      ", hit = " +
                      mysql.escape(hit) +
                      ", miss = " +
                      mysql.escape(miss) +
                      ", size = " +
                      mysql.escape(size) +
                      ", uniq_ip = " +
                      mysql.escape(uniqIp) +
                      ", uniq_referrer = " +
                      mysql.escape(uniqReferrer);
                    let post = {
                      date_tag: dateTagBase,
                      api_key: targetKey,
                      uid: targetUid,
                      total: total,
                      hit: hit,
                      miss: miss,
                      size: size,
                      uniq_ip: uniqIp,
                      uniq_referrer: uniqReferrer,
                    };
  
                    db.doEasyQueryPost(qry, post, function (error, results) {
                      if (error) {
                        utils.log(
                          "error",
                          "insert, error=",
                          error,
                          utils.__where()
                        );
                      }
                      subCallback(error, results);
                    });
                  },
                  function (error, subResults) {
                    if (error) {
                      utils.log(
                        "error",
                        "error on makeStatisticsFromScrach(), error : " +
                          JSON.stringify(error) +
                          " " +
                          utils.__where()
                      );
                      cb(error, resultData);
                    } else {
                      async.mapSeries(
                        uids,
                        function (records, subCallback) {
                          let targetUid = records[0].uid;
                          let total = records.length;
                          let hit = records.length;
                          let miss = 0;
                          let size = 0;
                          let ipPool = {};
                          let referrerPool = {};
  
                          for (var i = 0; i < records.length; i++) {
                            if (records[i].hit == 0) {
                              hit--;
                              miss++;
                            }
                            size += records[i].size;
                            if (ipPool[records[i].ip] == undefined) {
                              ipPool[records[i].ip] = records[i].ip;
                            }
                            if (referrerPool[records[i].referrer] == undefined) {
                              referrerPool[records[i].referrer] =
                                records[i].referrer;
                            }
                          }
                          let uniqIp = Object.keys(ipPool).length;
                          let uniqReferrer = Object.keys(referrerPool).length;
  
                          let qry =
                            "INSERT INTO " +
                            db.options.database +
                            ".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                            mysql.escape(total) +
                            ", hit = " +
                            mysql.escape(hit) +
                            ", miss = " +
                            mysql.escape(miss) +
                            ", size = " +
                            mysql.escape(size) +
                            ", uniq_ip = " +
                            mysql.escape(uniqIp) +
                            ", uniq_referrer = " +
                            mysql.escape(uniqReferrer);
                          let post = {
                            date_tag: dateTagBase,
                            uid: targetUid,
                            total: total,
                            hit: hit,
                            miss: miss,
                            size: size,
                            uniq_ip: uniqIp,
                            uniq_referrer: uniqReferrer,
                          };
  
                          db.doEasyQueryPost(qry, post, function (
                            error,
                            results
                          ) {
                            if (error) {
                              utils.log(
                                "error",
                                "insert, error=",
                                error,
                                utils.__where()
                              );
                            }
                            subCallback(error, results);
                          });
                        },
                        function (error, subResults) {
                          if (error) {
                            utils.log(
                              "error",
                              "error on makeStatisticsFromScrach(), error : " +
                                JSON.stringify(error) +
                                " " +
                                utils.__where()
                            );
                            cb(error, resultData);
                          } else {
                            flushWordStatistics(targetTag, dateTagBase);
                            cb(error, resultData);
                          }
                        }
                      );
                    }
                  }
                );
              }
            }
          );
        }
        
      }
    }
  );
}
exports.makeStatisticsFromScrach = makeStatisticsFromScrach;


var tagJarPool = {};

function initTagValHashs() {
  tagJarPool = {};
}

function getHashFromTagVal(tag, val) {
  let tmpJar = tagJarPool[tag]

  if (tmpJar == undefined) {
    tmpJar = tagJarPool[tag] = {
      valPool: {},
      vals: []
    }    
  } 

  if (tmpJar.valPool[val] == undefined) {
    tmpJar.vals.push(val);
    tmpJar.valPool[val] = tmpJar.vals.length - 1;
  }

  return tmpJar.valPool[val];  
}

function getValFromTagHash(tag, hash) {
  let tmpJar = tagJarPool[tag]

  if (tmpJar == undefined) {
    return undefined;
  } 

  if (hash < 0 || tmpJar.vals.length -1 < hash) {
    return undefined;
  }

  return tmpJar.vals[hash];  
}




var statisticsJar = {};

function initStatisticsJar() {
  statisticsJar = {
    boomCount: 0,
    eventCount: 0,
    uidPool: {},
    keyPool: {}
  }
}

function doCalcOnJar(tag, dateBase, jar, log) {

  jar.total++;

  if (log.hit == 0) {
    jar.miss++;
  } else {
    jar.hit++;
  }

  jar.size += log.size;

  if (jar.ipPool[log.ip] == undefined) {
    jar.ipPool[log.ip] = 1;
  } else {
    jar.ipPool[log.ip]++;
  }
  if (jar.referrerPool[log.rferrer] == undefined) {
    jar.referrerPool[log.rferrer] = 1;
  } else {
    jar.referrerPool[log.rferrer]++;
  }

  if (log.words != undefined) {  
    makeWordStatistics(
      tag, dateBase, 
      log.uid, 
      log.key, 
      log.words
    );
  }
}

function doWorkOnStatisticsJar(tag, dateBase, data) {

  statisticsJar.boomCount++;

  let tmpAry = data.split("\n");

  statisticsJar.eventCount += tmpAry.length;


  if (statisticsJar.boomCount % 1000 == 0) {
    console.log('statisticsJar.boomCount = ', statisticsJar.boomCount, ', statisticsJar.eventCount = ', statisticsJar.eventCount);
  }
  
  for (var i = 0; i < tmpAry.length; i++) {
    if (tmpAry[i].trim() != "") {
      let tmpJSONLogObj = utils.safeJSON(tmpAry[i]);
/*
      tmpJSONLogObj.uid,
      tmpJSONLogObj.key,
      tmpJSONLogObj.hit,
      tmpJSONLogObj.words,
      tmpJSONLogObj.size,
      tmpJSONLogObj.rferrer,
      tmpJSONLogObj.ip,
*/                        

      if (statisticsJar.uidPool[tmpJSONLogObj.uid] == undefined) {
        statisticsJar.uidPool[tmpJSONLogObj.uid] = {
          uid: tmpJSONLogObj.uid,
          key: tmpJSONLogObj.key,          
          total: 0,
          hit: 0,
          miss: 0,
          size: 0,
          ipPool: {},
          referrerPool: {},
        }
      }
      if (statisticsJar.keyPool[tmpJSONLogObj.key] == undefined) {
        statisticsJar.keyPool[tmpJSONLogObj.key] = {
          uid: tmpJSONLogObj.uid,
          key: tmpJSONLogObj.key,          
          total: 0,
          hit: 0,
          miss: 0,
          size: 0,
          ipPool: {},
          referrerPool: {},
        }
      }

      doCalcOnJar(tag, dateBase, statisticsJar.uidPool[tmpJSONLogObj.uid], tmpJSONLogObj)
      doCalcOnJar(tag, dateBase, statisticsJar.keyPool[tmpJSONLogObj.key], tmpJSONLogObj)

    }
  }  
}


function makeStatisticsFromScrachDebug(targetTag, dateTagBase, cb) {

  let resultData = {boom: 0, event: 0};

  initStatisticsJar();

  initWordStatistics(targetTag, dateTagBase);

  let statisticsResultPool = {
    key: {},
    uid: {},
  };
  
  crossCtl.s3.listObjectsDebug(
    crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
    function (error, data) {
      if (error) {
        utils.log(
          "error",
          "error on makeStatisticsFromScrachDebug(), error : " +
            JSON.stringify(error) +
            " " +
            utils.__where()
        );
        cb(error, resultData);
      } else {
        
        resultData.boom = data.Contents.length;
        if (data.Contents.length == 0) {
          flushWordStatistics(targetTag, dateTagBase);
          cb(error, resultData);
        } else {
          console.log('start to fetch logs...');
          var tmpBoomCount = 0;
          var tmpEventCount = 0;
          async.mapSeries(
            data.Contents,
            function (key, subCallback) {
              // console.log('key = ', key);
              // console.log('data.Prefix + key.Key = ', data.Prefix + key.Key);
              let parts = (data.Prefix + key.Key).split("/");
              let nameOnly =
                crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
              // console.log('nameOnly=', nameOnly);
              crossCtl.s3.readTextFile(nameOnly, function (error, data) {
                if (error) {
                  utils.log(
                    "error",
                    "error on makeStatisticsFromScrachDebug(), error : " +
                      JSON.stringify(error) +
                      " " +
                      utils.__where()
                  );
                } else {
                  doWorkOnStatisticsJar(targetTag, dateTagBase, data);
                }
                subCallback(error, key);
              });
            },
            function (error, subResults) {
              if (error) {
                utils.log(
                  "error",
                  "error on makeStatisticsFromScrachDebug(), error : " +
                    JSON.stringify(error) +
                    " " +
                    utils.__where()
                );
              } else {

                resultData.event = statisticsJar.eventCount;

                let keys = [];
                Object.keys(statisticsJar.keyPool).forEach(function (key) {
                  keys.push(statisticsJar.keyPool[key]);
                });
                // console.log('keys = ', keys);
                let uids = [];
                Object.keys(statisticsJar.uidPool).forEach(function (uid) {
                  uids.push(statisticsJar.uidPool[uid]);
                });
                // console.log('uids = ', uids);
  
                async.mapSeries(
                  keys,
                  function (record, subCallback) {

                    let uniqIp = Object.keys(record.ipPool).length;
                    let uniqReferrer = Object.keys(record.referrerPool).length;
  
                    let qry =
                      "INSERT INTO " +
                      db.options.database +
                      ".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                      mysql.escape(record.total) +
                      ", hit = " +
                      mysql.escape(record.hit) +
                      ", miss = " +
                      mysql.escape(record.miss) +
                      ", size = " +
                      mysql.escape(record.size) +
                      ", uniq_ip = " +
                      mysql.escape(uniqIp) +
                      ", uniq_referrer = " +
                      mysql.escape(uniqReferrer);
                    let post = {
                      date_tag: dateTagBase,
                      api_key: record.key,
                      uid: record.uid,
                      total: record.total,
                      hit: record.hit,
                      miss: record.miss,
                      size: record.size,
                      uniq_ip: uniqIp,
                      uniq_referrer: uniqReferrer,
                    };
  
                    db.doEasyQueryPost(qry, post, function (error, results) {
                      if (error) {
                        utils.log(
                          "error",
                          "insert, error=",
                          error,
                          utils.__where()
                        );
                      }
                      subCallback(error, results);
                    });
                  },
                  function (error, subResults) {
                    if (error) {
                      utils.log(
                        "error",
                        "error on makeStatisticsFromScrachDebug(), error : " +
                          JSON.stringify(error) +
                          " " +
                          utils.__where()
                      );
                      cb(error, resultData);
                    } else {
                      async.mapSeries(
                        uids,
                        function (record, subCallback) {

                          let uniqIp = Object.keys(record.ipPool).length;
                          let uniqReferrer = Object.keys(record.referrerPool).length;
  
                          let qry =
                            "INSERT INTO " +
                            db.options.database +
                            ".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                            mysql.escape(record.total) +
                            ", hit = " +
                            mysql.escape(record.hit) +
                            ", miss = " +
                            mysql.escape(record.miss) +
                            ", size = " +
                            mysql.escape(record.size) +
                            ", uniq_ip = " +
                            mysql.escape(uniqIp) +
                            ", uniq_referrer = " +
                            mysql.escape(uniqReferrer);
                          let post = {
                            date_tag: dateTagBase,
                            uid: record.uid,
                            total: record.total,
                            hit: record.hit,
                            miss: record.miss,
                            size: record.size,
                            uniq_ip: uniqIp,
                            uniq_referrer: uniqReferrer,
                          };
  
                          db.doEasyQueryPost(qry, post, function (
                            error,
                            results
                          ) {
                            if (error) {
                              utils.log(
                                "error",
                                "insert, error=",
                                error,
                                utils.__where()
                              );
                            }
                            subCallback(error, results);
                          });
                        },
                        function (error, subResults) {
                          if (error) {
                            utils.log(
                              "error",
                              "error on makeStatisticsFromScrachDebug(), error : " +
                                JSON.stringify(error) +
                                " " +
                                utils.__where()
                            );
                            cb(error, resultData);
                          } else {
                            flushWordStatistics(targetTag, dateTagBase);
                            cb(error, resultData);
                          }
                        }
                      );
                    }
                  }
                );
              }
            }
          );
        }
        
      }
    }
  );
}
exports.makeStatisticsFromScrachDebug = makeStatisticsFromScrachDebug;





function makeStatisticsFromScrachDebugBakup(targetTag, dateTagBase, cb) {

  let resultData = {boom: 0, event: 0};

  initTagValHashs();

  initWordStatistics(targetTag, dateTagBase);

  let statisticsResultPool = {
    key: {},
    uid: {},
  };
  
  crossCtl.s3.listObjectsDebug(
    crossCtl.sConfig.boom.basePath + "boom_success_" + dateTagBase,
    function (error, data) {
      if (error) {
        utils.log(
          "error",
          "error on makeStatisticsFromScrachDebug(), error : " +
            JSON.stringify(error) +
            " " +
            utils.__where()
        );
        cb(error, resultData);
      } else {
        
        resultData.boom = data.Contents.length;
        if (data.Contents.length == 0) {
          flushWordStatistics(targetTag, dateTagBase);
          cb(error, resultData);
        } else {
          console.log('start to fetch logs...');
          var tmpBoomCount = 0;
          var tmpEventCount = 0;
          async.mapSeries(
            data.Contents,
            function (key, subCallback) {
              // console.log('key = ', key);
              // console.log('data.Prefix + key.Key = ', data.Prefix + key.Key);
              let parts = (data.Prefix + key.Key).split("/");
              let nameOnly =
                crossCtl.sConfig.boom.basePath + parts[parts.length - 1];
              // console.log('nameOnly=', nameOnly);
              crossCtl.s3.readTextFile(nameOnly, function (error, data) {
                if (error) {
                  utils.log(
                    "error",
                    "error on makeStatisticsFromScrachDebug(), error : " +
                      JSON.stringify(error) +
                      " " +
                      utils.__where()
                  );
                  subCallback(error, key);
                } else {
                  // console.log('data = ', data);
                  let tmpAry = data.split("\n");
                  // console.log('tmpAry=', tmpAry);
                  resultData.event += tmpAry.length;

                  tmpEventCount = resultData.event;

                  tmpBoomCount+=1;

                  if (tmpBoomCount % 1000 == 0) {
                    console.log('tmpBoomCount = ', tmpBoomCount, ', tmpEventCount = ', tmpEventCount);
                  } else {
                    // console.log('skip boom display..., condition is ', tmpBoomCount % 10)
                    // console.log('skip boom display..., tmpBoomCount is ', tmpBoomCount)
                  }
                  
                  // console.log('resultData.event=', resultData.event);
                  for (var i = 0; i < tmpAry.length; i++) {
                    if (tmpAry[i].trim() != "") {
                      let tmpJSONLogObj = utils.safeJSON(tmpAry[i]);
                      let tmpLogObj = [
                        getHashFromTagVal('uid', tmpJSONLogObj.uid),
                        getHashFromTagVal('key', tmpJSONLogObj.key),
                        tmpJSONLogObj.hit,
                        tmpJSONLogObj.words,
                        tmpJSONLogObj.size,
                        getHashFromTagVal('rferrer', tmpJSONLogObj.rferrer),
                        getHashFromTagVal('ip', tmpJSONLogObj.ip),

/*
                        tmpJSONLogObj.uid,
                        tmpJSONLogObj.key,
                        tmpJSONLogObj.hit,
                        tmpJSONLogObj.words,
                        tmpJSONLogObj.size,
                        tmpJSONLogObj.rferrer,
                        tmpJSONLogObj.ip,
*/                        
                      ];
                      if (
                        statisticsResultPool["key"][tmpJSONLogObj.key] == undefined
                      ) {
                        statisticsResultPool["key"][tmpJSONLogObj.key] = [];
                      }
                      if (
                        statisticsResultPool["uid"][tmpJSONLogObj.uid] == undefined
                      ) {
                        statisticsResultPool["uid"][tmpJSONLogObj.uid] = [];
                      }
                      statisticsResultPool["key"][tmpJSONLogObj.key].push(tmpLogObj);
                      statisticsResultPool["uid"][tmpJSONLogObj.uid].push(tmpLogObj);
                      
                      delete tmpJSONLogObj;
                    }
                  }
                  subCallback(error, key);
                }
              });
            },
            function (error, subResults) {
              if (error) {
                utils.log(
                  "error",
                  "error on makeStatisticsFromScrachDebug(), error : " +
                    JSON.stringify(error) +
                    " " +
                    utils.__where()
                );
              } else {

                // let make real statistics and save to db
                // statisticsResultPool['key'][tmpLogObj.key].push(tmpLogObj)
                // statisticsResultPool['uid'][tmpLogObj.uid].push(tmpLogObj)
                let keys = [];
                Object.keys(statisticsResultPool["key"]).forEach(function (key) {
                  keys.push(statisticsResultPool["key"][key]);
                });
                // console.log('keys = ', keys);
                let uids = [];
                Object.keys(statisticsResultPool["uid"]).forEach(function (uid) {
                  uids.push(statisticsResultPool["uid"][uid]);
                });
                // console.log('uids = ', uids);
  
                async.mapSeries(
                  keys,
                  function (records, subCallback) {
                    // let targetKey = records[0].key;
                    // let targetKey = records[0][1];
                    let targetKey = getValFromTagHash('key', records[0][1]);
                    // let targetUid = records[0].uid;
                    // let targetUid = records[0][0];
                    let targetUid = getValFromTagHash('uid', records[0][0]);
                    let total = records.length;
                    let hit = records.length;
                    let miss = 0;
                    let size = 0;
                    let ipPool = {};
                    let referrerPool = {};
  
                    for (var i = 0; i < records.length; i++) {
                      // if (records[i].hit == 0) {
                        if (records[i][2] == 0) {  
                        hit--;
                        miss++;
                      }
                      // size += records[i].size;
                      size += records[i][4];
                      // if (ipPool[records[i].ip] == undefined) {
                      if (ipPool[records[i][6]] == undefined) {
                        // ipPool[records[i].ip] = records[i].ip;
                        ipPool[records[i][6]] = records[i][6];
                      }
                      // if (referrerPool[records[i].referrer] == undefined) {
                      if (referrerPool[records[i][7]] == undefined) {  
                        // referrerPool[records[i].referrer] = records[i].referrer;
                        referrerPool[records[i][7]] = records[i][7];
                      }
  
                      // if (records[i].words != undefined) {
                      if (records[i][3] != undefined) {  
                        // makeWordStatistics(targetTag, dateTagBase, records[i].uid, records[i].key, records[i].words);
                        // makeWordStatistics(targetTag, dateTagBase, records[i][0], records[i][1], records[i][3]);
                        makeWordStatistics(
                          targetTag, dateTagBase, 
                          getValFromTagHash('uid', records[i][0]), 
                          getValFromTagHash('key', records[i][1]), 
                          records[i][3]
                        );
                      }
                    }
                    let uniqIp = Object.keys(ipPool).length;
                    let uniqReferrer = Object.keys(referrerPool).length;
  
                    let qry =
                      "INSERT INTO " +
                      db.options.database +
                      ".tbl_key_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                      mysql.escape(total) +
                      ", hit = " +
                      mysql.escape(hit) +
                      ", miss = " +
                      mysql.escape(miss) +
                      ", size = " +
                      mysql.escape(size) +
                      ", uniq_ip = " +
                      mysql.escape(uniqIp) +
                      ", uniq_referrer = " +
                      mysql.escape(uniqReferrer);
                    let post = {
                      date_tag: dateTagBase,
                      api_key: targetKey,
                      uid: targetUid,
                      total: total,
                      hit: hit,
                      miss: miss,
                      size: size,
                      uniq_ip: uniqIp,
                      uniq_referrer: uniqReferrer,
                    };
  
                    db.doEasyQueryPost(qry, post, function (error, results) {
                      if (error) {
                        utils.log(
                          "error",
                          "insert, error=",
                          error,
                          utils.__where()
                        );
                      }
                      subCallback(error, results);
                    });
                  },
                  function (error, subResults) {
                    if (error) {
                      utils.log(
                        "error",
                        "error on makeStatisticsFromScrachDebug(), error : " +
                          JSON.stringify(error) +
                          " " +
                          utils.__where()
                      );
                      cb(error, resultData);
                    } else {
                      async.mapSeries(
                        uids,
                        function (records, subCallback) {
                          // let targetUid = records[0].uid;
                          // let targetUid = records[0][0];
                          let targetUid = getHashFromTagVal('uid', records[0][0]);
                          let total = records.length;
                          let hit = records.length;
                          let miss = 0;
                          let size = 0;
                          let ipPool = {};
                          let referrerPool = {};
  
                          for (var i = 0; i < records.length; i++) {
                            // if (records[i].hit == 0) {
                              if (records[i][2] == 0) {  
                              hit--;
                              miss++;
                            }
                            // size += records[i].size;
                            size += records[i][4];
                            // if (ipPool[records[i].ip] == undefined) {
                            if (ipPool[records[i][6]] == undefined) {  
                              // ipPool[records[i].ip] = records[i].ip;
                              ipPool[records[i][6]] = records[i][6];
                            }
                            // if (referrerPool[records[i].referrer] == undefined) {
                            if (referrerPool[records[i][5]] == undefined) {  
                              // referrerPool[records[i].referrer] = records[i].referrer;
                              referrerPool[records[i][5]] = records[i][5];
                            }
                          }
                          let uniqIp = Object.keys(ipPool).length;
                          let uniqReferrer = Object.keys(referrerPool).length;
  
                          let qry =
                            "INSERT INTO " +
                            db.options.database +
                            ".tbl_uid_statistics SET ? ON DUPLICATE KEY UPDATE total = " +
                            mysql.escape(total) +
                            ", hit = " +
                            mysql.escape(hit) +
                            ", miss = " +
                            mysql.escape(miss) +
                            ", size = " +
                            mysql.escape(size) +
                            ", uniq_ip = " +
                            mysql.escape(uniqIp) +
                            ", uniq_referrer = " +
                            mysql.escape(uniqReferrer);
                          let post = {
                            date_tag: dateTagBase,
                            uid: targetUid,
                            total: total,
                            hit: hit,
                            miss: miss,
                            size: size,
                            uniq_ip: uniqIp,
                            uniq_referrer: uniqReferrer,
                          };
  
                          db.doEasyQueryPost(qry, post, function (
                            error,
                            results
                          ) {
                            if (error) {
                              utils.log(
                                "error",
                                "insert, error=",
                                error,
                                utils.__where()
                              );
                            }
                            subCallback(error, results);
                          });
                        },
                        function (error, subResults) {
                          if (error) {
                            utils.log(
                              "error",
                              "error on makeStatisticsFromScrachDebug(), error : " +
                                JSON.stringify(error) +
                                " " +
                                utils.__where()
                            );
                            cb(error, resultData);
                          } else {
                            flushWordStatistics(targetTag, dateTagBase);
                            cb(error, resultData);
                          }
                        }
                      );
                    }
                  }
                );
              }
            }
          );
        }
        
      }
    }
  );
}
exports.makeStatisticsFromScrachDebugBakup = makeStatisticsFromScrachDebugBakup;




function makeStatisticsFromDbInChain(termTag, cb) {

  makeStatisticsFromDb(termTag, 'uid', function(error, results) {

    makeStatisticsFromDb(termTag, 'key', function(error, results) {
    
      makeStatisticsFromDb(termTag, 'word', function(error, results) {

        cb(error, results);
    
      })

    })    

  })
}


function makeStatisticsFromDb(termTag, targetTag, cb) {

  let qry = '';

  switch (targetTag) {
    case 'uid':
      qry = "INSERT INTO " + db.options.database + ".tbl_uid_statistics (date_tag, uid, total, hit, miss, `error`, size) SELECT date_tag, uid, total, hit, miss, `error`, size FROM (SELECT '" + termTag + "' AS date_tag, uid, sum(total) as total, sum(hit) as hit, sum(miss) as miss, sum(`error`) as `error`, sum(size) as size FROM " + db.options.database + ".tbl_uid_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY uid) as t ON DUPLICATE KEY UPDATE total = t.total, hit = t.hit, miss = t.miss, `error` = t.`error`, size = t.size";
      break;
    case 'key':
      qry = "INSERT INTO " + db.options.database + ".tbl_key_statistics (date_tag, api_key, uid, total, hit, miss, `error`, size) SELECT date_tag, api_key, uid, total, hit, miss, `error`, size FROM (SELECT '" + termTag + "' AS date_tag, api_key, uid, sum(total) as total, sum(hit) as hit, sum(miss) as miss, sum(`error`) as `error`, sum(size) as size FROM " + db.options.database + ".tbl_key_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY api_key) as t ON DUPLICATE KEY UPDATE total = t.total, hit = t.hit, miss = t.miss, `error` = t.`error`, size = t.size";
      break;
    case 'word':
      qry = "INSERT INTO " + db.options.database + ".tbl_word_statistics (date_tag, uid, api_key, word, count) SELECT date_tag, uid, api_key, word, count FROM (SELECT '" + termTag + "' AS date_tag, uid, api_key, word, sum(count) as count FROM " + db.options.database + ".tbl_word_statistics WHERE date_tag LIKE '" + termTag + "%' AND LENGTH(date_tag) = LENGTH('" + termTag + "') + 2 GROUP BY api_key, word) as t ON DUPLICATE KEY UPDATE count = t.count";
      break;
  }

  db.doEasyQuery(qry, function (error, results) {
    if (error) {
      utils.log(
        "error",
        "makeStatisticsFromDb(), error=",
        error,
        utils.__where()
      );
    }
    cb(error, {boom: 0, event: results.affectedRows});
  });
}
exports.makeStatisticsFromDb = makeStatisticsFromDb;


function enqueueStatisticsWork(termTag, dateTag, cb) {

  // let currentDateTagByTerm = moment().subtract(1, 'hours').format("YYYYMMDDHH");
  /*
  let currentDateTagByTerm = utils.getDateTimeTag('M');

  switch (termTag) {
    case 'minute':
      currentDateTagByTerm = utils.getDateTimeTag('M');
      break;
    case 'hour':
      currentDateTagByTerm = utils.getDateTimeTag('h');
      break;
    case 'day':
      currentDateTagByTerm = utils.getDateTimeTag('d');
      break;
    case 'month':
      currentDateTagByTerm = utils.getDateTimeTag('m');
      break;
    case 'year':
      currentDateTagByTerm = utils.getDateTimeTag('y');
      break;
  }
  */

  let qry = "INSERT INTO " + config.db.database + ".tbl_statistics_queue SET ? ";
  let post = {
    term_tag: termTag,
    date_tag: dateTag,
  }
  db.doEasyQueryPost(qry, post, function (error, results) {
    cb(error);
  });

}


function initializeStatisticsWork(cb) {
  if (crossCtl.sConfig.master == true) {
    let qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET status = 0, retry_count = retry_count + 1 WHERE status = 1";
    db.doEasyQuery(qry, function (error, results) {
      if (error) {
        utils.log("debug", "initializeStatisticsWork(), error=", error);
      } else {
        if (results.affectedRows != 0) {
          utils.log("info", "initializeStatisticsWork(), retry = ", results.affectedRows);
        }
      }
      cb(error);
    });
  } else {
    cb(null);
  }
}

function handleStatisticsWork (termTag, dateTag, cb) {

  switch (termTag) {

    case 'all':
      // makeStatisticsFromScrach('hour', dateTag.substring(0, 10), cb);
      makeStatisticsFromScrachDebug(termTag, dateTag, cb);
      break;


    case 'minute':
      // makeStatisticsFromScrach('hour', dateTag.substring(0, 10), cb);
      makeStatisticsFromScrach(termTag, dateTag, cb);
      break;

    case 'hour':
      makeStatisticsFromDbInChain(dateTag, cb);
      break;

    case 'day':
      makeStatisticsFromDbInChain(dateTag, cb);
      // makeStatisticsFromScrach(termTag, dateTag, cb);
      break;

    case 'month':
      makeStatisticsFromDbInChain(dateTag, cb);
      // makeStatisticsFromScrach(termTag, dateTag, cb);
      break;

    case 'year':
      makeStatisticsFromDbInChain(dateTag, cb);
      
      break;

    default:
      cb(new Error('bad termTag = ' + termTag), {boom: 0, event: 0})
            
/*      
    case 'day':
      makeStatisticsFromDbInChain(
        date_tag,
        function(error) {
          makeStatisticsFromDbInChain(
            date_tag.substring(0, 6),
            function(error) {
              makeStatisticsFromDbInChain(
                date_tag.substring(0, 4),
                function(error) {
                  cb(error)
                }
              );
            }
          );
        }
      );
      break;

    case 'month':
      makeStatisticsFromDbInChain(
        date_tag,
        function(error) {
          makeStatisticsFromDbInChain(
            date_tag.substring(0, 6),
            function(error) {
              makeStatisticsFromDbInChain(
                date_tag.substring(0, 4),
                function(error) {
                  cb(error)
                }
              );
            }
          );
        }
      );
      break;

    case 'year':
      makeStatisticsFromDbInChain(
        date_tag,
        cb
      );
      break;

*/      
  }
}

var statisticsInProgressFlag = false;

function dequeueStatisticsWork() {

  if (statisticsInProgressFlag == true) {
    console.log('dequeueStatisticsWork(), skip....')
  } else {
    statisticsInProgressFlag = true;
    // console.log('dequeueStatisticsWork(), ok.')
    let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE status = 0 ORDER BY serial ASC LIMIT 1";
    db.doEasyQuery(qry, function (error, results) {
      if (error) {
        utils.log("debug", "dequeueStatisticsWork(), error=", error);
        statisticsInProgressFlag = false;
      } else {
        if (results.length == 0) {
          console.log('dequeueStatisticsWork(), no target');

          handlePostQueueWork(function (error) {
            setImmediate((arg) => {
              // console.log(`executing immediate: ${arg}`);
              statisticsInProgressFlag = false;

            }, 'dummyParam');            
          })


        } else {
          let {serial, term_tag, date_tag} = results[0];
          console.log('dequeueStatisticsWork(), got one. ', serial, term_tag, date_tag)
          qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET started = now(), status = 1 WHERE serial = " + mysql.escape(serial);
          db.doEasyQuery(qry, function (error, results) {
            if (error) {
              utils.log("debug", "dequeueStatisticsWork(), error=", error);
              setImmediate((arg) => {
                // console.log(`executing immediate: ${arg}`);
                statisticsInProgressFlag = false;
                dequeueStatisticsWork();
              }, 'dummyParam');            
            } else {
              if (results.affectedRows != 1) {
                utils.log("debug", "dequeueStatisticsWork(), error= bad affted count. affectedRows = ", results.affectedRows);
                setImmediate((arg) => {
                  // console.log(`executing immediate: ${arg}`);
                  statisticsInProgressFlag = false;
                  dequeueStatisticsWork();
                }, 'dummyParam');            
              } else {
                handleStatisticsWork(term_tag, date_tag, function(error, results) {
                  if (error) {
                    utils.log("debug", "dequeueStatisticsWork(), error=", error);

                    qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), status = 3 WHERE serial = " + mysql.escape(serial);
                    db.doEasyQuery(qry, function (error, results) {
                      if (error) {
                        utils.log("debug", "dequeueStatisticsWork(), error=", error);
                      }
                      setImmediate((arg) => {
                        // console.log(`executing immediate: ${arg}`);
                        statisticsInProgressFlag = false;
                        dequeueStatisticsWork();
                      }, 'dummyParam');            
                  });
                  } else {
                    qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), ? WHERE serial = " + mysql.escape(serial);
                    let post = {
                      status: 2,
                      boom_count: results.boom, 
                      event_count: results.event
                    }
                    db.doEasyQueryPost(qry, post, function (error, results) {
                      if (error) {
                        utils.log("debug", "dequeueStatisticsWork(), error=", error);
                      }
                      handlePostQueueWork(function (error) {
                        setImmediate((arg) => {
                          // console.log(`executing immediate: ${arg}`);
                          statisticsInProgressFlag = false;
                          dequeueStatisticsWork();
                        }, 'dummyParam');            
                      })
                    });
                  }
                });
              }
            }
          });
        }
      }
    });
  }
}

function handlePostQueueWork(cb) {

  let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE term_tag = 'minute' ORDER BY serial DESC LIMIT 1";
  db.doEasyQuery(qry, function (error, results) {
    if (error) {
      utils.log("debug", "handlePostQueueWork(), error=", error);
      cb(error)
    } else {
      if (results.length == 0) {
        enqueueStatisticsWork('minute', '202306190800', function(error) {
          cb(error);
        });
      } else {
        if (results[0].status != 2) {
          // just wait... 
          cb(null);
        } else {
          let currentDateTag = results[0].date_tag;
          let nextDateTag = moment(currentDateTag, 'YYYYMMDDHHmm').add(1, 'minutes').format("YYYYMMDDHHmm");
          console.log('currentDateTag=', currentDateTag)
          console.log('nextDateTag=', nextDateTag)
          if (moment(currentDateTag, 'YYYYMMDDHHmm').isBefore(moment().subtract(5, 'minutes'))) {
            // if hour diff
            if (currentDateTag.substring(0, 10) != nextDateTag.substring(0, 10)) {
              enqueueStatisticsWork('hour', currentDateTag.substring(0, 10), function(error) {
                enqueueStatisticsWork('day', currentDateTag.substring(0, 8), function(error) {
                  enqueueStatisticsWork('month', currentDateTag.substring(0, 6), function(error) {
                    enqueueStatisticsWork('year', currentDateTag.substring(0, 4), function(error) {
                      enqueueStatisticsWork('minute', nextDateTag, function(error) {
                          cb(error);
                      });
                    });
                  });
                });
              });
            } else {
              if (moment(currentDateTag, 'YYYYMMDDHHmm').isBefore(moment().subtract(5, 'minutes'))) {
                enqueueStatisticsWork('minute', nextDateTag, function(error) {
                    cb(error);
                });
              } else {
                cb(error);
              }
            }
          } else {
            cb(error);
          }
        }
      }
    }
  });
}


function dequeueStatisticsWorkBack() {

  if (statisticsInProgressFlag == true) {
    console.log('dequeueStatisticsWork(), skip....')
  } else {
    statisticsInProgressFlag = true;
    // console.log('dequeueStatisticsWork(), ok.')
    let qry = "SELECT * FROM " + config.db.database + ".tbl_statistics_queue WHERE status = 0 ORDER BY serial ASC LIMIT 1";
    db.doEasyQuery(qry, function (error, results) {
      if (error) {
        utils.log("debug", "dequeueStatisticsWork(), error=", error);
        statisticsInProgressFlag = false;
      }
      else {
        if (results.length == 0) {
          console.log('dequeueStatisticsWork(), no target')
          statisticsInProgressFlag = false;
        } else {

          let {serial, term_tag, date_tag} = results[0];

          console.log('dequeueStatisticsWork(), got one. ', serial, term_tag, date_tag)

          qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET started = now(), status = 1 WHERE serial = " + mysql.escape(serial);
          db.doEasyQuery(qry, function (error, results) {
            if (error) {
              utils.log("debug", "dequeueStatisticsWork(), error=", error);
              statisticsInProgressFlag = false;
            } else {
              if (results.affectedRows != 1) {
                utils.log("debug", "dequeueStatisticsWork(), error= bad affted count. affectedRows = ", results.affectedRows);
                statisticsInProgressFlag = false;
              } else {
                handleStatisticsWork(term_tag, date_tag, function(error, results) {
                  if (error) {
                    utils.log("debug", "dequeueStatisticsWork(), error=", error);

                    qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), status = 3 WHERE serial = " + mysql.escape(serial);
                    db.doEasyQuery(qry, function (error, results) {
                      if (error) {
                        utils.log("debug", "dequeueStatisticsWork(), error=", error);
                      }
                      statisticsInProgressFlag = false;
                      dequeueStatisticsWork();
                    });
                  } else {
                    qry = "UPDATE " + config.db.database + ".tbl_statistics_queue SET finished = now(), ? WHERE serial = " + mysql.escape(serial);
                    let post = {
                      status: 2,
                      boom_count: results.boom, 
                      event_count: results.event
                    }
                    db.doEasyQueryPost(qry, post, function (error, results) {
                      if (error) {
                        utils.log("debug", "dequeueStatisticsWork(), error=", error);
                      }
                      statisticsInProgressFlag = false;
                      dequeueStatisticsWork();
                    });
                  }
                });
              }
            }
          });
        }
      }
    });
  }
}

function handleEvent(eventTag, params, cb) {
  // utils.log('debug', 'local handleEvent(), ', eventTag, params)

  switch (eventTag) {

    case "safekiso:filter:sync":
      syncFilterWithDbAll(function (error) {
        cb(error)
      });

        break;


    case "safekiso:key:update":

      updateAPIKeyInfo(params.target, function (error, result) {
        cb(error)
      })
        break;


    case "cron":
      // console.log('event cron, params=', params)
      // let statisticsTag = "";

      dequeueStatisticsWork();
      cb(null);
/*
      switch (params.interval) {

        case "minute":
          enqueueStatisticsWork('minute', moment().subtract(0, 'hours').format("YYYYMMDDHHmm"), function(error) {
            cb(error);
            dequeueStatisticsWork();
          });
          break;

        case "hour":
          enqueueStatisticsWork('hour', moment().subtract(1, 'hours').format("YYYYMMDDHH"), function(error) {
            enqueueStatisticsWork('day', moment().subtract(1, 'hours').format("YYYYMMDD"), function(error) {
              cb(error);
              dequeueStatisticsWork();
            });
          });
          break;


        case "day":
          enqueueStatisticsWork('day', moment().subtract(1, 'days').format("YYYYMMDD"), function(error) {
            enqueueStatisticsWork('month', moment().subtract(1, 'days').format("YYYYMM"), function(error) {
              enqueueStatisticsWork('year', moment().subtract(1, 'days').format("YYYY"), function(error) {
                cb(error);
                dequeueStatisticsWork();
              });
            });
          });
          break;
  

        default:
          cb(null);
      }
*/      
      break;

    case "addUser":
      /*
      var userNick = "";
      utils.log("debug", "user=", params);
      if (params.user_info.provider === "facebook") {
        userNick = "f " + params.user_info.displayName;
      } else if (params.user_info.provider === "google") {
        userNick = "g+ " + params.account_info.name;
      } else if (params.user_info.provider === "kakao") {
        userNick = "k " + params.user_info.displayName;
      } else {
        userNick = params.user_info.displayName;
      }

      var post = {
        uid: params.uid,
        nick: userNick,
      };

      var qry = "INSERT INTO " + config.db.database + ".tbl_user_list SET ? ";
      //utils.log('debug', 'qry=', qry)
      //ut//ils.log('debug', 'post=', post)
      db.doEasyQueryPost(qry, post, function (error, results) {
        //utils.log('debug', 'error=', error)
        cb(error);
      });
      */
      cb(null);
      break;

    case "withdrawal":

        handleWithdrawal(params, cb);
        break;
  

    default:
      cb(new Error("unhandled event: " + eventTag));
  }
}
exports.handleEvent = handleEvent;


function handleWithdrawal(params, cb) {
  let {req, uid} = params;

  let qry = "SELECT * FROM " + config.db.database + ".tbl_key_list WHERE uid = " + mysql.escape(uid) + " AND status = 0";
  db.doEasyQuery(qry, function (error, results) {
    if (error) {
      cb(error);
    } else {
      if (results.length == 0) {
        cb(null);
      } else {
        async.mapSeries(
          results,
          function (record, subCallback) {

            logUserAction(
              uid,
              'system',
              "key:delete",
              record.serial,
              "system",
              "API Key serial " +
              record.serial +
                "을 계정탈퇴로 인해 삭제",
              req,
              record
            );

    
            let qry =
              "UPDATE " +
              db.options.database +
              ".tbl_key_list SET status = 4 WHERE serial = " + record.serial;
            db.doEasyQuery(qry, function (
              error,
              results
            ) {
              if (error) {
                utils.log(
                  "error",
                  "update, error=",
                  error,
                  utils.__where()
                );
              }
              subCallback(error, record);
            });
          },
          function (error, subResults) {
            if (error) {
              utils.log(
                "error",
                "error on handleWithdrawal(), error : " +
                  JSON.stringify(error) +
                  " " +
                  utils.__where()
              );
    
            }
            // console.log('flush word Statistics done!');
            cb(error);
    
          }
        );
  
      }




    }
  });
}

function logUserAction(uid, name, tag, targetKey, by, description, req, infos) {
  let dumpedReq = {
    url: req.protocol + "://" + req.get("host") + req.originalUrl,
    headers: req.headers,
    query: req.query,
    body: req.body,
    referrer: utils.getReferrerFromReq(req),
    ip: utils.getIPFromReq(req),
  };

  var post = {
    uid: uid,
    name: name,
    tag: tag,
    target_key: targetKey,
    by: by,
    description: description,
    req: JSON.stringify(dumpedReq),
    infos: JSON.stringify(infos),
  };

  var qry = "INSERT INTO " + config.db.database + ".tbl_user_log SET ? ";
  //utils.log('debug', 'qry=', qry)
  //ut//ils.log('debug', 'post=', post)
  db.doEasyQueryPost(qry, post, function (error, results) {
    if (error) {
      utils.log("debug", "logUserAction(), error=", error);
    }
  });
}
exports.logUserAction = logUserAction;

function getPagedList(
  selectClause,
  tableName,
  joinClause,
  whereClause,
  groupByClause,
  orderByClause,
  targetPageSize,
  targetPageNumber,
  cb
) {
  var pageResults = {};
  var countAllQuery =
    "SELECT COUNT(*) as totalCount FROM " +
    tableName +
    (whereClause === "" ? "" : " WHERE " + whereClause);
  // console.log('countAllQuery = ', countAllQuery)
  // console.log('targetPageNumber = ', targetPageNumber)
  db.doEasyQuery(countAllQuery, function (error, results) {
    if (error) {
      cb(error, null);
    } else {
      var adjustedTargetPageSize =
        parseInt(targetPageSize) === -1
          ? results[0].totalCount
          : targetPageSize;
      // console.log('adjustedTargetPageSize = ', adjustedTargetPageSize)
      pageResults.pageSize = adjustedTargetPageSize;
      pageResults.totalCount = results[0].totalCount;
      pageResults.totalPageCount =
        Math.ceil(pageResults.totalCount / adjustedTargetPageSize) < 1
          ? 1
          : Math.ceil(pageResults.totalCount / adjustedTargetPageSize);

      if (pageResults.totalPageCount < targetPageNumber) {
        pageResults.currentPageNumber = pageResults.totalPageCount;
      } else {
        pageResults.currentPageNumber = targetPageNumber;
      }

      // console.log('pageResults', pageResults)

      // get page contents
      var contentQuery =
        "SELECT " +
        selectClause +
        " FROM " +
        tableName +
        (joinClause === "" ? "" : " " + joinClause) +
        (whereClause === "" ? "" : " WHERE " + whereClause) +
        " ORDER BY " +
        orderByClause +
        " LIMIT " +
        (pageResults.currentPageNumber - 1) * adjustedTargetPageSize +
        ", " +
        adjustedTargetPageSize;
      console.log("contentQuery = ", contentQuery);
      db.doEasyQuery(contentQuery, function (error, results) {
        if (error) {
          cb(error, null);
        } else {
          pageResults.fileList = results;
          cb(null, pageResults);
        }
      });
    }
  });
}
exports.getPagedList = getPagedList;


var wordStatisticsPool = {};

function initWordStatistics(termTag, dateTag) {
/*
    console.log('init word Statistics!');
    console.log('termTag = ', termTag);
    console.log('dateTag = ', dateTag);
*/
    let tmpBucketKey = termTag + "::" + dateTag;
    wordStatisticsPool[tmpBucketKey] = {}

}


function makeWordStatistics(termTag, dateTag, uid, key, words) {

  /*
    console.log('make word Statistics!');
    console.log('termTag = ', termTag);
    console.log('dateTag = ', dateTag);
    console.log('uid = ', uid);
    console.log('key = ', key);
    console.log('words = ', words);
  */
    let tmpBucketKey = termTag + "::" + dateTag;

    for (var i = 0; i < words.length; i++) {
      let tmpPoolKey = dateTag + "::" + uid + "::" + key + "::" + words[i][1]
      // console.log('hit : ', tmpPoolKey)
      let tmpAry = wordStatisticsPool[tmpBucketKey]
      let tmpHit = tmpAry[tmpPoolKey]
      if (tmpHit == undefined) {
        tmpAry[tmpPoolKey] = {
          uid: uid, 
          key: key, 
          word: words[i][1],
          hit: 1
        }
        tmpHit = tmpAry[tmpPoolKey]
      } else {
        tmpAry[tmpPoolKey].hit++;
      }
    }
}

function flushWordStatistics(termTag, dateTag) {
/*  
    console.log('flush word Statistics!');
    console.log('termTag = ', termTag);
    console.log('dateTag = ', dateTag);
*/
    let tmpBucketKey = termTag + "::" + dateTag;
    // console.log('flush', wordStatisticsPool[tmpBucketKey])


    let uids = []

    Object.keys(wordStatisticsPool[tmpBucketKey]).forEach(function(key) {
      var val = wordStatisticsPool[tmpBucketKey][key];
      uids.push(val);
    });

    // console.log('uids=', uids);



    async.mapSeries(
      uids,
      function (record, subCallback) {

        let qry =
          "INSERT INTO " +
          db.options.database +
          ".tbl_word_statistics SET ? ON DUPLICATE KEY UPDATE count = " +
          mysql.escape(record.hit);
        let post = {
          date_tag: dateTag,
          uid: record.uid,
          api_key: record.key,
          word: record.word,
          count: record.hit
        };

        db.doEasyQueryPost(qry, post, function (
          error,
          results
        ) {
          if (error) {
            utils.log(
              "error",
              "insert, error=",
              error,
              utils.__where()
            );
          }
          subCallback(error, results);
        });
      },
      function (error, subResults) {
        if (error) {
          utils.log(
            "error",
            "error on flushWordStatistics(), error : " +
              JSON.stringify(error) +
              " " +
              utils.__where()
          );

        }
        // console.log('flush word Statistics done!');

      }
    );


}