UI: DB: Add denormalized tables.

src/tests/db/Db.pro

@@ -2,11 +2,13 @@ include(../common/Test.pri)
 
 SOURCES += \
     $$UIPATH/db/databasemanager.cpp \
+    $$UIPATH/db/databasesql.cpp \
     $$UIPATH/fortcommon.cpp \
     $$UIPATH/util/fileutil.cpp
 
 HEADERS += \
     $$UIPATH/db/databasemanager.h \
+    $$UIPATH/db/databasesql.h \
     $$UIPATH/fortcommon.h \
     $$UIPATH/util/fileutil.h
 

src/tests/db/test.cpp

@@ -1,6 +1,7 @@
 #include "test.h"
 
 #include <QtTest>
+#include <QElapsedTimer>
 
 #include "db/databasemanager.h"
 #include "db/sqlite/sqlitedb.h"
@@ -28,6 +29,9 @@ void Test::dbWriteRead()
 
     debugProcNew(databaseManager.sqliteDb());
 
+    QElapsedTimer timer;
+    timer.start();
+
     // Add app traffics
     {
         const quint8 procBits = 0xFF;
@@ -39,7 +43,7 @@ void Test::dbWriteRead()
         databaseManager.handleStatTraf(procCount, &procBits, trafBytes);
     }
 
-    debugStatTraf(databaseManager.sqliteDb());
+    qDebug() << "elapsed>" << timer.restart() << "msec";
 
     // Delete apps
     {
@@ -51,6 +55,8 @@ void Test::dbWriteRead()
         databaseManager.handleStatTraf(procCount, &procBits, trafBytes);
     }
 
+    qDebug() << "elapsed>" << timer.elapsed() << "msec";
+
     debugStatTraf(databaseManager.sqliteDb());
 }
 
@@ -71,18 +77,40 @@ void Test::debugProcNew(SqliteDb *sqliteDb)
 }
 
 void Test::debugStatTraf(SqliteDb *sqliteDb)
+{
+    debugStatTrafStep(sqliteDb, "traffic_app_hour",
+                      "SELECT app_id, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_app_hour;");
+    debugStatTrafStep(sqliteDb, "traffic_app_day",
+                      "SELECT app_id, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_app_day;");
+    debugStatTrafStep(sqliteDb, "traffic_app_month",
+                      "SELECT app_id, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_app_month;");
+
+    debugStatTrafStep(sqliteDb, "traffic_hour",
+                      "SELECT 0, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_hour;");
+    debugStatTrafStep(sqliteDb, "traffic_day",
+                      "SELECT 0, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_day;");
+    debugStatTrafStep(sqliteDb, "traffic_month",
+                      "SELECT 0, unix_time, in_bytes, out_bytes"
+                      "  FROM traffic_month;");
+}
+
+void Test::debugStatTrafStep(SqliteDb *sqliteDb, const char *name,
+                             const char *sql)
 {
     SqliteStmt stmt;
 
-    QVERIFY(stmt.prepare(sqliteDb->db(),
+    QVERIFY(stmt.prepare(sqliteDb->db(), sql));
-                         "SELECT app_id, unix_hour, in_bytes, out_bytes"
-                         "  FROM traffic_app_hour;"));
 
-    qDebug() << "> traf <";
+    qDebug() << '>' << name << '<';
     while (stmt.step() == SqliteStmt::StepRow) {
         const qint64 unixTime = stmt.columnInt64(1) * 3600;
 
-        qDebug() << ">"
+        qDebug() << '>'
                  << stmt.columnInt64(0)
                  << QDateTime::fromSecsSinceEpoch(unixTime).toString()
                  << stmt.columnInt64(2)

src/tests/db/test.h

@@ -14,7 +14,10 @@ private slots:
 
 private:
     void debugProcNew(SqliteDb *sqliteDb);
+
     void debugStatTraf(SqliteDb *sqliteDb);
+    void debugStatTrafStep(SqliteDb *sqliteDb, const char *name,
+                           const char *sql);
 };
 
 #endif // TEST_H

src/ui/FortFirewall.pro

@@ -11,6 +11,7 @@ SOURCES += \
     conf/appgroup.cpp \
     conf/firewallconf.cpp \
     db/databasemanager.cpp \
+    db/databasesql.cpp \
     driver/drivermanager.cpp \
     driver/driverworker.cpp \
     fortcommon.cpp \
@@ -51,6 +52,7 @@ HEADERS += \
     conf/appgroup.h \
     conf/firewallconf.h \
     db/databasemanager.h \
+    db/databasesql.h \
     driver/drivermanager.h \
     driver/driverworker.h \
     fortcommon.h \

src/ui/db/databasemanager.cpp

@@ -3,55 +3,17 @@
 #include <QDateTime>
 
 #include "../util/fileutil.h"
+#include "databasesql.h"
 #include "sqlite/sqliteengine.h"
 #include "sqlite/sqlitedb.h"
 #include "sqlite/sqlitestmt.h"
 
-static const char * const sqlPragmas =
-        "PRAGMA locking_mode=EXCLUSIVE;"
-        "PRAGMA journal_mode=WAL;"
-        "PRAGMA synchronous=NORMAL;"
-        ;
-
-static const char * const sqlCreateTables =
-        "CREATE TABLE app("
-        "  id INTEGER PRIMARY KEY,"
-        "  path TEXT UNIQUE NOT NULL"
-        ");"
-
-        "CREATE TABLE traffic_app_hour("
-        "  app_id INTEGER NOT NULL,"
-        "  unix_hour INTEGER NOT NULL,"
-        "  in_bytes INTEGER NOT NULL,"
-        "  out_bytes INTEGER NOT NULL,"
-        "  PRIMARY KEY (app_id, unix_hour)"
-        ") WITHOUT ROWID;"
-        ;
-
-static const char * const sqlSelectAppId =
-        "SELECT id FROM app WHERE path = ?1;"
-        ;
-
-static const char * const sqlInsertAppId =
-        "INSERT INTO app(path) VALUES(?1);"
-        ;
-
-static const char * const sqlUpsertAppTraffic =
-        "WITH new(app_id, unix_hour, in_bytes, out_bytes)"
-        "    AS ( VALUES(?1, ?2, ?3, ?4) )"
-        "  INSERT OR REPLACE INTO traffic_app_hour("
-        "    app_id, unix_hour, in_bytes, out_bytes)"
-        "  SELECT new.app_id, new.unix_hour,"
-        "    new.in_bytes + ifnull(old.in_bytes, 0),"
-        "    new.out_bytes + ifnull(old.out_bytes, 0)"
-        "  FROM new LEFT JOIN traffic_app_hour AS old"
-        "    ON new.app_id = old.app_id"
-        "      AND new.unix_hour = old.unix_hour;"
-        ;
-
 DatabaseManager::DatabaseManager(const QString &filePath,
                                  QObject *parent) :
     QObject(parent),
+    m_lastUnixHour(0),
+    m_lastUnixDay(0),
+    m_lastUnixMonth(0),
     m_filePath(filePath),
     m_sqliteDb(new SqliteDb())
 {
@@ -74,7 +36,7 @@ bool DatabaseManager::initialize()
     if (!m_sqliteDb->open(m_filePath))
         return false;
 
-    m_sqliteDb->execute(sqlPragmas);
+    m_sqliteDb->execute(DatabaseSql::sqlPragmas);
 
     return fileExists || createTables();
 }
@@ -92,11 +54,72 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
 {
     QVector<quint16> delProcIndexes;
 
-    const qint32 unixHour = qint32(QDateTime::currentSecsSinceEpoch() / 3600);
+    const qint64 unixTime = QDateTime::currentSecsSinceEpoch();
 
-    SqliteStmt *stmtUpsert = getSqliteStmt(sqlUpsertAppTraffic);
+    const qint32 unixHour = qint32(unixTime / 3600);
+    const bool isNewHour = (unixHour != m_lastUnixHour);
 
-    stmtUpsert->bindInt(2, unixHour);
+    const qint32 unixDay = isNewHour ? getUnixDay(unixTime)
+                                     : m_lastUnixDay;
+    const bool isNewDay = (unixDay != m_lastUnixDay);
+
+    const qint32 unixMonth = isNewDay ? getUnixMonth(unixTime)
+                                      : m_lastUnixMonth;
+    const bool isNewMonth = (unixMonth != m_lastUnixMonth);
+
+    SqliteStmt *stmtInsertAppHour = nullptr;
+    SqliteStmt *stmtInsertAppDay = nullptr;
+    SqliteStmt *stmtInsertAppMonth = nullptr;
+
+    SqliteStmt *stmtInsertHour = nullptr;
+    SqliteStmt *stmtInsertDay = nullptr;
+    SqliteStmt *stmtInsertMonth = nullptr;
+
+    if (isNewHour) {
+        m_lastUnixHour = unixHour;
+
+        stmtInsertAppHour = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppHour);
+        stmtInsertHour = getSqliteStmt(DatabaseSql::sqlInsertTrafficHour);
+
+        stmtInsertAppHour->bindInt(1, unixHour);
+        stmtInsertHour->bindInt(1, unixHour);
+
+        if (isNewDay) {
+            m_lastUnixDay = unixDay;
+
+            stmtInsertAppDay = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppDay);
+            stmtInsertDay = getSqliteStmt(DatabaseSql::sqlInsertTrafficDay);
+
+            stmtInsertAppDay->bindInt(1, unixDay);
+            stmtInsertDay->bindInt(1, unixDay);
+
+            if (isNewMonth) {
+                m_lastUnixMonth = unixMonth;
+
+                stmtInsertAppMonth = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppMonth);
+                stmtInsertMonth = getSqliteStmt(DatabaseSql::sqlInsertTrafficMonth);
+
+                stmtInsertAppMonth->bindInt(1, unixMonth);
+                stmtInsertMonth->bindInt(1, unixMonth);
+            }
+        }
+    }
+
+    SqliteStmt *stmtUpdateAppHour = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppHour);
+    SqliteStmt *stmtUpdateAppDay = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppDay);
+    SqliteStmt *stmtUpdateAppMonth = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppMonth);
+
+    SqliteStmt *stmtUpdateHour = getSqliteStmt(DatabaseSql::sqlUpdateTrafficHour);
+    SqliteStmt *stmtUpdateDay = getSqliteStmt(DatabaseSql::sqlUpdateTrafficDay);
+    SqliteStmt *stmtUpdateMonth = getSqliteStmt(DatabaseSql::sqlUpdateTrafficMonth);
+
+    stmtUpdateAppHour->bindInt(1, unixHour);
+    stmtUpdateAppDay->bindInt(1, unixDay);
+    stmtUpdateAppMonth->bindInt(1, unixMonth);
+
+    stmtUpdateHour->bindInt(1, unixHour);
+    stmtUpdateDay->bindInt(1, unixDay);
+    stmtUpdateMonth->bindInt(1, unixMonth);
 
     m_sqliteDb->beginTransaction();
 
@@ -110,16 +133,35 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
         const quint32 inBytes = procTrafBytes[0];
         const quint32 outBytes = procTrafBytes[1];
 
-        if (inBytes != 0 || outBytes != 0) {
+        if (!(isNewHour || inBytes || outBytes))
-            const qint64 appId = m_appIds.at(procIndex);
+            continue;
 
-            stmtUpsert->bindInt64(1, appId);
+        const qint64 appId = m_appIds.at(procIndex);
-            stmtUpsert->bindInt64(3, inBytes);
-            stmtUpsert->bindInt64(4, outBytes);
 
-            stmtUpsert->step();
+        // Insert zero bytes
-            stmtUpsert->reset();
+        if (isNewHour) {
+            insertTraffic(stmtInsertAppHour, appId);
+            insertTraffic(stmtInsertHour);
+
+            if (isNewDay) {
+                insertTraffic(stmtInsertAppDay, appId);
+                insertTraffic(stmtInsertDay);
+
+                if (isNewMonth) {
+                    insertTraffic(stmtInsertAppMonth, appId);
+                    insertTraffic(stmtInsertMonth);
+                }
+            }
         }
+
+        // Update bytes
+        updateTraffic(stmtUpdateAppHour, inBytes, outBytes, appId);
+        updateTraffic(stmtUpdateAppDay, inBytes, outBytes, appId);
+        updateTraffic(stmtUpdateAppMonth, inBytes, outBytes, appId);
+
+        updateTraffic(stmtUpdateHour, inBytes, outBytes);
+        updateTraffic(stmtUpdateDay, inBytes, outBytes);
+        updateTraffic(stmtUpdateMonth, inBytes, outBytes);
     }
 
     m_sqliteDb->commitTransaction();
@@ -137,7 +179,7 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
 
 bool DatabaseManager::createTables()
 {
-    return m_sqliteDb->execute(sqlCreateTables);
+    return m_sqliteDb->execute(DatabaseSql::sqlCreateTables);
 }
 
 qint64 DatabaseManager::getAppId(const QString &appPath)
@@ -146,7 +188,7 @@ qint64 DatabaseManager::getAppId(const QString &appPath)
 
     // Check existing
     {
-        SqliteStmt *stmt = getSqliteStmt(sqlSelectAppId);
+        SqliteStmt *stmt = getSqliteStmt(DatabaseSql::sqlSelectAppId);
 
         stmt->bindText(1, appPath);
         if (stmt->step() == SqliteStmt::StepRow) {
@@ -157,7 +199,7 @@ qint64 DatabaseManager::getAppId(const QString &appPath)
 
     // Create new one
     if (!appId) {
-        SqliteStmt *stmt = getSqliteStmt(sqlInsertAppId);
+        SqliteStmt *stmt = getSqliteStmt(DatabaseSql::sqlInsertAppId);
 
         stmt->bindText(1, appPath);
         if (stmt->step() == SqliteStmt::StepDone) {
@@ -182,3 +224,42 @@ SqliteStmt *DatabaseManager::getSqliteStmt(const char *sql)
 
     return stmt;
 }
+
+void DatabaseManager::insertTraffic(SqliteStmt *stmt, qint64 appId)
+{
+    if (appId != 0) {
+        stmt->bindInt64(2, appId);
+    }
+
+    stmt->step();
+    stmt->reset();
+}
+
+void DatabaseManager::updateTraffic(SqliteStmt *stmt, quint32 inBytes,
+                                       quint32 outBytes, qint64 appId)
+{
+    stmt->bindInt64(2, inBytes);
+    stmt->bindInt64(3, outBytes);
+
+    if (appId != 0) {
+        stmt->bindInt64(4, appId);
+    }
+
+    stmt->step();
+    stmt->reset();
+}
+
+qint32 DatabaseManager::getUnixDay(qint64 unixTime)
+{
+    const QDate date = QDateTime::fromSecsSinceEpoch(unixTime).date();
+
+    return qint32(QDateTime(date).toSecsSinceEpoch() / 3600);
+}
+
+qint32 DatabaseManager::getUnixMonth(qint64 unixTime)
+{
+    const QDate date = QDateTime::fromSecsSinceEpoch(unixTime).date();
+
+    return qint32(QDateTime(QDate(date.year(), date.month(), 1))
+                  .toSecsSinceEpoch() / 3600);
+}

src/ui/db/databasemanager.h

@@ -37,7 +37,18 @@ private:
 
     SqliteStmt *getSqliteStmt(const char *sql);
 
+    void insertTraffic(SqliteStmt *stmt, qint64 appId = 0);
+    void updateTraffic(SqliteStmt *stmt, quint32 inBytes,
+                          quint32 outBytes, qint64 appId = 0);
+
+    static qint32 getUnixDay(qint64 unixTime);
+    static qint32 getUnixMonth(qint64 unixTime);
+
 private:
+    qint32 m_lastUnixHour;
+    qint32 m_lastUnixDay;
+    qint32 m_lastUnixMonth;
+
     QString m_filePath;
 
     SqliteDb *m_sqliteDb;

src/ui/db/databasesql.cpp

@@ -0,0 +1,136 @@
+#include "databasesql.h"
+
+const char * const DatabaseSql::sqlPragmas =
+        "PRAGMA locking_mode=EXCLUSIVE;"
+        "PRAGMA journal_mode=WAL;"
+        "PRAGMA synchronous=NORMAL;"
+        ;
+
+const char * const DatabaseSql::sqlCreateTables =
+        "CREATE TABLE app("
+        "  id INTEGER PRIMARY KEY,"
+        "  path TEXT UNIQUE NOT NULL"
+        ");"
+
+        "CREATE TABLE traffic_app_hour("
+        "  app_id INTEGER NOT NULL,"
+        "  unix_time INTEGER NOT NULL,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL,"
+        "  PRIMARY KEY (app_id, unix_time)"
+        ") WITHOUT ROWID;"
+
+        "CREATE TABLE traffic_app_day("
+        "  app_id INTEGER NOT NULL,"
+        "  unix_time INTEGER NOT NULL,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL,"
+        "  PRIMARY KEY (app_id, unix_time)"
+        ") WITHOUT ROWID;"
+
+        "CREATE TABLE traffic_app_month("
+        "  app_id INTEGER NOT NULL,"
+        "  unix_time INTEGER NOT NULL,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL,"
+        "  PRIMARY KEY (app_id, unix_time)"
+        ") WITHOUT ROWID;"
+
+        "CREATE TABLE traffic_hour("
+        "  unix_time INTEGER PRIMARY KEY,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL"
+        ") WITHOUT ROWID;"
+
+        "CREATE TABLE traffic_day("
+        "  unix_time INTEGER PRIMARY KEY,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL"
+        ") WITHOUT ROWID;"
+
+        "CREATE TABLE traffic_month("
+        "  unix_time INTEGER PRIMARY KEY,"
+        "  in_bytes INTEGER NOT NULL,"
+        "  out_bytes INTEGER NOT NULL"
+        ") WITHOUT ROWID;"
+        ;
+
+const char * const DatabaseSql::sqlSelectAppId =
+        "SELECT id FROM app WHERE path = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlInsertAppId =
+        "INSERT INTO app(path) VALUES(?1);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficAppHour =
+        "INSERT INTO traffic_app_hour(app_id, unix_time, in_bytes, out_bytes)"
+        "  VALUES(?2, ?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficAppDay =
+        "INSERT INTO traffic_app_day(app_id, unix_time, in_bytes, out_bytes)"
+        "  VALUES(?2, ?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficAppMonth =
+        "INSERT INTO traffic_app_month(app_id, unix_time, in_bytes, out_bytes)"
+        "  VALUES(?2, ?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficHour =
+        "INSERT INTO traffic_hour(unix_time, in_bytes, out_bytes)"
+        "  VALUES(?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficDay =
+        "INSERT INTO traffic_day(unix_time, in_bytes, out_bytes)"
+        "  VALUES(?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlInsertTrafficMonth =
+        "INSERT INTO traffic_month(unix_time, in_bytes, out_bytes)"
+        "  VALUES(?1, 0, 0);"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficAppHour =
+        "UPDATE traffic_app_hour"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE app_id = ?4 and unix_time = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficAppDay =
+        "UPDATE traffic_app_day"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE app_id = ?4 and unix_time = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficAppMonth =
+        "UPDATE traffic_app_month"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE app_id = ?4 and unix_time = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficHour =
+        "UPDATE traffic_hour"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE unix_time = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficDay =
+        "UPDATE traffic_day"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE unix_time = ?1;"
+        ;
+
+const char * const DatabaseSql::sqlUpdateTrafficMonth =
+        "UPDATE traffic_month"
+        "  SET in_bytes = in_bytes + ?2,"
+        "    out_bytes = out_bytes + ?3"
+        "  WHERE unix_time = ?1;"
+        ;

src/ui/db/databasesql.h

@@ -0,0 +1,30 @@
+#ifndef DATABASESQL_H
+#define DATABASESQL_H
+
+class DatabaseSql
+{
+public:
+    static const char * const sqlPragmas;
+    static const char * const sqlCreateTables;
+
+    static const char * const sqlSelectAppId;
+    static const char * const sqlInsertAppId;
+
+    static const char * const sqlInsertTrafficAppHour;
+    static const char * const sqlInsertTrafficAppDay;
+    static const char * const sqlInsertTrafficAppMonth;
+
+    static const char * const sqlInsertTrafficHour;
+    static const char * const sqlInsertTrafficDay;
+    static const char * const sqlInsertTrafficMonth;
+
+    static const char * const sqlUpdateTrafficAppHour;
+    static const char * const sqlUpdateTrafficAppDay;
+    static const char * const sqlUpdateTrafficAppMonth;
+
+    static const char * const sqlUpdateTrafficHour;
+    static const char * const sqlUpdateTrafficDay;
+    static const char * const sqlUpdateTrafficMonth;
+};
+
+#endif // DATABASESQL_H