mirrors/fort
1
0
Fork 0
mirror of https://github.com/tnodir/fort synced 2024-11-15 07:45:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

UI: DB: Add denormalized tables.

Browse Source
This commit is contained in:
Nodir Temirkhodjaev 2017-12-04 18:55:03 +05:00
parent 4d5346bc57
commit 5cea967304
8 changed files with 355 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/tests/db/Db.pro
View File

@ -2,11 +2,13 @@ include(../common/Test.pri)
SOURCES += \
$$UIPATH/db/databasemanager.cpp \
$$UIPATH/db/databasesql.cpp \
$$UIPATH/fortcommon.cpp \
$$UIPATH/util/fileutil.cpp
HEADERS += \
$$UIPATH/db/databasemanager.h \
$$UIPATH/db/databasesql.h \
$$UIPATH/fortcommon.h \
$$UIPATH/util/fileutil.h

40
src/tests/db/test.cpp
View File

@ -1,6 +1,7 @@
#include "test.h"
#include <QtTest>
#include <QElapsedTimer>
#include "db/databasemanager.h"
#include "db/sqlite/sqlitedb.h"
@ -28,6 +29,9 @@ void Test::dbWriteRead()
debugProcNew(databaseManager.sqliteDb());
QElapsedTimer timer;
timer.start();
// Add app traffics
{
const quint8 procBits = 0xFF;
@ -39,7 +43,7 @@ void Test::dbWriteRead()
databaseManager.handleStatTraf(procCount, &procBits, trafBytes);
}
debugStatTraf(databaseManager.sqliteDb());
qDebug() << "elapsed>" << timer.restart() << "msec";
// Delete apps
{
@ -51,6 +55,8 @@ void Test::dbWriteRead()
databaseManager.handleStatTraf(procCount, &procBits, trafBytes);
}
qDebug() << "elapsed>" << timer.elapsed() << "msec";
debugStatTraf(databaseManager.sqliteDb());
}
@ -71,18 +77,40 @@ void Test::debugProcNew(SqliteDb *sqliteDb)
}
void Test::debugStatTraf(SqliteDb *sqliteDb)
{
debugStatTrafStep(sqliteDb, "traffic_app_hour",
"SELECT app_id, unix_time, in_bytes, out_bytes"
" FROM traffic_app_hour;");
debugStatTrafStep(sqliteDb, "traffic_app_day",
"SELECT app_id, unix_time, in_bytes, out_bytes"
" FROM traffic_app_day;");
debugStatTrafStep(sqliteDb, "traffic_app_month",
"SELECT app_id, unix_time, in_bytes, out_bytes"
" FROM traffic_app_month;");
debugStatTrafStep(sqliteDb, "traffic_hour",
"SELECT 0, unix_time, in_bytes, out_bytes"
" FROM traffic_hour;");
debugStatTrafStep(sqliteDb, "traffic_day",
"SELECT 0, unix_time, in_bytes, out_bytes"
" FROM traffic_day;");
debugStatTrafStep(sqliteDb, "traffic_month",
"SELECT 0, unix_time, in_bytes, out_bytes"
" FROM traffic_month;");
}
void Test::debugStatTrafStep(SqliteDb *sqliteDb, const char *name,
const char *sql)
{
SqliteStmt stmt;
QVERIFY(stmt.prepare(sqliteDb->db(),
"SELECT app_id, unix_hour, in_bytes, out_bytes"
" FROM traffic_app_hour;"));
QVERIFY(stmt.prepare(sqliteDb->db(), sql));
qDebug() << "> traf <";
qDebug() << '>' << name << '<';
while (stmt.step() == SqliteStmt::StepRow) {
const qint64 unixTime = stmt.columnInt64(1) * 3600;
qDebug() << ">"
qDebug() << '>'
<< stmt.columnInt64(0)
<< QDateTime::fromSecsSinceEpoch(unixTime).toString()
<< stmt.columnInt64(2)

3
src/tests/db/test.h
View File

@ -14,7 +14,10 @@ private slots:
private:
void debugProcNew(SqliteDb *sqliteDb);
void debugStatTraf(SqliteDb *sqliteDb);
void debugStatTrafStep(SqliteDb *sqliteDb, const char *name,
const char *sql);
};
#endif // TEST_H

2
src/ui/FortFirewall.pro
View File

@ -11,6 +11,7 @@ SOURCES += \
conf/appgroup.cpp \
conf/firewallconf.cpp \
db/databasemanager.cpp \
db/databasesql.cpp \
driver/drivermanager.cpp \
driver/driverworker.cpp \
fortcommon.cpp \
@ -51,6 +52,7 @@ HEADERS += \
conf/appgroup.h \
conf/firewallconf.h \
db/databasemanager.h \
db/databasesql.h \
driver/drivermanager.h \
driver/driverworker.h \
fortcommon.h \

193
src/ui/db/databasemanager.cpp
View File

@ -3,55 +3,17 @@
#include <QDateTime>
#include "../util/fileutil.h"
#include "databasesql.h"
#include "sqlite/sqliteengine.h"
#include "sqlite/sqlitedb.h"
#include "sqlite/sqlitestmt.h"
static const char * const sqlPragmas =
"PRAGMA locking_mode=EXCLUSIVE;"
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
;
static const char * const sqlCreateTables =
"CREATE TABLE app("
" id INTEGER PRIMARY KEY,"
" path TEXT UNIQUE NOT NULL"
");"
"CREATE TABLE traffic_app_hour("
" app_id INTEGER NOT NULL,"
" unix_hour INTEGER NOT NULL,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL,"
" PRIMARY KEY (app_id, unix_hour)"
") WITHOUT ROWID;"
;
static const char * const sqlSelectAppId =
"SELECT id FROM app WHERE path = ?1;"
;
static const char * const sqlInsertAppId =
"INSERT INTO app(path) VALUES(?1);"
;
static const char * const sqlUpsertAppTraffic =
"WITH new(app_id, unix_hour, in_bytes, out_bytes)"
" AS ( VALUES(?1, ?2, ?3, ?4) )"
" INSERT OR REPLACE INTO traffic_app_hour("
" app_id, unix_hour, in_bytes, out_bytes)"
" SELECT new.app_id, new.unix_hour,"
" new.in_bytes + ifnull(old.in_bytes, 0),"
" new.out_bytes + ifnull(old.out_bytes, 0)"
" FROM new LEFT JOIN traffic_app_hour AS old"
" ON new.app_id = old.app_id"
" AND new.unix_hour = old.unix_hour;"
;
DatabaseManager::DatabaseManager(const QString &filePath,
QObject *parent) :
QObject(parent),
m_lastUnixHour(0),
m_lastUnixDay(0),
m_lastUnixMonth(0),
m_filePath(filePath),
m_sqliteDb(new SqliteDb())
{
@ -74,7 +36,7 @@ bool DatabaseManager::initialize()
if (!m_sqliteDb->open(m_filePath))
return false;
m_sqliteDb->execute(sqlPragmas);
m_sqliteDb->execute(DatabaseSql::sqlPragmas);
return fileExists || createTables();
}
@ -92,11 +54,72 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
{
QVector<quint16> delProcIndexes;
const qint32 unixHour = qint32(QDateTime::currentSecsSinceEpoch() / 3600);
const qint64 unixTime = QDateTime::currentSecsSinceEpoch();
SqliteStmt *stmtUpsert = getSqliteStmt(sqlUpsertAppTraffic);
const qint32 unixHour = qint32(unixTime / 3600);
const bool isNewHour = (unixHour != m_lastUnixHour);
stmtUpsert->bindInt(2, unixHour);
const qint32 unixDay = isNewHour ? getUnixDay(unixTime)
: m_lastUnixDay;
const bool isNewDay = (unixDay != m_lastUnixDay);
const qint32 unixMonth = isNewDay ? getUnixMonth(unixTime)
: m_lastUnixMonth;
const bool isNewMonth = (unixMonth != m_lastUnixMonth);
SqliteStmt *stmtInsertAppHour = nullptr;
SqliteStmt *stmtInsertAppDay = nullptr;
SqliteStmt *stmtInsertAppMonth = nullptr;
SqliteStmt *stmtInsertHour = nullptr;
SqliteStmt *stmtInsertDay = nullptr;
SqliteStmt *stmtInsertMonth = nullptr;
if (isNewHour) {
m_lastUnixHour = unixHour;
stmtInsertAppHour = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppHour);
stmtInsertHour = getSqliteStmt(DatabaseSql::sqlInsertTrafficHour);
stmtInsertAppHour->bindInt(1, unixHour);
stmtInsertHour->bindInt(1, unixHour);
if (isNewDay) {
m_lastUnixDay = unixDay;
stmtInsertAppDay = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppDay);
stmtInsertDay = getSqliteStmt(DatabaseSql::sqlInsertTrafficDay);
stmtInsertAppDay->bindInt(1, unixDay);
stmtInsertDay->bindInt(1, unixDay);
if (isNewMonth) {
m_lastUnixMonth = unixMonth;
stmtInsertAppMonth = getSqliteStmt(DatabaseSql::sqlInsertTrafficAppMonth);
stmtInsertMonth = getSqliteStmt(DatabaseSql::sqlInsertTrafficMonth);
stmtInsertAppMonth->bindInt(1, unixMonth);
stmtInsertMonth->bindInt(1, unixMonth);
}
}
}
SqliteStmt *stmtUpdateAppHour = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppHour);
SqliteStmt *stmtUpdateAppDay = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppDay);
SqliteStmt *stmtUpdateAppMonth = getSqliteStmt(DatabaseSql::sqlUpdateTrafficAppMonth);
SqliteStmt *stmtUpdateHour = getSqliteStmt(DatabaseSql::sqlUpdateTrafficHour);
SqliteStmt *stmtUpdateDay = getSqliteStmt(DatabaseSql::sqlUpdateTrafficDay);
SqliteStmt *stmtUpdateMonth = getSqliteStmt(DatabaseSql::sqlUpdateTrafficMonth);
stmtUpdateAppHour->bindInt(1, unixHour);
stmtUpdateAppDay->bindInt(1, unixDay);
stmtUpdateAppMonth->bindInt(1, unixMonth);
stmtUpdateHour->bindInt(1, unixHour);
stmtUpdateDay->bindInt(1, unixDay);
stmtUpdateMonth->bindInt(1, unixMonth);
m_sqliteDb->beginTransaction();
@ -110,16 +133,35 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
const quint32 inBytes = procTrafBytes[0];
const quint32 outBytes = procTrafBytes[1];
if (inBytes != 0 || outBytes != 0) {
const qint64 appId = m_appIds.at(procIndex);
if (!(isNewHour || inBytes || outBytes))
continue;
stmtUpsert->bindInt64(1, appId);
stmtUpsert->bindInt64(3, inBytes);
stmtUpsert->bindInt64(4, outBytes);
const qint64 appId = m_appIds.at(procIndex);
stmtUpsert->step();
stmtUpsert->reset();
// Insert zero bytes
if (isNewHour) {
insertTraffic(stmtInsertAppHour, appId);
insertTraffic(stmtInsertHour);
if (isNewDay) {
insertTraffic(stmtInsertAppDay, appId);
insertTraffic(stmtInsertDay);
if (isNewMonth) {
insertTraffic(stmtInsertAppMonth, appId);
insertTraffic(stmtInsertMonth);
}
}
}
// Update bytes
updateTraffic(stmtUpdateAppHour, inBytes, outBytes, appId);
updateTraffic(stmtUpdateAppDay, inBytes, outBytes, appId);
updateTraffic(stmtUpdateAppMonth, inBytes, outBytes, appId);
updateTraffic(stmtUpdateHour, inBytes, outBytes);
updateTraffic(stmtUpdateDay, inBytes, outBytes);
updateTraffic(stmtUpdateMonth, inBytes, outBytes);
}
m_sqliteDb->commitTransaction();
@ -137,7 +179,7 @@ void DatabaseManager::handleStatTraf(quint16 procCount, const quint8 *procBits,
bool DatabaseManager::createTables()
{
return m_sqliteDb->execute(sqlCreateTables);
return m_sqliteDb->execute(DatabaseSql::sqlCreateTables);
}
qint64 DatabaseManager::getAppId(const QString &appPath)
@ -146,7 +188,7 @@ qint64 DatabaseManager::getAppId(const QString &appPath)
// Check existing
{
SqliteStmt *stmt = getSqliteStmt(sqlSelectAppId);
SqliteStmt *stmt = getSqliteStmt(DatabaseSql::sqlSelectAppId);
stmt->bindText(1, appPath);
if (stmt->step() == SqliteStmt::StepRow) {
@ -157,7 +199,7 @@ qint64 DatabaseManager::getAppId(const QString &appPath)
// Create new one
if (!appId) {
SqliteStmt *stmt = getSqliteStmt(sqlInsertAppId);
SqliteStmt *stmt = getSqliteStmt(DatabaseSql::sqlInsertAppId);
stmt->bindText(1, appPath);
if (stmt->step() == SqliteStmt::StepDone) {
@ -182,3 +224,42 @@ SqliteStmt *DatabaseManager::getSqliteStmt(const char *sql)
return stmt;
}
void DatabaseManager::insertTraffic(SqliteStmt *stmt, qint64 appId)
{
if (appId != 0) {
stmt->bindInt64(2, appId);
}
stmt->step();
stmt->reset();
}
void DatabaseManager::updateTraffic(SqliteStmt *stmt, quint32 inBytes,
quint32 outBytes, qint64 appId)
{
stmt->bindInt64(2, inBytes);
stmt->bindInt64(3, outBytes);
if (appId != 0) {
stmt->bindInt64(4, appId);
}
stmt->step();
stmt->reset();
}
qint32 DatabaseManager::getUnixDay(qint64 unixTime)
{
const QDate date = QDateTime::fromSecsSinceEpoch(unixTime).date();
return qint32(QDateTime(date).toSecsSinceEpoch() / 3600);
}
qint32 DatabaseManager::getUnixMonth(qint64 unixTime)
{
const QDate date = QDateTime::fromSecsSinceEpoch(unixTime).date();
return qint32(QDateTime(QDate(date.year(), date.month(), 1))
.toSecsSinceEpoch() / 3600);
}

11
src/ui/db/databasemanager.h
View File

@ -37,7 +37,18 @@ private:
SqliteStmt *getSqliteStmt(const char *sql);
void insertTraffic(SqliteStmt *stmt, qint64 appId = 0);
void updateTraffic(SqliteStmt *stmt, quint32 inBytes,
quint32 outBytes, qint64 appId = 0);
static qint32 getUnixDay(qint64 unixTime);
static qint32 getUnixMonth(qint64 unixTime);
private:
qint32 m_lastUnixHour;
qint32 m_lastUnixDay;
qint32 m_lastUnixMonth;
QString m_filePath;
SqliteDb *m_sqliteDb;

136
src/ui/db/databasesql.cpp Normal file
View File

@ -0,0 +1,136 @@
#include "databasesql.h"
const char * const DatabaseSql::sqlPragmas =
"PRAGMA locking_mode=EXCLUSIVE;"
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
;
const char * const DatabaseSql::sqlCreateTables =
"CREATE TABLE app("
" id INTEGER PRIMARY KEY,"
" path TEXT UNIQUE NOT NULL"
");"
"CREATE TABLE traffic_app_hour("
" app_id INTEGER NOT NULL,"
" unix_time INTEGER NOT NULL,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL,"
" PRIMARY KEY (app_id, unix_time)"
") WITHOUT ROWID;"
"CREATE TABLE traffic_app_day("
" app_id INTEGER NOT NULL,"
" unix_time INTEGER NOT NULL,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL,"
" PRIMARY KEY (app_id, unix_time)"
") WITHOUT ROWID;"
"CREATE TABLE traffic_app_month("
" app_id INTEGER NOT NULL,"
" unix_time INTEGER NOT NULL,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL,"
" PRIMARY KEY (app_id, unix_time)"
") WITHOUT ROWID;"
"CREATE TABLE traffic_hour("
" unix_time INTEGER PRIMARY KEY,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL"
") WITHOUT ROWID;"
"CREATE TABLE traffic_day("
" unix_time INTEGER PRIMARY KEY,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL"
") WITHOUT ROWID;"
"CREATE TABLE traffic_month("
" unix_time INTEGER PRIMARY KEY,"
" in_bytes INTEGER NOT NULL,"
" out_bytes INTEGER NOT NULL"
") WITHOUT ROWID;"
;
const char * const DatabaseSql::sqlSelectAppId =
"SELECT id FROM app WHERE path = ?1;"
;
const char * const DatabaseSql::sqlInsertAppId =
"INSERT INTO app(path) VALUES(?1);"
;
const char * const DatabaseSql::sqlInsertTrafficAppHour =
"INSERT INTO traffic_app_hour(app_id, unix_time, in_bytes, out_bytes)"
" VALUES(?2, ?1, 0, 0);"
;
const char * const DatabaseSql::sqlInsertTrafficAppDay =
"INSERT INTO traffic_app_day(app_id, unix_time, in_bytes, out_bytes)"
" VALUES(?2, ?1, 0, 0);"
;
const char * const DatabaseSql::sqlInsertTrafficAppMonth =
"INSERT INTO traffic_app_month(app_id, unix_time, in_bytes, out_bytes)"
" VALUES(?2, ?1, 0, 0);"
;
const char * const DatabaseSql::sqlInsertTrafficHour =
"INSERT INTO traffic_hour(unix_time, in_bytes, out_bytes)"
" VALUES(?1, 0, 0);"
;
const char * const DatabaseSql::sqlInsertTrafficDay =
"INSERT INTO traffic_day(unix_time, in_bytes, out_bytes)"
" VALUES(?1, 0, 0);"
;
const char * const DatabaseSql::sqlInsertTrafficMonth =
"INSERT INTO traffic_month(unix_time, in_bytes, out_bytes)"
" VALUES(?1, 0, 0);"
;
const char * const DatabaseSql::sqlUpdateTrafficAppHour =
"UPDATE traffic_app_hour"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE app_id = ?4 and unix_time = ?1;"
;
const char * const DatabaseSql::sqlUpdateTrafficAppDay =
"UPDATE traffic_app_day"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE app_id = ?4 and unix_time = ?1;"
;
const char * const DatabaseSql::sqlUpdateTrafficAppMonth =
"UPDATE traffic_app_month"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE app_id = ?4 and unix_time = ?1;"
;
const char * const DatabaseSql::sqlUpdateTrafficHour =
"UPDATE traffic_hour"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE unix_time = ?1;"
;
const char * const DatabaseSql::sqlUpdateTrafficDay =
"UPDATE traffic_day"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE unix_time = ?1;"
;
const char * const DatabaseSql::sqlUpdateTrafficMonth =
"UPDATE traffic_month"
" SET in_bytes = in_bytes + ?2,"
" out_bytes = out_bytes + ?3"
" WHERE unix_time = ?1;"
;

30
src/ui/db/databasesql.h Normal file
View File

@ -0,0 +1,30 @@
#ifndef DATABASESQL_H
#define DATABASESQL_H
class DatabaseSql
{
public:
static const char * const sqlPragmas;
static const char * const sqlCreateTables;
static const char * const sqlSelectAppId;
static const char * const sqlInsertAppId;
static const char * const sqlInsertTrafficAppHour;
static const char * const sqlInsertTrafficAppDay;
static const char * const sqlInsertTrafficAppMonth;
static const char * const sqlInsertTrafficHour;
static const char * const sqlInsertTrafficDay;
static const char * const sqlInsertTrafficMonth;
static const char * const sqlUpdateTrafficAppHour;
static const char * const sqlUpdateTrafficAppDay;
static const char * const sqlUpdateTrafficAppMonth;
static const char * const sqlUpdateTrafficHour;
static const char * const sqlUpdateTrafficDay;
static const char * const sqlUpdateTrafficMonth;
};
#endif // DATABASESQL_H