From b8446181d3e0a676a73dd6169f0e7325df8b89cc Mon Sep 17 00:00:00 2001 From: panariga Date: Wed, 18 Dec 2024 10:26:40 +0200 Subject: [PATCH] added timestampOverride for manual log process --- ...fa076808377b28b01c478_2.file_index.tpl.php | 55 +-- smarty/template/index.tpl | 34 +- src/Classes/Report.php | 423 +++++++++++++----- src/InitTables.php | 2 +- src/Request.php | 8 +- 5 files changed, 352 insertions(+), 170 deletions(-) diff --git a/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php b/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php index bda19ff..01c48f2 100644 --- a/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php +++ b/smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php @@ -1,18 +1,18 @@ getCompiled()->isFresh($_smarty_tpl, array ( 'version' => '5.4.2', - 'unifunc' => 'content_676160989e3635_07407148', + 'unifunc' => 'content_6761e5219d7cf3_55254235', 'has_nocache_code' => false, 'file_dependency' => array ( 'affb24851ed623b62affa076808377b28b01c478' => array ( 0 => 'index.tpl', - 1 => 1734434957, + 1 => 1734468879, 2 => 'file', ), ), @@ -20,7 +20,7 @@ if ($_smarty_tpl->getCompiled()->isFresh($_smarty_tpl, array ( array ( ), ))) { -function content_676160989e3635_07407148 (\Smarty\Template $_smarty_tpl) { +function content_6761e5219d7cf3_55254235 (\Smarty\Template $_smarty_tpl) { $_smarty_current_dir = '/home/l/public_html/xbotcontrol/smarty/template'; ?> @@ -76,43 +76,26 @@ $_smarty_current_dir = '/home/l/public_html/xbotcontrol/smarty/template'; Latest
  • - getConfigVariable('linkContacts')), ENT_QUOTES, 'UTF-8');?> - + Top by IP
  • - getConfigVariable('linkLists')), ENT_QUOTES, 'UTF-8');?> - + Top by UA +
    • +
  • + IP+UA+Path +
    • +
  • + IP+Load +
    • +
  • + IP+RPS +
    • +
  • + IP+RPS
    • - -
  • - getConfigVariable('linkMembership')), ENT_QUOTES, 'UTF-8');?> - -
    • - -
  • - getConfigVariable('linkUsers')), ENT_QUOTES, 'UTF-8');?> - -
    • - -
  • - getConfigVariable('linkLog')), ENT_QUOTES, 'UTF-8');?> - -
    • - - - getConfigVariable('logOutbtn')), ENT_QUOTES, 'UTF-8');?> - - ( -) + diff --git a/smarty/template/index.tpl b/smarty/template/index.tpl index 02b13cd..e78cc3f 100644 --- a/smarty/template/index.tpl +++ b/smarty/template/index.tpl @@ -42,30 +42,26 @@ Latest
  • - {#linkContacts#} + Top by IP
  • - {#linkLists#} + Top by UA +
    • +
  • + IP+UA+Path +
    • +
  • + IP+Load +
    • +
  • + IP+RPS +
    • +
  • + IP+RPS
    • - {if $smarty.session.user_role == 'admin'} -
  • - {#linkMembership#} -
    • - -
  • - {#linkUsers#} -
    • - -
  • - {#linkLog#} -
    • - {/if} - - {#logOutbtn#} - ({$smarty.session.username}) + diff --git a/src/Classes/Report.php b/src/Classes/Report.php index 8bb677a..2a533f2 100644 --- a/src/Classes/Report.php +++ b/src/Classes/Report.php @@ -4,129 +4,330 @@ declare(strict_types=1); namespace XBotControl\Classes; - use React\Promise\PromiseInterface; use Psr\Http\Message\ServerRequestInterface; class Report { - - public static function latest_requests(ServerRequestInterface $request): PromiseInterface + private static function generateColumns(array $definitions): array { - $columnsDefinition = [ - [ - 'title' => 'id', - 'field' => 'id', - 'visible' => false, - 'sortable' => true, - 'filterControl' => 'input', - 'widthUnit' => 'input', - 'width' => 'input', - ], - [ - 'sortable' => true, - 'title' => 'ip', - 'field' => 'ip', - 'sortable' => true, - 'filterControl' => 'input', - - ], - [ - 'sortable' => true, - 'title' => 'domain', - 'field' => 'domain', - 'sortable' => true, - 'visible' => false, - 'filterControl' => 'input', - - ], - [ - 'sortable' => true, - 'title' => 'path', - 'field' => 'path', - 'sortable' => true, - 'filterControl' => 'input', - - ], - [ - 'sortable' => true, - 'title' => 'useragent', - 'field' => 'useragent', - 'sortable' => true, - 'filterControl' => 'input', - - ], - [ - 'sortable' => true, - 'title' => 'load', - 'field' => 'load', - 'sortable' => true, - 'filterControl' => 'input', - - ], - [ - 'sortable' => true, - 'title' => 'datetime', - 'field' => 'datetime', - 'sortable' => true, - 'filterControl' => 'input', - - ], - ]; - - - $sql = "SELECT - req.rowid AS id, - ip.data AS ip, - domain.data AS domain, - path.data AS path, - useragent.data AS useragent, - headers.data AS headers , - (SELECT load.load1 - FROM load - WHERE load.rowid >= req.timestamp - ORDER BY load.rowid DESC LIMIT 1) AS load, - datetime(req.timestamp, 'auto') AS datetime - - FROM - request req - LEFT JOIN - ip ON req.id_ip = ip.rowid - LEFT JOIN - domain ON req.id_domain = domain.rowid - LEFT JOIN - path ON req.id_path = path.rowid - LEFT JOIN - useragent ON req.id_useragent = useragent.rowid - LEFT JOIN - headers ON req.id_headers = headers.rowid - WHERE 1=1 "; - - - $params = []; - $query = $request->getQueryParams(); - if (isset($query['filter'])) { - $filter = json_decode($request->getQueryParams()['filter'], true); - } else { - $filter = []; + $columns = []; + foreach ($definitions as $definition) { + $columns[] = array_merge( + [ + 'sortable' => true, + 'visible' => true, + 'filterControl' => 'input', + ], + $definition + ); } + return $columns; + } - foreach ($filter as $field => $value) { - $sql .= 'AND ' . $field . ' LIKE ? '; - $params[] = '%' . $value . '%'; - } - $sql .= " AND req.timestamp BETWEEN ? AND ? "; - $sql .= ' ORDER BY req.rowid DESC '; - $sql .= ' LIMIT ? ;'; - $params[] = strtotime($request->getQueryParams()['from'] ?? 'yesterday'); - $params[] = strtotime($request->getQueryParams()['to'] ?? 'now'); - $params[] = (int)$request->getQueryParams()['limit'] ?? 100; - - return \XBotControl\Storage::getInstance()->db->query($sql, $params)->then(function ($result) use ($columnsDefinition) { + private static function executeQuery(string $sql, array $params, array $columnsDefinition): PromiseInterface + { + return \XBotControl\Storage::getInstance()->db->query($sql, $params)->then(function ($result) use ($columnsDefinition) { return [ "columns" => $columnsDefinition, "rows" => $result->rows, ]; }); } + + private static function parseQueryParams(ServerRequestInterface $request): array + { + $query = $request->getQueryParams(); + return [ + 'from' => strtotime($query['from'] ?? 'yesterday'), + 'to' => strtotime($query['to'] ?? 'now'), + 'limit' => (int)($query['limit'] ?? 100), + 'filter' => isset($query['filter']) ? json_decode($query['filter'], true) : [] + ]; + } + + private static function prepareFilterClauses(array $filter): array + { + $sql = ''; + $params = []; + foreach ($filter as $field => $value) { + $sql .= 'AND ' . $field . ' LIKE ? '; + $params[] = '%' . $value . '%'; + } + return [$sql, $params]; + } + + public static function latest_requests(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "id", "field" => "id", "visible" => false], + ["title" => "ip", "field" => "ip"], + ["title" => "domain", "field" => "domain", "visible" => false], + ["title" => "path", "field" => "path"], + ["title" => "useragent", "field" => "useragent"], + ["title" => "load", "field" => "load"], + ["title" => "datetime", "field" => "datetime"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + SELECT + req.rowid AS id, ip.data AS ip, domain.data AS domain, + path.data AS path, useragent.data AS useragent, + headers.data AS headers, + (SELECT load.load1 FROM load WHERE load.rowid >= req.timestamp ORDER BY load.rowid DESC LIMIT 1) AS load, + datetime(req.timestamp, 'auto') AS datetime + FROM + request req + LEFT JOIN ip ON req.id_ip = ip.rowid + LEFT JOIN domain ON req.id_domain = domain.rowid + LEFT JOIN path ON req.id_path = path.rowid + LEFT JOIN useragent ON req.id_useragent = useragent.rowid + LEFT JOIN headers ON req.id_headers = headers.rowid + WHERE 1=1 + "; + + list($filterSQL, $filterParams) = self::prepareFilterClauses($queryParams['filter']); + $sql .= $filterSQL . " AND req.timestamp BETWEEN ? AND ? ORDER BY req.rowid DESC LIMIT ?;"; + $params = array_merge($filterParams, [$queryParams['from'], $queryParams['to'], $queryParams['limit']]); + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function count_requests_by_ip(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "ip", "field" => "ip_address"], + ["title" => "request_count", "field" => "request_count"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + SELECT + ip.data AS ip_address, + COUNT(request.id_ip) AS request_count + FROM + request + INNER JOIN + ip ON request.id_ip = ip.rowid + WHERE + request.timestamp BETWEEN ? AND ? + GROUP BY + ip.data + ORDER BY + request_count DESC + LIMIT ?; + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function count_requests_by_ua(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "useragent", "field" => "id_useragent"], + ["title" => "request_count", "field" => "request_count"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + SELECT + useragent.data AS id_useragent, + COUNT(request.id_useragent) AS request_count + FROM + request + INNER JOIN + useragent ON request.id_useragent = useragent.rowid + WHERE + request.timestamp BETWEEN ? AND ? + GROUP BY + useragent.data + ORDER BY + request_count DESC + LIMIT ?; + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function top_ip_ua_path(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "ip", "field" => "ip"], + ["title" => "useragent", "field" => "user_agent"], + ["title" => "path", "field" => "path"], + ["title" => "count", "field" => "count"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + SELECT + ip.data AS ip, + useragent.data AS user_agent, + path.data AS path, + COUNT(request.rowid) AS count + FROM + request + JOIN ip ON request.id_ip = ip.rowid + JOIN useragent ON request.id_useragent = useragent.rowid + JOIN path ON request.id_path = path.rowid + WHERE + request.timestamp BETWEEN ? AND ? + GROUP BY + ip.data, useragent.data, path.data + ORDER BY + count DESC + LIMIT ?; + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function top_ip_by_load(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "ip", "field" => "data"], + ["title" => "avg_load", "field" => "avg_load"], + ["title" => "request_count", "field" => "request_count"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + SELECT + ip.data, + COUNT(request.rowid) AS request_count, + AVG(load.load1) AS avg_load + FROM + request + JOIN ip ON request.id_ip = ip.rowid + JOIN load ON load.rowid = ( + SELECT MIN(load_sub.rowid) + FROM load AS load_sub + WHERE load_sub.rowid > request.timestamp + ) + WHERE + load.load1 > 1 + AND request.timestamp BETWEEN ? AND ? + GROUP BY + ip.data + ORDER BY + avg_load DESC, request_count DESC + LIMIT ?; + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function top_ip_by_rps(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "ip", "field" => "ip_address"], + ["title" => "avg_request_per_second", "field" => "avg_request_per_second"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " + WITH TimestampIPRequests AS ( + SELECT + id_ip, + timestamp, + COUNT(*) AS request_count + FROM + request + WHERE + + request.timestamp BETWEEN ? AND ? + GROUP BY + id_ip, timestamp + HAVING + COUNT(*) > 1 +), +IPRequestPerSecond AS ( + SELECT + id_ip, + AVG(request_count) AS avg_request_per_second + FROM + TimestampIPRequests + + GROUP BY + id_ip +) +SELECT + ip.data as ip_address, + avg_request_per_second +FROM + IPRequestPerSecond + JOIN ip ON IPRequestPerSecond.id_ip = ip.rowid +ORDER BY + avg_request_per_second DESC +LIMIT ?; + + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } + + public static function top_net_28_by_rps(ServerRequestInterface $request): PromiseInterface + { + $columnsDefinition = self::generateColumns([ + ["title" => "ip", "field" => "network"], + ["title" => "avg_request_per_second", "field" => "avg_request_per_second"], + ]); + + $queryParams = self::parseQueryParams($request); + $sql = " +CREATE FUNCTION cidr_to_network(cidr VARCHAR(30), prefix INT) RETURNS VARCHAR(30) +BEGIN + RETURN inet_ntoa(inet_aton(substring_index(cidr, '/', 1)) & ((2 ^ (32 - prefix)) - 1 ^ 0xFFFFFFFF)) || '/' || prefix; +END; + +WITH TimestampNetworkRequests AS ( + SELECT + CAST(cidr_to_network(ip.data, 28) AS TEXT) AS network, + timestamp, + COUNT(*) AS request_count + FROM + request + JOIN + ip ON request.id_ip = ip.rowid + WHERE + request.timestamp BETWEEN ? AND ? + GROUP BY + network, timestamp + HAVING + COUNT(*) > 1 +), +NetworkRequestPerSecond AS ( + SELECT + network, + AVG(request_count) AS avg_request_per_second + FROM + TimestampNetworkRequests + GROUP BY + network +) +SELECT + network AS network_address, + avg_request_per_second +FROM + NetworkRequestPerSecond +ORDER BY + avg_request_per_second DESC +LIMIT ?; + "; + + $params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']]; + + return self::executeQuery($sql, $params, $columnsDefinition); + } } diff --git a/src/InitTables.php b/src/InitTables.php index 83448da..99d6a21 100644 --- a/src/InitTables.php +++ b/src/InitTables.php @@ -23,7 +23,7 @@ class InitTables })->then(function () use ($db) { return $db->exec('CREATE TABLE IF NOT EXISTS headers ( data TEXT UNIQUE NOT NULL) STRICT ;'); })->then(function () use ($db) { - return $db->exec("CREATE TABLE IF NOT EXISTS networkwhitelist ( data TEXT UNIQUE NOT NULL CHECK (data LIKE '%/%'), CONSTRAINT valid_network CHECK ( data LIKE '%.%/%' OR data LIKE '%:%/%' )) STRICT ;"); + return $db->exec("CREATE TABLE IF NOT EXISTS networkwhitelist ( data TEXT UNIQUE NOT NULL CHECK (data LIKE '%/%'), source TEXT NULL , CONSTRAINT valid_network CHECK ( data LIKE '%.%/%' OR data LIKE '%:%/%' )) STRICT ;"); })->then(function () use ($db) { return $db->exec('CREATE TABLE IF NOT EXISTS request ( id_ip INTEGER NOT NULL, id_method INTEGER NOT NULL, id_domain INTEGER NOT NULL, id_path INTEGER NOT NULL, id_useragent INTEGER NOT NULL, id_headers INTEGER NOT NULL, timestamp INTEGER NOT NULL, FOREIGN KEY (id_ip) REFERENCES ip(rowid), FOREIGN KEY (id_domain) REFERENCES domain(rowid), FOREIGN KEY (id_path) REFERENCES path(rowid), FOREIGN KEY (id_useragent) REFERENCES useragent(rowid), FOREIGN KEY (id_headers) REFERENCES headers(rowid) ) STRICT ;'); })->then(function () use ($db) { diff --git a/src/Request.php b/src/Request.php index 5516cfc..16de98c 100644 --- a/src/Request.php +++ b/src/Request.php @@ -30,7 +30,7 @@ class Request - public static function save(ServerRequestInterface $request): PromiseInterface + public static function save(ServerRequestInterface $request, ?int $timestampOverride = null): PromiseInterface { $realIp = self::getRealIP($request); $userAgent = $request->getHeaderLine('User-Agent') ?: 'Unknown'; @@ -45,6 +45,8 @@ class Request 'id_path' => $storage::getId('path', '/' . $request->getAttribute('original_uri', '')), 'id_useragent' => $storage::getId('useragent', $userAgent), 'id_headers' => 0, + 'id_method' => self::METHOD[$request->getMethod()] ?? 0, + 'timestamp' => $timestampOverride ?? time(), ]; if ($_ENV['SAVE_HEADERS'] === true) { @@ -56,8 +58,8 @@ class Request return \React\Promise\all($idPromises) ->then(function ($resolvedValues) use ($request, $storage) { // Set resolved values efficiently - $resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0; - $resolvedValues['timestamp'] = time(); + // $resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0; + // $resolvedValues['timestamp'] = time(); // Directly save data asynchronously return $storage::insert('request', $resolvedValues);