starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update dashboard widgets

Browse Source
This commit is contained in:
DariusIII
2026-07-20 13:07:56 +02:00
parent c2789fb333
commit 7906148f7f
16 changed files with 375 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/BackfillUserActivityStats.php
+3 -43
View File
@@ -5,8 +5,6 @@ declare(strict_types=1);
namespace App\Console\Commands;
use App\Models\UserActivityStat;
use App\Models\UserDownload;
use App\Models\UserRequest;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
@@ -29,7 +27,7 @@ class BackfillUserActivityStats extends Command
*
* @var string
*/
protected $description = 'Backfill user activity stats (daily or hourly) from existing user_downloads and user_requests data';
protected $description = 'Backfill hourly activity from raw logs or daily activity from retained hourly aggregates';
/**
* Execute the console command.
@@ -77,24 +75,7 @@ class BackfillUserActivityStats extends Command
continue;
}
// Count downloads for the date
$downloadsCount = UserDownload::query()
->whereRaw('DATE(timestamp) = ?', [$date])
->count();
// Count API hits for the date
$apiHitsCount = UserRequest::query()
->whereRaw('DATE(timestamp) = ?', [$date])
->count();
// Store or update the stats
UserActivityStat::updateOrCreate(
['stat_date' => $date],
[
'downloads_count' => $downloadsCount,
'api_hits_count' => $apiHitsCount,
]
);
UserActivityStat::collectDailyStats($date);
$statsCollected++;
$progressBar->advance();
@@ -137,28 +118,7 @@ class BackfillUserActivityStats extends Command
continue;
}
// Count downloads for the hour
$downloadsCount = UserDownload::query()
->where('timestamp', '>=', $hour)
->where('timestamp', '<', Carbon::parse($hour)->addHour()->format('Y-m-d H:00:00'))
->count();
// Count API hits for the hour
$apiHitsCount = UserRequest::query()
->where('timestamp', '>=', $hour)
->where('timestamp', '<', Carbon::parse($hour)->addHour()->format('Y-m-d H:00:00'))
->count();
// Store or update the stats
DB::table('user_activity_stats_hourly')->updateOrInsert(
['stat_hour' => $hour],
[
'downloads_count' => $downloadsCount,
'api_hits_count' => $apiHitsCount,
'updated_at' => now(),
'created_at' => DB::raw('COALESCE(created_at, NOW())'),
]
);
UserActivityStat::collectHourlyStats($hour);
$statsCollected++;
$progressBar->advance();
app/Console/Commands/CollectStats.php
+2 -2
View File
@@ -44,10 +44,10 @@ class CollectStats extends Command
$this->info('New users by month collected.');
RoleStat::insertUsersByRole();
$this->info('Users by role collected.');
UserActivityStat::collectDailyStats();
$this->info('User activity daily stats collected.');
UserActivityStat::collectHourlyStats();
$this->info('User activity hourly stats collected.');
UserActivityStat::collectDailyStats();
$this->info('User activity daily stats collected.');
$this->info('Site stats collected.');
}
}
app/Jobs/RecordApiUsage.php
+2 -5
View File
@@ -37,11 +37,8 @@ final class RecordApiUsage implements ShouldQueue
public function handle(): void
{
UserRequest::query()->insert([
'users_id' => $this->userId,
'request' => $this->requestUri,
'timestamp' => $this->occurredAt,
]);
// Legacy job retained while pre-deployment audit jobs drain.
UserRequest::recordApiRequest($this->userId, $this->requestUri, $this->occurredAt);
$lockKey = 'api_access_update:'.$this->userId;
$interval = max(1, (int) config('nntmux.api.access_update_interval', 60));
app/Jobs/UpdateUserApiAccess.php
+56
View File
@@ -0,0 +1,56 @@
<?php
declare(strict_types=1);
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\Attributes\Backoff;
use Illuminate\Queue\Attributes\FailOnTimeout;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Throwable;
#[Backoff([1, 5, 30])]
#[FailOnTimeout]
final class UpdateUserApiAccess implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 4;
public int $timeout = 15;
public function __construct(
public readonly int $userId,
public readonly ?string $ip,
public readonly string $occurredAt,
) {
$this->onQueue((string) config('nntmux.api.audit_queue', 'api-audit'));
}
public function handle(): void
{
$lockKey = 'api_access_update:'.$this->userId;
$interval = max(1, (int) config('nntmux.api.access_update_interval', 60));
try {
if (! Cache::add($lockKey, true, $interval)) {
return;
}
} catch (Throwable) {
// Cache degradation must not prevent the metadata update.
}
$update = ['apiaccess' => $this->occurredAt];
if ($this->ip !== null) {
$update['host'] = $this->ip;
}
DB::table('users')->where('id', $this->userId)->update($update);
}
}
app/Models/UserActivityStat.php
+26 -29
View File
@@ -22,29 +22,34 @@ class UserActivityStat extends Model
/**
* Collect and store user activity stats for a specific date
* This aggregates data from user_downloads and user_requests tables
* This rolls up the retained hourly aggregates so the result cannot shrink
* after the raw user_downloads and user_requests rows are pruned.
*/
public static function collectDailyStats(?string $date = null): void
{
$statDate = $date ? Carbon::parse($date)->format('Y-m-d') : Carbon::yesterday()->format('Y-m-d');
$dayStart = Carbon::parse($statDate)->startOfDay();
$dayEnd = $dayStart->copy()->addDay();
// Count downloads for the date
$downloadsCount = UserDownload::query()
->whereRaw('DATE(timestamp) = ?', [$statDate])
->count();
$totals = DB::table('user_activity_stats_hourly')
->where('stat_hour', '>=', $dayStart->format('Y-m-d H:00:00'))
->where('stat_hour', '<', $dayEnd->format('Y-m-d H:00:00'))
->selectRaw('COALESCE(SUM(downloads_count), 0) as downloads, COALESCE(SUM(api_hits_count), 0) as api_hits')
->first();
// Count API hits for the date
$apiHitsCount = UserRequest::query()
->whereRaw('DATE(timestamp) = ?', [$statDate])
->count();
$values = [
'downloads_count' => (int) ($totals->downloads ?? 0),
'api_hits_count' => (int) ($totals->api_hits ?? 0),
'updated_at' => now(),
];
// Store or update the stats
self::updateOrCreate(
if (! DB::table('user_activity_stats')->where('stat_date', $statDate)->exists()) {
$values['created_at'] = now();
}
DB::table('user_activity_stats')->updateOrInsert(
['stat_date' => $statDate],
[
'downloads_count' => $downloadsCount,
'api_hits_count' => $apiHitsCount,
]
$values,
);
}
@@ -111,9 +116,8 @@ class UserActivityStat extends Model
/**
* Get total downloads across the last N days inclusive of today.
*
* Sums closed days from `user_activity_stats` and today's closed hours
* from `user_activity_stats_hourly`. The current in-progress hour is not
* included here — callers needing live "today" totals should use
* Sums closed hours from `user_activity_stats_hourly`. The current
* in-progress hour is not included here — callers needing live totals use
* {@see UserStatsService::getSummaryStats()} which layers
* the live tables on top.
*/
@@ -134,20 +138,13 @@ class UserActivityStat extends Model
private static function sumLastNDays(int $days, string $column): int
{
$today = Carbon::now()->startOfDay();
$weekStart = Carbon::now()->subDays($days - 1)->startOfDay();
$currentHour = Carbon::now()->startOfHour();
$closedDays = (int) self::query()
->where('stat_date', '>=', $weekStart->format('Y-m-d'))
->where('stat_date', '<', $today->format('Y-m-d'))
return (int) DB::table('user_activity_stats_hourly')
->where('stat_hour', '>=', $weekStart->format('Y-m-d H:00:00'))
->where('stat_hour', '<', $currentHour->format('Y-m-d H:00:00'))
->sum($column);
$todayClosedHours = (int) DB::table('user_activity_stats_hourly')
->where('stat_hour', '>=', $today->format('Y-m-d H:00:00'))
->where('stat_hour', '<', Carbon::now()->startOfHour()->format('Y-m-d H:00:00'))
->sum($column);
return $closedDays + $todayClosedHours;
}
/**
app/Models/UserRequest.php
+10 -1
View File
@@ -132,7 +132,16 @@ class UserRequest extends Model
} else {
$userID = User::query()->select(['id'])->where('api_token', $tokenOrUserId)->value('id');
}
self::query()->insert(['users_id' => $userID, 'request' => $request, 'timestamp' => now()]);
self::recordApiRequest((int) $userID, $request, now()->toDateTimeString());
}
public static function recordApiRequest(int $userId, string $request, string $occurredAt): void
{
self::query()->insert([
'users_id' => $userId,
'request' => $request,
'timestamp' => $occurredAt,
]);
}
/**
app/Services/Api/ApiUsageService.php
+14 -4
View File
@@ -4,8 +4,9 @@ declare(strict_types=1);
namespace App\Services\Api;
use App\Jobs\RecordApiUsage;
use App\Jobs\UpdateUserApiAccess;
use App\Models\User;
use App\Models\UserRequest;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
@@ -30,18 +31,27 @@ final class ApiUsageService
public function record(User $user, Request $request): void
{
$job = new RecordApiUsage(
$occurredAt = now()->toDateTimeString();
// Request accounting drives quotas and admin statistics, so it must be
// durable before the response returns even when no queue worker runs.
UserRequest::recordApiRequest(
$user->id,
$request->getRequestUri(),
$occurredAt,
);
$job = new UpdateUserApiAccess(
$user->id,
$request->ip(),
now()->toDateTimeString(),
$occurredAt,
);
if ((bool) config('nntmux.api.async_audit', true)) {
try {
dispatch($job);
} catch (Throwable) {
// Preserve audit and quota correctness when the queue backend is unavailable.
// Keep access metadata current when the queue backend is unavailable.
$job->handle();
}
app/Services/UserStatsService.php
+63 -64
View File
@@ -38,11 +38,8 @@ class UserStatsService
/**
* Get downloads per day for the last N days (inclusive of today).
*
* Source layering — designed to avoid the 1-day rolling purge of
* `user_downloads` (see routes/console.php cleanup-api-request-logs):
* - Closed days (today-(N-1) … yesterday) come from `user_activity_stats`.
* - Today is composed of closed hours from `user_activity_stats_hourly`
* + the in-progress hour from the live `user_downloads` table.
* Closed hours come from `user_activity_stats_hourly`; the current hour
* comes from the live `user_downloads` table.
*
* @return list<array<string, int|string>>
*/
@@ -65,8 +62,15 @@ class UserStatsService
*/
public function getDownloadsPerHour(int $hours = 168): array
{
// Use the aggregated hourly stats from UserActivityStat model
return UserActivityStat::getDownloadsPerHour($hours);
$result = UserActivityStat::getDownloadsPerHour($hours);
if ($result !== []) {
$result[count($result) - 1]['count'] = UserDownload::query()
->where('timestamp', '>=', Carbon::now()->startOfHour())
->count();
}
return $result;
}
/**
@@ -77,14 +81,12 @@ class UserStatsService
public function getDownloadsPerMinute(int $minutes = 60): array
{
$startTime = Carbon::now()->subMinutes($minutes);
$minuteExpression = $this->minuteBucketExpression();
$downloads = UserDownload::query()
->select(
DB::raw('DATE_FORMAT(timestamp, "%Y-%m-%d %H:%i:00") as minute'),
DB::raw('COUNT(*) as count')
)
->selectRaw($minuteExpression.' as minute, COUNT(*) as count')
->where('timestamp', '>=', $startTime)
->groupBy(DB::raw('DATE_FORMAT(timestamp, "%Y-%m-%d %H:%i:00")'))
->groupByRaw($minuteExpression)
->orderBy('minute', 'asc')
->get()
->keyBy('minute');
@@ -107,9 +109,8 @@ class UserStatsService
/**
* Get API hits per day for the last N days (inclusive of today).
*
* Mirrors {@see self::getDownloadsPerDay()} — closed days from
* `user_activity_stats`, today composed of `user_activity_stats_hourly`
* + the in-progress hour from `user_requests`.
* Mirrors {@see self::getDownloadsPerDay()} using combined authenticated
* API v1/v2 and RSS requests from `user_requests`.
*
* @return list<array<string, int|string>>
*/
@@ -132,8 +133,15 @@ class UserStatsService
*/
public function getApiHitsPerHour(int $hours = 168): array
{
// Use the aggregated hourly stats from UserActivityStat model
return UserActivityStat::getApiHitsPerHour($hours);
$result = UserActivityStat::getApiHitsPerHour($hours);
if ($result !== []) {
$result[count($result) - 1]['count'] = UserRequest::query()
->where('timestamp', '>=', Carbon::now()->startOfHour())
->count();
}
return $result;
}
/**
@@ -144,15 +152,13 @@ class UserStatsService
public function getApiHitsPerMinute(int $minutes = 60): array
{
$startTime = Carbon::now()->subMinutes($minutes);
$minuteExpression = $this->minuteBucketExpression();
// Track actual API requests from user_requests table
$apiHits = UserRequest::query()
->select(
DB::raw('DATE_FORMAT(timestamp, "%Y-%m-%d %H:%i:00") as minute'),
DB::raw('COUNT(*) as count')
)
->selectRaw($minuteExpression.' as minute, COUNT(*) as count')
->where('timestamp', '>=', $startTime)
->groupBy(DB::raw('DATE_FORMAT(timestamp, "%Y-%m-%d %H:%i:00")'))
->groupByRaw($minuteExpression)
->orderBy('minute', 'asc')
->get()
->keyBy('minute');
@@ -177,12 +183,8 @@ class UserStatsService
*
* Window: "today" and "(7d) = last 7 calendar days inclusive of today".
*
* The live `user_downloads` / `user_requests` tables are pruned hourly to
* the last ~24h (see routes/console.php), so we layer sources to keep the
* totals consistent regardless of when the purge ran:
* - Closed days (today-6 … yesterday) → `user_activity_stats`.
* - Today's closed hours → `user_activity_stats_hourly`.
* - Current in-progress hour → live `user_downloads` / `user_requests`.
* Closed hours come from `user_activity_stats_hourly`; the current
* in-progress hour comes from live `user_downloads` / `user_requests`.
*
* @return array{total_users: int, downloads_today: int, downloads_week: int, api_hits_today: int, api_hits_week: int}
*/
@@ -192,10 +194,9 @@ class UserStatsService
$weekStart = Carbon::now()->subDays(6)->startOfDay();
$currentHourStart = Carbon::now()->startOfHour();
// Closed days: [today-6 .. yesterday]
$historical = UserActivityStat::query()
->where('stat_date', '>=', $weekStart->format('Y-m-d'))
->where('stat_date', '<', $today->format('Y-m-d'))
$weekClosed = DB::table('user_activity_stats_hourly')
->where('stat_hour', '>=', $weekStart->format('Y-m-d H:00:00'))
->where('stat_hour', '<', $currentHourStart->format('Y-m-d H:00:00'))
->selectRaw('COALESCE(SUM(downloads_count), 0) as downloads, COALESCE(SUM(api_hits_count), 0) as api_hits')
->first();
@@ -220,9 +221,9 @@ class UserStatsService
return [
'total_users' => User::whereNull('deleted_at')->count(),
'downloads_today' => $downloadsToday,
'downloads_week' => (int) ($historical->downloads ?? 0) + $downloadsToday,
'downloads_week' => (int) ($weekClosed->downloads ?? 0) + $downloadsCurrentHour,
'api_hits_today' => $apiHitsToday,
'api_hits_week' => (int) ($historical->api_hits ?? 0) + $apiHitsToday,
'api_hits_week' => (int) ($weekClosed->api_hits ?? 0) + $apiHitsCurrentHour,
];
}
@@ -247,10 +248,8 @@ class UserStatsService
}
/**
* Build a "last N days inclusive of today" series, layering closed days
* (from `user_activity_stats`), today's closed hours (from
* `user_activity_stats_hourly`) and the current in-progress hour from a
* caller-supplied live counter.
* Build a "last N days inclusive of today" series from hourly aggregates
* plus the current in-progress hour from a caller-supplied live counter.
*
* @param callable(Carbon): int $liveCurrentHourCounter
* @return list<array{date: string, count: int}>
@@ -260,42 +259,42 @@ class UserStatsService
$today = Carbon::now()->startOfDay();
$startDate = $today->copy()->subDays($days - 1);
$historical = UserActivityStat::query()
->select('stat_date', $statColumn)
->where('stat_date', '>=', $startDate->format('Y-m-d'))
->where('stat_date', '<', $today->format('Y-m-d'))
->orderBy('stat_date', 'asc')
->get()
->keyBy(static fn (UserActivityStat $s): string => $s->stat_date instanceof Carbon
? $s->stat_date->format('Y-m-d')
: (string) $s->stat_date);
$currentHourStart = Carbon::now()->startOfHour();
$hourly = DB::table('user_activity_stats_hourly')
->select('stat_hour', $statColumn)
->where('stat_hour', '>=', $startDate->format('Y-m-d H:00:00'))
->where('stat_hour', '<', $currentHourStart->format('Y-m-d H:00:00'))
->orderBy('stat_hour')
->get();
$dailyCounts = [];
foreach ($hourly as $stat) {
$date = Carbon::parse((string) $stat->stat_hour)->format('Y-m-d');
$dailyCounts[$date] = ($dailyCounts[$date] ?? 0) + (int) $stat->{$statColumn};
}
$result = [];
$cursor = $startDate->copy();
while ($cursor->lt($today)) {
$stat = $historical->get($cursor->format('Y-m-d'));
while ($cursor->lte($today)) {
$date = $cursor->format('Y-m-d');
$result[] = [
'date' => $cursor->format('M d'),
'count' => $stat ? (int) $stat->{$statColumn} : 0,
'count' => $dailyCounts[$date] ?? 0,
];
$cursor->addDay();
}
// Today: closed hours from hourly aggregate + the current hour from live.
$currentHourStart = Carbon::now()->startOfHour();
$hourlyColumn = $statColumn === 'api_hits_count' ? 'api_hits_count' : 'downloads_count';
$todayClosed = (int) DB::table('user_activity_stats_hourly')
->where('stat_hour', '>=', $today->format('Y-m-d H:00:00'))
->where('stat_hour', '<', $currentHourStart->format('Y-m-d H:00:00'))
->sum($hourlyColumn);
$todayLive = (int) $liveCurrentHourCounter($currentHourStart);
$result[] = [
'date' => $today->format('M d'),
'count' => $todayClosed + $todayLive,
];
if ($result !== []) {
$result[count($result) - 1]['count'] += (int) $liveCurrentHourCounter($currentHourStart);
}
return $result;
}
private function minuteBucketExpression(): string
{
return DB::connection()->getDriverName() === 'sqlite'
? "strftime('%Y-%m-%d %H:%M:00', timestamp)"
: 'DATE_FORMAT(timestamp, "%Y-%m-%d %H:%i:00")';
}
}