turkey/cv4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.1.3

Browse Source
This commit is contained in:
User
2025-01-18 19:05:23 -05:00
parent 3f282d6ecd
commit 14938f63df
23 changed files with 652 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
app/Http/Controllers/ComicController.php
View File

@@ -3,11 +3,14 @@
namespace App\Http\Controllers;
use App\Helper\ZhConversion;
use App\Jobs\ComicInsert;
use App\Jobs\ComicUpsert;
use App\Jobs\ImageUpsert;
use App\Jobs\RemotePrefetch;
use App\Models\Author;
use App\Models\Chapter;
use App\Models\Comic;
use App\Models\ReadingHistory;
use App\Remote\CopyManga;
use App\Remote\ImageFetcher;
use GuzzleHttp\Exception\GuzzleException;
@@ -19,6 +22,7 @@ use Illuminate\Http\JsonResponse;
use Illuminate\Http\RedirectResponse;
use Illuminate\Http\Request;
use Illuminate\Http\Response as IlluminateHttpResponse;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Inertia\Inertia;
use Inertia\Response;
@@ -30,12 +34,15 @@ class ComicController extends Controller
{
/**
* Autoload classes
*
* @param CopyManga $copyManga
* @param ZhConversion $zhConversion
*/
public function __construct(private readonly CopyManga $copyManga, private readonly ZhConversion $zhConversion)
{
}
public function __construct(
private readonly CopyManga $copyManga,
private readonly ZhConversion $zhConversion
) {}
/**
* Convert SC to ZH
@@ -61,10 +68,8 @@ class ComicController extends Controller
*/
public function favourites(Request $request): Response
{
$favourites = $this->scToZh($request->user()->favourites()->with(['authors'])->orderBy('upstream_updated_at', 'desc')->get());
return Inertia::render('Comic/Favourites', [
'favourites' => $favourites,
'favourites' => $this->scToZh($request->user()->favourites()->with(['authors'])->orderBy('upstream_updated_at', 'desc')->get()),
]);
}
@@ -84,16 +89,16 @@ class ComicController extends Controller
// Fetch from remote
$remoteComic = $this->copyManga->comic($request->pathword);
$comic = new Comic;
$comic->pathword = $remoteComic['comic']['path_word'];
$comic->name = $remoteComic['comic']['name'];
$comic->cover = $remoteComic['comic']['cover'];
$comic->upstream_updated_at = $remoteComic['comic']['datetime_updated'];
$comic->uuid = $remoteComic['comic']['uuid'];
$comic->alias = explode(',', $remoteComic['comic']['alias']);
$comic->description = $remoteComic['comic']['brief'];
$comic->metadata = $remoteComic;
$comic->save();
$comic = Comic::create([
'pathword' => $remoteComic['comic']['path_word'],
'name' => $remoteComic['comic']['name'],
'cover' => $remoteComic['comic']['cover'],
'upstream_updated_at' => $remoteComic['comic']['datetime_updated'],
'uuid' => $remoteComic['comic']['uuid'],
'alias' => explode(',', $remoteComic['comic']['alias']),
'description' => $remoteComic['comic']['brief'],
'metadata' => $remoteComic,
]);
}
// Set favourite
@@ -125,47 +130,6 @@ class ComicController extends Controller
]);
}
// Internal function for Upsert comics to db
protected function comicsUpsert($comics): void
{
// Prep the array for upsert
$comicsUpsertArray = [];
$authorsUpsertArray = [];
foreach ($comics['list'] as $comic) {
$comicsUpsertArray[] = [
'pathword' => $comic['path_word'],
'uuid' => '',
'name' => $comic['name'],
'alias' => '{}',
'description' => '',
'cover' => $comic['cover'],
'upstream_updated_at' => $comic['datetime_updated'] ?? null,
];
foreach ($comic['author'] as $author) {
$authorsUpsertArray[] = [
'name' => $author['name']
];
}
}
// Do an upsert for comics
Comic::upsert($comicsUpsertArray, uniqueBy: 'pathword', update: ['upstream_updated_at']);
Author::upsert($authorsUpsertArray, uniqueBy: 'name');
// Had to do a second pass to insert the relationships
foreach ($comics['list'] as $comic) {
// Get the comic id
$comicObj = Comic::where('pathword', $comic['path_word'])->first();
foreach ($comic['author'] as $author) {
$authorObj = Author::where('name', $author['name'])->first();
$comicObj->authors()->sync($authorObj);
}
}
}
/**
* Index / Tags for comic listing
*
@@ -180,12 +144,23 @@ class ComicController extends Controller
$params['theme'] = $request->get('tag');
}
$comics = $this->copyManga->comics(30, $request->header('offset', 0), $request->get('top', 'all'), $params);
$this->comicsUpsert($comics);
$offset = $request->header('offset', 0);
$comics = $this->copyManga->comics(30, $offset, $request->get('top', 'all'), $params);
// Upsert into DB
ComicUpsert::dispatch($comics);
// Prefetch next page
RemotePrefetch::dispatch('comics', [
'limit' => 30,
'offset' => $offset + 30,
'top' => $request->get('top', 'all'),
'params' => $params
])->delay(now()->addSecond());
return Inertia::render('Comic/Index', [
'comics' => $this->scToZh($comics),
'offset' => $request->header('offset', 0)
'offset' => $offset
]);
}
@@ -199,15 +174,15 @@ class ComicController extends Controller
*/
public function author(Request $request, string $author): Response
{
$params = [];
$params['author'] = $author;
$offset = $request->header('offset', 0);
$comics = $this->copyManga->comics(30, $offset, $request->get('top', 'all'), ['author' => $author]);
$comics = $this->copyManga->comics(30, $request->header('offset', 0), $request->get('top', 'all'), $params);
$this->comicsUpsert($comics);
// Upsert into DB
ComicUpsert::dispatch($comics);
return Inertia::render('Comic/Index', [
'comics' => $this->scToZh($comics),
'offset' => $request->header('offset', 0)
'offset' => $offset
]);
}
@@ -221,13 +196,14 @@ class ComicController extends Controller
*/
public function search(Request $request, string $search): Response
{
$comics = $this->copyManga->search($search, 30, $request->header('offset', 0));
$offset = $request->header('offset', 0);
$comics = $this->copyManga->search($search, 30, $offset);
// Search API is limited, no upsert
return Inertia::render('Comic/Index', [
'comics' => $this->scToZh($comics),
'offset' => $request->header('offset', 0)
'offset' => $offset
]);
}
@@ -245,8 +221,10 @@ class ComicController extends Controller
return to_route('comics.index');
}
$offset = $request->header('offset', 0);
$comic = $this->copyManga->comic($pathword);
$chapters = $this->copyManga->chapters($pathword, 200, $request->header('offset', 0), [], $request->get('group', 'default'));
$chapters = $this->copyManga->chapters($pathword, 200, $offset, [], $request->get('group', 'default'));
// Get the comic object and fill other parameters
try {
@@ -297,15 +275,20 @@ class ComicController extends Controller
// Do an upsert
Chapter::upsert($arrayForUpsert, uniqueBy: 'chapter_uuid');
// Get history
// Get user history
$histories = $request->user()->readingHistories()->where('reading_histories.comic_id', $comicObject->id)
->distinct()->select('chapter_uuid')->get()->pluck('chapter_uuid');
// Check if the first chapter is already fetched or not, if not, do a prefetch
if (!$histories->contains($chapters['list'][0]['uuid'])) {
RemotePrefetch::dispatch('chapter', ['pathword' => $pathword, 'uuid' => $chapters['list'][0]['uuid']]);
}
return Inertia::render('Comic/Chapters', [
'comic' => $this->scToZh($comic),
'chapters' => $this->scToZh($chapters),
'histories' => $histories,
'offset' => $request->header('offset', 0)
'offset' => $offset
]);
}
@@ -351,7 +334,7 @@ class ComicController extends Controller
try {
Chapter::where('chapter_uuid', $chapter['chapter']['next'])->firstOrFail()->images()->firstOrFail();
} catch (ModelNotFoundException $e) {
RemotePrefetch::dispatch('chapter', ['pathword' => $pathword, 'uuid' => $chapter['chapter']['next']]);
RemotePrefetch::dispatch('chapter', ['pathword' => $pathword, 'uuid' => $chapter['chapter']['next']]);
}
}
@@ -370,7 +353,48 @@ class ComicController extends Controller
*/
public function histories(Request $request): Response
{
// Get history
// Group by comic
if ($request->get('group', '') === 'comic') {
$readingHistory = ReadingHistory::query()
->join('comics', 'reading_histories.comic_id', '=', 'comics.id')
->join('chapters', 'reading_histories.chapter_id', '=', 'chapters.id')
->select(
'comics.id as comic_id',
'comics.name as comic_name',
'comics.pathword as comic_pathword',
'comics.upstream_updated_at as comic_upstream_updated_at',
'chapters.name as chapter_name',
'chapters.chapter_uuid as chapter_uuid',
'reading_histories.created_at as read_at'
)
->where('reading_histories.user_id', $request->user()->id)
->orderBy('comics.name')->orderByDesc('reading_histories.created_at')->paginate(2000);
// Transform the paginated data into a 2D array grouped by comic_id, including comic data and histories
$groupedData = $readingHistory->getCollection()
->groupBy('comic_id')->map(function (Collection $records, $comicId) {
$firstRecord = $records->first();
return [
'comic' => [
'comic_id' => $comicId,
'comic_name' => $firstRecord->comic_name,
'comic_pathword' => $firstRecord->comic_pathword,
'comic_upstream_updated_at' => $firstRecord->comic_upstream_updated_at,
],
'histories' => $records->map(fn($record) => [
'chapter_name' => $record->chapter_name,
'chapter_uuid' => $record->chapter_uuid,
'read_at' => $record->read_at,
])->toArray(),
];
})->values()->toArray(); // Use values() to reset numeric keys after grouping
return Inertia::render('Comic/HistoriesByComic', [
'histories' => $this->scToZh($groupedData)
]);
}
// Only order by chapter
$histories = $request->user()->readingHistories()->with(['comic:id,name,pathword'])->orderByDesc('reading_histories.created_at')
->select(['reading_histories.id as hid', 'chapters.comic_id', 'chapters.name'])->paginate(50)->toArray();
@@ -425,7 +449,7 @@ class ComicController extends Controller
*/
public function destroyHistories(Request $request): RedirectResponse
{
$result = $request->user()->cleanUpReadingHistories();
$request->user()->cleanUpReadingHistories();
return redirect()->route('comics.histories');
}

71
app/Jobs/ComicUpsert.php Normal file
View File

@@ -0,0 +1,71 @@
<?php
namespace App\Jobs;
use App\Models\Author;
use App\Models\Comic;
use App\Models\Image;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Support\Facades\Log;
use romanzipp\QueueMonitor\Traits\IsMonitored;
class ComicUpsert implements ShouldQueue
{
use IsMonitored, Queueable;
/**
* Create a new job instance.
*/
public function __construct(
public array $comic,
) {}
/**
* Execute the job.
*/
public function handle(): void
{
$this->queueProgress(0);
Log::info("JOB ComicUpsert START");
$comicsUpsertArray = [];
$authorsUpsertArray = [];
foreach ($this->comic['list'] as $comic) {
$comicsUpsertArray[] = [
'pathword' => $comic['path_word'],
'uuid' => '',
'name' => $comic['name'],
'alias' => '{}',
'description' => '',
'cover' => $comic['cover'],
'upstream_updated_at' => $comic['datetime_updated'] ?? null,
];
foreach ($comic['author'] as $author) {
$authorsUpsertArray[] = [
'name' => $author['name']
];
}
}
// Do an upsert for comics
Comic::upsert($comicsUpsertArray, uniqueBy: 'pathword', update: ['upstream_updated_at']);
Author::upsert($authorsUpsertArray, uniqueBy: 'name');
// Had to do a second pass to insert the relationships
foreach ($this->comic['list'] as $comic) {
// Get the comic id
$comicObj = Comic::where('pathword', $comic['path_word'])->first();
foreach ($comic['author'] as $author) {
$authorObj = Author::where('name', $author['name'])->first();
$comicObj->authors()->sync($authorObj);
}
}
Log::info('JOB ComicUpsert END');
$this->queueProgress(100);
}
}

9
app/Jobs/ImageUpsert.php
View File

@@ -6,10 +6,11 @@ use App\Models\Image;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Support\Facades\Log;
use romanzipp\QueueMonitor\Traits\IsMonitored;
class ImageUpsert implements ShouldQueue
{
use Queueable;
use IsMonitored, Queueable;
/**
* Create a new job instance.
@@ -25,7 +26,12 @@ class ImageUpsert implements ShouldQueue
*/
public function handle(): void
{
$this->queueProgress(0);
Log::info("JOB ImageUpsert START, comicId: {$this->comicId}, chapterId: {$this->chapterId}");
$this->queueData([
'comicId' => $this->comicId,
'chapterId' => $this->chapterId,
]);
$arrayForUpsert = [];
@@ -47,5 +53,6 @@ class ImageUpsert implements ShouldQueue
Image::upsert($arrayForUpsert, uniqueBy: 'url');
Log::info('JOB ImageUpsert END');
$this->queueProgress(100);
}
}

29
app/Jobs/RemotePrefetch.php
View File

@@ -6,10 +6,11 @@ use App\Remote\CopyManga;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Support\Facades\Log;
use romanzipp\QueueMonitor\Traits\IsMonitored;
class RemotePrefetch implements ShouldQueue
{
use Queueable;
use IsMonitored, Queueable;
/**
* Create a new job instance.
@@ -24,11 +25,17 @@ class RemotePrefetch implements ShouldQueue
*/
public function handle(): void
{
$this->queueProgress(0);
$copyManga = new CopyManga();
switch ($this->action) {
case 'chapter':
Log::info("JOB RemotePrefetch START, action '{$this->action}', Pathword: {$this->parameters['pathword']}, UUID: {$this->parameters['uuid']}");
$this->queueData([
'action' => $this->action,
'pathword' => $this->parameters['pathword'],
'uuid' => $this->parameters['uuid']
]);
$copyManga->chapter($this->parameters['pathword'], $this->parameters['uuid']);
@@ -37,15 +44,25 @@ class RemotePrefetch implements ShouldQueue
case 'chapters':
// TODO:
break;
case 'index':
// TODO
break;
case 'tags':
// TODO
case 'comics':
Log::info("JOB RemotePrefetch START, action '{$this->action}', Offset: {$this->parameters['offset']}");
$this->queueData([
'action' => $this->action,
'offset' => $this->parameters['offset'],
'limit' => $this->parameters['limit'],
'top' => $this->parameters['top'],
'params' => $this->parameters['params']
]);
$copyManga->comics($this->parameters['offset'], $this->parameters['limit'], $this->parameters['top'], $this->parameters['params']);
Log::info("JOB RemotePrefetch END, action '{$this->action}'");
break;
default:
Log::info("JOB RemotePrefetch Unknown action '{$this->action}'");
break;
}
$this->queueProgress(100);
}
}

4
app/Remote/CopyManga.php
View File

@@ -296,7 +296,7 @@ class CopyManga
public function legacyChapter(string $comic, string $chapter): array
{
$userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36";
$responses = $this->execute($this->legacyBuildUrl("comic/{$comic}/chapter/{$chapter}"), "GET", $userAgent, ttl: 24 * 60 * 60);
$responses = $this->execute($this->legacyBuildUrl("comic/{$comic}/chapter/{$chapter}"), "GET", $userAgent, ttl: 24 * 60 * 60 * 30);
// Get Content Key
$dom = new DOMDocument();
@@ -346,7 +346,7 @@ class CopyManga
*/
public function chapter(string $comic, string $chapter, array $parameters = []): array
{
$responses = $this->execute($this->buildUrl("comic/{$comic}/chapter2/{$chapter}", $parameters), ttl: 24 * 60 * 60);
$responses = $this->execute($this->buildUrl("comic/{$comic}/chapter2/{$chapter}", $parameters), ttl: 24 * 60 * 60 * 30);
if ($this->legacyImagesFetch) {
$responses['sorted'] = $this->legacyChapter($comic, $chapter);

2
app/Remote/ImageFetcher.php
View File

@@ -14,7 +14,7 @@ class ImageFetcher
'cache' => [
'enabled' => true,
'prefix' => 'image_fetcher_',
'ttl' => 3600,
'ttl' => 60 * 60 * 24 * 30,
],
'http' => [
'timeout' => 60,