Skip to content

Commit

Permalink
Update AnalyzeAllArtworks command [WEB-2995]
Browse files Browse the repository at this point in the history
  • Loading branch information
@web-dev-trev
web-dev-trev committed Jan 14, 2025
1 parent 9723969 commit e01fcd1
Showing 1 changed file with 151 additions and 59 deletions.
210 changes: 151 additions & 59 deletions app/Console/Commands/AI/AnalyzeAllArtworks.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@
use App\Services\DescriptionService;
use App\Models\Collections\Artwork;
use Illuminate\Support\Facades\Log;
use Symfony\Component\Console\Output\OutputInterface;
use Carbon\Carbon;
use Exception;

class AnalyzeAllArtworks extends BaseCommand
{
protected $signature = 'ai:analyze-all {--days=30 : Number of days before re-analyzing artwork}';
protected $signature = 'ai:analyze-all {--days=30 : Number of days before re-analyzing artwork}
{--start-id= : Start processing from this artwork ID}';
protected $description = 'Analyze all artworks that need embeddings or haven\'t been analyzed recently';

protected EmbeddingService $embeddingService;
Expand All @@ -29,63 +31,136 @@ public function __construct(

public function handle(): int
{
$this->getAicLogo();
$this->info($this->getAicLogo(), OutputInterface::VERBOSITY_VERBOSE);

try {
$daysThreshold = $this->option('days');
$startId = $this->option('start-id');
$cutoffDate = Carbon::now()->subDays($daysThreshold);

$this->info("Starting artwork analysis...");
$this->info("Re-analyzing artworks last processed before: " . $cutoffDate->toDateTimeString());
$this->info("Starting artwork analysis...", OutputInterface::VERBOSITY_VERBOSE);
$this->info(
"Re-analyzing artworks last processed before: " . $cutoffDate->toDateTimeString(),
OutputInterface::VERBOSITY_VERBOSE
);

if ($startId) {
$this->info("Starting from artwork ID: " . $startId, OutputInterface::VERBOSITY_VERBOSE);
}

$artworks = $this->getArtworksToProcess($cutoffDate);
$total = $artworks->count();
// Get total count first
$total = $this->getArtworksCount($startId);

if ($total === 0) {
$this->info("No artworks need processing at this time.");
return 0;
}

$this->info("Found {$total} artworks to process");
$this->info("Found {$total} artworks to process", OutputInterface::VERBOSITY_VERBOSE);
$bar = $this->output->createProgressBar($total);

// Define custom placeholder names
$bar->setMessage('Processing artworks...');
$bar->setMessage('Starting...', 'title');

$bar->setFormat(
" %message%\n" .
" %current%/%max% [%bar%] %percent:3s%%\n" .
" Current: %title%\n" .
" Elapsed: %elapsed:6s%"
);

$processed = 0;
$skipped = 0;
$errors = 0;

foreach ($artworks as $artwork) {
try {
if ($this->shouldProcessArtwork($artwork, $cutoffDate)) {
$this->processArtwork($artwork);
$errors = [];
$startTime = now();
$chunkSize = 100; // Process 100 artworks at a time

$query = $this->getArtworksToProcess($startId);
$query->chunk($chunkSize, function ($artworks) use (
&$processed,
&$skipped,
&$errors,
$bar,
$total
) {
foreach ($artworks as $artwork) {
// Update the progress bar with current artwork title
$bar->setMessage($artwork->title ?? "Artwork #{$artwork->id}", 'title');

try {
$this->info(
"\nProcessing artwork: {$artwork->title} (ID: {$artwork->id})",
OutputInterface::VERBOSITY_VERBOSE
);

$imageUrl = $this->buildImageUrl($artwork);
$this->info("Image URL: {$imageUrl}", OutputInterface::VERBOSITY_VERBOSE);

$analysisResults = $this->analyzeImage($artwork, $imageUrl);
$this->processEmbeddings($artwork, $imageUrl, $analysisResults);

$processed++;
} else {
$skipped++;
} catch (Exception $e) {
$errors[] = [
'id' => $artwork->id,
'title' => $artwork->title,
'error' => $e->getMessage()
];

Log::error('Error processing artwork:', [
'artwork_id' => $artwork->id,
'message' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);

$this->error(
"\nFailed processing artwork ID {$artwork->id}: {$e->getMessage()}",
OutputInterface::VERBOSITY_VERBOSE
);
}
} catch (Exception $e) {
$errors++;
Log::error('Error processing artwork:', [
'artwork_id' => $artwork->id,
'message' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
}

$bar->advance();
}
$bar->advance();

// Update progress message with current statistics
$completedPercentage = ($processed + count($errors)) / $total * 100;
$errorPercentage = count($errors) / $total * 100;
$bar->setMessage(sprintf(
'Progress: %.1f%% complete, %.1f%% failed',
$completedPercentage,
$errorPercentage
));
}
});

$bar->finish();
$endTime = now();
$duration = $endTime->diffForHumans($startTime, ['parts' => 2]);

$this->newLine(2);
$this->info("Analysis completed in {$duration}!", OutputInterface::VERBOSITY_VERBOSE);

$this->table(
['Metric', 'Count'],
['Component', 'Status'],
[
['Total Artworks', $total],
['Processed', $processed],
['Skipped', $skipped],
['Errors', $errors]
['Successfully Processed', sprintf("%d (%.1f%%)", $processed, $processed / $total * 100)],
['Skipped', sprintf("%d (%.1f%%)", $skipped, $skipped / $total * 100)],
['Failed', sprintf("%d (%.1f%%)", count($errors), count($errors) / $total * 100)]
]
);

if (count($errors) > 0) {
$this->newLine();
$this->error('Failed Artworks:');
$this->table(
['Artwork ID', 'Title', 'Error'],
$errors
);
$this->info("To resume processing from the last failed artwork, run:");
$this->line("php artisan ai:analyze-all --start-id=" . end($errors)['id']);
}

return 0;
} catch (Exception $e) {
$this->error("Error: " . $e->getMessage());
Expand All @@ -97,40 +172,26 @@ public function handle(): int
}
}

protected function getArtworksToProcess(Carbon $cutoffDate): \Illuminate\Database\Eloquent\Collection
protected function getArtworksCount(?int $startId = null): int
{
return Artwork::query()
->whereHas('image')
->where(function ($query) use ($cutoffDate) {
$query->whereDoesntHave('embeddings')
->orWhereHas('embeddings', function ($q) use ($cutoffDate) {
$q->where('generated_at', '<', $cutoffDate);
});
})
->get();
}
$query = Artwork::query();

protected function shouldProcessArtwork(Artwork $artwork, Carbon $cutoffDate): bool
{
// Check if artwork has any embeddings
if (!$artwork->embeddings()->exists()) {
return true;
if ($startId) {
$query->where('id', '>=', $startId);
}

// Get the most recent embedding
$latestEmbedding = $artwork->embeddings()
->latest('generated_at')
->first();

// Process if the latest embedding is older than the cutoff date
return $latestEmbedding && Carbon::parse($latestEmbedding->generated_at)->lt($cutoffDate);
return $query->count();
}

protected function processArtwork(Artwork $artwork): void
protected function getArtworksToProcess(?int $startId = null): \Illuminate\Database\Eloquent\Builder
{
$imageUrl = $this->buildImageUrl($artwork);
$analysisResults = $this->analyzeImage($artwork, $imageUrl);
$this->processEmbeddings($artwork, $imageUrl, $analysisResults);
$query = Artwork::query();

if ($startId) {
$query->where('id', '>=', $startId);
}

return $query->orderBy('id');
}

protected function buildImageUrl(Artwork $artwork): string
Expand All @@ -147,8 +208,11 @@ protected function buildImageUrl(Artwork $artwork): string

protected function analyzeImage(Artwork $artwork, string $imageUrl): array
{
$this->info("\nPerforming image analysis...", OutputInterface::VERBOSITY_VERBOSE);

// Get image description
$generatedDescription = $this->embeddingService->getImageDescription($imageUrl);
$this->info("Generated base description", OutputInterface::VERBOSITY_VERBOSE);

// Get AIC description if available
$aicDescription = $artwork->description;
Expand All @@ -158,6 +222,7 @@ protected function analyzeImage(Artwork $artwork, string $imageUrl): array
$aicDescription,
$generatedDescription
);
$this->info("Generated summarized description", OutputInterface::VERBOSITY_VERBOSE);

return [
'generated' => $generatedDescription,
Expand All @@ -171,13 +236,40 @@ protected function processEmbeddings(
string $imageUrl,
array $analysisResults
): void {
$this->info("\nProcessing embeddings...", OutputInterface::VERBOSITY_VERBOSE);

// Get and save image embeddings
$imageEmbeddingArray = $this->embeddingService->getImageEmbeddings($imageUrl);
$this->saveImageEmbeddings($artwork, $imageEmbeddingArray, $imageUrl, $analysisResults);

$this->info(
"Image embedding response type: " . gettype($imageEmbeddingArray),
OutputInterface::VERBOSITY_VERBOSE
);

if (is_array($imageEmbeddingArray)) {
$this->info(
"Image embedding array count: " . count($imageEmbeddingArray),
OutputInterface::VERBOSITY_VERBOSE
);
}

try {
$this->saveImageEmbeddings($artwork, $imageEmbeddingArray, $imageUrl, $analysisResults);
$this->info("Saved image embeddings", OutputInterface::VERBOSITY_VERBOSE);
} catch (\Exception $e) {
throw new Exception("Failed to save image embeddings: " . $e->getMessage());
}

// Get and save text embeddings
$this->info("\nGetting text embeddings...", OutputInterface::VERBOSITY_VERBOSE);
$textEmbeddingArray = $this->embeddingService->getEmbeddings($analysisResults['summarized']);
$this->saveTextEmbeddings($artwork, $textEmbeddingArray, $imageUrl, $analysisResults);

try {
$this->saveTextEmbeddings($artwork, $textEmbeddingArray, $imageUrl, $analysisResults);
$this->info("Saved text embeddings", OutputInterface::VERBOSITY_VERBOSE);
} catch (\Exception $e) {
throw new Exception("Failed to save text embeddings: " . $e->getMessage());
}
}

protected function saveImageEmbeddings(
Expand Down Expand Up @@ -221,4 +313,4 @@ protected function saveTextEmbeddings(
]
);
}
}
}

0 comments on commit e01fcd1

Please sign in to comment.