ROBOTSTXT/robotstxt-mediaaudit
Watch
2
0
Fork
You've already forked robotstxt-mediaaudit
 0
Code Issues Pull requests Projects Releases 11 Packages Wiki Activity Actions

v1.3.0

Browse source
This commit is contained in:
Javier Casares 2026-06-03 06:26:23 +00:00
commit 672d0f295f
11 changed files with 438 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

389
includes/External/ExternalScanner.php vendored
View file

@ -13,6 +13,9 @@ use MediaRightsAudit\Core\Queue\Scheduler;
* Processes the external scan queue: runs each pending attachment through
* all active providers and stores results in mra_external_results.
*
* Per-provider scan state is tracked in mra_provider_status. The aggregate
* external_status in mra_media_index is updated after each scan for display.
*
* Active providers are resolved at runtime via the mra/external/providers filter.
* Scanning is triggered manually by an administrator (Run External Scan bulk action
* or wp mra scan-external CLI) — never automatically, to control API costs.
@ -27,11 +30,14 @@ class ExternalScanner {
const AS_HOOK = 'mra_external_scan_batch';
/**
* Scans one batch of pending attachments with all active providers.
* Scans one batch of queued attachments with their pending providers.
*
* @param int $batch_size Maximum number of attachments to process.
* Queries mra_provider_status for queued entries and processes each
* attachment with the intersection of active providers and queued slugs.
*
* @return int Number of attachments attempted.
* @param int $batch_size Maximum number of attachment rows to process.
*
* @return int Number of attachment rows attempted.
*/
public static function scan_batch( int $batch_size = 10 ): int {
global $wpdb;
@ -41,13 +47,20 @@ class ExternalScanner {
return 0;
}
// Index providers by slug for fast intersection lookup.
$provider_map = array();
foreach ( $providers as $provider ) {
$provider_map[ $provider->provider_slug() ] = $provider;
}
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$rows = $wpdb->get_results(
$wpdb->prepare(
"SELECT attachment_id, file_url
FROM {$wpdb->prefix}mra_media_index
WHERE external_status = 'queued'
ORDER BY attachment_id ASC
"SELECT DISTINCT ps.attachment_id, mi.file_url
FROM {$wpdb->prefix}mra_provider_status ps
JOIN {$wpdb->prefix}mra_media_index mi ON mi.attachment_id = ps.attachment_id
WHERE ps.status = 'queued'
ORDER BY ps.attachment_id ASC
LIMIT %d",
$batch_size
),
@ -69,14 +82,41 @@ class ExternalScanner {
continue;
}
self::process_attachment( $aid, $url, $providers );
// Get queued provider slugs for this attachment.
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$queued_slugs = $wpdb->get_col(
$wpdb->prepare(
"SELECT provider FROM {$wpdb->prefix}mra_provider_status
WHERE attachment_id = %d AND status = 'queued'",
$aid
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
if ( empty( $queued_slugs ) ) {
continue;
}
// Intersect queued slugs with currently active providers.
$batch_providers = array();
foreach ( $queued_slugs as $slug ) {
if ( is_string( $slug ) && isset( $provider_map[ $slug ] ) ) {
$batch_providers[] = $provider_map[ $slug ];
}
}
if ( empty( $batch_providers ) ) {
continue;
}
self::process_attachment( $aid, $url, $batch_providers );
}
return count( $rows );
}
/**
* Returns the count of attachments still waiting for external scanning.
* Returns the count of provider-attachment rows still waiting for external scanning.
*
* @return int
*/
@ -84,7 +124,7 @@ class ExternalScanner {
global $wpdb;
$result = $wpdb->get_var( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
"SELECT COUNT(*) FROM {$wpdb->prefix}mra_media_index WHERE external_status = 'queued'"
"SELECT COUNT(*) FROM {$wpdb->prefix}mra_provider_status WHERE status = 'queued'"
);
return is_numeric( $result ) ? (int) $result : 0;
@ -106,40 +146,22 @@ class ExternalScanner {
}
/**
* Promotes all attachments with external_status = 'pending' to 'queued'.
* Marks all pending provider rows as queued and schedules a background scan batch.
*
* Does not schedule an Action Scheduler action. Useful in CLI context where
* processing runs synchronously in a loop rather than via background jobs.
*
* @return void
*/
public static function queue_all_pending(): void {
global $wpdb;
$wpdb->query( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
"UPDATE {$wpdb->prefix}mra_media_index
SET external_status = 'queued'
WHERE external_status = 'pending'"
);
}
/**
* Marks all pending attachments as queued and schedules a background scan batch.
*
* Moves every attachment with external_status = 'pending' to 'queued' so the
* status change is visible immediately in the admin list. Only schedules the
* Action Scheduler action if there are now queued items and no batch is already running.
* Calls queue_provider_attachments() for each active provider to populate
* mra_provider_status, then syncs the aggregate status on mra_media_index.
* Only schedules the Action Scheduler action if there are queued items and
* no batch is already running.
*
* @return void
*/
public static function schedule(): void {
global $wpdb;
$providers = self::get_active_providers();
foreach ( $providers as $provider ) {
self::queue_provider_attachments( $provider->provider_slug() );
}
$wpdb->query( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
"UPDATE {$wpdb->prefix}mra_media_index
SET external_status = 'queued'
WHERE external_status = 'pending'"
);
self::sync_aggregate_queued_status();
if ( self::get_pending_count() > 0 && ! Scheduler::has_pending( self::AS_HOOK ) ) {
Scheduler::schedule_single( self::AS_HOOK );
@ -147,10 +169,75 @@ class ExternalScanner {
}
/**
* Marks specific attachments as pending and schedules a scan batch.
* Promotes all pending provider rows to queued for all active providers.
*
* Idempotent: already-pending attachments are left unchanged;
* previously-scanned or error attachments are reset to pending.
* Does not schedule an Action Scheduler action. Useful in CLI context where
* processing runs synchronously in a loop rather than via background jobs.
*
* @return void
*/
public static function queue_all_pending(): void {
$providers = self::get_active_providers();
foreach ( $providers as $provider ) {
self::queue_provider_attachments( $provider->provider_slug() );
}
self::sync_aggregate_queued_status();
}
/**
* Creates scan entries for newly registered providers.
*
* For each active provider, inserts mra_provider_status rows for attachments
* that have no existing row for that provider. Already-scanned or error rows
* are not touched. If new rows are created, syncs aggregate status and
* schedules a background scan batch.
*
* @return int Total number of new rows inserted.
*/
public static function sync_providers(): int {
global $wpdb;
$providers = self::get_active_providers();
$total = 0;
foreach ( $providers as $provider ) {
$slug = $provider->provider_slug();
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$wpdb->query(
$wpdb->prepare(
"INSERT IGNORE INTO {$wpdb->prefix}mra_provider_status (attachment_id, provider, status)
SELECT mi.attachment_id, %s, 'queued'
FROM {$wpdb->prefix}mra_media_index mi
LEFT JOIN {$wpdb->prefix}mra_provider_status ps
ON ps.attachment_id = mi.attachment_id AND ps.provider = %s
WHERE ps.attachment_id IS NULL",
$slug,
$slug
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$total += (int) $wpdb->rows_affected;
}
if ( $total > 0 ) {
self::sync_aggregate_queued_status();
if ( ! Scheduler::has_pending( self::AS_HOOK ) ) {
Scheduler::schedule_single( self::AS_HOOK );
}
}
return $total;
}
/**
* Marks specific attachments as queued for all active providers and schedules a scan batch.
*
* Idempotent: already-queued or scanned rows are left unchanged;
* error rows are reset to queued.
*
* @param array<int, int> $attachment_ids Attachment IDs to queue. Must be non-empty.
*
@ -165,49 +252,58 @@ class ExternalScanner {
$ids = array_map( 'intval', $attachment_ids );
$placeholders = implode( ',', array_fill( 0, count( $ids ), '%d' ) );
$providers = self::get_active_providers();
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared,WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare
$wpdb->query(
$wpdb->prepare(
"UPDATE {$wpdb->prefix}mra_media_index
SET external_status = 'queued', external_scanned_at = NULL
WHERE attachment_id IN ({$placeholders})
AND external_status != 'queued'",
...$ids
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared,WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare
foreach ( $providers as $provider ) {
$slug = $provider->provider_slug();
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared,WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare
$wpdb->query(
$wpdb->prepare(
"INSERT INTO {$wpdb->prefix}mra_provider_status (attachment_id, provider, status)
SELECT attachment_id, %s, 'queued'
FROM {$wpdb->prefix}mra_media_index
WHERE attachment_id IN ({$placeholders})
ON DUPLICATE KEY UPDATE status = IF(status NOT IN ('scanned','queued'),'queued',status)",
array_merge( array( $slug ), $ids )
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared,WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare
}
self::sync_aggregate_queued_status();
// Schedule directly — do not call schedule() which would also mark all 'pending' items.
if ( ! Scheduler::has_pending( self::AS_HOOK ) ) {
Scheduler::schedule_single( self::AS_HOOK );
}
}
/**
* Re-queues all attachments that previously failed with external_status = 'error'.
* Re-queues all provider rows that previously failed with status = 'error'.
*
* Does not touch pending, queued, scanned, or matched attachments.
* Does not touch pending, queued, or scanned rows.
* Schedules a background scan batch if any items were re-queued and
* no batch is already pending.
*
* @return int Number of attachments re-queued.
* @return int Number of provider rows re-queued.
*/
public static function requeue_errors(): int {
global $wpdb;
// phpcs:disable WordPress.DB.DirectDatabaseQuery
$count = (int) $wpdb->get_var(
"SELECT COUNT(*) FROM {$wpdb->prefix}mra_media_index WHERE external_status = 'error'"
"SELECT COUNT(*) FROM {$wpdb->prefix}mra_provider_status WHERE status = 'error'"
);
if ( $count > 0 ) {
$wpdb->query(
"UPDATE {$wpdb->prefix}mra_media_index
SET external_status = 'queued', external_scanned_at = NULL
WHERE external_status = 'error'"
"UPDATE {$wpdb->prefix}mra_provider_status
SET status = 'queued', scanned_at = NULL
WHERE status = 'error'"
);
self::sync_aggregate_queued_status();
if ( ! Scheduler::has_pending( self::AS_HOOK ) ) {
Scheduler::schedule_single( self::AS_HOOK );
}
@ -243,6 +339,14 @@ class ExternalScanner {
)
);
$wpdb->query(
$wpdb->prepare(
"DELETE FROM {$wpdb->prefix}mra_provider_status
WHERE attachment_id IN ({$placeholders})",
...$ids
)
);
$wpdb->query(
$wpdb->prepare(
"UPDATE {$wpdb->prefix}mra_media_index
@ -257,7 +361,7 @@ class ExternalScanner {
/**
* Deletes all external scan results and resets every attachment's external_status to pending.
*
* @return int Number of result rows deleted.
* @return int Number of result rows deleted from mra_external_results.
*/
public static function purge_all(): int {
global $wpdb;
@ -265,6 +369,7 @@ class ExternalScanner {
// phpcs:disable WordPress.DB.DirectDatabaseQuery
$count = (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->prefix}mra_external_results" );
$wpdb->query( "DELETE FROM {$wpdb->prefix}mra_external_results" );
$wpdb->query( "DELETE FROM {$wpdb->prefix}mra_provider_status" );
$wpdb->query(
"UPDATE {$wpdb->prefix}mra_media_index
SET external_status = 'pending', external_scanned_at = NULL"
@ -277,6 +382,8 @@ class ExternalScanner {
/**
* Returns a count of attachments grouped by external_status.
*
* Reads from the mra_media_index aggregate column for display purposes.
*
* @return array<string, int>
*/
public static function get_status_counts(): array {
@ -315,35 +422,55 @@ class ExternalScanner {
// -------------------------------------------------------------------------
/**
* Scans a single attachment with all providers and updates the index.
* Scans a single attachment with specified providers and updates per-provider status.
*
* After scanning all providers, calls update_aggregate_status() to sync
* the mra_media_index aggregate column.
*
* @param int $attachment_id WordPress attachment ID.
* @param string $file_url Public URL of the image.
* @param array<AbstractProvider> $providers Resolved provider list.
* @param array<AbstractProvider> $providers Resolved provider list for this attachment.
*
* @return void
*/
private static function process_attachment( int $attachment_id, string $file_url, array $providers ): void {
global $wpdb;
$any_success = false;
$any_match = false;
foreach ( $providers as $provider ) {
$slug = $provider->provider_slug();
try {
$result = $provider->scan( $attachment_id, $file_url );
self::save_result( $attachment_id, $provider->provider_slug(), $result );
$any_success = true;
if ( $result->match_count() > 0 ) {
$any_match = true;
}
self::save_result( $attachment_id, $slug, $result );
$wpdb->query( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
$wpdb->prepare(
// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
"UPDATE {$wpdb->prefix}mra_provider_status
SET status = 'scanned', scanned_at = NOW()
WHERE attachment_id = %d AND provider = %s",
$attachment_id,
$slug
)
);
} catch ( \RuntimeException $e ) {
$wpdb->query( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
$wpdb->prepare(
// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
"UPDATE {$wpdb->prefix}mra_provider_status
SET status = 'error', scanned_at = NOW()
WHERE attachment_id = %d AND provider = %s",
$attachment_id,
$slug
)
);
if ( defined( 'WP_DEBUG_LOG' ) && WP_DEBUG_LOG ) {
// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log
error_log(
sprintf(
'[MRA] %s scan failed for attachment %d: %s',
$provider->provider_slug(),
$slug,
$attachment_id,
$e->getMessage()
)
@ -352,14 +479,69 @@ class ExternalScanner {
}
}
if ( ! $any_success ) {
$new_status = 'error';
} elseif ( $any_match ) {
$new_status = 'matches';
} else {
$new_status = 'scanned';
self::update_aggregate_status( $attachment_id );
}
/**
* Recomputes and persists the aggregate external_status for a single attachment.
*
* Logic:
* - Any provider still queued → 'queued'
* - No provider succeeded → 'error'
* - At least one match found → 'matches'
* - Otherwise → 'scanned'
*
* @param int $attachment_id WordPress attachment ID.
*
* @return void
*/
private static function update_aggregate_status( int $attachment_id ): void {
global $wpdb;
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$row = $wpdb->get_row(
$wpdb->prepare(
"SELECT
SUM(status = 'queued') AS queued_count,
SUM(status = 'error') AS error_count,
SUM(status = 'scanned') AS scanned_count
FROM {$wpdb->prefix}mra_provider_status
WHERE attachment_id = %d",
$attachment_id
),
ARRAY_A
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
if ( ! is_array( $row ) ) {
return;
}
$queued_count = isset( $row['queued_count'] ) && is_numeric( $row['queued_count'] ) ? (int) $row['queued_count'] : 0;
$error_count = isset( $row['error_count'] ) && is_numeric( $row['error_count'] ) ? (int) $row['error_count'] : 0;
$scanned_count = isset( $row['scanned_count'] ) && is_numeric( $row['scanned_count'] ) ? (int) $row['scanned_count'] : 0;
if ( $queued_count > 0 ) {
$new_status = 'queued';
} elseif ( 0 === $scanned_count ) {
$new_status = 'error';
} else {
// Check for matches across scanned providers.
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$match_sum = $wpdb->get_var(
$wpdb->prepare(
"SELECT SUM(match_count) FROM {$wpdb->prefix}mra_external_results WHERE attachment_id = %d",
$attachment_id
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$new_status = ( is_numeric( $match_sum ) && (int) $match_sum > 0 ) ? 'matches' : 'scanned';
}
// Suppress unused variable inspection: $error_count is available for future use.
unset( $error_count );
$wpdb->update( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
$wpdb->prefix . 'mra_media_index',
array(
@ -372,6 +554,57 @@ class ExternalScanner {
);
}
/**
* Updates mra_media_index.external_status to 'queued' for all attachments
* that have at least one queued row in mra_provider_status.
*
* Only updates rows not already marked 'queued', to avoid unnecessary writes.
*
* @return void
*/
private static function sync_aggregate_queued_status(): void {
global $wpdb;
$wpdb->query( // phpcs:ignore WordPress.DB.DirectDatabaseQuery
// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
"UPDATE {$wpdb->prefix}mra_media_index mi
INNER JOIN (
SELECT DISTINCT attachment_id
FROM {$wpdb->prefix}mra_provider_status
WHERE status = 'queued'
) AS q ON q.attachment_id = mi.attachment_id
SET mi.external_status = 'queued'
WHERE mi.external_status != 'queued'"
);
}
/**
* Inserts or updates mra_provider_status rows for all indexed attachments.
*
* For attachments without an existing row for this provider, inserts 'queued'.
* For attachments with an existing non-scanned, non-queued row, resets to 'queued'.
* Scanned and already-queued rows are left unchanged.
*
* @param string $provider_slug Provider machine-readable slug.
*
* @return void
*/
private static function queue_provider_attachments( string $provider_slug ): void {
global $wpdb;
// phpcs:disable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
$wpdb->query(
$wpdb->prepare(
"INSERT INTO {$wpdb->prefix}mra_provider_status (attachment_id, provider, status)
SELECT attachment_id, %s, 'queued'
FROM {$wpdb->prefix}mra_media_index
ON DUPLICATE KEY UPDATE status = IF(status NOT IN ('scanned','queued'),'queued',status)",
$provider_slug
)
);
// phpcs:enable WordPress.DB.DirectDatabaseQuery,WordPress.DB.PreparedSQL.InterpolatedNotPrepared
}
/**
* Persists a provider's scan result (upsert via REPLACE INTO).
*