<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Aditya Prajapati aditya@jkprajapati.tech
* @package seek_quarry
* @subpackage library
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\media_jobs;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\MediaConstants;
/**
* MediaJob to handle audio file transcription using Whisper
*
* This job processes audio files uploaded to group pages and wiki pages
* by generating text transcripts using OpenAI's Whisper speech recognition
* model. Transcripts are stored alongside the original audio files and
* can be displayed to users via the web interface.
*/
class AudioTranscriptionJob extends MediaJob
{
/**
* Maximum number of concurrent transcription tasks
*/
const MAX_CONCURRENT_TASKS = 3;
/**
* Maximum retry attempts for failed transcriptions
*/
const MAX_RETRY_ATTEMPTS = 3;
/**
* Supported audio file types for transcription
* @var array
*/
public $supported_audio_types = [
'audio/mpeg', 'audio/mp3', 'audio/wav', 'audio/ogg',
'audio/mp4', 'audio/webm', 'audio/m4a'
];
/**
* Initialize the job configuration
*/
public function init()
{
$this->name_server_does_client_tasks = true;
$this->name_server_does_client_tasks_only = true;
}
/**
* Check if prerequisites for transcription are met
* Verifies Whisper installation and checks for pending tasks
*
* @return bool whether prerequisites are satisfied
*/
public function checkPrerequisites()
{
// Check if Whisper is installed and configured
if (!C\nsdefined('WHISPER') || !function_exists("exec")) {
L\crawlLog("AudioTranscriptionJob: Whisper not configured or " .
"exec() not available");
return false;
}
// Verify Whisper binary exists
$whisper_path = C\WHISPER;
if (!is_executable($whisper_path)) {
L\crawlLog("AudioTranscriptionJob: Whisper binary not found or " .
"not executable at: $whisper_path");
return false;
}
// Check for pending transcription tasks
$task_dir = C\SCHEDULES_DIR . "/audio_transcribe";
if (!file_exists($task_dir)) {
return false;
}
$pending_files = glob($task_dir . "/*.txt");
if (empty($pending_files)) {
return false;
}
// Check for tasks that are actually pending
foreach ($pending_files as $file) {
$task_data = unserialize(file_get_contents($file));
if ($task_data) {
$status_file = $task_dir . "/" . $task_data['task_id'] .
".status";
if (file_exists($status_file)) {
$status = file_get_contents($status_file);
if ($status === 'pending' || $status === 'failed') {
return true;
}
}
}
}
return false;
}
/**
* Handle nondistributed mode by directly processing tasks
*/
public function nondistributedTasks()
{
L\crawlLog("AudioTranscriptionJob: Running in nondistributed mode");
$this->prepareTasks();
if (!empty($this->tasks)) {
foreach ($this->tasks as $task) {
$result = $this->doTasks($task);
if (!$result['success']) {
L\crawlLog("AudioTranscriptionJob: Task failed - " .
$result['error']);
}
}
}
}
/**
* Prepare tasks for audio transcription
* Scans for audio files that need transcription and filters by status
*/
public function prepareTasks()
{
$task_dir = C\SCHEDULES_DIR . "/audio_transcribe";
if (!file_exists($task_dir)) {
return;
}
$tasks = [];
$files = glob($task_dir . "/*.txt");
$current_processing = $this->countProcessingTasks($task_dir);
foreach ($files as $file) {
// Limit concurrent processing
if (count($tasks) + $current_processing >=
self::MAX_CONCURRENT_TASKS) {
break;
}
$task_data = unserialize(file_get_contents($file));
if (!$task_data || !isset($task_data['task_id'])) {
continue;
}
$status_file = $task_dir . "/" . $task_data['task_id'] . ".status";
if (file_exists($status_file)) {
$status = file_get_contents($status_file);
// Process pending tasks or retry failed tasks
if ($status === 'pending' ||
($status === 'failed' && $this->shouldRetry($task_data))) {
// Validate task data
if ($this->validateTaskData($task_data)) {
$tasks[] = $task_data;
} else {
L\crawlLog("AudioTranscriptionJob: Invalid task data, " .
"removing task " . $task_data['task_id']);
$this->cleanupTask($task_data['task_id'], $task_dir);
}
} else if ($status === 'failed' &&
!$this->shouldRetry($task_data)) {
// Handle permanently failed tasks (max retries exceeded)
L\crawlLog("AudioTranscriptionJob: Task " .
$task_data['task_id'] . " permanently failed after " .
($task_data['retry_count'] ?? 0) . " attempts");
$this->handlePermanentFailure($task_data, $task_dir);
}
}
}
$this->tasks = $tasks;
L\crawlLog("AudioTranscriptionJob: Prepared " . count($tasks) .
" tasks for processing");
}
/**
* Process audio file and generate transcript
*
* @param array $task Audio file task data
* @return array Result of transcription
*/
public function doTasks($task)
{
$task_id = $task['task_id'];
$filename = $task['filename'];
L\crawlLog("AudioTranscriptionJob: Starting transcription for " .
"task $task_id: $filename");
$task_dir = C\SCHEDULES_DIR . "/audio_transcribe";
$status_file = $task_dir . "/" . $task_id . ".status";
$task_file = $task_dir . "/" . $task_id . ".txt";
// Update status to processing
file_put_contents($status_file, 'processing');
$audio_path = $task['audio_path'];
// Validate audio file exists and is readable
if (!file_exists($audio_path) || !is_readable($audio_path)) {
$error = "Audio file not found or not readable: $audio_path";
L\crawlLog("AudioTranscriptionJob: $error");
$task['retry_count'] = ($task['retry_count'] ?? 0) + 1;
file_put_contents($task_file, serialize($task));
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => $error,
'file' => $filename
];
}
// Check if audio format is supported, convert if needed
$working_audio_path = $audio_path;
$converted_file = null;
if (!$this->isSupportedAudioFormat($filename)) {
L\crawlLog("AudioTranscriptionJob: Unsupported format detected: " .
"$filename, attempting conversion");
$converted_file = dirname($audio_path) . "/" .
pathinfo($filename, PATHINFO_FILENAME) . "_converted.m4a";
if (!$this->convertAudioFormat($audio_path, $converted_file)) {
$error = "Failed to convert unsupported audio format: " .
$filename;
L\crawlLog("AudioTranscriptionJob: $error");
$task['retry_count'] = ($task['retry_count'] ?? 0) + 1;
file_put_contents($task_file, serialize($task));
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => $error,
'file' => $filename
];
}
$working_audio_path = $converted_file;
L\crawlLog("AudioTranscriptionJob: Successfully converted to: " .
$converted_file);
}
// Check file size
$file_size = filesize($audio_path);
if ($file_size > C\MAX_AUDIO_TRANSCRIBE_SIZE) {
$error = "Audio file too large: " .
round($file_size / 1024 / 1024, 2) . "MB";
L\crawlLog("AudioTranscriptionJob: $error");
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => $error,
'file' => $filename
];
}
$transcript_path = dirname($audio_path) . "/" .
pathinfo($filename, PATHINFO_FILENAME) . ".txt";
$whisper_cmd = C\WHISPER;
$output_dir = dirname($transcript_path);
$output_format = "txt";
$model = "turbo"; // Use turbo model for faster processing
$command = sprintf(
"%s %s --model %s --output_dir %s --output_format %s 2>&1",
escapeshellcmd($whisper_cmd),
escapeshellarg($working_audio_path),
escapeshellarg($model),
escapeshellarg($output_dir),
escapeshellarg($output_format)
);
L\crawlLog("AudioTranscriptionJob: Executing: $command");
$start_time = time();
exec($command, $output, $return_var);
$duration = time() - $start_time;
L\crawlLog("AudioTranscriptionJob: Command completed in " .
"{$duration}s with return code: $return_var");
if ($return_var !== 0) {
$error_msg = "Whisper transcription failed (exit code: " .
"$return_var): " . implode("\n", $output);
L\crawlLog("AudioTranscriptionJob: $error_msg");
// Increment retry count
$task['retry_count'] = ($task['retry_count'] ?? 0) + 1;
file_put_contents($task_file, serialize($task));
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => "Failed to transcribe audio: " .
(end($output) ?: "Unknown error"),
'file' => $filename
];
}
// Find and move the generated transcript
// Whisper generates output based on the input filename
$whisper_output = $output_dir . "/" .
pathinfo($working_audio_path, PATHINFO_FILENAME) . ".txt";
if (file_exists($whisper_output)) {
// Validate transcript content
$transcript_content = file_get_contents($whisper_output);
if (empty(trim($transcript_content))) {
$error = "Generated transcript is empty";
L\crawlLog("AudioTranscriptionJob: $error");
file_put_contents($status_file, 'failed');
unlink($whisper_output);
return [
'success' => false,
'error' => $error,
'file' => $filename
];
}
// Move to final location if different
if ($whisper_output !== $transcript_path) {
if (!rename($whisper_output, $transcript_path)) {
$error = "Failed to move transcript to final location";
L\crawlLog("AudioTranscriptionJob: $error");
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => $error,
'file' => $filename
];
}
}
} else {
$error = "Transcript file not generated at expected location: " .
$whisper_output;
L\crawlLog("AudioTranscriptionJob: $error");
file_put_contents($status_file, 'failed');
return [
'success' => false,
'error' => "Transcript file not found after processing",
'file' => $filename
];
}
// Mark as completed and cleanup
file_put_contents($status_file, 'completed');
unlink($task_file);
// Clean up converted file if it was created
if ($converted_file && file_exists($converted_file)) {
unlink($converted_file);
L\crawlLog("AudioTranscriptionJob: Cleaned up converted file: " .
$converted_file);
}
L\crawlLog("AudioTranscriptionJob: Successfully completed " .
"transcription for: $filename");
return [
'success' => true,
'file' => $filename,
'transcript_path' => $transcript_path,
'duration' => $duration
];
}
/**
* Count currently processing tasks to limit concurrency
*
* @param string $task_dir Task directory path
* @return int Number of tasks currently being processed
*/
private function countProcessingTasks($task_dir)
{
$processing_count = 0;
$status_files = glob($task_dir . "/*.status");
foreach ($status_files as $status_file) {
$status = file_get_contents($status_file);
if ($status === 'processing') {
$processing_count++;
}
}
return $processing_count;
}
/**
* Validate task data structure and file availability
*
* @param array $task_data Task data to validate
* @return bool true if task data is valid
*/
private function validateTaskData($task_data)
{
$required_fields = ['task_id', 'audio_path', 'filename'];
foreach ($required_fields as $field) {
if (!isset($task_data[$field]) || empty($task_data[$field])) {
return false;
}
}
// Validate audio file exists
if (!file_exists($task_data['audio_path'])) {
return false;
}
// Validate file size
$file_size = filesize($task_data['audio_path']);
if ($file_size > C\MAX_AUDIO_TRANSCRIBE_SIZE || $file_size === 0) {
return false;
}
return true;
}
/**
* Determine if a failed task should be retried
*
* @param array $task_data Task data
* @return bool true if task should be retried
*/
private function shouldRetry($task_data)
{
$retry_count = $task_data['retry_count'] ?? 0;
return $retry_count < self::MAX_RETRY_ATTEMPTS;
}
/**
* Clean up task files for a given task ID
*
* @param string $task_id Task identifier
* @param string $task_dir Task directory path
*/
private function cleanupTask($task_id, $task_dir)
{
$task_file = $task_dir . "/" . $task_id . ".txt";
$status_file = $task_dir . "/" . $task_id . ".status";
if (file_exists($task_file)) {
unlink($task_file);
}
if (file_exists($status_file)) {
unlink($status_file);
}
}
/**
* Handle permanently failed tasks that have exceeded max retry attempts
*
* @param array $task_data Task data
* @param string $task_dir Task directory path
*/
private function handlePermanentFailure($task_data, $task_dir)
{
$task_id = $task_data['task_id'];
$audio_path = $task_data['audio_path'] ?? '';
$filename = $task_data['filename'] ?? '';
// Update transcript file to show permanent failure
if (!empty($audio_path) && file_exists($audio_path)) {
$transcript_path = dirname($audio_path) . "/" .
pathinfo($filename, PATHINFO_FILENAME) . ".txt";
$error_message = "Transcription failed permanently after " .
($task_data['retry_count'] ?? 0) . " attempts. " .
"The audio format may not be supported or the " .
"file may be corrupted.";
file_put_contents($transcript_path, $error_message);
L\crawlLog("AudioTranscriptionJob: Updated transcript file with " .
"error message: $transcript_path");
}
// Update status to permanently failed
$status_file = $task_dir . "/" . $task_id . ".status";
file_put_contents($status_file, 'permanently_failed');
// Clean up task file but keep status for debugging
$task_file = $task_dir . "/" . $task_id . ".txt";
if (file_exists($task_file)) {
unlink($task_file);
}
L\crawlLog("AudioTranscriptionJob: Marked task $task_id as " .
"permanently failed");
}
/**
* Convert unsupported audio formats to supported ones using ffmpeg
*
* @param string $input_path Path to input audio file
* @param string $output_path Path for converted output file
* @return bool true if conversion succeeded
*/
private function convertAudioFormat($input_path, $output_path)
{
// Check if ffmpeg is available
if (!C\nsdefined('FFMPEG')) {
L\crawlLog("AudioTranscriptionJob: FFMPEG not configured, " .
"cannot convert audio format");
return false;
}
$ffmpeg_cmd = C\FFMPEG;
$command = sprintf(
"%s -i %s -c:a aac -b:a 128k %s 2>&1",
escapeshellcmd($ffmpeg_cmd),
escapeshellarg($input_path),
escapeshellarg($output_path)
);
L\crawlLog("AudioTranscriptionJob: Converting audio format: $command");
exec($command, $output, $return_var);
if ($return_var === 0 && file_exists($output_path)) {
L\crawlLog("AudioTranscriptionJob: Successfully converted " .
"audio format");
return true;
} else {
L\crawlLog("AudioTranscriptionJob: Audio conversion failed: " .
implode("\n", $output));
return false;
}
}
/**
* Check if audio file format is supported by Whisper
*
* @param string $filename Audio filename
* @return bool true if format is supported
*/
private function isSupportedAudioFormat($filename)
{
$supported_extensions = [ 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav',
'webm', 'flac', 'ogg', 'oga' ];
$extension = strtolower(pathinfo($filename, PATHINFO_EXTENSION));
return in_array($extension, $supported_extensions);
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software