Keep AI jobs alive during processing
This commit is contained in:
@@ -113,6 +113,9 @@ func (w *Worker) tick(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, job *model.Job) {
|
||||
stopHeartbeat := w.startHeartbeat(ctx, job)
|
||||
defer stopHeartbeat()
|
||||
|
||||
if job.TaskType == TaskTranscription {
|
||||
w.processTranscription(ctx, job)
|
||||
return
|
||||
@@ -168,6 +171,41 @@ func (w *Worker) fail(ctx context.Context, job *model.Job, code, message string)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) startHeartbeat(ctx context.Context, job *model.Job) func() {
|
||||
heartbeatCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
ticker := time.NewTicker(w.heartbeatInterval())
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-heartbeatCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := w.store.HeartbeatJob(heartbeatCtx, job.ID); err != nil {
|
||||
slog.Warn("heartbeat job failed", "job_id", job.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) heartbeatInterval() time.Duration {
|
||||
interval := w.leaseTimeout / 3
|
||||
if interval < 10*time.Second {
|
||||
return 10 * time.Second
|
||||
}
|
||||
if interval > time.Minute {
|
||||
return time.Minute
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
func classifyTranscriptionError(err error) string {
|
||||
if err == nil {
|
||||
return "unknown"
|
||||
|
||||
Reference in New Issue
Block a user