| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034 |
- const content = `--[[
- Move job from active to a finished status (completed o failed)
- A job can only be moved to completed if it was active.
- The job must be locked before it can be moved to a finished status,
- and the lock must be released in this script.
- Input:
- KEYS[1] wait key
- KEYS[2] active key
- KEYS[3] prioritized key
- KEYS[4] event stream key
- KEYS[5] stalled key
- -- Rate limiting
- KEYS[6] rate limiter key
- KEYS[7] delayed key
- KEYS[8] paused key
- KEYS[9] meta key
- KEYS[10] pc priority counter
- KEYS[11] completed/failed key
- KEYS[12] jobId key
- KEYS[13] metrics key
- KEYS[14] marker key
- ARGV[1] jobId
- ARGV[2] timestamp
- ARGV[3] msg property returnvalue / failedReason
- ARGV[4] return value / failed reason
- ARGV[5] target (completed/failed)
- ARGV[6] fetch next?
- ARGV[7] keys prefix
- ARGV[8] opts
- ARGV[9] job fields to update
- opts - token - lock token
- opts - keepJobs
- opts - lockDuration - lock duration in milliseconds
- opts - attempts max attempts
- opts - maxMetricsSize
- opts - fpof - fail parent on fail
- opts - cpof - continue parent on fail
- opts - idof - ignore dependency on fail
- opts - rdof - remove dependency on fail
- opts - name - worker name
- Output:
- 0 OK
- -1 Missing key.
- -2 Missing lock.
- -3 Job not in active set
- -4 Job has pending children
- -6 Lock is not owned by this client
- -9 Job has failed children
- Events:
- 'completed/failed'
- ]]
- local rcall = redis.call
- --- Includes
- --[[
- Functions to collect metrics based on a current and previous count of jobs.
- Granualarity is fixed at 1 minute.
- ]]
- --[[
- Function to loop in batches.
- Just a bit of warning, some commands as ZREM
- could receive a maximum of 7000 parameters per call.
- ]]
- local function batches(n, batchSize)
- local i = 0
- return function()
- local from = i * batchSize + 1
- i = i + 1
- if (from <= n) then
- local to = math.min(from + batchSize - 1, n)
- return from, to
- end
- end
- end
- local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
- timestamp)
- -- Increment current count
- local count = rcall("HINCRBY", metaKey, "count", 1) - 1
- -- Compute how many data points we need to add to the list, N.
- local prevTS = rcall("HGET", metaKey, "prevTS")
- if not prevTS then
- -- If prevTS is nil, set it to the current timestamp
- rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
- return
- end
- local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
- if N > 0 then
- local delta = count - rcall("HGET", metaKey, "prevCount")
- -- If N > 1, add N-1 zeros to the list
- if N > 1 then
- local points = {}
- points[1] = delta
- for i = 2, N do
- points[i] = 0
- end
- for from, to in batches(#points, 7000) do
- rcall("LPUSH", dataPointsList, unpack(points, from, to))
- end
- else
- -- LPUSH delta to the list
- rcall("LPUSH", dataPointsList, delta)
- end
- -- LTRIM to keep list to its max size
- rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
- -- update prev count with current count
- rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
- end
- end
- --[[
- Function to fetch the next job to process.
- Tries to get the next job to avoid an extra roundtrip if the queue is
- not closing and not rate limited.
- Input:
- waitKey - wait list key
- activeKey - active list key
- prioritizedKey - prioritized sorted set key
- eventStreamKey - event stream key
- rateLimiterKey - rate limiter key
- delayedKey - delayed sorted set key
- pausedKey - paused list key
- metaKey - meta hash key
- pcKey - priority counter key
- markerKey - marker key
- prefix - keys prefix
- timestamp - current timestamp
- opts - options table:
- token (required) - lock token used when locking jobs
- lockDuration (required) - lock duration for acquired jobs
- limiter (optional) - rate limiter options table (e.g. { max = number })
- ]]
- -- Includes
- --[[
- Function to return the next delayed job timestamp.
- ]]
- local function getNextDelayedTimestamp(delayedKey)
- local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
- if #result then
- local nextTimestamp = tonumber(result[2])
- if nextTimestamp ~= nil then
- return nextTimestamp / 0x1000
- end
- end
- end
- --[[
- Function to get current rate limit ttl.
- ]]
- local function getRateLimitTTL(maxJobs, rateLimiterKey)
- if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
- local pttl = rcall("PTTL", rateLimiterKey)
- if pttl == 0 then
- rcall("DEL", rateLimiterKey)
- end
- if pttl > 0 then
- return pttl
- end
- end
- return 0
- end
- --[[
- Function to check for the meta.paused key to decide if we are paused or not
- (since an empty list and !EXISTS are not really the same).
- ]]
- local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
- if queueAttributes[1] then
- return pausedKey, true, queueAttributes[3], queueAttributes[4]
- else
- if queueAttributes[2] then
- local activeCount = rcall("LLEN", activeKey)
- if activeCount >= tonumber(queueAttributes[2]) then
- return waitKey, true, queueAttributes[3], queueAttributes[4]
- else
- return waitKey, false, queueAttributes[3], queueAttributes[4]
- end
- end
- end
- return waitKey, false, queueAttributes[3], queueAttributes[4]
- end
- --[[
- Function to move job from prioritized state to active.
- ]]
- local function moveJobFromPrioritizedToActive(priorityKey, activeKey, priorityCounterKey)
- local prioritizedJob = rcall("ZPOPMIN", priorityKey)
- if #prioritizedJob > 0 then
- rcall("LPUSH", activeKey, prioritizedJob[1])
- return prioritizedJob[1]
- else
- rcall("DEL", priorityCounterKey)
- end
- end
- --[[
- Function to move job from wait state to active.
- Input:
- opts - token - lock token
- opts - lockDuration
- opts - limiter
- ]]
- -- Includes
- --[[
- Add marker if needed when a job is available.
- ]]
- local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
- if not isPausedOrMaxed then
- rcall("ZADD", markerKey, 0, "0")
- end
- end
- local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
- jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
- local jobKey = keyPrefix .. jobId
- -- Check if we need to perform rate limiting.
- if maxJobs then
- local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
- if jobCounter == 1 then
- local integerDuration = math.floor(math.abs(limiterDuration))
- rcall("PEXPIRE", rateLimiterKey, integerDuration)
- end
- end
- -- get a lock
- if opts['token'] ~= "0" then
- local lockKey = jobKey .. ':lock'
- rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
- end
- local optionalValues = {}
- if opts['name'] then
- -- Set "processedBy" field to the worker name
- table.insert(optionalValues, "pb")
- table.insert(optionalValues, opts['name'])
- end
- rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
- rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
- rcall("HINCRBY", jobKey, "ats", 1)
- addBaseMarkerIfNeeded(markerKey, false)
- -- rate limit delay must be 0 in this case to prevent adding more delay
- -- when job that is moved to active needs to be processed
- return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
- end
- --[[
- Updates the delay set, by moving delayed jobs that should
- be processed now to "wait".
- Events:
- 'waiting'
- ]]
- -- Includes
- --[[
- Function to add job in target list and add marker if needed.
- ]]
- -- Includes
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
- rcall(pushCmd, targetKey, jobId)
- addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
- end
- --[[
- Function to add job considering priority.
- ]]
- -- Includes
- --[[
- Function to get priority score.
- ]]
- local function getPriorityScore(priority, priorityCounterKey)
- local prioCounter = rcall("INCR", priorityCounterKey)
- return priority * 0x100000000 + prioCounter % 0x100000000
- end
- local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
- isPausedOrMaxed)
- local score = getPriorityScore(priority, priorityCounterKey)
- rcall("ZADD", prioritizedKey, score, jobId)
- addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
- end
- -- Try to get as much as 1000 jobs at once
- local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
- eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
- local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
- if (#jobs > 0) then
- rcall("ZREM", delayedKey, unpack(jobs))
- for _, jobId in ipairs(jobs) do
- local jobKey = prefix .. jobId
- local priority =
- tonumber(rcall("HGET", jobKey, "priority")) or 0
- if priority == 0 then
- -- LIFO or FIFO
- rcall("LPUSH", targetKey, jobId)
- else
- local score = getPriorityScore(priority, priorityCounterKey)
- rcall("ZADD", prioritizedKey, score, jobId)
- end
- -- Emit waiting event
- rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
- jobId, "prev", "delayed")
- rcall("HSET", jobKey, "delay", 0)
- end
- addBaseMarkerIfNeeded(markerKey, isPaused)
- end
- end
- local function fetchNextJob(waitKey, activeKey, prioritizedKey, eventStreamKey,
- rateLimiterKey, delayedKey, pausedKey, metaKey, pcKey, markerKey, prefix,
- timestamp, opts)
- local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration =
- getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
- -- Check if there are delayed jobs that can be promoted
- promoteDelayedJobs(delayedKey, markerKey, target, prioritizedKey,
- eventStreamKey, prefix, timestamp, pcKey, isPausedOrMaxed)
- local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
- -- Check if we are rate limited first.
- local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
- if expireTime > 0 then
- return {0, 0, expireTime, 0}
- end
- -- paused or maxed queue
- if isPausedOrMaxed then
- return {0, 0, 0, 0}
- end
- local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
- local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
- if jobId then
- -- Markers in waitlist DEPRECATED in v5: Remove in v6.
- if string.sub(jobId, 1, 2) == "0:" then
- rcall("LREM", activeKey, 1, jobId)
- -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
- -- but if ID is 0:0, then there is at least 1 prioritized job to process
- if jobId == "0:0" then
- jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
- return prepareJobForProcessing(prefix, rateLimiterKey,
- eventStreamKey, jobId, timestamp, maxJobs,
- limiterDuration, markerKey, opts)
- end
- else
- return prepareJobForProcessing(prefix, rateLimiterKey,
- eventStreamKey, jobId, timestamp, maxJobs,
- limiterDuration, markerKey, opts)
- end
- else
- jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
- if jobId then
- return prepareJobForProcessing(prefix, rateLimiterKey,
- eventStreamKey, jobId, timestamp, maxJobs,
- limiterDuration, markerKey, opts)
- end
- end
- -- Return the timestamp for the next delayed job if any.
- local nextTimestamp = getNextDelayedTimestamp(delayedKey)
- if nextTimestamp ~= nil then
- -- The result is guaranteed to be positive, since the
- -- ZRANGEBYSCORE command would have return a job otherwise.
- return {0, 0, 0, nextTimestamp}
- end
- end
- --[[
- Function to recursively move from waitingChildren to failed.
- ]]
- -- Includes
- --[[
- Validate and move parent to a wait status (waiting, delayed or prioritized)
- if no pending dependencies.
- ]]
- -- Includes
- --[[
- Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
- ]]
- -- Includes
- --[[
- Move parent to a wait status (wait, prioritized or delayed)
- ]]
- -- Includes
- --[[
- Add delay marker if needed.
- ]]
- -- Includes
- local function addDelayMarkerIfNeeded(markerKey, delayedKey)
- local nextTimestamp = getNextDelayedTimestamp(delayedKey)
- if nextTimestamp ~= nil then
- -- Replace the score of the marker with the newest known
- -- next timestamp.
- rcall("ZADD", markerKey, nextTimestamp, "1")
- end
- end
- --[[
- Function to check if queue is paused or maxed
- (since an empty list and !EXISTS are not really the same).
- ]]
- local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
- if queueAttributes[1] then
- return true
- else
- if queueAttributes[2] then
- local activeCount = rcall("LLEN", activeKey)
- return activeCount >= tonumber(queueAttributes[2])
- end
- end
- return false
- end
- local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
- local parentWaitKey = parentQueueKey .. ":wait"
- local parentPausedKey = parentQueueKey .. ":paused"
- local parentActiveKey = parentQueueKey .. ":active"
- local parentMetaKey = parentQueueKey .. ":meta"
- local parentMarkerKey = parentQueueKey .. ":marker"
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
- local priority = tonumber(jobAttributes[1]) or 0
- local delay = tonumber(jobAttributes[2]) or 0
- if delay > 0 then
- local delayedTimestamp = tonumber(timestamp) + delay
- local score = delayedTimestamp * 0x1000
- local parentDelayedKey = parentQueueKey .. ":delayed"
- rcall("ZADD", parentDelayedKey, score, parentId)
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
- delayedTimestamp)
- addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
- else
- if priority == 0 then
- local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
- parentWaitKey, parentPausedKey)
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
- else
- local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
- addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
- parentQueueKey .. ":pc", isPausedOrMaxed)
- end
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
- "waiting-children")
- end
- end
- local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
- if rcall("EXISTS", parentKey) == 1 then
- local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
- if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
- rcall("ZREM", parentWaitingChildrenKey, parentId)
- moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
- end
- end
- end
- local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
- parentId, timestamp)
- local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
- if doNotHavePendingDependencies then
- moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
- end
- end
- local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
- if rcall("EXISTS", parentKey) == 1 then
- local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
- local parentDelayedKey = parentQueueKey .. ":delayed"
- local parentWaitingChildrenOrDelayedKey
- if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
- parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
- elseif rcall("ZSCORE", parentDelayedKey, parentId) then
- parentWaitingChildrenOrDelayedKey = parentDelayedKey
- rcall("HSET", parentKey, "delay", 0)
- end
- if parentWaitingChildrenOrDelayedKey then
- rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
- local deferredFailure = "child " .. jobIdKey .. " failed"
- rcall("HSET", parentKey, "defa", deferredFailure)
- moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
- else
- if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
- local deferredFailure = "child " .. jobIdKey .. " failed"
- rcall("HSET", parentKey, "defa", deferredFailure)
- end
- end
- end
- end
- local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
- if rawParentData then
- local parentData = cjson.decode(rawParentData)
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
- local parentDependenciesChildrenKey = parentKey .. ":dependencies"
- if parentData['fpof'] then
- if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
- local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful"
- rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey)
- handleChildFailureAndMoveParentToWait(
- parentData['queueKey'],
- parentKey,
- parentData['id'],
- childKey,
- timestamp
- )
- end
- elseif parentData['cpof'] then
- if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
- local parentFailedChildrenKey = parentKey .. ":failed"
- rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
- moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
- end
- elseif parentData['idof'] or parentData['rdof'] then
- if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
- moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
- parentKey, parentData['id'], timestamp)
- if parentData['idof'] then
- local parentFailedChildrenKey = parentKey .. ":failed"
- rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
- end
- end
- end
- end
- end
- --[[
- Function to remove deduplication key if needed
- when a job is moved to completed or failed states.
- ]]
- local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
- deduplicationId, jobId)
- if deduplicationId then
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
- local pttl = rcall("PTTL", deduplicationKey)
- if pttl == 0 then
- return rcall("DEL", deduplicationKey)
- end
- if pttl == -1 then
- local currentJobId = rcall('GET', deduplicationKey)
- if currentJobId and currentJobId == jobId then
- return rcall("DEL", deduplicationKey)
- end
- end
- end
- end
- --[[
- Function to remove job keys.
- ]]
- local function removeJobKeys(jobKey)
- return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
- jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
- end
- --[[
- Functions to remove jobs by max age.
- ]]
- -- Includes
- --[[
- Function to remove job.
- ]]
- -- Includes
- --[[
- Function to remove deduplication key if needed
- when a job is being removed.
- ]]
- local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
- jobId, deduplicationId)
- if deduplicationId then
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
- local currentJobId = rcall('GET', deduplicationKey)
- if currentJobId and currentJobId == jobId then
- rcall("DEL", deduplicationKey)
- -- Also clean up any pending dedup-next data for this dedup ID
- rcall("DEL", prefixKey .. "dn:" .. deduplicationId)
- return 1
- end
- end
- end
- --[[
- Check if this job has a parent. If so we will just remove it from
- the parent child list, but if it is the last child we should move the parent to "wait/paused"
- which requires code from "moveToFinished"
- ]]
- -- Includes
- --[[
- Functions to destructure job key.
- Just a bit of warning, these functions may be a bit slow and affect performance significantly.
- ]]
- local getJobIdFromKey = function (jobKey)
- return string.match(jobKey, ".*:(.*)")
- end
- local getJobKeyPrefix = function (jobKey, jobId)
- return string.sub(jobKey, 0, #jobKey - #jobId)
- end
- local function _moveParentToWait(parentPrefix, parentId, emitEvent)
- local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
- parentPrefix .. "wait", parentPrefix .. "paused")
- addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
- if emitEvent then
- local parentEventStream = parentPrefix .. "events"
- rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
- end
- end
- local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
- if parentKey then
- local parentDependenciesKey = parentKey .. ":dependencies"
- local result = rcall("SREM", parentDependenciesKey, jobKey)
- if result > 0 then
- local pendingDependencies = rcall("SCARD", parentDependenciesKey)
- if pendingDependencies == 0 then
- local parentId = getJobIdFromKey(parentKey)
- local parentPrefix = getJobKeyPrefix(parentKey, parentId)
- local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
- if numRemovedElements == 1 then
- if hard then -- remove parent in same queue
- if parentPrefix == baseKey then
- removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
- removeJobKeys(parentKey)
- if debounceId then
- rcall("DEL", parentPrefix .. "de:" .. debounceId)
- end
- else
- _moveParentToWait(parentPrefix, parentId)
- end
- else
- _moveParentToWait(parentPrefix, parentId, true)
- end
- end
- end
- return true
- end
- else
- local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
- local missedParentKey = parentAttributes[1]
- if( (type(missedParentKey) == "string") and missedParentKey ~= ""
- and (rcall("EXISTS", missedParentKey) == 1)) then
- local parentDependenciesKey = missedParentKey .. ":dependencies"
- local result = rcall("SREM", parentDependenciesKey, jobKey)
- if result > 0 then
- local pendingDependencies = rcall("SCARD", parentDependenciesKey)
- if pendingDependencies == 0 then
- local parentId = getJobIdFromKey(missedParentKey)
- local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
- local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
- if numRemovedElements == 1 then
- if hard then
- if parentPrefix == baseKey then
- removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
- removeJobKeys(missedParentKey)
- if parentAttributes[2] then
- rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
- end
- else
- _moveParentToWait(parentPrefix, parentId)
- end
- else
- _moveParentToWait(parentPrefix, parentId, true)
- end
- end
- end
- return true
- end
- end
- end
- return false
- end
- local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
- local jobKey = baseKey .. jobId
- removeParentDependencyKey(jobKey, hard, nil, baseKey)
- if shouldRemoveDeduplicationKey then
- local deduplicationId = rcall("HGET", jobKey, "deid")
- removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
- end
- removeJobKeys(jobKey)
- end
- local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
- local start = timestamp - maxAge * 1000
- local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit)
- for i, jobId in ipairs(jobIds) do
- removeJob(jobId, false, prefix, false --[[remove debounce key]])
- end
- if #jobIds > 0 then
- if #jobIds < maxLimit then
- rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
- else
- for from, to in batches(#jobIds, 7000) do
- rcall("ZREM", targetSet, unpack(jobIds, from, to))
- end
- end
- end
- end
- --[[
- Functions to remove jobs by max count.
- ]]
- -- Includes
- local function removeJobsByMaxCount(maxCount, targetSet, prefix)
- local start = maxCount
- local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
- for i, jobId in ipairs(jobIds) do
- removeJob(jobId, false, prefix, false --[[remove debounce key]])
- end
- rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
- end
- local function removeLock(jobKey, stalledKey, token, jobId)
- if token ~= "0" then
- local lockKey = jobKey .. ':lock'
- local lockToken = rcall("GET", lockKey)
- if lockToken == token then
- rcall("DEL", lockKey)
- rcall("SREM", stalledKey, jobId)
- else
- if lockToken then
- -- Lock exists but token does not match
- return -6
- else
- -- Lock is missing completely
- return -2
- end
- end
- end
- return 0
- end
- --[[
- Function to create a new job from stored dedup-next data
- when a deduplicated job with keepLastIfActive finishes.
- At most one next job is created per deduplication ID.
- Multiple triggers while active overwrite the dedup-next data,
- so only the latest data is used.
- ]]
- -- Includes
- --[[
- Function to get max events value or set by default 10000.
- ]]
- local function getOrSetMaxEvents(metaKey)
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
- if not maxEvents then
- maxEvents = 10000
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
- end
- return maxEvents
- end
- --[[
- Function to set the deduplication key for a job.
- Uses TTL from deduplication opts if provided.
- ]]
- local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
- local ttl = deduplicationOpts and deduplicationOpts['ttl']
- if ttl and ttl > 0 then
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
- else
- rcall('SET', deduplicationKey, jobId)
- end
- end
- --[[
- Shared helper to store a job and enqueue it into the appropriate list/set.
- Handles delayed, prioritized, and standard (LIFO/FIFO) jobs.
- Emits the appropriate event after enqueuing ("delayed" or "waiting").
- Returns delay, priority from storeJob.
- ]]
- -- Includes
- --[[
- Adds a delayed job to the queue by doing the following:
- - Creates a new job key with the job data.
- - adds to delayed zset.
- - Emits a global event 'delayed' if the job is delayed.
- ]]
- -- Includes
- --[[
- Bake in the job id first 12 bits into the timestamp
- to guarantee correct execution order of delayed jobs
- (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
- WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
- ]]
- local function getDelayedScore(delayedKey, timestamp, delay)
- local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
- local minScore = delayedTimestamp * 0x1000
- local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
- local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
- minScore, "WITHSCORES","LIMIT", 0, 1)
- if #result then
- local currentMaxScore = tonumber(result[2])
- if currentMaxScore ~= nil then
- if currentMaxScore >= maxScore then
- return maxScore, delayedTimestamp
- else
- return currentMaxScore + 1, delayedTimestamp
- end
- end
- end
- return minScore, delayedTimestamp
- end
- local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
- maxEvents, markerKey, delay)
- local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
- rcall("ZADD", delayedKey, score, jobId)
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
- "jobId", jobId, "delay", delayedTimestamp)
- -- mark that a delayed job is available
- addDelayMarkerIfNeeded(markerKey, delayedKey)
- end
- --[[
- Function to store a job
- ]]
- local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
- parentKey, parentData, repeatJobKey)
- local jsonOpts = cjson.encode(opts)
- local delay = opts['delay'] or 0
- local priority = opts['priority'] or 0
- local debounceId = opts['de'] and opts['de']['id']
- local optionalValues = {}
- if parentKey ~= nil then
- table.insert(optionalValues, "parentKey")
- table.insert(optionalValues, parentKey)
- table.insert(optionalValues, "parent")
- table.insert(optionalValues, parentData)
- end
- if repeatJobKey then
- table.insert(optionalValues, "rjk")
- table.insert(optionalValues, repeatJobKey)
- end
- if debounceId then
- table.insert(optionalValues, "deid")
- table.insert(optionalValues, debounceId)
- end
- rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
- "timestamp", timestamp, "delay", delay, "priority", priority,
- unpack(optionalValues))
- rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
- return delay, priority
- end
- local function storeAndEnqueueJob(eventsKey, jobIdKey, jobId, name, data, opts,
- timestamp, parentKey, parentData, repeatJobKey, maxEvents,
- waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
- priorityCounterKey, delayedKey, markerKey)
- local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
- opts, timestamp, parentKey, parentData, repeatJobKey)
- if delay ~= 0 and delayedKey then
- addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
- else
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
- if priority > 0 then
- addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
- priorityCounterKey, isPausedOrMaxed)
- else
- local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
- addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
- end
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
- "jobId", jobId)
- end
- return delay, priority
- end
- local function requeueDeduplicatedJob(prefix, deduplicationId, eventStreamKey,
- metaKey, activeKey, waitKey, pausedKey, markerKey, prioritizedKey,
- priorityCounterKey, delayedKey, timestamp)
- local deduplicationNextKey = prefix .. "dn:" .. deduplicationId
- if rcall("EXISTS", deduplicationNextKey) == 1 then
- local nextData = rcall("HMGET", deduplicationNextKey,
- "name", "data", "opts", "pk", "pd", "pdk", "rjk")
- local newJobId = rcall("INCR", prefix .. "id") .. ""
- local newJobIdKey = prefix .. newJobId
- local newOpts = cjson.decode(nextData[3])
- local deduplicationKey = prefix .. "de:" .. deduplicationId
- local parentKey = nextData[4] or nil
- local parentData = nextData[5] or nil
- local parentDependenciesKey = nextData[6] or nil
- local repeatJobKey = nextData[7] or nil
- -- Set dedup key for the new job (without TTL when keepLastIfActive,
- -- so the key outlives the job's active duration)
- local deOpts = newOpts['de']
- if deOpts and deOpts['keepLastIfActive'] then
- rcall('SET', deduplicationKey, newJobId)
- else
- setDeduplicationKey(deduplicationKey, newJobId, deOpts)
- end
- -- Store and enqueue using the shared helper (handles priority/lifo/delayed)
- local maxEvents = getOrSetMaxEvents(metaKey)
- storeAndEnqueueJob(eventStreamKey, newJobIdKey, newJobId, nextData[1], nextData[2],
- newOpts, timestamp, parentKey, parentData, repeatJobKey, maxEvents,
- waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
- priorityCounterKey, delayedKey, markerKey)
- -- Register as child dependency if the job has a parent
- if parentDependenciesKey then
- rcall("SADD", parentDependenciesKey, newJobIdKey)
- end
- -- Only delete the dedup-next hash after the job is fully created,
- -- so that if any step above errors, the data is not permanently lost.
- rcall("DEL", deduplicationNextKey)
- end
- end
- --[[
- Function to trim events, default 10000.
- ]]
- -- Includes
- local function trimEvents(metaKey, eventStreamKey)
- local maxEvents = getOrSetMaxEvents(metaKey)
- if maxEvents then
- rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
- else
- rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
- end
- end
- --[[
- Validate and move or add dependencies to parent.
- ]]
- -- Includes
- local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
- parentId, jobIdKey, returnvalue, timestamp )
- local processedSet = parentKey .. ":processed"
- rcall("HSET", processedSet, jobIdKey, returnvalue)
- moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
- end
- --[[
- Function to update a bunch of fields in a job.
- ]]
- local function updateJobFields(jobKey, msgpackedFields)
- if msgpackedFields and #msgpackedFields > 0 then
- local fieldsToUpdate = cmsgpack.unpack(msgpackedFields)
- if fieldsToUpdate then
- rcall("HMSET", jobKey, unpack(fieldsToUpdate))
- end
- end
- end
- local jobIdKey = KEYS[12]
- if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
- -- Make sure it does not have pending dependencies
- -- It must happen before removing lock
- if ARGV[5] == "completed" then
- if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
- return -4
- end
- if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then
- return -9
- end
- end
- local opts = cmsgpack.unpack(ARGV[8])
- local token = opts['token']
- local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
- if errorCode < 0 then
- return errorCode
- end
- updateJobFields(jobIdKey, ARGV[9]);
- local attempts = opts['attempts']
- local maxMetricsSize = opts['maxMetricsSize']
- local maxCount = opts['keepJobs']['count']
- local maxAge = opts['keepJobs']['age']
- local maxLimit = opts['keepJobs']['limit'] or 1000
- local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
- local parentKey = jobAttributes[1] or ""
- local parentId = ""
- local parentQueueKey = ""
- if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
- local jsonDecodedParent = cjson.decode(jobAttributes[2])
- parentId = jsonDecodedParent['id']
- parentQueueKey = jsonDecodedParent['queueKey']
- end
- local jobId = ARGV[1]
- local timestamp = ARGV[2]
- -- Remove from active list (if not active we shall return error)
- local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
- if (numRemovedElements < 1) then
- return -3
- end
- local eventStreamKey = KEYS[4]
- local metaKey = KEYS[9]
- -- Trim events before emiting them to avoid trimming events emitted in this script
- trimEvents(metaKey, eventStreamKey)
- local prefix = ARGV[7]
- removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
- -- Check if there is requeue data for this dedup ID (keepLastIfActive mode)
- if jobAttributes[3] then
- requeueDeduplicatedJob(prefix, jobAttributes[3], eventStreamKey,
- metaKey, KEYS[2], KEYS[1], KEYS[8], KEYS[14], KEYS[3], KEYS[10],
- KEYS[7], timestamp)
- end
- -- If job has a parent we need to
- -- 1) remove this job id from parents dependencies
- -- 2) move the job Id to parent "processed" set
- -- 3) push the results into parent "results" list
- -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
- if parentId == "" and parentKey ~= "" then
- parentId = getJobIdFromKey(parentKey)
- parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
- end
- if parentId ~= "" then
- if ARGV[5] == "completed" then
- local dependenciesSet = parentKey .. ":dependencies"
- if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
- updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
- timestamp)
- end
- else
- moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
- end
- end
- local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
- -- Remove job?
- if maxCount ~= 0 then
- local targetSet = KEYS[11]
- -- Add to complete/failed set
- rcall("ZADD", targetSet, timestamp, jobId)
- rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
- -- "returnvalue" / "failedReason" and "finishedOn"
- if ARGV[5] == "failed" then
- rcall("HDEL", jobIdKey, "defa")
- end
- -- Remove old jobs?
- if maxAge ~= nil then
- removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
- end
- if maxCount ~= nil and maxCount > 0 then
- removeJobsByMaxCount(maxCount, targetSet, prefix)
- end
- else
- removeJobKeys(jobIdKey)
- if parentKey ~= "" then
- -- TODO: when a child is removed when finished, result or failure in parent
- -- must not be deleted, those value references should be deleted when the parent
- -- is deleted
- removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
- end
- end
- rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
- if ARGV[5] == "failed" then
- if tonumber(attemptsMade) >= tonumber(attempts) then
- rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
- attemptsMade)
- end
- end
- -- Collect metrics
- if maxMetricsSize ~= "" then
- collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
- end
- -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
- -- and not rate limited.
- if (ARGV[6] == "1") then
- local result = fetchNextJob(KEYS[1], KEYS[2], KEYS[3], eventStreamKey,
- KEYS[6], KEYS[7], KEYS[8], metaKey, KEYS[10], KEYS[14], prefix,
- timestamp, opts)
- if result then
- return result
- end
- end
- local waitLen = rcall("LLEN", KEYS[1])
- if waitLen == 0 then
- local activeLen = rcall("LLEN", KEYS[2])
- if activeLen == 0 then
- local prioritizedLen = rcall("ZCARD", KEYS[3])
- if prioritizedLen == 0 then
- rcall("XADD", eventStreamKey, "*", "event", "drained")
- end
- end
- end
- return 0
- else
- return -1
- end
- `;
- export const moveToFinished = {
- name: 'moveToFinished',
- content,
- keys: 14,
- };
- //# sourceMappingURL=moveToFinished-14.js.map
|