moveToFinished-14.js 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034
  1. const content = `--[[
  2. Move job from active to a finished status (completed o failed)
  3. A job can only be moved to completed if it was active.
  4. The job must be locked before it can be moved to a finished status,
  5. and the lock must be released in this script.
  6. Input:
  7. KEYS[1] wait key
  8. KEYS[2] active key
  9. KEYS[3] prioritized key
  10. KEYS[4] event stream key
  11. KEYS[5] stalled key
  12. -- Rate limiting
  13. KEYS[6] rate limiter key
  14. KEYS[7] delayed key
  15. KEYS[8] paused key
  16. KEYS[9] meta key
  17. KEYS[10] pc priority counter
  18. KEYS[11] completed/failed key
  19. KEYS[12] jobId key
  20. KEYS[13] metrics key
  21. KEYS[14] marker key
  22. ARGV[1] jobId
  23. ARGV[2] timestamp
  24. ARGV[3] msg property returnvalue / failedReason
  25. ARGV[4] return value / failed reason
  26. ARGV[5] target (completed/failed)
  27. ARGV[6] fetch next?
  28. ARGV[7] keys prefix
  29. ARGV[8] opts
  30. ARGV[9] job fields to update
  31. opts - token - lock token
  32. opts - keepJobs
  33. opts - lockDuration - lock duration in milliseconds
  34. opts - attempts max attempts
  35. opts - maxMetricsSize
  36. opts - fpof - fail parent on fail
  37. opts - cpof - continue parent on fail
  38. opts - idof - ignore dependency on fail
  39. opts - rdof - remove dependency on fail
  40. opts - name - worker name
  41. Output:
  42. 0 OK
  43. -1 Missing key.
  44. -2 Missing lock.
  45. -3 Job not in active set
  46. -4 Job has pending children
  47. -6 Lock is not owned by this client
  48. -9 Job has failed children
  49. Events:
  50. 'completed/failed'
  51. ]]
  52. local rcall = redis.call
  53. --- Includes
  54. --[[
  55. Functions to collect metrics based on a current and previous count of jobs.
  56. Granualarity is fixed at 1 minute.
  57. ]]
  58. --[[
  59. Function to loop in batches.
  60. Just a bit of warning, some commands as ZREM
  61. could receive a maximum of 7000 parameters per call.
  62. ]]
  63. local function batches(n, batchSize)
  64. local i = 0
  65. return function()
  66. local from = i * batchSize + 1
  67. i = i + 1
  68. if (from <= n) then
  69. local to = math.min(from + batchSize - 1, n)
  70. return from, to
  71. end
  72. end
  73. end
  74. local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
  75. timestamp)
  76. -- Increment current count
  77. local count = rcall("HINCRBY", metaKey, "count", 1) - 1
  78. -- Compute how many data points we need to add to the list, N.
  79. local prevTS = rcall("HGET", metaKey, "prevTS")
  80. if not prevTS then
  81. -- If prevTS is nil, set it to the current timestamp
  82. rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
  83. return
  84. end
  85. local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
  86. if N > 0 then
  87. local delta = count - rcall("HGET", metaKey, "prevCount")
  88. -- If N > 1, add N-1 zeros to the list
  89. if N > 1 then
  90. local points = {}
  91. points[1] = delta
  92. for i = 2, N do
  93. points[i] = 0
  94. end
  95. for from, to in batches(#points, 7000) do
  96. rcall("LPUSH", dataPointsList, unpack(points, from, to))
  97. end
  98. else
  99. -- LPUSH delta to the list
  100. rcall("LPUSH", dataPointsList, delta)
  101. end
  102. -- LTRIM to keep list to its max size
  103. rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
  104. -- update prev count with current count
  105. rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
  106. end
  107. end
  108. --[[
  109. Function to fetch the next job to process.
  110. Tries to get the next job to avoid an extra roundtrip if the queue is
  111. not closing and not rate limited.
  112. Input:
  113. waitKey - wait list key
  114. activeKey - active list key
  115. prioritizedKey - prioritized sorted set key
  116. eventStreamKey - event stream key
  117. rateLimiterKey - rate limiter key
  118. delayedKey - delayed sorted set key
  119. pausedKey - paused list key
  120. metaKey - meta hash key
  121. pcKey - priority counter key
  122. markerKey - marker key
  123. prefix - keys prefix
  124. timestamp - current timestamp
  125. opts - options table:
  126. token (required) - lock token used when locking jobs
  127. lockDuration (required) - lock duration for acquired jobs
  128. limiter (optional) - rate limiter options table (e.g. { max = number })
  129. ]]
  130. -- Includes
  131. --[[
  132. Function to return the next delayed job timestamp.
  133. ]]
  134. local function getNextDelayedTimestamp(delayedKey)
  135. local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
  136. if #result then
  137. local nextTimestamp = tonumber(result[2])
  138. if nextTimestamp ~= nil then
  139. return nextTimestamp / 0x1000
  140. end
  141. end
  142. end
  143. --[[
  144. Function to get current rate limit ttl.
  145. ]]
  146. local function getRateLimitTTL(maxJobs, rateLimiterKey)
  147. if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
  148. local pttl = rcall("PTTL", rateLimiterKey)
  149. if pttl == 0 then
  150. rcall("DEL", rateLimiterKey)
  151. end
  152. if pttl > 0 then
  153. return pttl
  154. end
  155. end
  156. return 0
  157. end
  158. --[[
  159. Function to check for the meta.paused key to decide if we are paused or not
  160. (since an empty list and !EXISTS are not really the same).
  161. ]]
  162. local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
  163. local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
  164. if queueAttributes[1] then
  165. return pausedKey, true, queueAttributes[3], queueAttributes[4]
  166. else
  167. if queueAttributes[2] then
  168. local activeCount = rcall("LLEN", activeKey)
  169. if activeCount >= tonumber(queueAttributes[2]) then
  170. return waitKey, true, queueAttributes[3], queueAttributes[4]
  171. else
  172. return waitKey, false, queueAttributes[3], queueAttributes[4]
  173. end
  174. end
  175. end
  176. return waitKey, false, queueAttributes[3], queueAttributes[4]
  177. end
  178. --[[
  179. Function to move job from prioritized state to active.
  180. ]]
  181. local function moveJobFromPrioritizedToActive(priorityKey, activeKey, priorityCounterKey)
  182. local prioritizedJob = rcall("ZPOPMIN", priorityKey)
  183. if #prioritizedJob > 0 then
  184. rcall("LPUSH", activeKey, prioritizedJob[1])
  185. return prioritizedJob[1]
  186. else
  187. rcall("DEL", priorityCounterKey)
  188. end
  189. end
  190. --[[
  191. Function to move job from wait state to active.
  192. Input:
  193. opts - token - lock token
  194. opts - lockDuration
  195. opts - limiter
  196. ]]
  197. -- Includes
  198. --[[
  199. Add marker if needed when a job is available.
  200. ]]
  201. local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
  202. if not isPausedOrMaxed then
  203. rcall("ZADD", markerKey, 0, "0")
  204. end
  205. end
  206. local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
  207. jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
  208. local jobKey = keyPrefix .. jobId
  209. -- Check if we need to perform rate limiting.
  210. if maxJobs then
  211. local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
  212. if jobCounter == 1 then
  213. local integerDuration = math.floor(math.abs(limiterDuration))
  214. rcall("PEXPIRE", rateLimiterKey, integerDuration)
  215. end
  216. end
  217. -- get a lock
  218. if opts['token'] ~= "0" then
  219. local lockKey = jobKey .. ':lock'
  220. rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
  221. end
  222. local optionalValues = {}
  223. if opts['name'] then
  224. -- Set "processedBy" field to the worker name
  225. table.insert(optionalValues, "pb")
  226. table.insert(optionalValues, opts['name'])
  227. end
  228. rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
  229. rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
  230. rcall("HINCRBY", jobKey, "ats", 1)
  231. addBaseMarkerIfNeeded(markerKey, false)
  232. -- rate limit delay must be 0 in this case to prevent adding more delay
  233. -- when job that is moved to active needs to be processed
  234. return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
  235. end
  236. --[[
  237. Updates the delay set, by moving delayed jobs that should
  238. be processed now to "wait".
  239. Events:
  240. 'waiting'
  241. ]]
  242. -- Includes
  243. --[[
  244. Function to add job in target list and add marker if needed.
  245. ]]
  246. -- Includes
  247. local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
  248. rcall(pushCmd, targetKey, jobId)
  249. addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
  250. end
  251. --[[
  252. Function to add job considering priority.
  253. ]]
  254. -- Includes
  255. --[[
  256. Function to get priority score.
  257. ]]
  258. local function getPriorityScore(priority, priorityCounterKey)
  259. local prioCounter = rcall("INCR", priorityCounterKey)
  260. return priority * 0x100000000 + prioCounter % 0x100000000
  261. end
  262. local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
  263. isPausedOrMaxed)
  264. local score = getPriorityScore(priority, priorityCounterKey)
  265. rcall("ZADD", prioritizedKey, score, jobId)
  266. addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
  267. end
  268. -- Try to get as much as 1000 jobs at once
  269. local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
  270. eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
  271. local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
  272. if (#jobs > 0) then
  273. rcall("ZREM", delayedKey, unpack(jobs))
  274. for _, jobId in ipairs(jobs) do
  275. local jobKey = prefix .. jobId
  276. local priority =
  277. tonumber(rcall("HGET", jobKey, "priority")) or 0
  278. if priority == 0 then
  279. -- LIFO or FIFO
  280. rcall("LPUSH", targetKey, jobId)
  281. else
  282. local score = getPriorityScore(priority, priorityCounterKey)
  283. rcall("ZADD", prioritizedKey, score, jobId)
  284. end
  285. -- Emit waiting event
  286. rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
  287. jobId, "prev", "delayed")
  288. rcall("HSET", jobKey, "delay", 0)
  289. end
  290. addBaseMarkerIfNeeded(markerKey, isPaused)
  291. end
  292. end
  293. local function fetchNextJob(waitKey, activeKey, prioritizedKey, eventStreamKey,
  294. rateLimiterKey, delayedKey, pausedKey, metaKey, pcKey, markerKey, prefix,
  295. timestamp, opts)
  296. local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration =
  297. getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
  298. -- Check if there are delayed jobs that can be promoted
  299. promoteDelayedJobs(delayedKey, markerKey, target, prioritizedKey,
  300. eventStreamKey, prefix, timestamp, pcKey, isPausedOrMaxed)
  301. local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
  302. -- Check if we are rate limited first.
  303. local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
  304. if expireTime > 0 then
  305. return {0, 0, expireTime, 0}
  306. end
  307. -- paused or maxed queue
  308. if isPausedOrMaxed then
  309. return {0, 0, 0, 0}
  310. end
  311. local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
  312. local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
  313. if jobId then
  314. -- Markers in waitlist DEPRECATED in v5: Remove in v6.
  315. if string.sub(jobId, 1, 2) == "0:" then
  316. rcall("LREM", activeKey, 1, jobId)
  317. -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
  318. -- but if ID is 0:0, then there is at least 1 prioritized job to process
  319. if jobId == "0:0" then
  320. jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
  321. return prepareJobForProcessing(prefix, rateLimiterKey,
  322. eventStreamKey, jobId, timestamp, maxJobs,
  323. limiterDuration, markerKey, opts)
  324. end
  325. else
  326. return prepareJobForProcessing(prefix, rateLimiterKey,
  327. eventStreamKey, jobId, timestamp, maxJobs,
  328. limiterDuration, markerKey, opts)
  329. end
  330. else
  331. jobId = moveJobFromPrioritizedToActive(prioritizedKey, activeKey, pcKey)
  332. if jobId then
  333. return prepareJobForProcessing(prefix, rateLimiterKey,
  334. eventStreamKey, jobId, timestamp, maxJobs,
  335. limiterDuration, markerKey, opts)
  336. end
  337. end
  338. -- Return the timestamp for the next delayed job if any.
  339. local nextTimestamp = getNextDelayedTimestamp(delayedKey)
  340. if nextTimestamp ~= nil then
  341. -- The result is guaranteed to be positive, since the
  342. -- ZRANGEBYSCORE command would have return a job otherwise.
  343. return {0, 0, 0, nextTimestamp}
  344. end
  345. end
  346. --[[
  347. Function to recursively move from waitingChildren to failed.
  348. ]]
  349. -- Includes
  350. --[[
  351. Validate and move parent to a wait status (waiting, delayed or prioritized)
  352. if no pending dependencies.
  353. ]]
  354. -- Includes
  355. --[[
  356. Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
  357. ]]
  358. -- Includes
  359. --[[
  360. Move parent to a wait status (wait, prioritized or delayed)
  361. ]]
  362. -- Includes
  363. --[[
  364. Add delay marker if needed.
  365. ]]
  366. -- Includes
  367. local function addDelayMarkerIfNeeded(markerKey, delayedKey)
  368. local nextTimestamp = getNextDelayedTimestamp(delayedKey)
  369. if nextTimestamp ~= nil then
  370. -- Replace the score of the marker with the newest known
  371. -- next timestamp.
  372. rcall("ZADD", markerKey, nextTimestamp, "1")
  373. end
  374. end
  375. --[[
  376. Function to check if queue is paused or maxed
  377. (since an empty list and !EXISTS are not really the same).
  378. ]]
  379. local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
  380. local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
  381. if queueAttributes[1] then
  382. return true
  383. else
  384. if queueAttributes[2] then
  385. local activeCount = rcall("LLEN", activeKey)
  386. return activeCount >= tonumber(queueAttributes[2])
  387. end
  388. end
  389. return false
  390. end
  391. local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
  392. local parentWaitKey = parentQueueKey .. ":wait"
  393. local parentPausedKey = parentQueueKey .. ":paused"
  394. local parentActiveKey = parentQueueKey .. ":active"
  395. local parentMetaKey = parentQueueKey .. ":meta"
  396. local parentMarkerKey = parentQueueKey .. ":marker"
  397. local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
  398. local priority = tonumber(jobAttributes[1]) or 0
  399. local delay = tonumber(jobAttributes[2]) or 0
  400. if delay > 0 then
  401. local delayedTimestamp = tonumber(timestamp) + delay
  402. local score = delayedTimestamp * 0x1000
  403. local parentDelayedKey = parentQueueKey .. ":delayed"
  404. rcall("ZADD", parentDelayedKey, score, parentId)
  405. rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
  406. delayedTimestamp)
  407. addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
  408. else
  409. if priority == 0 then
  410. local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
  411. parentWaitKey, parentPausedKey)
  412. addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
  413. else
  414. local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
  415. addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
  416. parentQueueKey .. ":pc", isPausedOrMaxed)
  417. end
  418. rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
  419. "waiting-children")
  420. end
  421. end
  422. local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
  423. if rcall("EXISTS", parentKey) == 1 then
  424. local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
  425. if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
  426. rcall("ZREM", parentWaitingChildrenKey, parentId)
  427. moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
  428. end
  429. end
  430. end
  431. local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
  432. parentId, timestamp)
  433. local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
  434. if doNotHavePendingDependencies then
  435. moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
  436. end
  437. end
  438. local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
  439. if rcall("EXISTS", parentKey) == 1 then
  440. local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
  441. local parentDelayedKey = parentQueueKey .. ":delayed"
  442. local parentWaitingChildrenOrDelayedKey
  443. if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
  444. parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
  445. elseif rcall("ZSCORE", parentDelayedKey, parentId) then
  446. parentWaitingChildrenOrDelayedKey = parentDelayedKey
  447. rcall("HSET", parentKey, "delay", 0)
  448. end
  449. if parentWaitingChildrenOrDelayedKey then
  450. rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
  451. local deferredFailure = "child " .. jobIdKey .. " failed"
  452. rcall("HSET", parentKey, "defa", deferredFailure)
  453. moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
  454. else
  455. if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
  456. local deferredFailure = "child " .. jobIdKey .. " failed"
  457. rcall("HSET", parentKey, "defa", deferredFailure)
  458. end
  459. end
  460. end
  461. end
  462. local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
  463. if rawParentData then
  464. local parentData = cjson.decode(rawParentData)
  465. local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
  466. local parentDependenciesChildrenKey = parentKey .. ":dependencies"
  467. if parentData['fpof'] then
  468. if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
  469. local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful"
  470. rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey)
  471. handleChildFailureAndMoveParentToWait(
  472. parentData['queueKey'],
  473. parentKey,
  474. parentData['id'],
  475. childKey,
  476. timestamp
  477. )
  478. end
  479. elseif parentData['cpof'] then
  480. if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
  481. local parentFailedChildrenKey = parentKey .. ":failed"
  482. rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
  483. moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
  484. end
  485. elseif parentData['idof'] or parentData['rdof'] then
  486. if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
  487. moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
  488. parentKey, parentData['id'], timestamp)
  489. if parentData['idof'] then
  490. local parentFailedChildrenKey = parentKey .. ":failed"
  491. rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
  492. end
  493. end
  494. end
  495. end
  496. end
  497. --[[
  498. Function to remove deduplication key if needed
  499. when a job is moved to completed or failed states.
  500. ]]
  501. local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
  502. deduplicationId, jobId)
  503. if deduplicationId then
  504. local deduplicationKey = prefixKey .. "de:" .. deduplicationId
  505. local pttl = rcall("PTTL", deduplicationKey)
  506. if pttl == 0 then
  507. return rcall("DEL", deduplicationKey)
  508. end
  509. if pttl == -1 then
  510. local currentJobId = rcall('GET', deduplicationKey)
  511. if currentJobId and currentJobId == jobId then
  512. return rcall("DEL", deduplicationKey)
  513. end
  514. end
  515. end
  516. end
  517. --[[
  518. Function to remove job keys.
  519. ]]
  520. local function removeJobKeys(jobKey)
  521. return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
  522. jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
  523. end
  524. --[[
  525. Functions to remove jobs by max age.
  526. ]]
  527. -- Includes
  528. --[[
  529. Function to remove job.
  530. ]]
  531. -- Includes
  532. --[[
  533. Function to remove deduplication key if needed
  534. when a job is being removed.
  535. ]]
  536. local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
  537. jobId, deduplicationId)
  538. if deduplicationId then
  539. local deduplicationKey = prefixKey .. "de:" .. deduplicationId
  540. local currentJobId = rcall('GET', deduplicationKey)
  541. if currentJobId and currentJobId == jobId then
  542. rcall("DEL", deduplicationKey)
  543. -- Also clean up any pending dedup-next data for this dedup ID
  544. rcall("DEL", prefixKey .. "dn:" .. deduplicationId)
  545. return 1
  546. end
  547. end
  548. end
  549. --[[
  550. Check if this job has a parent. If so we will just remove it from
  551. the parent child list, but if it is the last child we should move the parent to "wait/paused"
  552. which requires code from "moveToFinished"
  553. ]]
  554. -- Includes
  555. --[[
  556. Functions to destructure job key.
  557. Just a bit of warning, these functions may be a bit slow and affect performance significantly.
  558. ]]
  559. local getJobIdFromKey = function (jobKey)
  560. return string.match(jobKey, ".*:(.*)")
  561. end
  562. local getJobKeyPrefix = function (jobKey, jobId)
  563. return string.sub(jobKey, 0, #jobKey - #jobId)
  564. end
  565. local function _moveParentToWait(parentPrefix, parentId, emitEvent)
  566. local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
  567. parentPrefix .. "wait", parentPrefix .. "paused")
  568. addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
  569. if emitEvent then
  570. local parentEventStream = parentPrefix .. "events"
  571. rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
  572. end
  573. end
  574. local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
  575. if parentKey then
  576. local parentDependenciesKey = parentKey .. ":dependencies"
  577. local result = rcall("SREM", parentDependenciesKey, jobKey)
  578. if result > 0 then
  579. local pendingDependencies = rcall("SCARD", parentDependenciesKey)
  580. if pendingDependencies == 0 then
  581. local parentId = getJobIdFromKey(parentKey)
  582. local parentPrefix = getJobKeyPrefix(parentKey, parentId)
  583. local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
  584. if numRemovedElements == 1 then
  585. if hard then -- remove parent in same queue
  586. if parentPrefix == baseKey then
  587. removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
  588. removeJobKeys(parentKey)
  589. if debounceId then
  590. rcall("DEL", parentPrefix .. "de:" .. debounceId)
  591. end
  592. else
  593. _moveParentToWait(parentPrefix, parentId)
  594. end
  595. else
  596. _moveParentToWait(parentPrefix, parentId, true)
  597. end
  598. end
  599. end
  600. return true
  601. end
  602. else
  603. local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
  604. local missedParentKey = parentAttributes[1]
  605. if( (type(missedParentKey) == "string") and missedParentKey ~= ""
  606. and (rcall("EXISTS", missedParentKey) == 1)) then
  607. local parentDependenciesKey = missedParentKey .. ":dependencies"
  608. local result = rcall("SREM", parentDependenciesKey, jobKey)
  609. if result > 0 then
  610. local pendingDependencies = rcall("SCARD", parentDependenciesKey)
  611. if pendingDependencies == 0 then
  612. local parentId = getJobIdFromKey(missedParentKey)
  613. local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
  614. local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
  615. if numRemovedElements == 1 then
  616. if hard then
  617. if parentPrefix == baseKey then
  618. removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
  619. removeJobKeys(missedParentKey)
  620. if parentAttributes[2] then
  621. rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
  622. end
  623. else
  624. _moveParentToWait(parentPrefix, parentId)
  625. end
  626. else
  627. _moveParentToWait(parentPrefix, parentId, true)
  628. end
  629. end
  630. end
  631. return true
  632. end
  633. end
  634. end
  635. return false
  636. end
  637. local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
  638. local jobKey = baseKey .. jobId
  639. removeParentDependencyKey(jobKey, hard, nil, baseKey)
  640. if shouldRemoveDeduplicationKey then
  641. local deduplicationId = rcall("HGET", jobKey, "deid")
  642. removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
  643. end
  644. removeJobKeys(jobKey)
  645. end
  646. local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
  647. local start = timestamp - maxAge * 1000
  648. local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit)
  649. for i, jobId in ipairs(jobIds) do
  650. removeJob(jobId, false, prefix, false --[[remove debounce key]])
  651. end
  652. if #jobIds > 0 then
  653. if #jobIds < maxLimit then
  654. rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
  655. else
  656. for from, to in batches(#jobIds, 7000) do
  657. rcall("ZREM", targetSet, unpack(jobIds, from, to))
  658. end
  659. end
  660. end
  661. end
  662. --[[
  663. Functions to remove jobs by max count.
  664. ]]
  665. -- Includes
  666. local function removeJobsByMaxCount(maxCount, targetSet, prefix)
  667. local start = maxCount
  668. local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
  669. for i, jobId in ipairs(jobIds) do
  670. removeJob(jobId, false, prefix, false --[[remove debounce key]])
  671. end
  672. rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
  673. end
  674. local function removeLock(jobKey, stalledKey, token, jobId)
  675. if token ~= "0" then
  676. local lockKey = jobKey .. ':lock'
  677. local lockToken = rcall("GET", lockKey)
  678. if lockToken == token then
  679. rcall("DEL", lockKey)
  680. rcall("SREM", stalledKey, jobId)
  681. else
  682. if lockToken then
  683. -- Lock exists but token does not match
  684. return -6
  685. else
  686. -- Lock is missing completely
  687. return -2
  688. end
  689. end
  690. end
  691. return 0
  692. end
  693. --[[
  694. Function to create a new job from stored dedup-next data
  695. when a deduplicated job with keepLastIfActive finishes.
  696. At most one next job is created per deduplication ID.
  697. Multiple triggers while active overwrite the dedup-next data,
  698. so only the latest data is used.
  699. ]]
  700. -- Includes
  701. --[[
  702. Function to get max events value or set by default 10000.
  703. ]]
  704. local function getOrSetMaxEvents(metaKey)
  705. local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
  706. if not maxEvents then
  707. maxEvents = 10000
  708. rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
  709. end
  710. return maxEvents
  711. end
  712. --[[
  713. Function to set the deduplication key for a job.
  714. Uses TTL from deduplication opts if provided.
  715. ]]
  716. local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
  717. local ttl = deduplicationOpts and deduplicationOpts['ttl']
  718. if ttl and ttl > 0 then
  719. rcall('SET', deduplicationKey, jobId, 'PX', ttl)
  720. else
  721. rcall('SET', deduplicationKey, jobId)
  722. end
  723. end
  724. --[[
  725. Shared helper to store a job and enqueue it into the appropriate list/set.
  726. Handles delayed, prioritized, and standard (LIFO/FIFO) jobs.
  727. Emits the appropriate event after enqueuing ("delayed" or "waiting").
  728. Returns delay, priority from storeJob.
  729. ]]
  730. -- Includes
  731. --[[
  732. Adds a delayed job to the queue by doing the following:
  733. - Creates a new job key with the job data.
  734. - adds to delayed zset.
  735. - Emits a global event 'delayed' if the job is delayed.
  736. ]]
  737. -- Includes
  738. --[[
  739. Bake in the job id first 12 bits into the timestamp
  740. to guarantee correct execution order of delayed jobs
  741. (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
  742. WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
  743. ]]
  744. local function getDelayedScore(delayedKey, timestamp, delay)
  745. local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
  746. local minScore = delayedTimestamp * 0x1000
  747. local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
  748. local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
  749. minScore, "WITHSCORES","LIMIT", 0, 1)
  750. if #result then
  751. local currentMaxScore = tonumber(result[2])
  752. if currentMaxScore ~= nil then
  753. if currentMaxScore >= maxScore then
  754. return maxScore, delayedTimestamp
  755. else
  756. return currentMaxScore + 1, delayedTimestamp
  757. end
  758. end
  759. end
  760. return minScore, delayedTimestamp
  761. end
  762. local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
  763. maxEvents, markerKey, delay)
  764. local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
  765. rcall("ZADD", delayedKey, score, jobId)
  766. rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
  767. "jobId", jobId, "delay", delayedTimestamp)
  768. -- mark that a delayed job is available
  769. addDelayMarkerIfNeeded(markerKey, delayedKey)
  770. end
  771. --[[
  772. Function to store a job
  773. ]]
  774. local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
  775. parentKey, parentData, repeatJobKey)
  776. local jsonOpts = cjson.encode(opts)
  777. local delay = opts['delay'] or 0
  778. local priority = opts['priority'] or 0
  779. local debounceId = opts['de'] and opts['de']['id']
  780. local optionalValues = {}
  781. if parentKey ~= nil then
  782. table.insert(optionalValues, "parentKey")
  783. table.insert(optionalValues, parentKey)
  784. table.insert(optionalValues, "parent")
  785. table.insert(optionalValues, parentData)
  786. end
  787. if repeatJobKey then
  788. table.insert(optionalValues, "rjk")
  789. table.insert(optionalValues, repeatJobKey)
  790. end
  791. if debounceId then
  792. table.insert(optionalValues, "deid")
  793. table.insert(optionalValues, debounceId)
  794. end
  795. rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
  796. "timestamp", timestamp, "delay", delay, "priority", priority,
  797. unpack(optionalValues))
  798. rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
  799. return delay, priority
  800. end
  801. local function storeAndEnqueueJob(eventsKey, jobIdKey, jobId, name, data, opts,
  802. timestamp, parentKey, parentData, repeatJobKey, maxEvents,
  803. waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
  804. priorityCounterKey, delayedKey, markerKey)
  805. local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
  806. opts, timestamp, parentKey, parentData, repeatJobKey)
  807. if delay ~= 0 and delayedKey then
  808. addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
  809. else
  810. local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
  811. if priority > 0 then
  812. addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
  813. priorityCounterKey, isPausedOrMaxed)
  814. else
  815. local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
  816. addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
  817. end
  818. rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
  819. "jobId", jobId)
  820. end
  821. return delay, priority
  822. end
  823. local function requeueDeduplicatedJob(prefix, deduplicationId, eventStreamKey,
  824. metaKey, activeKey, waitKey, pausedKey, markerKey, prioritizedKey,
  825. priorityCounterKey, delayedKey, timestamp)
  826. local deduplicationNextKey = prefix .. "dn:" .. deduplicationId
  827. if rcall("EXISTS", deduplicationNextKey) == 1 then
  828. local nextData = rcall("HMGET", deduplicationNextKey,
  829. "name", "data", "opts", "pk", "pd", "pdk", "rjk")
  830. local newJobId = rcall("INCR", prefix .. "id") .. ""
  831. local newJobIdKey = prefix .. newJobId
  832. local newOpts = cjson.decode(nextData[3])
  833. local deduplicationKey = prefix .. "de:" .. deduplicationId
  834. local parentKey = nextData[4] or nil
  835. local parentData = nextData[5] or nil
  836. local parentDependenciesKey = nextData[6] or nil
  837. local repeatJobKey = nextData[7] or nil
  838. -- Set dedup key for the new job (without TTL when keepLastIfActive,
  839. -- so the key outlives the job's active duration)
  840. local deOpts = newOpts['de']
  841. if deOpts and deOpts['keepLastIfActive'] then
  842. rcall('SET', deduplicationKey, newJobId)
  843. else
  844. setDeduplicationKey(deduplicationKey, newJobId, deOpts)
  845. end
  846. -- Store and enqueue using the shared helper (handles priority/lifo/delayed)
  847. local maxEvents = getOrSetMaxEvents(metaKey)
  848. storeAndEnqueueJob(eventStreamKey, newJobIdKey, newJobId, nextData[1], nextData[2],
  849. newOpts, timestamp, parentKey, parentData, repeatJobKey, maxEvents,
  850. waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
  851. priorityCounterKey, delayedKey, markerKey)
  852. -- Register as child dependency if the job has a parent
  853. if parentDependenciesKey then
  854. rcall("SADD", parentDependenciesKey, newJobIdKey)
  855. end
  856. -- Only delete the dedup-next hash after the job is fully created,
  857. -- so that if any step above errors, the data is not permanently lost.
  858. rcall("DEL", deduplicationNextKey)
  859. end
  860. end
  861. --[[
  862. Function to trim events, default 10000.
  863. ]]
  864. -- Includes
  865. local function trimEvents(metaKey, eventStreamKey)
  866. local maxEvents = getOrSetMaxEvents(metaKey)
  867. if maxEvents then
  868. rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
  869. else
  870. rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
  871. end
  872. end
  873. --[[
  874. Validate and move or add dependencies to parent.
  875. ]]
  876. -- Includes
  877. local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
  878. parentId, jobIdKey, returnvalue, timestamp )
  879. local processedSet = parentKey .. ":processed"
  880. rcall("HSET", processedSet, jobIdKey, returnvalue)
  881. moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
  882. end
  883. --[[
  884. Function to update a bunch of fields in a job.
  885. ]]
  886. local function updateJobFields(jobKey, msgpackedFields)
  887. if msgpackedFields and #msgpackedFields > 0 then
  888. local fieldsToUpdate = cmsgpack.unpack(msgpackedFields)
  889. if fieldsToUpdate then
  890. rcall("HMSET", jobKey, unpack(fieldsToUpdate))
  891. end
  892. end
  893. end
  894. local jobIdKey = KEYS[12]
  895. if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
  896. -- Make sure it does not have pending dependencies
  897. -- It must happen before removing lock
  898. if ARGV[5] == "completed" then
  899. if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
  900. return -4
  901. end
  902. if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then
  903. return -9
  904. end
  905. end
  906. local opts = cmsgpack.unpack(ARGV[8])
  907. local token = opts['token']
  908. local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
  909. if errorCode < 0 then
  910. return errorCode
  911. end
  912. updateJobFields(jobIdKey, ARGV[9]);
  913. local attempts = opts['attempts']
  914. local maxMetricsSize = opts['maxMetricsSize']
  915. local maxCount = opts['keepJobs']['count']
  916. local maxAge = opts['keepJobs']['age']
  917. local maxLimit = opts['keepJobs']['limit'] or 1000
  918. local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
  919. local parentKey = jobAttributes[1] or ""
  920. local parentId = ""
  921. local parentQueueKey = ""
  922. if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
  923. local jsonDecodedParent = cjson.decode(jobAttributes[2])
  924. parentId = jsonDecodedParent['id']
  925. parentQueueKey = jsonDecodedParent['queueKey']
  926. end
  927. local jobId = ARGV[1]
  928. local timestamp = ARGV[2]
  929. -- Remove from active list (if not active we shall return error)
  930. local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
  931. if (numRemovedElements < 1) then
  932. return -3
  933. end
  934. local eventStreamKey = KEYS[4]
  935. local metaKey = KEYS[9]
  936. -- Trim events before emiting them to avoid trimming events emitted in this script
  937. trimEvents(metaKey, eventStreamKey)
  938. local prefix = ARGV[7]
  939. removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
  940. -- Check if there is requeue data for this dedup ID (keepLastIfActive mode)
  941. if jobAttributes[3] then
  942. requeueDeduplicatedJob(prefix, jobAttributes[3], eventStreamKey,
  943. metaKey, KEYS[2], KEYS[1], KEYS[8], KEYS[14], KEYS[3], KEYS[10],
  944. KEYS[7], timestamp)
  945. end
  946. -- If job has a parent we need to
  947. -- 1) remove this job id from parents dependencies
  948. -- 2) move the job Id to parent "processed" set
  949. -- 3) push the results into parent "results" list
  950. -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
  951. if parentId == "" and parentKey ~= "" then
  952. parentId = getJobIdFromKey(parentKey)
  953. parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
  954. end
  955. if parentId ~= "" then
  956. if ARGV[5] == "completed" then
  957. local dependenciesSet = parentKey .. ":dependencies"
  958. if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
  959. updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
  960. timestamp)
  961. end
  962. else
  963. moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
  964. end
  965. end
  966. local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
  967. -- Remove job?
  968. if maxCount ~= 0 then
  969. local targetSet = KEYS[11]
  970. -- Add to complete/failed set
  971. rcall("ZADD", targetSet, timestamp, jobId)
  972. rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
  973. -- "returnvalue" / "failedReason" and "finishedOn"
  974. if ARGV[5] == "failed" then
  975. rcall("HDEL", jobIdKey, "defa")
  976. end
  977. -- Remove old jobs?
  978. if maxAge ~= nil then
  979. removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
  980. end
  981. if maxCount ~= nil and maxCount > 0 then
  982. removeJobsByMaxCount(maxCount, targetSet, prefix)
  983. end
  984. else
  985. removeJobKeys(jobIdKey)
  986. if parentKey ~= "" then
  987. -- TODO: when a child is removed when finished, result or failure in parent
  988. -- must not be deleted, those value references should be deleted when the parent
  989. -- is deleted
  990. removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
  991. end
  992. end
  993. rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
  994. if ARGV[5] == "failed" then
  995. if tonumber(attemptsMade) >= tonumber(attempts) then
  996. rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
  997. attemptsMade)
  998. end
  999. end
  1000. -- Collect metrics
  1001. if maxMetricsSize ~= "" then
  1002. collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
  1003. end
  1004. -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
  1005. -- and not rate limited.
  1006. if (ARGV[6] == "1") then
  1007. local result = fetchNextJob(KEYS[1], KEYS[2], KEYS[3], eventStreamKey,
  1008. KEYS[6], KEYS[7], KEYS[8], metaKey, KEYS[10], KEYS[14], prefix,
  1009. timestamp, opts)
  1010. if result then
  1011. return result
  1012. end
  1013. end
  1014. local waitLen = rcall("LLEN", KEYS[1])
  1015. if waitLen == 0 then
  1016. local activeLen = rcall("LLEN", KEYS[2])
  1017. if activeLen == 0 then
  1018. local prioritizedLen = rcall("ZCARD", KEYS[3])
  1019. if prioritizedLen == 0 then
  1020. rcall("XADD", eventStreamKey, "*", "event", "drained")
  1021. end
  1022. end
  1023. end
  1024. return 0
  1025. else
  1026. return -1
  1027. end
  1028. `;
  1029. export const moveToFinished = {
  1030. name: 'moveToFinished',
  1031. content,
  1032. keys: 14,
  1033. };
  1034. //# sourceMappingURL=moveToFinished-14.js.map