moveToFinished-14.lua 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. --[[
  2. Move job from active to a finished status (completed o failed)
  3. A job can only be moved to completed if it was active.
  4. The job must be locked before it can be moved to a finished status,
  5. and the lock must be released in this script.
  6. Input:
  7. KEYS[1] wait key
  8. KEYS[2] active key
  9. KEYS[3] prioritized key
  10. KEYS[4] event stream key
  11. KEYS[5] stalled key
  12. -- Rate limiting
  13. KEYS[6] rate limiter key
  14. KEYS[7] delayed key
  15. KEYS[8] paused key
  16. KEYS[9] meta key
  17. KEYS[10] pc priority counter
  18. KEYS[11] completed/failed key
  19. KEYS[12] jobId key
  20. KEYS[13] metrics key
  21. KEYS[14] marker key
  22. ARGV[1] jobId
  23. ARGV[2] timestamp
  24. ARGV[3] msg property returnvalue / failedReason
  25. ARGV[4] return value / failed reason
  26. ARGV[5] target (completed/failed)
  27. ARGV[6] fetch next?
  28. ARGV[7] keys prefix
  29. ARGV[8] opts
  30. ARGV[9] job fields to update
  31. opts - token - lock token
  32. opts - keepJobs
  33. opts - lockDuration - lock duration in milliseconds
  34. opts - attempts max attempts
  35. opts - maxMetricsSize
  36. opts - fpof - fail parent on fail
  37. opts - cpof - continue parent on fail
  38. opts - idof - ignore dependency on fail
  39. opts - rdof - remove dependency on fail
  40. opts - name - worker name
  41. Output:
  42. 0 OK
  43. -1 Missing key.
  44. -2 Missing lock.
  45. -3 Job not in active set
  46. -4 Job has pending children
  47. -6 Lock is not owned by this client
  48. -9 Job has failed children
  49. Events:
  50. 'completed/failed'
  51. ]]
  52. local rcall = redis.call
  53. --- Includes
  54. --- @include "includes/collectMetrics"
  55. --- @include "includes/fetchNextJob"
  56. --- @include "includes/moveChildFromDependenciesIfNeeded"
  57. --- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
  58. --- @include "includes/removeJobKeys"
  59. --- @include "includes/removeJobsByMaxAge"
  60. --- @include "includes/removeJobsByMaxCount"
  61. --- @include "includes/removeLock"
  62. --- @include "includes/removeParentDependencyKey"
  63. --- @include "includes/requeueDeduplicatedJob"
  64. --- @include "includes/trimEvents"
  65. --- @include "includes/updateParentDepsIfNeeded"
  66. --- @include "includes/updateJobFields"
  67. local jobIdKey = KEYS[12]
  68. if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
  69. -- Make sure it does not have pending dependencies
  70. -- It must happen before removing lock
  71. if ARGV[5] == "completed" then
  72. if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
  73. return -4
  74. end
  75. if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then
  76. return -9
  77. end
  78. end
  79. local opts = cmsgpack.unpack(ARGV[8])
  80. local token = opts['token']
  81. local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
  82. if errorCode < 0 then
  83. return errorCode
  84. end
  85. updateJobFields(jobIdKey, ARGV[9]);
  86. local attempts = opts['attempts']
  87. local maxMetricsSize = opts['maxMetricsSize']
  88. local maxCount = opts['keepJobs']['count']
  89. local maxAge = opts['keepJobs']['age']
  90. local maxLimit = opts['keepJobs']['limit'] or 1000
  91. local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
  92. local parentKey = jobAttributes[1] or ""
  93. local parentId = ""
  94. local parentQueueKey = ""
  95. if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
  96. local jsonDecodedParent = cjson.decode(jobAttributes[2])
  97. parentId = jsonDecodedParent['id']
  98. parentQueueKey = jsonDecodedParent['queueKey']
  99. end
  100. local jobId = ARGV[1]
  101. local timestamp = ARGV[2]
  102. -- Remove from active list (if not active we shall return error)
  103. local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
  104. if (numRemovedElements < 1) then
  105. return -3
  106. end
  107. local eventStreamKey = KEYS[4]
  108. local metaKey = KEYS[9]
  109. -- Trim events before emiting them to avoid trimming events emitted in this script
  110. trimEvents(metaKey, eventStreamKey)
  111. local prefix = ARGV[7]
  112. removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
  113. -- Check if there is requeue data for this dedup ID (keepLastIfActive mode)
  114. if jobAttributes[3] then
  115. requeueDeduplicatedJob(prefix, jobAttributes[3], eventStreamKey,
  116. metaKey, KEYS[2], KEYS[1], KEYS[8], KEYS[14], KEYS[3], KEYS[10],
  117. KEYS[7], timestamp)
  118. end
  119. -- If job has a parent we need to
  120. -- 1) remove this job id from parents dependencies
  121. -- 2) move the job Id to parent "processed" set
  122. -- 3) push the results into parent "results" list
  123. -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
  124. if parentId == "" and parentKey ~= "" then
  125. parentId = getJobIdFromKey(parentKey)
  126. parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
  127. end
  128. if parentId ~= "" then
  129. if ARGV[5] == "completed" then
  130. local dependenciesSet = parentKey .. ":dependencies"
  131. if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
  132. updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
  133. timestamp)
  134. end
  135. else
  136. moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
  137. end
  138. end
  139. local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
  140. -- Remove job?
  141. if maxCount ~= 0 then
  142. local targetSet = KEYS[11]
  143. -- Add to complete/failed set
  144. rcall("ZADD", targetSet, timestamp, jobId)
  145. rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
  146. -- "returnvalue" / "failedReason" and "finishedOn"
  147. if ARGV[5] == "failed" then
  148. rcall("HDEL", jobIdKey, "defa")
  149. end
  150. -- Remove old jobs?
  151. if maxAge ~= nil then
  152. removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
  153. end
  154. if maxCount ~= nil and maxCount > 0 then
  155. removeJobsByMaxCount(maxCount, targetSet, prefix)
  156. end
  157. else
  158. removeJobKeys(jobIdKey)
  159. if parentKey ~= "" then
  160. -- TODO: when a child is removed when finished, result or failure in parent
  161. -- must not be deleted, those value references should be deleted when the parent
  162. -- is deleted
  163. removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
  164. end
  165. end
  166. rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
  167. if ARGV[5] == "failed" then
  168. if tonumber(attemptsMade) >= tonumber(attempts) then
  169. rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
  170. attemptsMade)
  171. end
  172. end
  173. -- Collect metrics
  174. if maxMetricsSize ~= "" then
  175. collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
  176. end
  177. -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
  178. -- and not rate limited.
  179. if (ARGV[6] == "1") then
  180. local result = fetchNextJob(KEYS[1], KEYS[2], KEYS[3], eventStreamKey,
  181. KEYS[6], KEYS[7], KEYS[8], metaKey, KEYS[10], KEYS[14], prefix,
  182. timestamp, opts)
  183. if result then
  184. return result
  185. end
  186. end
  187. local waitLen = rcall("LLEN", KEYS[1])
  188. if waitLen == 0 then
  189. local activeLen = rcall("LLEN", KEYS[2])
  190. if activeLen == 0 then
  191. local prioritizedLen = rcall("ZCARD", KEYS[3])
  192. if prioritizedLen == 0 then
  193. rcall("XADD", eventStreamKey, "*", "event", "drained")
  194. end
  195. end
  196. end
  197. return 0
  198. else
  199. return -1
  200. end