Mike/Kordant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement background job system with queue, worker, scheduler, and handlers

Browse Source 
- Add job queue abstraction (InMemoryQueue and Redis/BullMQ adapter)
- Add polling worker with retry logic and exponential backoff
- Add 6 job handlers: darkwatch.scan, voiceprint.batch, hometitle.scan,
  removebrokers.process, reports.generate, notifications.send
- Add cron-based scheduler with tier-appropriate frequencies
  (Basic/Plus/Premium)
- Add tRPC scheduler router for admin (runJobNow, getJobStatus, etc.)
- Add entry point with graceful shutdown support
- Achieve 100% test pass rate for new job system
This commit is contained in:
Michael Freno
2026-05-25 17:16:21 -04:00
parent 659ab9b71a
commit eb8e57c674
19 changed files with 1429 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
pnpm-lock.yaml generated
View File

@@ -55,15 +55,24 @@ importers:
bcryptjs:
specifier: ^3.0.3
version: 3.0.3
bullmq:
specifier: ^5.77.3
version: 5.77.3
drizzle-orm:
specifier: ^0.45.2
version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0)
firebase-admin:
specifier: ^13.10.0
version: 13.10.0
ioredis:
specifier: ^5.10.1
version: 5.10.1
jose:
specifier: ^5
version: 5.10.0
node-cron:
specifier: ^4.2.1
version: 4.2.1
pg:
specifier: ^8.21.0
version: 8.21.0
@@ -92,6 +101,9 @@ importers:
specifier: ^7.0.0
version: 7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.48.0)
devDependencies:
'@types/node-cron':
specifier: ^3.0.11
version: 3.0.11
'@types/pg':
specifier: ^8.20.0
version: 8.20.0
@@ -1040,6 +1052,36 @@ packages:
engines: {node: '>=18'}
hasBin: true
'@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4':
resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==}
cpu: [arm64]
os: [darwin]
'@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4':
resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==}
cpu: [x64]
os: [darwin]
'@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4':
resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==}
cpu: [arm64]
os: [linux]
'@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4':
resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==}
cpu: [arm]
os: [linux]
'@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4':
resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==}
cpu: [x64]
os: [linux]
'@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4':
resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==}
cpu: [x64]
os: [win32]
'@nodable/entities@2.1.0':
resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}
@@ -1650,6 +1692,9 @@ packages:
'@types/ms@2.1.0':
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
'@types/node-cron@3.0.11':
resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==}
'@types/node@25.9.1':
resolution: {integrity: sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==}
@@ -1943,6 +1988,15 @@ packages:
buffer@6.0.3:
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
bullmq@5.77.3:
resolution: {integrity: sha512-ccGjnRF5Bbu21CfzDG7toIewoAj5W9DUHjqhFnppt8YBtoQplynzBGuSzn7yYQobpyYeHEqrYhiJ1NZInrKbyA==}
engines: {node: '>=12.22.0'}
peerDependencies:
redis: '>=5.0.0'
peerDependenciesMeta:
redis:
optional: true
bundle-name@4.1.0:
resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==}
engines: {node: '>=18'}
@@ -2085,6 +2139,10 @@ packages:
resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==}
engines: {node: '>= 14'}
cron-parser@4.9.0:
resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}
engines: {node: '>=12.0.0'}
croner@10.0.1:
resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}
engines: {node: '>=18.0'}
@@ -3051,6 +3109,10 @@ packages:
lru-memoizer@2.3.0:
resolution: {integrity: sha512-GXn7gyHAMhO13WSKrIiNfztwxodVsP8IoZ3XfrJV4yH2x0/OeTO/FIaAHTY5YekdGgW94njfuKmyyt1E0mR6Ug==}
luxon@3.7.2:
resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==}
engines: {node: '>=12'}
magic-string@0.30.21:
resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==}
@@ -3153,6 +3215,13 @@ packages:
ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
msgpackr-extract@3.0.4:
resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==}
hasBin: true
msgpackr@2.0.1:
resolution: {integrity: sha512-9J+tqTEsbHqY8YohazYgty7LgerFIWxvMLpUjqETSmjHojtJm2WnX2kK/2a1fLI7CO7ERP1YSEUXMucz4j+yBA==}
nanoid@3.3.12:
resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -3168,9 +3237,16 @@ packages:
xml2js:
optional: true
node-abort-controller@3.1.1:
resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
node-addon-api@7.1.1:
resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==}
node-cron@4.2.1:
resolution: {integrity: sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==}
engines: {node: '>=6.0.0'}
node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
@@ -3196,6 +3272,10 @@ packages:
resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==}
engines: {node: '>= 6.13.0'}
node-gyp-build-optional-packages@5.2.2:
resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==}
hasBin: true
node-gyp-build@4.8.4:
resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}
hasBin: true
@@ -3576,6 +3656,11 @@ packages:
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
hasBin: true
semver@7.8.0:
resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==}
engines: {node: '>=10'}
hasBin: true
semver@7.8.1:
resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==}
engines: {node: '>=10'}
@@ -5055,6 +5140,24 @@ snapshots:
- encoding
- supports-color
'@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4':
optional: true
'@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4':
optional: true
'@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4':
optional: true
'@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4':
optional: true
'@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4':
optional: true
'@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4':
optional: true
'@nodable/entities@2.1.0':
optional: true
@@ -5658,6 +5761,8 @@ snapshots:
'@types/ms@2.1.0': {}
'@types/node-cron@3.0.11': {}
'@types/node@25.9.1':
dependencies:
undici-types: 7.24.6
@@ -5982,6 +6087,17 @@ snapshots:
base64-js: 1.5.1
ieee754: 1.2.1
bullmq@5.77.3:
dependencies:
cron-parser: 4.9.0
ioredis: 5.10.1
msgpackr: 2.0.1
node-abort-controller: 3.1.1
semver: 7.8.0
tslib: 2.8.1
transitivePeerDependencies:
- supports-color
bundle-name@4.1.0:
dependencies:
run-applescript: 7.1.0
@@ -6115,6 +6231,10 @@ snapshots:
crc-32: 1.2.2
readable-stream: 4.7.0
cron-parser@4.9.0:
dependencies:
luxon: 3.7.2
croner@10.0.1: {}
cross-spawn@7.0.6:
@@ -7118,6 +7238,8 @@ snapshots:
lodash.clonedeep: 4.5.0
lru-cache: 6.0.0
luxon@3.7.2: {}
magic-string@0.30.21:
dependencies:
'@jridgewell/sourcemap-codec': 1.5.5
@@ -7222,6 +7344,22 @@ snapshots:
ms@2.1.3: {}
msgpackr-extract@3.0.4:
dependencies:
node-gyp-build-optional-packages: 5.2.2
optionalDependencies:
'@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4
'@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4
'@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4
'@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4
'@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4
'@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4
optional: true
msgpackr@2.0.1:
optionalDependencies:
msgpackr-extract: 3.0.4
nanoid@3.3.12: {}
nitropack@2.13.4:
@@ -7328,8 +7466,12 @@ snapshots:
- supports-color
- uploadthing
node-abort-controller@3.1.1: {}
node-addon-api@7.1.1: {}
node-cron@4.2.1: {}
node-domexception@1.0.0: {}
node-fetch-native@1.6.7: {}
@@ -7346,6 +7488,11 @@ snapshots:
node-forge@1.4.0: {}
node-gyp-build-optional-packages@5.2.2:
dependencies:
detect-libc: 2.1.2
optional: true
node-gyp-build@4.8.4: {}
node-mock-http@1.0.4: {}
@@ -7762,6 +7909,8 @@ snapshots:
semver@6.3.1: {}
semver@7.8.0: {}
semver@7.8.1: {}
send@1.2.1: