diff --git a/packages/core/src/infrastructure/bull.types.ts b/packages/core/src/infrastructure/bull.types.ts index d5586922..8bd4c257 100644 --- a/packages/core/src/infrastructure/bull.types.ts +++ b/packages/core/src/infrastructure/bull.types.ts @@ -378,6 +378,12 @@ export const OneTimeCodeBullJob = z.discriminatedUnion('name', [ ]); export const OpportunityBullJob = z.discriminatedUnion('name', [ + z.object({ + name: z.literal('opportunity.check_expired'), + data: z.object({ + opportunityId: z.string().trim().min(1).optional(), + }), + }), z.object({ name: z.literal('opportunity.create'), data: z.object({ diff --git a/packages/core/src/modules/opportunities.ts b/packages/core/src/modules/opportunities.ts index 68d2489f..5aabdfc2 100644 --- a/packages/core/src/modules/opportunities.ts +++ b/packages/core/src/modules/opportunities.ts @@ -142,7 +142,114 @@ export async function checkForDeletedOpportunity({ } } -// "Create Opportunity" +const EXPIRED_PHRASES = [ + '404', + 'closed', + 'does not exist', + "doesn't exist", + 'expired', + 'filled', + 'no longer accepting', + 'no longer available', + 'no longer exists', + 'no longer open', + 'not accepting', + 'not available', + 'not be found', + 'not currently accepting', + 'not found', + 'not open', + 'oops', + 'removed', + 'sorry', +]; + +type CheckForExpiredOpportunityInput = { + link?: string | null; + opportunityId: string; +}; + +/** + * This function uses puppeteer to scrape the opportunity's website and + * determine whether or not the opportunity has closed or not. If it has, + * the opportunity will be marked as "expired" and thus will no longer appear + * in the opportunities board. + * + * Returns `true` if the opportunity has expired, `false` otherwise. + * + * @param input - The opportunity to check for expiration. + */ +async function checkForExpiredOpportunity({ + link = null, + opportunityId, +}: CheckForExpiredOpportunityInput): Promise> { + // If the link is passed in, we'll use that. Otherwise, we'll scrape the + // opportunity's website to get the link. + if (!link) { + link = await getLinkFromOpportunity(opportunityId); + } + + if (!link) { + return success(false); + } + + const content = await getPageContent(link); + + const hasExpired = EXPIRED_PHRASES.some((phrase) => { + return content.toLowerCase().includes(phrase); + }); + + if (hasExpired) { + await db + .updateTable('opportunities') + .set({ expiresAt: new Date() }) + .where('id', '=', opportunityId) + .executeTakeFirst(); + } + + return success(hasExpired); +} + +/** + * Checks for expired opportunities and marks them as expired. This can be + * triggered via a Bull job. This is limited to 100 opportunities at a time + * to prevent overwhelming our server with too many puppeteer instances. + * + * @returns The number of opportunities marked as expired. + */ +async function checkForExpiredOpportunities(): Promise> { + const opportunities = await db + .selectFrom('opportunities') + .leftJoin('slackMessages', (join) => { + return join + .onRef('slackMessages.channelId', '=', 'opportunities.slackChannelId') + .onRef('slackMessages.id', '=', 'opportunities.slackMessageId'); + }) + .select(['opportunities.id', 'slackMessages.text']) + .where('expiresAt', '>', new Date()) + .orderBy('createdAt', 'asc') + .limit(100) + .execute(); + + let count = 0; + + for (const opportunity of opportunities) { + const result = await checkForExpiredOpportunity({ + link: getFirstLinkInMessage(opportunity.text!), + opportunityId: opportunity.id, + }); + + if (result.ok && result.data === true) { + count += 1; + } + } + + console.log( + `Checked ${opportunities.length} opportunities and found ${count} expired opportunities.` + ); + + return success(count); +} type CreateOpportunityInput = { sendNotification?: boolean; @@ -563,6 +670,12 @@ export async function refineOpportunity( }); } + // If the AI didn't return a title, then we don't want to finish the process + // because there was no opportunity to refine. We exit gracefully. + if (!data.title || !data.description) { + return success({}); + } + const opportunity = await db.transaction().execute(async (trx) => { const companyId = data.company ? await getMostRelevantCompany(trx, data.company) @@ -659,7 +772,7 @@ export async function refineOpportunity( * @param message - Slack message to extract the URL from. * @returns First URL found in the message or `null` if it doesn't exist. */ -function getFirstLinkInMessage(message: string) { +function getFirstLinkInMessage(message: string): string | undefined { return message.match(/<(https?:\/\/[^\s|>]+)(?:\|[^>]+)?>/)?.[1]; } @@ -878,6 +991,11 @@ export const opportunityWorker = registerWorker( OpportunityBullJob, async (job) => { const result = await match(job) + .with({ name: 'opportunity.check_expired' }, async ({ data }) => { + return data.opportunityId + ? checkForExpiredOpportunity({ opportunityId: data.opportunityId }) + : checkForExpiredOpportunities(); + }) .with({ name: 'opportunity.create' }, async ({ data }) => { return createOpportunity(data); })