From 6c9a5012589423a6e030d802d4086c60225f7e77 Mon Sep 17 00:00:00 2001 From: dslucas <55889299+dslucas@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:54:26 -0500 Subject: [PATCH] Add support for Twitter API v2 Full-Archive Search (#18) As-is, Harassment Manager leverages the Enterprise Full-Archive Search API to fetch tweets directed at the logged-in user. This change enables developers to use either Enterprise or Twitter API v2 Full-Archive Search, which is the latest version of the Twitter API. To implement this, we add the relevant request logic to `twitter.middleware.ts`. We "pack" the v2 response format into the Enterprise response format to enable developers to switch between Enterprise and v2 forwards and backwards, without breaking usage for existing users of the application. Additional changes include: - Removing unused types and defining new ones for the v2 format - Changing `fromDate` and `toDate` to `startDateTimeMs` and `endDateTimeMs` and using formatting functions to accommodate differences in the timestamps expected by Enterprise vs. v2 - Modifying `server_config_template.json` with a `bearerToken` field, which is necessary for using v2 Full-Archive Search - Updating the AppEngine runtime to `nodejs12` to support some new operators, like `flatMap` - Updating documentation We deployed and tested an instance of Harassment Manager with each API and the application's behavior is largely identical between both versions, with a couple minor differences: 1. The APIs return slightly different sets of tweets because of differences in the granularity of the timestamp format expected by each API. This difference is usually no more than an additional 1-3 tweets in the v2 instance. 2. The tweets are displayed in slightly different orders when sorted by "Priority". This is due to small differences in how we parse the tweet text, which causes some variation in the Perspective API scores for the text. We opened issue #19 to investigate. We also noticed the Enterprise instance does not render some images that the v2 instance does. This seems more like an implementation issue on our end, rather than an API difference. We opened issue #17 to investigate. --- .gcloudignore | 5 +- app.yaml | 4 +- docs/1_setup.md | 65 +++-- docs/2_development.md | 40 ++- package-lock.json | 14 +- package.json | 2 +- src/app/social-media-item.service.ts | 29 +-- src/app/test_constants.ts | 4 - .../tweet-image/tweet-image.component.html | 6 +- src/common-types.ts | 100 ++++++-- src/server/middleware/twitter.middleware.ts | 238 +++++++++++++++++- src/server/server_config.template.json | 5 +- src/server/serving.ts | 12 +- 13 files changed, 420 insertions(+), 104 deletions(-) diff --git a/.gcloudignore b/.gcloudignore index a3f0c76..3d36e62 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -14,4 +14,7 @@ .gitignore # Node.js dependencies: -node_modules/ \ No newline at end of file +node_modules/ + +# Miscellaneous +.angular/cache \ No newline at end of file diff --git a/app.yaml b/app.yaml index c0a65ed..8f6a197 100644 --- a/app.yaml +++ b/app.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -runtime: nodejs10 +runtime: nodejs12 service: default instance_class: F4_1G @@ -25,7 +25,7 @@ automatic_scaling: # Required for min_idle_instances! inbound_services: -- warmup + - warmup env_variables: NODE_ENV: production diff --git a/docs/1_setup.md b/docs/1_setup.md index b2b6122..c7ce3ee 100644 --- a/docs/1_setup.md +++ b/docs/1_setup.md @@ -18,40 +18,61 @@ of developers. ## 1. Get access to Twitter APIs -**NOTE: The full suite of Twitter APIs the app uses require additional access -beyond the default Twitter API [Essential access -level](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api). -The [Enterprise -search](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) -additionally requires an [enterprise -account](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview).** - -**NOTE: We plan to migrate the Enterprise Full-Archive Search API to the v2 Search Tweets -in the future. We will update this documentation accordingly.** - The app makes use of several Twitter APIs, including: -- [The Enterprise Full-Archive Search - API](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) to fetch - tweets directed at the logged in user -- The v2 [blocks](https://developer.twitter.com/en/docs/twitter-api/users/blocks/introduction) +- The [Enterprise Full-Archive Search + API](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) + **or** the [v2 Full-Archive Search + endpoint](https://developer.twitter.com/en/docs/twitter-api/tweets/search/quick-start/full-archive-search) + to fetch tweets directed at the logged in user +- The v2 + [blocks](https://developer.twitter.com/en/docs/twitter-api/users/blocks/introduction) endpoint to block users on behalf of the authenticated user -- The v2 [mutes](https://developer.twitter.com/en/docs/twitter-api/users/mutes/introduction) +- The v2 + [mutes](https://developer.twitter.com/en/docs/twitter-api/users/mutes/introduction) endpoint to mute users on behalf of the authenticated user - The v2 [hide replies](https://developer.twitter.com/en/docs/twitter-api/tweets/hide-replies/introduction) endpoint to hide replies on behalf of the authenticated user To support all this functionality, you'll need to [get access to the Twitter -API](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api) -and the Enterprise Full-Archive Search API. +API](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api). -Once granted, take note of the: +**NOTE: The full suite of Twitter APIs the app uses require additional access +beyond the default Twitter API [Essential access +level](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api). +The [Enterprise +search](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview) +API requires an [enterprise +account](https://developer.twitter.com/en/docs/twitter-api/enterprise/search-api/overview), +while v2 Full-Archive Search requires [academic +access](https://developer.twitter.com/en/products/twitter-api/academic-research).** -- Account name, app key, and app secret for your Twitter API developer account -- Username and password for your Enterpise Full-Archive Search API account +Once granted access, take note of the: -You'll need both sets of credentials later on. +- Account name, app key, and app secret for your Twitter API developer account +- If using the Enterprise Full-Archive Search API, the Username and password for + your enterprise account +- If using the v2 Full-Archive Search endpoint, the Twitter bearer token for + your app + +You'll need these credentials later on. + +### Enterprise Full-Archive Search vs. v2 Full-Archive Search + +The tool is implemented in a way that either API can be used. While both APIs +offer similar functionality, there are key differences in rate limits. We refer +users to [Twitter's +comparison](https://developer.twitter.com/en/docs/twitter-api/tweets/search/migrate) +for more details. You may also see minute differences in: + +- Which tweets are fetched. This is due to differences in granularity for + timestamp format each API supports (YYYYMMDD for Enterprise and + YYYY-MM-DDTHH:mm:ssZ for v2). +- The order the tweets are displayed when sorted by "Priority". This is due to + small differences in how we parse out the tweet text, which causes some + variation in the Perspective API scores for the text. See issue #19 for more + details. ## 2. Create a Google Cloud Platform (GCP) project diff --git a/docs/2_development.md b/docs/2_development.md index 918dc71..53827e4 100644 --- a/docs/2_development.md +++ b/docs/2_development.md @@ -145,16 +145,16 @@ The required fields are: be the server-side key created in [setup](1_setup.md) in GCP [Credentials](https://console.cloud.google.com/apis/credentials) - `cloudProjectId`: Your Google Cloud project ID +- `twitterApiCredentials`: Your credentials for the Twitter APIs. For Enterprise + Full-Archive search, Twitter will provide you with the credentials. All other + API credentials should be available on the Twitter [Developer + Portal](https://developer.twitter.com/portal) under "Keys and Tokens" for your + app and project. -The optional fields are: +All together, your config should look something like one of the two configs +below, with the relevant credentials and key values replaced. -- `twitterApiCredentials`: Your credentials for the Twitter APIs. The server - expect this field to be an object with `accountName`, `username`, and - `password` fields for the Enterprise Search API and `appKey` and `appToken` - for the Standard API. - -All together, your config should look something like the config below, with the -relevant credentials and key values replaced. +### If using the Enteprise Full-Archive Search API: ```json { @@ -166,8 +166,24 @@ relevant credentials and key values replaced. "accountName": "{TWITTER_API_ACCOUNT_NAME}", "username": "{TWITTER_API_USERNAME}", "password": "{TWITTER_API_PASSWORD}", - "appKey": "{APP_KEY}", - "appToken": "{APP_TOKEN}" + "appKey": "{TWITTER_APP_KEY}", + "appToken": "{TWITTER_APP_TOKEN}" + } +} +``` + +### If using the v2 Full-Archive Search endpoint: + +```json +{ + "port": "3000", + "staticPath": "dist/harassment-manager", + "googleCloudApiKey": "{YOUR_GOOGLE_CLOUD_API_KEY}", + "cloudProjectId": "{YOUR_GOOGLE_CLOUD_PROJECTID}", + "twitterApiCredentials": { + "appKey": "{TWITTER_APP_KEY}", + "appToken": "{TWITTER_APP_TOKEN}", + "bearerToken": "{TWITTER_APP_BEARER_TOKEN}" } } ``` @@ -226,3 +242,7 @@ We maintain a [CircleCI](https://circleci.com/) configuration in is pushed to this GitHub repository. You can choose to use the same configuration for your own CircleCI setup if you'd like or remove the configuration in favor of another CI solution or none at all. + +``` + +``` diff --git a/package-lock.json b/package-lock.json index 2a0fbff..cc00250 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,7 +88,7 @@ "@types/jasmine": "^3.10.3", "@types/jasminewd2": "~2.0.10", "@types/jspdf": "^1.3.3", - "@types/node": "^17.0.21", + "@types/node": "17.0.10", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "5.14.0", "@typescript-eslint/parser": "5.14.0", @@ -6000,9 +6000,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + "version": "17.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.10.tgz", + "integrity": "sha512-S/3xB4KzyFxYGCppyDt68yzBU9ysL88lSdIah4D6cptdcltc4NCPCAMc0+PCpg/lLIyC7IPvj2Z52OJWeIUkog==" }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -29138,9 +29138,9 @@ "dev": true }, "@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==" + "version": "17.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.10.tgz", + "integrity": "sha512-S/3xB4KzyFxYGCppyDt68yzBU9ysL88lSdIah4D6cptdcltc4NCPCAMc0+PCpg/lLIyC7IPvj2Z52OJWeIUkog==" }, "@types/normalize-package-data": { "version": "2.4.1", diff --git a/package.json b/package.json index 1aed3a1..a3c6529 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "@types/jasmine": "^3.10.3", "@types/jasminewd2": "~2.0.10", "@types/jspdf": "^1.3.3", - "@types/node": "^17.0.21", + "@types/node": "17.0.10", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "5.14.0", "@typescript-eslint/parser": "5.14.0", diff --git a/src/app/social-media-item.service.ts b/src/app/social-media-item.service.ts index 1150056..5a5cb38 100644 --- a/src/app/social-media-item.service.ts +++ b/src/app/social-media-item.service.ts @@ -126,11 +126,11 @@ export class SocialMediaItemService { ): Observable { return from( this.twitterApiService.getTweets({ - fromDate: formatTimestamp(startDateTimeMs), - // Subtract 1 minute from the end time because the Twitter API - // sometimes returns an error if we request data for the most recent - // minute of time. - toDate: formatTimestamp(endDateTimeMs - 60000), + startDateTimeMs, + // Subtract 1 minute from the end time because the Twitter API sometimes + // returns an error if we request data for the most recent minute of + // time. + endDateTimeMs: endDateTimeMs - 60000, }) ).pipe(map((response: GetTweetsResponse) => response.tweets)); } @@ -166,22 +166,3 @@ export class SocialMediaItemService { .pipe(map(scores => ({ item, scores }))); } } - -// Format a millisecond-based timestamp into a date format suitable for the -// Twitter API, as defined in: -// https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search -function formatTimestamp(ms: number): string { - const date = new Date(ms); - const MM = date.getUTCMonth() + 1; // getMonth() is zero-based - const dd = date.getUTCDate(); - const hh = date.getUTCHours(); - const mm = date.getUTCMinutes(); - - return ( - `${date.getFullYear()}` + - `${(MM > 9 ? '' : '0') + MM}` + - `${(dd > 9 ? '' : '0') + dd}` + - `${(hh > 9 ? '' : '0') + hh}` + - `${(mm > 9 ? '' : '0') + mm}` - ); -} diff --git a/src/app/test_constants.ts b/src/app/test_constants.ts index ccda5dc..ded7ba9 100644 --- a/src/app/test_constants.ts +++ b/src/app/test_constants.ts @@ -37,7 +37,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 279], entities: { hashtags: [ { @@ -110,7 +109,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 213], entities: { hashtags: [ { @@ -189,7 +187,6 @@ export const TWITTER_ENTRIES: Array> = [ ], }, extended_tweet: { - display_text_range: [0, 203], entities: { hashtags: [], symbols: [], @@ -274,7 +271,6 @@ export const TWITTER_ENTRIES: Array> = [ user_mentions: [], }, extended_tweet: { - display_text_range: [0, 274], entities: { hashtags: [], symbols: [], diff --git a/src/app/tweet-image/tweet-image.component.html b/src/app/tweet-image/tweet-image.component.html index c8bd696..e2dd1f6 100644 --- a/src/app/tweet-image/tweet-image.component.html +++ b/src/app/tweet-image/tweet-image.component.html @@ -16,7 +16,7 @@ + *ngIf="tweet.extended_entities?.media?.length"> @@ -24,9 +24,9 @@
+ *ngIf="tweet.extended_entities?.media?.length">
diff --git a/src/common-types.ts b/src/common-types.ts index 4128094..33b63c5 100644 --- a/src/common-types.ts +++ b/src/common-types.ts @@ -79,8 +79,8 @@ export interface CreatePdfResponse { export interface GetTweetsRequest { credentials?: firebase.auth.UserCredential; nextPageToken?: string; - fromDate: string; // yyyymmddhhmm format is expected here. - toDate: string; // yyyymmddhhmm format is expected here. + startDateTimeMs: number; + endDateTimeMs: number; } export interface GetTweetsResponse { @@ -125,16 +125,11 @@ export interface HideRepliesTwitterResponse { export interface TwitterApiResponse { next: string; - requestParameters: TwitterApiRequestParams; results: TweetObject[]; } -interface TwitterApiRequestParams { - fromDate: string; - toDate: string; - maxResults: number; -} - +// Tweet object format returned by the Enterprise Twitter API. +// // From twitter documentation: When ingesting Tweet data the main object is the // Tweet Object, which is a parent object to several child objects. For // example, all Tweets include a User object that describes who authored the @@ -158,7 +153,6 @@ export interface TweetObject { // directed to the extended_entities section. entities?: TweetEntities; - display_text_range?: number[]; truncated?: boolean; extended_tweet?: ExtendedTweet; @@ -174,6 +168,85 @@ export interface TweetObject { source?: string; } +// Tweet object format returned by the v2 Twitter API. +// +// See +// https://developer.twitter.com/en/docs/twitter-api/data-dictionary/introduction +// for more details. +export interface V2TweetObject { + attachments?: V2Attachments; + author_id: string; + created_at: string; + entities?: V2Entities; + id: string; + lang: string; + public_metrics: V2PublicMetrics; + referenced_tweets?: V2ReferencedTweet[]; + source?: string; + text: string; +} + +interface V2Entities { + hashtags?: V2Hashtags[]; + mentions?: V2Mentions[]; + referenced_tweets?: V2ReferencedTweet[]; + urls?: V2Url[]; +} + +interface V2PublicMetrics { + like_count: number; + quote_count: number; + reply_count: number; + retweet_count: number; +} + +interface V2ReferencedTweet { + id: string; + type: string; +} + +interface V2Mentions { + start: number; + end: number; + username: string; + id: string; +} + +export interface V2Hashtags { + start: number; + end: number; + tag: string; +} + +interface V2Url { + display_url: string; + extended_url: string; + start: number; + end: number; +} + +export interface V2Includes { + media?: V2Media[]; + users?: V2Users[]; +} + +interface V2Media { + media_key: string; + type: string; + url: string; +} + +interface V2Users { + profile_image_url: string; + name: string; + username: string; + verified: boolean; + id: string; +} +interface V2Attachments { + media_keys: string[]; +} + export interface TwitterUser { id_str: string; screen_name: string; @@ -206,7 +279,7 @@ interface Symbols { text: string; } -interface TweetUserMention { +export interface TweetUserMention { id?: number; id_str?: string; indices: Indices; @@ -214,7 +287,7 @@ interface TweetUserMention { screen_name: string; } -interface TweetMedia { +export interface TweetMedia { id_str?: string; media_url: string; type: string; @@ -234,7 +307,7 @@ interface TweetMediaDimensions { resize: string; } -interface TweetUrl { +export interface TweetUrl { display_url?: string; expanded_url?: string; indices: Indices; @@ -249,7 +322,6 @@ export interface TweetHashtag { // For tweets above 140 characters. interface ExtendedTweet { full_text: string; - display_text_range: number[]; entities: TweetEntities; } diff --git a/src/server/middleware/twitter.middleware.ts b/src/server/middleware/twitter.middleware.ts index a702240..42f784e 100644 --- a/src/server/middleware/twitter.middleware.ts +++ b/src/server/middleware/twitter.middleware.ts @@ -30,8 +30,16 @@ import { MuteTwitterUsersRequest, MuteTwitterUsersResponse, Tweet, + TweetEntities, + TweetHashtag, + TweetMedia, TweetObject, + TweetUrl, + TweetUserMention, TwitterApiResponse, + TwitterUser, + V2Includes, + V2TweetObject, } from '../../common-types'; import { TwitterApiCredentials } from '../serving'; @@ -55,12 +63,15 @@ export async function getTweets( if (fs.existsSync('src/server/twitter_sample_results.json')) { twitterDataPromise = loadLocalTwitterData(); - } else { - if (!enterpriseSearchCredentialsAreValid(apiCredentials)) { - res.send(new Error('Invalid Twitter Enterprise Search API credentials')); - return; - } + } else if (v2SearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the Enterprise Full-Archive Search API'); + twitterDataPromise = loadTwitterDataV2(apiCredentials, req.body); + } else if (enterpriseSearchCredentialsAreValid(apiCredentials)) { + console.log('Fetching tweets using the v2 Full-Archive Search API'); twitterDataPromise = loadTwitterData(apiCredentials, req.body); + } else { + res.send(new Error('No valid Twitter API credentials')); + return; } try { @@ -278,19 +289,19 @@ function loadTwitterData( maxResults: BATCH_SIZE, }; - if (request.fromDate) { - twitterApiRequest.fromDate = request.fromDate; + if (request.startDateTimeMs) { + twitterApiRequest.fromDate = formatTimestamp(request.startDateTimeMs); } - if (request.toDate) { - twitterApiRequest.toDate = request.toDate; + if (request.endDateTimeMs) { + twitterApiRequest.toDate = formatTimestamp(request.endDateTimeMs); } if (request.nextPageToken) { twitterApiRequest.next = request.nextPageToken; } const auth: AxiosBasicCredentials = { - username: credentials!.username, - password: credentials!.password, + username: credentials.username!, + password: credentials.password!, }; return axios @@ -304,6 +315,175 @@ function loadTwitterData( }); } +function loadTwitterDataV2( + credentials: TwitterApiCredentials, + request: GetTweetsRequest +): Promise { + const requestUrl = 'https://api.twitter.com/2/tweets/search/all'; + const user = request.credentials?.additionalUserInfo?.username; + if (!user) { + throw new Error('No user credentials in GetTweetsRequest'); + } + + const params = { + // Include next_token if it's part of the request. + ...(request.nextPageToken && { next_token: request.nextPageToken }), + ...{ + query: `(@${user} OR url:twitter.com/${user}) -from:${user} -is:retweet`, + max_results: BATCH_SIZE, + 'user.fields': 'id,name,username,profile_image_url,verified', + expansions: 'author_id,attachments.media_keys,referenced_tweets.id', + start_time: formatTimestampForV2(request.startDateTimeMs), + end_time: formatTimestampForV2(request.endDateTimeMs), + 'media.fields': 'url,type', + 'tweet.fields': + 'attachments,created_at,id,entities,lang,public_metrics,source,text', + }, + }; + + return axios + .get(requestUrl, { + headers: { + authorization: `Bearer ${credentials.bearerToken}`, + }, + params, + transformResponse: [ + (data, _) => { + const parsed = JSON.parse(data); + const tweets: V2TweetObject[] = parsed.data ?? []; + const includes: V2Includes = parsed.includes ?? {}; + return { + results: tweets.map((tweet) => + packV2TweetAsEnterprise(tweet, includes) + ), + next: parsed.meta.next_token, + }; + }, + ], + }) + .then((response) => response.data); +} + +// Packs a Tweet response object from the v2 Search API format into the +// Enterprise Search API format. +function packV2TweetAsEnterprise( + tweet: V2TweetObject, + includes: V2Includes +): TweetObject { + const entities = packEntities(tweet, includes); + + const tweetObject: TweetObject = { + created_at: tweet.created_at, + id_str: tweet.id, + text: tweet.text, + // The Enterprise API entities field is not always complete, but the v2 + // entities field is. + entities: entities, + extended_entities: entities, + favorite_count: tweet.public_metrics.like_count, + // `favorited` omitted because it is not available in v2.. + in_reply_to_status_id: tweet.entities?.referenced_tweets?.find( + (tweet) => tweet.type === 'replied_to' + )?.id, + lang: tweet.lang, + reply_count: tweet.public_metrics.reply_count, + retweet_count: tweet.public_metrics.retweet_count, + source: tweet.source, + truncated: tweet.text.length > 140, + user: getUser(tweet.author_id, includes), + }; + + if (tweetObject.truncated) { + // Enteprise populates the extended_tweet field if the tweet is truncated, + // so we add it manually here for consistency. + tweetObject.extended_tweet = { + full_text: tweetObject.text, + entities, + }; + } + + return tweetObject; +} + +function packEntities( + tweet: V2TweetObject, + includes: V2Includes +): TweetEntities { + const entities: TweetEntities = {}; + + if (tweet.entities?.hashtags) { + entities.hashtags = tweet.entities.hashtags.map( + (hashtag) => + { + indices: [hashtag.start, hashtag.end], + text: hashtag.tag, + } + ); + } + + if (tweet.entities?.urls) { + entities.urls = tweet.entities.urls.map( + (url) => + { + display_url: url.display_url, + expanded_url: url.extended_url, + indices: [url.start, url.end], + } + ); + } + + if (tweet.entities?.mentions) { + entities.user_mentions = tweet.entities.mentions.map( + (mention) => + { + id_str: mention.id, + indices: [mention.start, mention.end], + screen_name: mention.username, + } + ); + } + + if (tweet.attachments?.media_keys) { + entities.media = tweet.attachments.media_keys + .flatMap((media_key) => { + // v2 includes the media fields (like media_url) in a separate + // `includes` object, so we search there for the media item in question. + const media = includes.media?.find( + (media) => media.media_key === media_key + ); + // This shouldn't happen, but we throw an error if it does. + if (!media) { + throw new Error('Unable to find media'); + } + return { + indices: [0, 0], // Indices not available in v2. + media_url: media.url, + type: media.type, + }; + }) + // Filter for photos because that's all we display. + .filter((media) => media.type === 'photo'); + } + + return entities; +} + +function getUser(id: string, includes: V2Includes): TwitterUser { + // v2 includes the user fields (like profile_image_url) in a separate + // `includes` object, so we search there for the media item in question. + const user = includes.users?.find((user) => user.id === id); + if (!user) { + throw new Error('Unable to find user'); + } + return { + id_str: user.id, + profile_image_url: user.profile_image_url, + name: user.name, + screen_name: user.username, + verified: user.verified, + }; +} + function loadLocalTwitterData(): Promise { return fs.promises .readFile('src/server/twitter_sample_results.json') @@ -326,6 +506,12 @@ function enterpriseSearchCredentialsAreValid( ); } +function v2SearchCredentialsAreValid( + credentials: TwitterApiCredentials +): boolean { + return !!credentials.bearerToken; +} + function standardApiCredentialsAreValid( credentials: TwitterApiCredentials ): boolean { @@ -368,7 +554,6 @@ function parseTweet(tweetObject: TweetObject): Tweet { const tweet: Tweet = { created_at: tweetObject.created_at, date: new Date(), - display_text_range: tweetObject.display_text_range, entities: tweetObject.entities, extended_entities: tweetObject.extended_entities, extended_tweet: tweetObject.extended_tweet, @@ -411,3 +596,32 @@ function getUserIdFromCredential(credential: firebase.auth.OAuthCredential) { const match = credential.accessToken?.match('[0-9]+'); return match && match.length ? match[0] : null; } + +// Format a millisecond-based timestamp into the yyyymmddhhmm date format +// suitable for the Enteprise Twitter API, as defined in: +// https://developer.twitter.com/en/docs/tweets/search/api-reference/enterprise-search +function formatTimestamp(ms: number): string { + const date = new Date(ms); + const month = date.getUTCMonth() + 1; // getMonth() is zero-based + const day = date.getUTCDate(); + const hours = date.getUTCHours(); + const minutes = date.getUTCMinutes(); + + return ( + `${date.getFullYear()}` + + `${(month > 9 ? '' : '0') + month}` + + `${(day > 9 ? '' : '0') + day}` + + `${(hours > 9 ? '' : '0') + hours}` + + `${(minutes > 9 ? '' : '0') + minutes}` + ); +} + +// Format a millisecond-based timestamp into the YYYY-MM-DDTHH:mm:ssZ date +// format suitable for the v2 Twitter API, as defined in: +// https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-all +function formatTimestampForV2(ms: number): string { + const date = new Date(ms); + // Remove milliseconds from the ISO string (e.g. + // from 2022-10-12T23:09:43.430Z to 2022-10-12T23:09:43Z. + return date.toISOString().substring(0, 19) + 'Z'; +} diff --git a/src/server/server_config.template.json b/src/server/server_config.template.json index ac780d1..3cc0dd5 100644 --- a/src/server/server_config.template.json +++ b/src/server/server_config.template.json @@ -6,6 +6,9 @@ "twitterApiCredentials": { "accountName": "", "username": "", - "password": "" + "password": "", + "appKey": "", + "appToken": "", + "bearerToken": "" } } diff --git a/src/server/serving.ts b/src/server/serving.ts index 6dd93a8..0961957 100644 --- a/src/server/serving.ts +++ b/src/server/serving.ts @@ -72,11 +72,17 @@ export interface Config { } export interface TwitterApiCredentials { - accountName: string; + // The below three fields are necessary if using Enterprise Full-Archive + // Search. + accountName?: string; + password?: string; + username?: string; + // The below two fields are necessary for the Blocks, Mutes, and Hide Replies + // APIs. appKey: string; appToken: string; - password: string; - username: string; + // Necessary if using v2 Full-Archive Search. + bearerToken?: string; } export interface WebAppCredentials {