Merge pull request #37 from betagouv/feature--hide-robot-stats

Feature to hide the stats from the detected user as bot
betagouv · Jan 20, 2025 · aabc7aa · aabc7aa
2 parents 2da4b53 + 0b3cb7a
commit aabc7aa
Show file tree

Hide file tree

Showing 13 changed files with 114 additions and 142 deletions.
diff --git a/api/src/controllers/redirect.ts b/api/src/controllers/redirect.ts
@@ -9,6 +9,7 @@ import { INVALID_PARAMS, INVALID_QUERY, NOT_FOUND, SERVER_ERROR, captureExceptio
 import CampaignModel from "../models/campaign";
 import MissionModel from "../models/mission";
 import PublisherModel from "../models/publisher";
+import StatsBotModel from "../models/stats-bot";
 import WidgetModel from "../models/widget";
 import { Mission, Stats } from "../types";
 import { identify, slugify } from "../utils";
@@ -63,6 +64,8 @@ router.get("/apply", cors({ origin: "*" }), async (req: Request, res: Response)
       if (!mission) captureMessage(`[Apply] Mission not found`, `mission ${query.data.mission}`);
     }
 
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+
     const obj = {
       referer: identity.referer,
       userAgent: identity.userAgent,
@@ -72,6 +75,7 @@ router.get("/apply", cors({ origin: "*" }), async (req: Request, res: Response)
       type: "apply",
       createdAt: new Date(),
       missionClientId: query.data.mission || "",
+      isBot: statBot ? true : false,
     } as Stats;
 
     if (mission) {
@@ -165,6 +169,8 @@ router.get("/account", cors({ origin: "*" }), async (req: Request, res: Response
       if (!mission) captureMessage(`[Account] Mission not found`, `mission ${query.data.mission}`);
     }
 
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+
     const obj = {
       referer: identity.referer,
       userAgent: identity.userAgent,
@@ -174,6 +180,7 @@ router.get("/account", cors({ origin: "*" }), async (req: Request, res: Response
       type: "account",
       createdAt: new Date(),
       missionClientId: query.data.mission || "",
+      isBot: statBot ? true : false,
     } as Stats;
 
     if (mission) {
@@ -252,7 +259,7 @@ router.get("/campaign/:id", cors({ origin: "*" }), async (req, res) => {
     const identity = identify(req);
     if (!identity) return res.redirect(302, campaign.url);
 
-    const obj1 = {
+    const obj = {
       type: "click",
       user: identity.user,
       referer: identity.referer,
@@ -267,9 +274,10 @@ router.get("/campaign/:id", cors({ origin: "*" }), async (req, res) => {
       toPublisherName: campaign.toPublisherName,
       fromPublisherId: campaign.fromPublisherId,
       fromPublisherName: campaign.fromPublisherName,
+      isBot: false,
     } as Stats;
-    const click = await esClient.index({ index: STATS_INDEX, body: obj1 });
 
+    const click = await esClient.index({ index: STATS_INDEX, body: obj });
     const url = new URL(campaign.url);
 
     if (!url.search) {
@@ -286,6 +294,10 @@ router.get("/campaign/:id", cors({ origin: "*" }), async (req, res) => {
     url.searchParams.set("apiengagement_id", click.body._id);
 
     res.redirect(302, url.href);
+
+    // Update stats just created to add isBot (do it after redirect to avoid delay)
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+    if (statBot) await esClient.update({ index: STATS_INDEX, id: click.body._id, body: { doc: { isBot: true } } });
   } catch (error) {
     captureException(error);
     return res.redirect(302, JVA_URL);
@@ -409,11 +421,11 @@ router.get("/widget/:id", cors({ origin: "*" }), async (req: Request, res: Respo
       missionDepartmentName: mission.departmentName || "",
       missionOrganizationName: mission.organizationName || "",
       missionOrganizationId: mission.organizationId || "",
-
       toPublisherId: mission.publisherId,
       toPublisherName: mission.publisherName,
       fromPublisherId: widget.fromPublisherId,
       fromPublisherName: widget.fromPublisherName,
+      isBot: false,
     } as Stats;
     const click = await esClient.index({ index: STATS_INDEX, body: obj });
 
@@ -436,6 +448,10 @@ router.get("/widget/:id", cors({ origin: "*" }), async (req: Request, res: Respo
     }
 
     res.redirect(302, url.href);
+
+    // Update stats just created to add isBot (do it after redirect to avoid delay)
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+    if (statBot) await esClient.update({ index: STATS_INDEX, id: click.body._id, body: { doc: { isBot: true } } });
   } catch (error: any) {
     captureException(error);
     res.status(500).send({ ok: false, code: SERVER_ERROR, message: error.message });
@@ -489,6 +505,7 @@ router.get("/seo/:id", cors({ origin: "*" }), async (req: Request, res: Response
 
       fromPublisherId: "63da29db7d356a87a4e35d4a",
       fromPublisherName: "API Engagement",
+      isBot: false,
     } as Stats;
 
     const click = await esClient.index({ index: STATS_INDEX, body: obj });
@@ -499,6 +516,10 @@ router.get("/seo/:id", cors({ origin: "*" }), async (req: Request, res: Response
     url.searchParams.set("utm_medium", "google");
     url.searchParams.set("utm_campaign", "seo");
     res.redirect(302, url.href);
+
+    // Update stats just created to add isBot (do it after redirect to avoid delay)
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+    if (statBot) await esClient.update({ index: STATS_INDEX, id: click.body._id, body: { doc: { isBot: true } } });
   } catch (error: any) {
     captureException(error);
     res.status(500).send({ ok: false, code: SERVER_ERROR, message: error.message });
@@ -586,6 +607,7 @@ router.get("/:missionId/:publisherId", cors({ origin: "*" }), async function tra
 
       fromPublisherId: fromPublisher && fromPublisher._id.toString(),
       fromPublisherName: fromPublisher && fromPublisher.name,
+      isBot: false,
     } as Stats;
 
     const click = await esClient.index({ index: STATS_INDEX, body: obj });
@@ -609,6 +631,10 @@ router.get("/:missionId/:publisherId", cors({ origin: "*" }), async function tra
     }
 
     res.redirect(302, url.href);
+
+    // Update stats just created to add isBot (do it after redirect to avoid delay)
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+    if (statBot) await esClient.update({ index: STATS_INDEX, id: click.body._id, body: { doc: { isBot: true } } });
   } catch (error: any) {
     captureException(error);
     res.status(500).send({ ok: false, code: SERVER_ERROR, message: error.message });
@@ -643,6 +669,8 @@ router.get("/impression/campaign/:campaignId", cors({ origin: "*" }), async (req
       return res.status(404).send({ ok: false, code: NOT_FOUND });
     }
 
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
+
     const obj = {
       type: "print",
       host: req.get("host") || "",
@@ -662,6 +690,7 @@ router.get("/impression/campaign/:campaignId", cors({ origin: "*" }), async (req
       source: "campaign",
       sourceId: campaign._id.toString(),
       sourceName: campaign.name,
+      isBot: statBot ? true : false,
     } as Stats;
 
     const print = await esClient.index({ index: STATS_INDEX, body: obj });
@@ -721,6 +750,7 @@ router.get("/impression/:missionId/:publisherId", cors({ origin: "*" }), async (
     const source = query.data.sourceId ? await WidgetModel.findById(query.data.sourceId) : null;
     if (!source && query.data.sourceId) captureMessage(`[Impression] Source not found`, `source ${query.data.sourceId}`);
 
+    const statBot = await StatsBotModel.findOne({ user: identity.user });
     const obj = {
       type: "print",
       requestId: query.data.requestId,
@@ -749,6 +779,8 @@ router.get("/impression/:missionId/:publisherId", cors({ origin: "*" }), async (
 
       fromPublisherId: fromPublisher._id.toString(),
       fromPublisherName: fromPublisher.name,
+
+      isBot: statBot ? true : false,
     } as Stats;
 
     const print = await esClient.index({ index: STATS_INDEX, body: obj });

diff --git a/api/src/controllers/stats-admin.ts b/api/src/controllers/stats-admin.ts
@@ -25,7 +25,7 @@ router.get("/views", passport.authenticate("user", { session: false }), async (r
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
     else if (query.data.from) where.bool.filter.push({ range: { createdAt: { gte: query.data.from } } });
@@ -283,7 +283,7 @@ router.get("/publishers-views", passport.authenticate("user", { session: false }
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const viewQuery = { bool: { filter: [] } as { [key: string]: any } };
+    const viewQuery = { bool: { must_not: [{ term: { "isBot.keyword": true } }], filter: [] } as { [key: string]: any } };
 
     if (query.data.from && query.data.to) viewQuery.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
     else if (query.data.from) viewQuery.bool.filter.push({ range: { createdAt: { gte: query.data.from } } });

diff --git a/api/src/controllers/stats-compare.ts b/api/src/controllers/stats-compare.ts
@@ -25,7 +25,7 @@ router.get("/", passport.authenticate("user", { session: false }), async (req: U
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });

diff --git a/api/src/controllers/stats-global.ts b/api/src/controllers/stats-global.ts
@@ -23,7 +23,7 @@ router.get("/broadcast-preview", passport.authenticate("user", { session: false
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { "fromPublisherId.keyword": query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -100,7 +100,7 @@ router.get("/announce-preview", async (req: UserRequest, res: Response, next: Ne
 
     if (!query.success) return res.status(400).send({ ok: false, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { "toPublisherId.keyword": query.data.publisherId } });
     if (query.data.from) where.bool.must.push({ range: { createdAt: { gte: query.data.from } } });
@@ -166,7 +166,7 @@ router.get("/distribution", passport.authenticate("user", { session: false }), a
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { "fromPublisherId.keyword": query.data.publisherId } });
     if (query.data.from) where.bool.filter.push({ range: { createdAt: { gte: query.data.from } } });
@@ -222,9 +222,7 @@ router.get("/evolution", async (req: UserRequest, res: Response, next: NextFunct
     const diff = (query.data.to.getTime() - query.data.from.getTime()) / (1000 * 60 * 60 * 24);
     const interval = diff < 1 ? "hour" : diff < 61 ? "day" : "month";
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
-
-    // where.bool.must_not.push({ term: { user: "d746d1caad9776c354db56fc433b165ae4b0161c" } });
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -297,7 +295,7 @@ router.get("/broadcast-publishers", passport.authenticate("user", { session: fal
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -375,7 +373,7 @@ router.get("/announce-publishers", passport.authenticate("user", { session: fals
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -420,7 +418,7 @@ router.get("/missions", async (req: UserRequest, res: Response, next: NextFuncti
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { "fromPublisherId.keyword": query.data.publisherId } });
     if (query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from } } });

diff --git a/api/src/controllers/stats-mean.ts b/api/src/controllers/stats-mean.ts
@@ -24,7 +24,7 @@ router.get("/", passport.authenticate("user", { session: false }), async (req: U
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (query.data.publisherId) where.bool.filter.push({ term: { "fromPublisherId.keyword": query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });

diff --git a/api/src/controllers/stats-mission.ts b/api/src/controllers/stats-mission.ts
@@ -28,8 +28,9 @@ router.get("/mission-performance", passport.authenticate("user", { session: fals
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [{ term: { "missionId.keyword": "" } }], should: [], filter: [{ exists: { field: "missionId" } }] } } as EsQuery;
-    // const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = {
+      bool: { must: [], must_not: [{ term: { "missionId.keyword": "" } }, { term: { "isBot.keyword": true } }], should: [], filter: [{ exists: { field: "missionId" } }] },
+    } as EsQuery;
 
     where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -137,7 +138,7 @@ router.get("/organisation-performance", passport.authenticate("user", { session:
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -218,7 +219,7 @@ router.get("/domain-performance", passport.authenticate("user", { session: false
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });
@@ -285,7 +286,7 @@ router.get("/domain-distribution", passport.authenticate("user", { session: fals
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     where.bool.filter.push({ term: { [`${query.data.flux}PublisherId.keyword`]: query.data.publisherId } });
     if (query.data.from && query.data.to) where.bool.filter.push({ range: { createdAt: { gte: query.data.from, lte: query.data.to } } });

diff --git a/api/src/controllers/stats-public.ts b/api/src/controllers/stats-public.ts
@@ -35,7 +35,7 @@ router.get("/graphs", async (req: Request, res: Response, next: NextFunction) =>
 
     if (!query.success) return res.status(400).send({ ok: false, code: INVALID_QUERY, error: query.error });
 
-    const viewQuery = { bool: { filter: [] } as { [key: string]: any } };
+    const viewQuery = { bool: { must_not: [{ term: { "isBot.keyword": true } }], filter: [] } as { [key: string]: any } };
     const whereMissions = {} as { [key: string]: any };
 
     if (query.data.department) {
@@ -186,6 +186,7 @@ router.get("/domains", async (req: Request, res: Response, next: NextFunction) =
       track_total_hits: true,
       query: {
         bool: {
+          must_not: [{ term: { "isBot.keyword": true } }],
           must: filters.length > 0 ? filters : [{ match_all: {} }],
         },
       },

diff --git a/api/src/controllers/stats.ts b/api/src/controllers/stats.ts
@@ -27,7 +27,7 @@ router.post("/search", passport.authenticate("user", { session: false }), async
     if (!body.success) return res.status(400).send({ ok: false, code: INVALID_BODY, error: body.error });
     if (!body.data.fromPublisherId && !body.data.toPublisherId) return res.status(400).send({ ok: false, code: INVALID_BODY, error: "Missing fromPublisherId or toPublisherId" });
 
-    const where = { bool: { must: [], must_not: [], should: [], filter: [] } } as EsQuery;
+    const where = { bool: { must: [], must_not: [{ term: { "isBot.keyword": true } }], should: [], filter: [] } } as EsQuery;
 
     if (body.data.fromPublisherId) where.bool.filter.push({ term: { fromPublisherId: body.data.fromPublisherId } });
     if (body.data.toPublisherId) where.bool.filter.push({ term: { toPublisherId: body.data.toPublisherId } });