Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix/claims-search #135

Merged
merged 3 commits into from
Aug 31, 2024
Merged

Fix/claims-search #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5dcb89
fix(claims-search): make sure there's no duplicate claims on fetching
omareloui Aug 28, 2024
36a3da2
chore(pagination): set a default limit value
omareloui Aug 28, 2024
3edcb97
docs(postman): export postman documentation
omareloui Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions docs/postman/trust_claim.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
{
"info": {
"_postman_id": "b4a29e5f-b329-4189-aa1b-94d465a372e0",
"name": "Trust Claim",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
"_exporter_id": "4736124"
},
"item": [
{
"name": "Claims",
"item": [
{
"name": "Feed",
"request": {
"method": "GET",
"header": [],
"url": {
"raw": "{{url}}/api/claimsfeed2?page=1&limit=15&search=magdi yacoub",
"host": [
"{{url}}"
],
"path": [
"api",
"claimsfeed2"
],
"query": [
{
"key": "page",
"value": "1"
},
{
"key": "limit",
"value": "15"
},
{
"key": "search",
"value": "magdi yacoub"
}
]
}
},
"response": []
}
]
}
],
"event": [
{
"listen": "prerequest",
"script": {
"type": "text/javascript",
"packages": {},
"exec": [
""
]
}
},
{
"listen": "test",
"script": {
"type": "text/javascript",
"packages": {},
"exec": [
""
]
}
}
],
"variable": [
{
"key": "url",
"value": "http://localhost:9000",
"type": "string"
}
]
}
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,5 +76,6 @@
"ts-node": "^10.9.1",
"tslib": "^2.5.0",
"typescript": "^4.9.5"
}
},
"packageManager": "[email protected]+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
}
40 changes: 27 additions & 13 deletions src/controllers/api.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const nodeDao = new NodeDao();
export const claimPost = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
let claim;
let claimData;
Expand All @@ -30,7 +30,7 @@ export const claimPost = async (
claimImages = await claimDao.createImages(
claim.id,
userId,
rawClaim.images
rawClaim.images,
);
} catch (err) {
passToExpressErrorHandler(err, next);
Expand All @@ -42,7 +42,7 @@ export const claimPost = async (
export const claimGetById = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
try {
const { claimId } = req.params;
Expand All @@ -67,7 +67,7 @@ export const claimGetById = async (
export const getAllClaims = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
try {
const claims = await prisma.claim.findMany();
Expand All @@ -88,7 +88,7 @@ export const getAllClaims = async (
export const claimSearch = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
try {
const { search, page = 1, limit = 10 } = req.query;
Expand All @@ -100,7 +100,7 @@ export const claimSearch = async (
const searchResult = await claimDao.searchClaims(
search as string,
Number(page),
Number(limit)
Number(limit),
);
claims = searchResult.claimData;
count = searchResult.count;
Expand All @@ -127,7 +127,7 @@ export const claimSearch = async (
export const claimsGet = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
try {
const { page = 0, limit = 0 } = req.query;
Expand All @@ -144,17 +144,31 @@ export const claimsGet = async (
export const claimsFeed = async (
req: Request,
res: Response,
next: NextFunction
next: NextFunction,
) => {
try {
// const { search } = req.query; // unused for now, TODO here search
const { page = 1, limit = 100, search = "" } = req.query; // defaults provided here
let { page = 1, limit = 100, search = "" } = req.query;

page = parseInt(page.toString());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can use page = +page here and the other fields instead of using all these methods as it costs somehow

Copy link
Contributor Author

@omareloui omareloui Aug 29, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The cost difference is nothing. I made it this way to avoid floating numbers as +page will include decimals

limit = parseInt(limit.toString());
search = decodeURIComponent(search.toString());

if (
Number.isNaN(page) ||
Number.isNaN(limit) ||
limit < 0 ||
page - 1 < 0
) {
throw new createError.UnprocessableEntity("Invalid query string value");
}

// Convert them to numbers
if (limit > 10000) {
throw new createError.UnprocessableEntity("The limit value is too high");
}

const offset = (Number(page) - 1) * Number(limit);
const offset = (page - 1) * limit;

const feed_entries = await nodeDao.getFeedEntries(offset, Number(limit), String(search));
const feed_entries = await nodeDao.getFeedEntries(offset, limit, search);
res.status(200).json(feed_entries);
return;
} catch (err) {
Expand Down
167 changes: 83 additions & 84 deletions src/dao/api.dao.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export class ClaimDao {
},
});
return image;
})
}),
);
}

Expand Down Expand Up @@ -141,7 +141,7 @@ export class ClaimDao {
});

const nodeIds = new Set(
edges.flatMap((edge) => [edge.startNodeId, edge.endNodeId])
edges.flatMap((edge) => [edge.startNodeId, edge.endNodeId]),
);

return prisma.node.findMany({
Expand Down Expand Up @@ -248,89 +248,88 @@ export class NodeDao {

getFeedEntries = async (offset: number, limit: number, search: string) => {
try {
if (typeof search === "string") {
search = decodeURIComponent(search);
}

const baseWhereClause = `
WHERE n1."entType" != 'CLAIM'
AND e.label != 'source'
AND c."effectiveDate" IS NOT NULL
AND n1.name IS NOT NULL
AND n1.name != ''
`;

const searchWhereClause = search
? `
AND (
c.statement ILIKE '%${search}%' OR
c."sourceURI" ILIKE '%${search}%' OR
c."subject" ILIKE '%${search}%' OR
n1.name ILIKE '%${search}%' OR
n3.name ILIKE '%${search}%' OR
n3."descrip" ILIKE '%${search}%'
)
`
: "";

const whereClause = baseWhereClause + searchWhereClause;

const rawQ = Prisma.sql`
SELECT
n1.name as name,
n1.thumbnail as thumbnail,
n1."nodeUri" as link,
n1."descrip" as description,
c.id as claim_id,
c.statement as statement,
c.stars as stars,
c.score as score,
c.amt as amt,
c."effectiveDate" as effective_date,
c."howKnown" as how_known,
c.aspect as aspect,
c.confidence as confidence,
e.label as claim,
e2.label as basis,
n3.name as source_name,
n3.thumbnail as source_thumbnail,
n3."nodeUri" as source_link,
n3."descrip" as source_description
FROM "Claim" c
INNER JOIN "Edge" AS e ON c.id = e."claimId"
INNER JOIN "Node" AS n1 ON e."startNodeId" = n1.id
INNER JOIN "Node" AS n2 ON e."endNodeId" = n2.id
LEFT JOIN "Edge" as e2 ON n2.id = e2."startNodeId"
LEFT JOIN "Node" as n3 ON e2."endNodeId" = n3.id
LEFT JOIN "Image" as i ON c.id = i."claimId"
LEFT JOIN "ClaimData" as cd ON c.id = cd."claimId"
${Prisma.raw(whereClause)}
ORDER BY c."effectiveDate" DESC
LIMIT ${limit}
WITH RankedClaims AS (
SELECT
n1.name AS name,
n1.thumbnail AS thumbnail,
n1."nodeUri" AS link,
n1."descrip" AS description,
c.id AS claim_id,
c.statement AS statement,
c.stars AS stars,
c.score AS score,
c.amt AS amt,
c."effectiveDate" AS effective_date,
c."howKnown" AS how_known,
c.aspect AS aspect,
c.confidence AS confidence,
e.label AS claim,
e2.label AS basis,
n3.name AS source_name,
n3.thumbnail AS source_thumbnail,
n3."nodeUri" AS source_link,
n3."descrip" AS source_description,
ROW_NUMBER() OVER (
PARTITION BY c.id
ORDER BY c."effectiveDate" DESC
) AS row_num
FROM "Claim" c
INNER JOIN "Edge" AS e ON c.id = e."claimId"
INNER JOIN "Node" AS n1 ON e."startNodeId" = n1.id
INNER JOIN "Node" AS n2 ON e."endNodeId" = n2.id
LEFT JOIN "Edge" AS e2 ON n2.id = e2."startNodeId"
LEFT JOIN "Node" AS n3 ON e2."endNodeId" = n3.id
LEFT JOIN "Image" AS i ON c.id = i."claimId"
LEFT JOIN "ClaimData" AS cd ON c.id = cd."claimId"
WHERE n1."entType" != 'CLAIM'
AND e.label != 'source'
AND c."effectiveDate" IS NOT NULL
AND n1.name IS NOT NULL
AND n1.name != ''
${Prisma.raw(
search
? `AND (
c.statement ILIKE '%${search}%' OR
c."sourceURI" ILIKE '%${search}%' OR
c."subject" ILIKE '%${search}%' OR
n1.name ILIKE '%${search}%' OR
n3.name ILIKE '%${search}%' OR
n3."descrip" ILIKE '%${search}%'
)`
: "",
)}
)
SELECT
name,
thumbnail,
link,
description,
claim_id,
statement,
stars,
score,
amt,
effective_date,
how_known,
aspect,
confidence,
claim,
basis,
source_name,
source_thumbnail,
source_link,
source_description

FROM RankedClaims
WHERE row_num = 1
ORDER BY effective_date DESC
OFFSET ${offset}
LIMIT ${limit}
`;

const feedEntries = await prisma.$queryRaw<FeedEntry[]>(rawQ);

const claimIdMap = new Map<number, FeedEntry>();
feedEntries.forEach((entry) => {
if (!claimIdMap.has(entry.claim_id)) {
claimIdMap.set(entry.claim_id, entry);
}
});

const uniqueEntriesByClaimId = Array.from(claimIdMap.values());

const nameMap = new Map<string, FeedEntry>();
uniqueEntriesByClaimId.forEach((entry) => {
if (!nameMap.has(entry.name)) {
nameMap.set(entry.name, entry);
}
});

const uniqueEntriesByName = Array.from(nameMap.values());

return uniqueEntriesByName;
const entries = await prisma.$queryRaw<FeedEntry[]>(rawQ);
return entries;
} catch (error) {
console.error("Error fetching feed entries:", error);
throw new Error("Failed to fetch feed entries");
Expand Down Expand Up @@ -453,7 +452,7 @@ export class NodeDao {
export const GetClaimReport = async (
claimId: any,
offset: number,
limit: number
limit: number,
) => {
const claimDao = new ClaimDao();

Expand Down Expand Up @@ -514,8 +513,8 @@ export const GetClaimReport = async (
const attestations = await prisma.$queryRaw<ReportI[]>`
${Prisma.raw(baseQuery)}
WHERE c."subject" = ${claimToGet?.subject} AND c."id" != ${Number(
claimId
)}
claimId,
)}
ORDER BY c.id DESC
LIMIT ${limit}
OFFSET ${offset}
Expand Down
Loading