Skip to content

Commit

Permalink
chore: staged changes
Browse files Browse the repository at this point in the history
  • Loading branch information
rayriffy committed Nov 12, 2024
1 parent 4172cd0 commit d681d40
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 3 deletions.
Binary file modified bun.lockb
Binary file not shown.
1 change: 1 addition & 0 deletions commons/src/@types/DatabaseCode.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export type DatabaseCode =
| string
| number
| {
code: number
Expand Down
5 changes: 3 additions & 2 deletions scraper/src/commands/fetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { Kysely } from 'kysely'
import { SQLDatabase } from '../@types/SQLDatabase'
import { createDBConnection } from '../functions/createDBConnection'
import { getRemoteHentai } from '../functions/getRemoteHentai'
import {parseUrl} from "../functions/parseUrl";

const fetchQueue = new PQueue({
concurrency: 8,
Expand Down Expand Up @@ -42,7 +43,7 @@ export const fetch = async (entryPoint: string) => {
const chunkFile = path.join(prebuiltChunkDirectory, `chunk-${i + 1}.json`)
await fs.promises.writeFile(
chunkFile,
JSON.stringify(chunk.map(o => (typeof o === 'number' ? o : o.code)))
JSON.stringify(chunk.map(o => (typeof o === 'number' ? o : typeof o === 'string' ? parseUrl(o) : o.code)))
)
})
)
Expand Down Expand Up @@ -87,7 +88,7 @@ export const fetch = async (entryPoint: string) => {
const orderedHentai = codes
.map(code => {
try {
const targetCode = typeof code === 'number' ? code : code.code
const targetCode = typeof code === 'number' ? code : typeof code === 'string' ? parseUrl(code) : code.code
const targetHentai: Hentai = JSON.parse(
fs
.readFileSync(path.join(hentaiDirectory, `${targetCode}.json`))
Expand Down
7 changes: 7 additions & 0 deletions scraper/src/functions/parseUrl.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export const parseUrl = (input: string) => {
const matcher = input.match(/https?:\/\/nhentai.net\/g\/(\d+)\/?/)
if (matcher !== null && Number.isInteger(Number(matcher[1])))
return Number(matcher[1])
else
throw new Error("Cannot parse " + input)
}
2 changes: 1 addition & 1 deletion web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"nanostores": "0.11.3",
"p-queue": "8.0.1",
"set-cookie-parser": "2.7.0",
"sharp": "0.33.1",
"sharp": "0.33.5",
"trpc-svelte-query": "2.1.0",
"zod": "3.23.8"
},
Expand Down

0 comments on commit d681d40

Please sign in to comment.