From 3f7741b4d1a8dcc52dc74c138a384ec4f4040db2 Mon Sep 17 00:00:00 2001 From: Dan Mindru Date: Fri, 15 Nov 2024 17:03:08 +0100 Subject: [PATCH] Add initial script WIP --- .github/workflows/parse-submission.yml | 38 ++ shipixen/TODO.md | 2 + shipixen/app/tag-data.json | 2 +- shipixen/data/sample-blog.mdx | 30 ++ shipixen/package-lock.json | 657 ++++++++++++++++++++++++- shipixen/package.json | 3 + shipixen/scripts/parse-app-info.js | 157 ++++++ 7 files changed, 878 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/parse-submission.yml create mode 100644 shipixen/TODO.md create mode 100644 shipixen/data/sample-blog.mdx create mode 100644 shipixen/scripts/parse-app-info.js diff --git a/.github/workflows/parse-submission.yml b/.github/workflows/parse-submission.yml new file mode 100644 index 00000000..3e9679b3 --- /dev/null +++ b/.github/workflows/parse-submission.yml @@ -0,0 +1,38 @@ +name: Update App Info + +on: + push: + branches: + - main + +jobs: + update-app-info: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' + + - name: Navigate to scripts directory + run: cd shipixen/scripts + + - name: Install dependencies + run: npm install + + - name: Run update script + run: node parse-app-info.js + + - name: Commit and push changes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git add . + git commit -m 'Update app info' + git push diff --git a/shipixen/TODO.md b/shipixen/TODO.md new file mode 100644 index 00000000..de51cd89 --- /dev/null +++ b/shipixen/TODO.md @@ -0,0 +1,2 @@ +- if there is a file for a given product name, only merge in some of the details like the price and website metadata, but keep the created date as it is and add a lastModified instead +- also create new table of contents in the main README.md file if there's a new category or subcategory diff --git a/shipixen/app/tag-data.json b/shipixen/app/tag-data.json index e2ad2fd1..bf75ea4d 100644 --- a/shipixen/app/tag-data.json +++ b/shipixen/app/tag-data.json @@ -1 +1 @@ -{"blog":1,"documentation":1,"guide":4,"markdown":3,"mdx":1,"images":1,"photos":1,"code":1,"syntax-highlight":1,"footnote":1,"katex":1,"math":1,"latex":1} \ No newline at end of file +{"release-notes":1,"updates":1,"changelog":1} \ No newline at end of file diff --git a/shipixen/data/sample-blog.mdx b/shipixen/data/sample-blog.mdx new file mode 100644 index 00000000..9d91df58 --- /dev/null +++ b/shipixen/data/sample-blog.mdx @@ -0,0 +1,30 @@ +--- +title: '{{Title From HTML}}' +date: '{{ Date Of Pull Request }}' +tags: + - '{{Tag From Markdown}}' + - '{{Tag From Markdown}}' + - '{{Tag From Markdown}}' + +images: + - '{{Image From HTML (optional)}}' + +logo: '{{Logo From HTML (optional)}}' +summary: '{{Summary From Markdown}}' +category: '{{Category from Markdown}}' +deal: '{{Deal from Markdown}}' +subcategory: '{{Subcategory from Markdown (optional)}}' +wesbsite: '{{Website from Markdown}}' +layout: PostLayout +--- + +## {{Title From HTML}} + +{{Title From Markdown}}
+{{Summary From Markdown}} + +## Rare Deal + +{{Deal from Markdown}} + + diff --git a/shipixen/package-lock.json b/shipixen/package-lock.json index 2c891ae7..755ecd79 100644 --- a/shipixen/package-lock.json +++ b/shipixen/package-lock.json @@ -1,11 +1,11 @@ { - "name": "~~packageName~~", + "name": "Rare Big Deal", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "~~packageName~~", + "name": "Rare Big Deal", "version": "1.0.0", "dependencies": { "@hookform/resolvers": "^3.1.0", @@ -47,6 +47,8 @@ "@types/react-dom": "^18.3.0", "@vercel/analytics": "^1.3.1", "@vercel/og": "^0.6.2", + "axios": "^1.7.7", + "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "^0.2.0", @@ -76,6 +78,7 @@ "remark": "^14.0.2", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "sharp": "^0.33.5", "shipixen-contentlayer": "1.0.2", "sonner": "^1.5.0", "tailwind-merge": "^1.14.0", @@ -2270,6 +2273,15 @@ "resolved": "https://registry.npmjs.org/@effect-ts/system/-/system-0.57.5.tgz", "integrity": "sha512-/crHGujo0xnuHIYNc1VgP0HGJGFSoSqq88JFXe6FmFyXPpWt8Xu39LyLg7rchsxfXFeEdA9CrIZvLV5eswXV5g==" }, + "node_modules/@emnapi/runtime": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", + "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emotion/is-prop-valid": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz", @@ -2854,6 +2866,348 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -7282,6 +7636,29 @@ "node": ">=4" } }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/axobject-query": { "version": "3.2.1", "dev": true, @@ -7651,6 +8028,57 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/cheerio": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=18.17" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio/node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/chokidar": { "version": "3.5.3", "funding": [ @@ -8088,6 +8516,18 @@ } } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "license": "MIT", @@ -8102,6 +8542,15 @@ "version": "1.1.4", "license": "MIT" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -8333,7 +8782,6 @@ }, "node_modules/css-select": { "version": "5.1.0", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "boolbase": "^1.0.0", @@ -8374,7 +8822,6 @@ }, "node_modules/css-what": { "version": "6.1.0", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">= 6" @@ -8706,6 +9153,14 @@ "node": ">=6" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -8770,7 +9225,6 @@ }, "node_modules/dom-serializer": { "version": "2.0.0", - "dev": true, "license": "MIT", "dependencies": { "domelementtype": "^2.3.0", @@ -8783,7 +9237,6 @@ }, "node_modules/domelementtype": { "version": "2.3.0", - "dev": true, "funding": [ { "type": "github", @@ -8794,7 +9247,6 @@ }, "node_modules/domhandler": { "version": "5.0.3", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "domelementtype": "^2.3.0" @@ -8808,7 +9260,6 @@ }, "node_modules/domutils": { "version": "3.1.0", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "dom-serializer": "^2.0.0", @@ -8892,6 +9343,18 @@ "iconv-lite": "^0.6.2" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", + "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, "node_modules/enhanced-resolve": { "version": "5.15.0", "dev": true, @@ -9872,6 +10335,25 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "dev": true, @@ -10743,6 +11225,24 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, "node_modules/human-signals": { "version": "4.3.1", "dev": true, @@ -10769,8 +11269,6 @@ "node_modules/iconv-lite": { "version": "0.6.3", "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -13823,6 +14321,51 @@ "version": "6.0.1", "license": "MIT" }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream/node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/pascal-case": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", @@ -14231,6 +14774,11 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.0", "dev": true, @@ -15624,6 +16172,55 @@ "node": ">= 0.4" } }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "dev": true, @@ -15680,6 +16277,19 @@ "dev": true, "license": "ISC" }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/sirv": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", @@ -16530,6 +17140,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", + "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", @@ -16942,6 +17560,25 @@ "node": ">= 10.13.0" } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "engines": { + "node": ">=18" + } + }, "node_modules/whatwg-url": { "version": "5.0.0", "license": "MIT", diff --git a/shipixen/package.json b/shipixen/package.json index 1dd96a1b..bc1d5f03 100644 --- a/shipixen/package.json +++ b/shipixen/package.json @@ -51,6 +51,8 @@ "@types/react-dom": "^18.3.0", "@vercel/analytics": "^1.3.1", "@vercel/og": "^0.6.2", + "axios": "^1.7.7", + "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "^0.2.0", @@ -80,6 +82,7 @@ "remark": "^14.0.2", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "sharp": "^0.33.5", "shipixen-contentlayer": "1.0.2", "sonner": "^1.5.0", "tailwind-merge": "^1.14.0", diff --git a/shipixen/scripts/parse-app-info.js b/shipixen/scripts/parse-app-info.js new file mode 100644 index 00000000..621bf71c --- /dev/null +++ b/shipixen/scripts/parse-app-info.js @@ -0,0 +1,157 @@ +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const cheerio = require('cheerio'); +const sharp = require('sharp'); // Image processing library + +const readmePath = 'README.md'; +const outputDir = 'shipixen/public/static/images'; + +const categoryTags = { + 'Developer Tools': ['Developer', 'Tools', 'macOS'], + 'AI Tools': ['AI', 'Machine Learning', 'Automation'], + 'Other AI tools': ['AI', 'Voice', 'Text to Speech'], + 'Design Tools': ['Design', 'Graphics', 'Marketing'], + 'Code Libraries': ['NextJs', 'React', 'SaaS'], + 'Productivity': ['Productivity', 'Efficiency', 'Tools'], + 'Marketing Tools': ['Marketing', 'SEO', 'Promotion'], + 'SEO Tools': ['SEO', 'Optimization', 'Marketing'], + 'Startup SaaS/Tools': ['Startup', 'SaaS', 'Business'], + 'Themes, Plugins': ['Themes', 'Plugins', 'Customization'], + 'Books': ['Books', 'Learning', 'Programming'], + 'Health and Fitness': ['Health', 'Fitness', 'Wellness'] +}; + +async function downloadImage(url, outputPath) { + try { + const response = await axios.get(url, { responseType: 'arraybuffer' }); + fs.writeFileSync(outputPath, response.data); + } catch (error) { + console.error(`Failed to download image from ${url}:`, error.message); + } +} + +async function extractAppInfo() { + const readmeContent = fs.readFileSync(readmePath, 'utf-8'); + const lines = readmeContent.split('\n'); + const apps = []; + + let currentCategory = ''; + let currentSubcategory = ''; + + for (const line of lines) { + if (line.startsWith('## ')) { + currentCategory = line.replace('## ', '').trim(); + } else if (line.startsWith('### ')) { + currentSubcategory = line.replace('### ', '').trim(); + } else if (line.startsWith('| ⭐ ')) { + const parts = line.split('|').map(part => part.trim()); + const name = parts[2].match(/\[(.*?)\]/)[1]; + const website = parts[2].match(/\((.*?)\)/)[1]; + const description = parts[3]; + const deal = parts[4]; + + apps.push({ + name, + website, + description, + deal, + category: currentCategory, + subcategory: currentSubcategory, + }); + } + } + + return apps; +} + +async function fetchAssets(app) { + const { website, name } = app; + const appDir = path.join(outputDir, name.replace(/\s+/g, '-').toLowerCase()); + fs.mkdirSync(appDir, { recursive: true }); + + try { + const response = await axios.get(website); + const $ = cheerio.load(response.data); + + const faviconUrl = $('link[rel="icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href'); + const ogImageUrl = $('meta[property="og:image"]').attr('content'); + + if (faviconUrl) { + const faviconPath = path.join(appDir, 'logo.png'); + const faviconExt = path.extname(faviconUrl).toLowerCase(); + + if (faviconExt === '.ico') { + const icoBuffer = await axios.get(new URL(faviconUrl, website).href, { responseType: 'arraybuffer' }); + const pngBuffer = await sharp(icoBuffer.data).png().toBuffer(); + fs.writeFileSync(faviconPath, pngBuffer); + } else { + await downloadImage(new URL(faviconUrl, website).href, faviconPath); + } + app.logo = faviconPath; + } + + if (ogImageUrl) { + const ogImagePath = path.join(appDir, 'og-image.png'); + await downloadImage(new URL(ogImageUrl, website).href, ogImagePath); + app.images = [ogImagePath]; + } + } catch (error) { + console.error(`Failed to fetch assets for ${name}:`, error.message); + } +} + +async function generateMarkdown(apps) { +try { + for (const app of apps) { + console.log(`👉 Generating markdown for ${app.name}`); + const tagsList = (categoryTags[app.category] || []).map(tag => ` - '${tag}'`).join('\n'); + const markdownContent = `--- +title: '${app.name}' +date: '${new Date().toISOString().split('T')[0]}' +tags: +${tagsList} +images: + - '${app.images ? app.images[0] : ''}' +logo: '${app.logo || ''}' +summary: '${app.description}' +category: '${app.category}' +deal: '${app.deal}' +subcategory: '${app.subcategory}' +website: '${app.website}' +layout: PostLayout +--- + +## [${app.name}](${app.website}) + +${app.name}
+${app.description} + +## Rare Deal + +${app.deal} +`; + + const markdownOutputPath = path.join('markdown', `${app.name.replace(/\s+/g, '-').toLowerCase()}.mdx`); + fs.writeFileSync(markdownOutputPath, markdownContent); + + console.log(`✅ Markdown generated for ${app.name}`); + } } catch (error) { + console.error(`💥 Could not generate markdown`, error.message); + } +} + + +async function main() { + const apps = await extractAppInfo(); + + for (const app of apps) { + await fetchAssets(app); + } + + await generateMarkdown(apps); +} + +main().catch(error => { + console.error('Error:', error.message); +});