From f90a43abdad74603510ab8122dab76aa88f3fca0 Mon Sep 17 00:00:00 2001
From: 21120447 <haonhat2729@gmail.com>
Date: Sun, 27 Oct 2024 17:33:59 +0700
Subject: [PATCH] :sparkles: feat: add feature image identification

---
 package.json                                  |   4 +
 .../LearnThroughImagesPage.tsx                | 191 +++++++++++++++++-
 yarn.lock                                     | 101 +++++++++
 3 files changed, 295 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index d9c1257..f7547bc 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,10 @@
     "@chakra-ui/react": "^2.10.3",
     "@emotion/react": "^11.13.3",
     "@emotion/styled": "^11.13.0",
+    "@tensorflow-models/mobilenet": "^2.1.1",
+    "@tensorflow/tfjs-backend-webgl": "^4.22.0",
+    "@tensorflow/tfjs-converter": "^4.22.0",
+    "@tensorflow/tfjs-core": "^4.22.0",
     "axios": "^1.7.7",
     "eslint-plugin-prettier": "^5.2.1",
     "eslint-plugin-react": "^7.35.0",
diff --git a/src/features/learn-through-images/LearnThroughImagesPage.tsx b/src/features/learn-through-images/LearnThroughImagesPage.tsx
index 721936b..9374581 100644
--- a/src/features/learn-through-images/LearnThroughImagesPage.tsx
+++ b/src/features/learn-through-images/LearnThroughImagesPage.tsx
@@ -1,5 +1,194 @@
+import "@tensorflow/tfjs-backend-webgl";
+import * as mobilenet from "@tensorflow-models/mobilenet";
+import { Button, Heading, HStack, Input, Text, VStack, Image, Box, Grid } from "@chakra-ui/react";
+import React from "react";
+import { Link } from "react-router-dom";
+
 const LearnThroughImagesPage = () => {
-  return <div>LearnThroughImagesPage</div>;
+  const imageRef = React.useRef<HTMLImageElement>(null);
+  const textInputRef = React.useRef<HTMLInputElement>(null);
+  const fileInputRef = React.useRef<HTMLInputElement>(null);
+
+  const [isModelLoading, setIsModelLoading] = React.useState(false);
+  const [model, setModel] = React.useState<mobilenet.MobileNet | null>(null);
+  const [history, setHistory] = React.useState<string[]>([]);
+  const [imageURL, setImageURL] = React.useState<string | null>(null);
+  const [results, setResults] = React.useState<
+    | {
+        className: string;
+        probability: number;
+      }[]
+    | undefined
+  >([]);
+
+  const handleOnChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    setImageURL(e.target.value);
+    setResults([]);
+  };
+
+  const uploadImage = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const { files } = e.target;
+    if (files!.length > 0) {
+      const url = URL.createObjectURL(files![0]);
+      setImageURL(url);
+    } else {
+      setImageURL(null);
+    }
+  };
+
+  const triggerUpload = () => {
+    fileInputRef.current?.click();
+  };
+
+  const loadModel = async () => {
+    setIsModelLoading(true);
+    try {
+      const model = await mobilenet.load();
+      setModel(model);
+      setIsModelLoading(false);
+    } catch (error) {
+      console.log(error);
+      setIsModelLoading(false);
+    }
+  };
+
+  React.useEffect(() => {
+    if (imageURL && !history.includes(imageURL)) {
+      setHistory([imageURL, ...history]);
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [imageURL]);
+
+  React.useEffect(() => {
+    loadModel();
+  }, []);
+
+  const identify = async () => {
+    if (isModelLoading) return;
+    textInputRef.current!.value = "";
+    const results = await model?.classify(imageRef.current!);
+    setResults(results);
+    console.log(results);
+  };
+
+  return (
+    <div>
+      <VStack marginTop={"10px"} marginBottom={"20px"}>
+        <Heading size="lg">Image identification</Heading>
+      </VStack>
+      <HStack gap={{ base: "10px", lg: "25px" }}>
+        <div>
+          <input
+            type="file"
+            accept="image/*"
+            className="w-0 h-0 hidden opacity-0"
+            onChange={uploadImage}
+            ref={fileInputRef}
+          />
+          <Button colorScheme="blue" onClick={triggerUpload}>
+            Upload Image
+          </Button>
+        </div>
+        <Text fontWeight="medium">OR</Text>
+        <Input placeholder="Paste image URL" className="max-w-[500px]" ref={textInputRef} onChange={handleOnChange} />
+      </HStack>
+      {imageURL && (
+        <div>
+          <VStack alignItems={"flex-start"} gap={"20px"} marginTop={"12px"} marginBottom={"6px"}>
+            <VStack alignItems={"flex-start"}>
+              <Box>
+                <Image
+                  src={imageURL}
+                  alt="Upload Preview"
+                  ref={imageRef}
+                  width={"full"}
+                  height={"400px"}
+                  objectFit={"cover"}
+                />
+              </Box>
+              <Button colorScheme="blue" onClick={identify} isLoading={isModelLoading}>
+                {isModelLoading ? "Model Loading..." : "Identify image"}
+              </Button>
+            </VStack>
+            {results && results.length > 0 && (
+              <VStack gap="10px" alignItems={"flex-start"}>
+                {results.map((result, index) => {
+                  // Splitting className by ","
+                  const classNames = result.className.split(",");
+
+                  return (
+                    <VStack
+                      alignItems={"flex-start"}
+                      padding={"10px"}
+                      borderWidth="1px"
+                      borderColor={"#333"}
+                      width={"full"}
+                      key={result.className}
+                      bg={index === 0 ? "#333" : "transparent"}
+                      color={index === 0 ? "white" : "#333"}
+                    >
+                      <HStack flexWrap={"wrap"} alignItems={"flex-start"}>
+                        {classNames.map((className, i) => (
+                          <Link
+                            to={`/dictionary?q=${className.trim()}`}
+                            target="_blank"
+                            key={i}
+                            style={{
+                              textDecoration: "none",
+                            }}
+                          >
+                            <Heading
+                              fontSize={"xl"}
+                              textTransform={"uppercase"}
+                              display={"inline-block"}
+                              _hover={{ color: index === 0 ? "yellow" : "blue" }}
+                            >
+                              {className.trim()}
+                              {i !== classNames.length - 1 && ", "}
+                            </Heading>
+                          </Link>
+                        ))}
+                      </HStack>
+                      <span className="confidence">
+                        Confidence level: {(result.probability * 100).toFixed(2)}%{" "}
+                        {index === 0 && <span className="bg-white text-[#333] px-1">Best Guess</span>}
+                      </span>
+                    </VStack>
+                  );
+                })}
+              </VStack>
+            )}
+          </VStack>
+        </div>
+      )}
+      {history.length > 0 && (
+        <VStack
+          marginTop={"30px"}
+          bg={"blue.200"}
+          paddingX={"10px"}
+          paddingY={"20px"}
+          gap={"10px"}
+          alignItems={"flex-start"}
+        >
+          <Heading size={"lg"}>Recent images</Heading>
+          <Grid templateColumns={{ base: "repeat(2, 1fr)", lg: "repeat(3, 1fr)", xl: "repeat(5, 1fr)" }} gap={"0px"}>
+            {history.map((image, index) => {
+              return (
+                <div key={`${image}${index}`}>
+                  <img
+                    src={image}
+                    alt="Recent Prediction"
+                    onClick={() => setImageURL(image)}
+                    className="h-[200px] w-full object-cover cursor-pointer"
+                  />
+                </div>
+              );
+            })}
+          </Grid>
+        </VStack>
+      )}
+    </div>
+  );
 };
 
 export default LearnThroughImagesPage;
diff --git a/yarn.lock b/yarn.lock
index ec384b2..ab0794c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -871,6 +871,47 @@
   dependencies:
     "@swc/counter" "^0.1.3"
 
+"@tensorflow-models/mobilenet@^2.1.1":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@tensorflow-models/mobilenet/-/mobilenet-2.1.1.tgz#5b02c3e99a19534e3b303f57dccf31f85fec7683"
+  integrity sha512-tv4s4UFzG74PkIwl4gT64AyRnCcNUq+s8wSzge+LN/Puc1VUuInZghrobvpNlWjZtVi1x1d1NsBD//TfOr2ssA==
+
+"@tensorflow/tfjs-backend-cpu@4.22.0":
+  version "4.22.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.22.0.tgz#72aeaab14f6f16bbd995c9e6751a8d094d5639a9"
+  integrity sha512-1u0FmuLGuRAi8D2c3cocHTASGXOmHc/4OvoVDENJayjYkS119fcTcQf4iHrtLthWyDIPy3JiPhRrZQC9EwnhLw==
+  dependencies:
+    "@types/seedrandom" "^2.4.28"
+    seedrandom "^3.0.5"
+
+"@tensorflow/tfjs-backend-webgl@^4.22.0":
+  version "4.22.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.22.0.tgz#c6ffb8c5e737b1b1ef7fab8f721328b5b2e658c0"
+  integrity sha512-H535XtZWnWgNwSzv538czjVlbJebDl5QTMOth4RXr2p/kJ1qSIXE0vZvEtO+5EC9b00SvhplECny2yDewQb/Yg==
+  dependencies:
+    "@tensorflow/tfjs-backend-cpu" "4.22.0"
+    "@types/offscreencanvas" "~2019.3.0"
+    "@types/seedrandom" "^2.4.28"
+    seedrandom "^3.0.5"
+
+"@tensorflow/tfjs-converter@^4.22.0":
+  version "4.22.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz#a5d727c1d97cf1fafda18b79be278e83b38a1ad3"
+  integrity sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==
+
+"@tensorflow/tfjs-core@^4.22.0":
+  version "4.22.0"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz#fc4b45d2377410fa3a42c88ca77d8f23d83cffc3"
+  integrity sha512-LEkOyzbknKFoWUwfkr59vSB68DMJ4cjwwHgicXN0DUi3a0Vh1Er3JQqCI1Hl86GGZQvY8ezVrtDIvqR1ZFW55A==
+  dependencies:
+    "@types/long" "^4.0.1"
+    "@types/offscreencanvas" "~2019.7.0"
+    "@types/seedrandom" "^2.4.28"
+    "@webgpu/types" "0.1.38"
+    long "4.0.0"
+    node-fetch "~2.6.1"
+    seedrandom "^3.0.5"
+
 "@types/conventional-commits-parser@^5.0.0":
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/@types/conventional-commits-parser/-/conventional-commits-parser-5.0.0.tgz#8c9d23e0b415b24b91626d07017303755d542dc8"
@@ -895,6 +936,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.12.tgz#25d71312bf66512105d71e55d42e22c36bcfc689"
   integrity sha512-sviUmCE8AYdaF/KIHLDJBQgeYzPBI0vf/17NaYehBJfYD1j6/L95Slh07NlyK2iNyBNaEkb3En2jRt+a8y3xZQ==
 
+"@types/long@^4.0.1":
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a"
+  integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==
+
 "@types/node@*":
   version "22.1.0"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-22.1.0.tgz#6d6adc648b5e03f0e83c78dc788c2b037d0ad94b"
@@ -902,6 +948,16 @@
   dependencies:
     undici-types "~6.13.0"
 
+"@types/offscreencanvas@~2019.3.0":
+  version "2019.3.0"
+  resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553"
+  integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==
+
+"@types/offscreencanvas@~2019.7.0":
+  version "2019.7.3"
+  resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz#90267db13f64d6e9ccb5ae3eac92786a7c77a516"
+  integrity sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==
+
 "@types/parse-json@^4.0.0":
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.2.tgz#5950e50960793055845e956c427fc2b0d70c5239"
@@ -927,6 +983,11 @@
     "@types/prop-types" "*"
     csstype "^3.0.2"
 
+"@types/seedrandom@^2.4.28":
+  version "2.4.34"
+  resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.34.tgz#c725cd0fc0442e2d3d0e5913af005686ffb7eb99"
+  integrity sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==
+
 "@typescript-eslint/eslint-plugin@^7.15.0":
   version "7.18.0"
   resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz#b16d3cf3ee76bf572fdf511e79c248bdec619ea3"
@@ -1020,6 +1081,11 @@
   dependencies:
     "@swc/core" "^1.5.7"
 
+"@webgpu/types@0.1.38":
+  version "0.1.38"
+  resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.38.tgz#6fda4b410edc753d3213c648320ebcf319669020"
+  integrity sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA==
+
 "@zag-js/dom-query@0.31.1":
   version "0.31.1"
   resolved "https://registry.yarnpkg.com/@zag-js/dom-query/-/dom-query-0.31.1.tgz#f40be43d0eb1eabdf51538abeeccad46c5b88ed6"
@@ -2826,6 +2892,11 @@ lodash.upperfirst@^4.3.1:
   resolved "https://registry.yarnpkg.com/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz#1365edf431480481ef0d1c68957a5ed99d49f7ce"
   integrity sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg==
 
+long@4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
+  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
+
 loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
@@ -2941,6 +3012,13 @@ next-themes@^0.3.0:
   resolved "https://registry.yarnpkg.com/next-themes/-/next-themes-0.3.0.tgz#b4d2a866137a67d42564b07f3a3e720e2ff3871a"
   integrity sha512-/QHIrsYpd6Kfk7xakK4svpDI5mmXP0gfvCoJdGpZQ2TOrQZmsW0QxjaiLn8wbIKjtm4BTSqLoix4lxYYOnLJ/w==
 
+node-fetch@~2.6.1:
+  version "2.6.13"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.13.tgz#a20acbbec73c2e09f9007de5cda17104122e0010"
+  integrity sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-releases@^2.0.18:
   version "2.0.18"
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f"
@@ -3496,6 +3574,11 @@ scheduler@^0.23.2:
   dependencies:
     loose-envify "^1.1.0"
 
+seedrandom@^3.0.5:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
+  integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
+
 semver@^6.3.1:
   version "6.3.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
@@ -3801,6 +3884,11 @@ toggle-selection@^1.0.6:
   resolved "https://registry.yarnpkg.com/toggle-selection/-/toggle-selection-1.0.6.tgz#6e45b1263f2017fa0acc7d89d78b15b8bf77da32"
   integrity sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==
 
+tr46@~0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
+  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
+
 ts-api-utils@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1"
@@ -3967,6 +4055,19 @@ vite@^5.3.4:
   optionalDependencies:
     fsevents "~2.3.3"
 
+webidl-conversions@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
+  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
+
+whatwg-url@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
+  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
+  dependencies:
+    tr46 "~0.0.3"
+    webidl-conversions "^3.0.0"
+
 which-boxed-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"