diff --git a/scrapping/vinted-db-feeder/.gitignore b/vinted-db-feeder/.gitignore similarity index 100% rename from scrapping/vinted-db-feeder/.gitignore rename to vinted-db-feeder/.gitignore diff --git a/scrapping/vinted-db-feeder/README.md b/vinted-db-feeder/README.md similarity index 100% rename from scrapping/vinted-db-feeder/README.md rename to vinted-db-feeder/README.md diff --git a/scrapping/vinted-db-feeder/data/raw/brands/A_elements.html b/vinted-db-feeder/data/raw/brands/A_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/A_elements.html rename to vinted-db-feeder/data/raw/brands/A_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/B_elements.html b/vinted-db-feeder/data/raw/brands/B_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/B_elements.html rename to vinted-db-feeder/data/raw/brands/B_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/C_elements.html b/vinted-db-feeder/data/raw/brands/C_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/C_elements.html rename to vinted-db-feeder/data/raw/brands/C_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/D_elements.html b/vinted-db-feeder/data/raw/brands/D_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/D_elements.html rename to vinted-db-feeder/data/raw/brands/D_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/E_elements.html b/vinted-db-feeder/data/raw/brands/E_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/E_elements.html rename to vinted-db-feeder/data/raw/brands/E_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/F_elements.html b/vinted-db-feeder/data/raw/brands/F_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/F_elements.html rename to vinted-db-feeder/data/raw/brands/F_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/G_elements.html b/vinted-db-feeder/data/raw/brands/G_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/G_elements.html rename to vinted-db-feeder/data/raw/brands/G_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/H_elements.html b/vinted-db-feeder/data/raw/brands/H_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/H_elements.html rename to vinted-db-feeder/data/raw/brands/H_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/I_elements.html b/vinted-db-feeder/data/raw/brands/I_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/I_elements.html rename to vinted-db-feeder/data/raw/brands/I_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/J_elements.html b/vinted-db-feeder/data/raw/brands/J_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/J_elements.html rename to vinted-db-feeder/data/raw/brands/J_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/K_elements.html b/vinted-db-feeder/data/raw/brands/K_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/K_elements.html rename to vinted-db-feeder/data/raw/brands/K_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/L_elements.html b/vinted-db-feeder/data/raw/brands/L_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/L_elements.html rename to vinted-db-feeder/data/raw/brands/L_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/M_elements.html b/vinted-db-feeder/data/raw/brands/M_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/M_elements.html rename to vinted-db-feeder/data/raw/brands/M_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/N_elements.html b/vinted-db-feeder/data/raw/brands/N_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/N_elements.html rename to vinted-db-feeder/data/raw/brands/N_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/O_elements.html b/vinted-db-feeder/data/raw/brands/O_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/O_elements.html rename to vinted-db-feeder/data/raw/brands/O_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/P_elements.html b/vinted-db-feeder/data/raw/brands/P_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/P_elements.html rename to vinted-db-feeder/data/raw/brands/P_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/R_elements.html b/vinted-db-feeder/data/raw/brands/R_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/R_elements.html rename to vinted-db-feeder/data/raw/brands/R_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/S_elements.html b/vinted-db-feeder/data/raw/brands/S_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/S_elements.html rename to vinted-db-feeder/data/raw/brands/S_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/T_elements.html b/vinted-db-feeder/data/raw/brands/T_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/T_elements.html rename to vinted-db-feeder/data/raw/brands/T_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/U_elements.html b/vinted-db-feeder/data/raw/brands/U_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/U_elements.html rename to vinted-db-feeder/data/raw/brands/U_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/V_elements.html b/vinted-db-feeder/data/raw/brands/V_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/V_elements.html rename to vinted-db-feeder/data/raw/brands/V_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/W_elements.html b/vinted-db-feeder/data/raw/brands/W_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/W_elements.html rename to vinted-db-feeder/data/raw/brands/W_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/Y_elements.html b/vinted-db-feeder/data/raw/brands/Y_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/Y_elements.html rename to vinted-db-feeder/data/raw/brands/Y_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/brands/Z_elements.html b/vinted-db-feeder/data/raw/brands/Z_elements.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/brands/Z_elements.html rename to vinted-db-feeder/data/raw/brands/Z_elements.html diff --git a/scrapping/vinted-db-feeder/data/raw/categories/catalogs.json b/vinted-db-feeder/data/raw/categories/catalogs.json similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/categories/catalogs.json rename to vinted-db-feeder/data/raw/categories/catalogs.json diff --git a/scrapping/vinted-db-feeder/data/raw/materials/EN.html b/vinted-db-feeder/data/raw/materials/EN.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/materials/EN.html rename to vinted-db-feeder/data/raw/materials/EN.html diff --git a/scrapping/vinted-db-feeder/data/raw/materials/ES.html b/vinted-db-feeder/data/raw/materials/ES.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/materials/ES.html rename to vinted-db-feeder/data/raw/materials/ES.html diff --git a/scrapping/vinted-db-feeder/data/raw/materials/FR.html b/vinted-db-feeder/data/raw/materials/FR.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/materials/FR.html rename to vinted-db-feeder/data/raw/materials/FR.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/en/home.html b/vinted-db-feeder/data/raw/sizes/en/home.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/en/home.html rename to vinted-db-feeder/data/raw/sizes/en/home.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/en/kids.html b/vinted-db-feeder/data/raw/sizes/en/kids.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/en/kids.html rename to vinted-db-feeder/data/raw/sizes/en/kids.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/en/men.html b/vinted-db-feeder/data/raw/sizes/en/men.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/en/men.html rename to vinted-db-feeder/data/raw/sizes/en/men.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/en/pet_care.html b/vinted-db-feeder/data/raw/sizes/en/pet_care.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/en/pet_care.html rename to vinted-db-feeder/data/raw/sizes/en/pet_care.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/en/women.html b/vinted-db-feeder/data/raw/sizes/en/women.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/en/women.html rename to vinted-db-feeder/data/raw/sizes/en/women.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/es/hogar.html b/vinted-db-feeder/data/raw/sizes/es/hogar.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/es/hogar.html rename to vinted-db-feeder/data/raw/sizes/es/hogar.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/es/hombre.html b/vinted-db-feeder/data/raw/sizes/es/hombre.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/es/hombre.html rename to vinted-db-feeder/data/raw/sizes/es/hombre.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/es/mascotas.html b/vinted-db-feeder/data/raw/sizes/es/mascotas.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/es/mascotas.html rename to vinted-db-feeder/data/raw/sizes/es/mascotas.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/es/mujer.html b/vinted-db-feeder/data/raw/sizes/es/mujer.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/es/mujer.html rename to vinted-db-feeder/data/raw/sizes/es/mujer.html diff --git "a/scrapping/vinted-db-feeder/data/raw/sizes/es/ni\303\261os.html" "b/vinted-db-feeder/data/raw/sizes/es/ni\303\261os.html" similarity index 100% rename from "scrapping/vinted-db-feeder/data/raw/sizes/es/ni\303\261os.html" rename to "vinted-db-feeder/data/raw/sizes/es/ni\303\261os.html" diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/fr/animaux.html b/vinted-db-feeder/data/raw/sizes/fr/animaux.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/fr/animaux.html rename to vinted-db-feeder/data/raw/sizes/fr/animaux.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/fr/enfants.html b/vinted-db-feeder/data/raw/sizes/fr/enfants.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/fr/enfants.html rename to vinted-db-feeder/data/raw/sizes/fr/enfants.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/fr/femme.html b/vinted-db-feeder/data/raw/sizes/fr/femme.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/fr/femme.html rename to vinted-db-feeder/data/raw/sizes/fr/femme.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/fr/hommes.html b/vinted-db-feeder/data/raw/sizes/fr/hommes.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/fr/hommes.html rename to vinted-db-feeder/data/raw/sizes/fr/hommes.html diff --git a/scrapping/vinted-db-feeder/data/raw/sizes/fr/maison.html b/vinted-db-feeder/data/raw/sizes/fr/maison.html similarity index 100% rename from scrapping/vinted-db-feeder/data/raw/sizes/fr/maison.html rename to vinted-db-feeder/data/raw/sizes/fr/maison.html diff --git a/scrapping/vinted-db-feeder/lib/__init__.py b/vinted-db-feeder/lib/__init__.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/__init__.py rename to vinted-db-feeder/lib/__init__.py diff --git a/scrapping/vinted-db-feeder/lib/constants.py b/vinted-db-feeder/lib/constants.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/constants.py rename to vinted-db-feeder/lib/constants.py diff --git a/scrapping/vinted-db-feeder/lib/get_brands_ids.py b/vinted-db-feeder/lib/get_brands_ids.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_brands_ids.py rename to vinted-db-feeder/lib/get_brands_ids.py diff --git a/scrapping/vinted-db-feeder/lib/get_brands_names.py b/vinted-db-feeder/lib/get_brands_names.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_brands_names.py rename to vinted-db-feeder/lib/get_brands_names.py diff --git a/scrapping/vinted-db-feeder/lib/get_categories.py b/vinted-db-feeder/lib/get_categories.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_categories.py rename to vinted-db-feeder/lib/get_categories.py diff --git a/scrapping/vinted-db-feeder/lib/get_colors.py b/vinted-db-feeder/lib/get_colors.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_colors.py rename to vinted-db-feeder/lib/get_colors.py diff --git a/scrapping/vinted-db-feeder/lib/get_countries.py b/vinted-db-feeder/lib/get_countries.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_countries.py rename to vinted-db-feeder/lib/get_countries.py diff --git a/scrapping/vinted-db-feeder/lib/get_materials.py b/vinted-db-feeder/lib/get_materials.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_materials.py rename to vinted-db-feeder/lib/get_materials.py diff --git a/scrapping/vinted-db-feeder/lib/get_sizes.py b/vinted-db-feeder/lib/get_sizes.py similarity index 100% rename from scrapping/vinted-db-feeder/lib/get_sizes.py rename to vinted-db-feeder/lib/get_sizes.py diff --git a/scrapping/vinted-db-feeder/main.py b/vinted-db-feeder/main.py similarity index 92% rename from scrapping/vinted-db-feeder/main.py rename to vinted-db-feeder/main.py index 73bed45..f04cace 100644 --- a/scrapping/vinted-db-feeder/main.py +++ b/vinted-db-feeder/main.py @@ -1,7 +1,8 @@ from lib import get_brands_ids, get_colors, get_materials, get_sizes, constants, get_categories, get_countries def debug(): - # Get brands ids from names [Readly slow scrap] + # Get brands ids from names [Really slow scrapping] + # get_brands_ids.exec(constants.BRANDS_FOLDER, constants.BRANDS_OUTFILE) # Get materials diff --git a/scrapping/vinted-db-feeder/requirements.txt b/vinted-db-feeder/requirements.txt similarity index 73% rename from scrapping/vinted-db-feeder/requirements.txt rename to vinted-db-feeder/requirements.txt index 0375fd8..7471af3 100644 --- a/scrapping/vinted-db-feeder/requirements.txt +++ b/vinted-db-feeder/requirements.txt @@ -1,8 +1,8 @@ beautifulsoup4==4.12.2 -bs4==0.0.1 +bs4==0.0.2 certifi==2024.7.4 charset-normalizer==3.1.0 -idna==3.4 +idna==3.7 requests==2.31.0 soupsieve==2.4.1 -urllib3==2.0.3 +urllib3==2.2.2 diff --git a/vinted-db-feeder/shell.nix b/vinted-db-feeder/shell.nix new file mode 100644 index 0000000..95c954f --- /dev/null +++ b/vinted-db-feeder/shell.nix @@ -0,0 +1,7 @@ +let + pkgs = import {}; +in pkgs.mkShell { + packages = [ + (pkgs.python3) + ]; +} \ No newline at end of file