From 9af30e8dbee559c9c867413b937373fdd38e8b10 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 24 Feb 2025 14:52:32 +0000 Subject: [PATCH 1/8] Revert "fix(backend): set new_geom_type default to Polygon as temp fix for #2164" This reverts commit 400ed112dc8406fe269b64506079ad85dd170dde. --- src/backend/app/central/central_crud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index cfe0ad9a2c..2d81005ebd 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -277,7 +277,7 @@ async def append_fields_to_user_xlsform( form_category: str = "buildings", additional_entities: list[str] = None, existing_id: str = None, - new_geom_type: DbGeomType = DbGeomType.POLYGON, + new_geom_type: DbGeomType = DbGeomType.POINT, ) -> tuple[str, BytesIO]: """Helper to return the intermediate XLSForm prior to convert.""" log.debug("Appending mandatory FMTM fields to XLSForm") @@ -295,7 +295,7 @@ async def validate_and_update_user_xlsform( form_category: str = "buildings", additional_entities: list[str] = None, existing_id: str = None, - new_geom_type: DbGeomType = DbGeomType.POLYGON, + new_geom_type: DbGeomType = DbGeomType.POINT, ) -> BytesIO: """Wrapper to append mandatory fields and validate user uploaded XLSForm.""" xform_id, updated_file_bytes = await append_fields_to_user_xlsform( From 68289d588fd36f9d51a944f5cc5c7f117987e673 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 24 Feb 2025 17:16:51 +0000 Subject: [PATCH 2/8] fix(backend): ignore missing token warning for auth checks --- src/backend/app/auth/roles.py | 5 ++++- src/backend/app/db/models.py | 10 ++++++++-- src/backend/app/projects/project_crud.py | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/backend/app/auth/roles.py b/src/backend/app/auth/roles.py index 7aab4ecd9c..0e6c660fa8 100644 --- a/src/backend/app/auth/roles.py +++ b/src/backend/app/auth/roles.py @@ -289,11 +289,14 @@ async def wrap_check_access( async def project_manager( - project: Annotated[DbProject, Depends(get_project)], + project_id: int, db: Annotated[Connection, Depends(db_conn)], current_user: Annotated[AuthUser, Depends(login_required)], ) -> ProjectUserDict: """A project manager for a specific project.""" + # NOTE here we get the project manually to avoid warnings before the project + # if fully created yet (about odk_token not existing) + project = await DbProject.one(db, project_id, warn_on_missing_token=False) return await wrap_check_access( project, db, diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index f999369185..66afb8de87 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -1087,7 +1087,13 @@ def set_odk_credentials_on_project( ) @classmethod - async def one(cls, db: Connection, project_id: int, minimal: bool = False) -> Self: + async def one( + cls, + db: Connection, + project_id: int, + minimal: bool = False, + warn_on_missing_token: bool = True, + ) -> Self: """Get project by ID, including all tasks and other details.""" # Simpler query without additional metadata if minimal: @@ -1231,7 +1237,7 @@ async def one(cls, db: Connection, project_id: int, minimal: bool = False) -> Se if db_project is None: raise KeyError(f"Project ({project_id}) not found.") - if db_project.odk_token is None: + if warn_on_missing_token and db_project.odk_token is None: log.warning( f"Project ({db_project.id}) has no 'odk_token' set. " "The QRCode will not work!" diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 27dcc7f28d..4e544a0979 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -529,7 +529,7 @@ async def generate_project_files( Returns: bool: True if success. """ - project = await project_deps.get_project_by_id(db, project_id) + project = await DbProject.one(db, project_id, warn_on_missing_token=False) log.info(f"Starting generate_project_files for project {project_id}") # Extract data extract from flatgeobuf From a7ac6b0e8d80d4b593f3d1408a83b0ef5dcf88fe Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 24 Feb 2025 17:17:16 +0000 Subject: [PATCH 3/8] build: migration to rename xform_category --> osm_category --- .../migrations/004-project-create-refine.sql | 15 +++++++++++++++ src/backend/migrations/init/fmtm_base_schema.sql | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 src/backend/migrations/004-project-create-refine.sql diff --git a/src/backend/migrations/004-project-create-refine.sql b/src/backend/migrations/004-project-create-refine.sql new file mode 100644 index 0000000000..919625701c --- /dev/null +++ b/src/backend/migrations/004-project-create-refine.sql @@ -0,0 +1,15 @@ +-- ## Migration to: +-- * Replace projects.xform_category with projects.osm_category + +-- Start a transaction +BEGIN; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'projects' AND column_name = 'xform_category') THEN + ALTER TABLE public.projects RENAME COLUMN xform_category TO osm_category; + END IF; +END $$; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 881c249848..8de3616c39 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -260,7 +260,7 @@ CREATE TABLE public.projects ( outline public.GEOMETRY (POLYGON, 4326), status public.projectstatus NOT NULL DEFAULT 'DRAFT', total_tasks integer, - xform_category character varying, + osm_category character varying, xlsform_content bytea, odk_form_id character varying, visibility public.projectvisibility NOT NULL DEFAULT 'PUBLIC', From c5b890da0a393e4f8427064ff79e9d585e88fa3d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 24 Feb 2025 18:36:25 +0000 Subject: [PATCH 4/8] refactor(createProject): set primary_geom_type and new_geom_type fields --- docs/manuals/project-managers.md | 5 +- docs/manuals/xlsform-design.md | 6 +- src/backend/app/central/central_crud.py | 24 +-- src/backend/app/central/central_deps.py | 15 +- src/backend/app/db/models.py | 6 +- src/backend/app/db/postgis_utils.py | 6 +- src/backend/app/helpers/helper_routes.py | 6 +- src/backend/app/projects/project_crud.py | 49 +---- src/backend/app/projects/project_deps.py | 2 +- src/backend/app/projects/project_routes.py | 66 ++----- src/backend/app/projects/project_schemas.py | 2 - .../app/submissions/submission_routes.py | 4 +- .../migrations/004-project-create-refine.sql | 30 ++- .../migrations/init/fmtm_base_schema.sql | 2 +- src/backend/tests/conftest.py | 27 +-- src/backend/tests/test_projects_routes.py | 17 +- src/frontend/src/api/CreateProjectService.ts | 9 +- src/frontend/src/api/Project.ts | 2 +- .../ManageProject/EditTab/FormUpdateTab.tsx | 16 +- .../createnewproject/DataExtract.tsx | 2 +- .../createnewproject/Description.tsx | 10 +- .../createnewproject/SelectForm.tsx | 174 ++++++++---------- .../createnewproject/SplitTasks.tsx | 8 +- .../validation/DataExtractValidation.tsx | 2 - .../validation/SelectFormValidation.tsx | 11 +- .../createproject/createProjectModel.ts | 2 +- .../src/models/project/projectModel.ts | 5 +- .../src/store/slices/CreateProjectSlice.ts | 8 +- .../src/store/types/ICreateProject.ts | 12 +- src/frontend/src/types/enums.ts | 2 +- src/frontend/src/views/CreateNewProject.tsx | 8 +- src/mapper/src/constants/enums.ts | 2 +- src/mapper/src/lib/components/map/main.svelte | 16 +- src/mapper/src/lib/types.ts | 4 +- .../src/routes/[projectId]/+page.svelte | 1 + 35 files changed, 242 insertions(+), 319 deletions(-) diff --git a/docs/manuals/project-managers.md b/docs/manuals/project-managers.md index cb116d0acf..1f76ba5719 100644 --- a/docs/manuals/project-managers.md +++ b/docs/manuals/project-managers.md @@ -138,9 +138,8 @@ Alternatively, request the creation of a new organisation for your team: ![image](https://github.com/user-attachments/assets/64aeda34-c682-4fdc-8c2f-1fd83e29c61f) -13. The step 3 is to choose the form category of the project. Meaning if you want - to survey each household or healthcare or educational institutes. - You can upload the custom XLS form by clicking on the checkbox. +13. Upload your XLSForm. Here you download pre-defined forms from FMTM. + Some are specifically designed to work with OpenStreetMap. Click on "Next" to proceed. ![image](https://github.com/user-attachments/assets/cdf1e050-42ec-4149-bf97-0d841bc5117f) diff --git a/docs/manuals/xlsform-design.md b/docs/manuals/xlsform-design.md index ecbaf34322..28c028d9fb 100644 --- a/docs/manuals/xlsform-design.md +++ b/docs/manuals/xlsform-design.md @@ -157,10 +157,10 @@ overview of the injected fields and their purposes: 4. If no feature is selected, the user would be prompted to take a GPS coordinate of new feature. Note: One of these two options must be filled up to proceed. -5. We also dedicate few rows for calculating form category, osm ID, - Task Id and mapping status used on FMTM. +5. We also dedicate few rows for calculating OSM ID, + Task ID and mapping status used on FMTM. 6. We then ask mappers to answer if the feature exist in reality? - If yes, the custom form uploaded by user is proceeded. + If yes, user proceeds with form submission. 7. If no, the user is prompted to capture an image (if available) and the form is terminated with a message: "You cannot proceed with data acquisition if the building does not exist." diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 2d81005ebd..8bf80cf222 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -274,16 +274,16 @@ def get_project_form_xml( async def append_fields_to_user_xlsform( xlsform: BytesIO, - form_category: str = "buildings", - additional_entities: list[str] = None, - existing_id: str = None, - new_geom_type: DbGeomType = DbGeomType.POINT, + form_name: str = "buildings", + additional_entities: Optional[list[str]] = None, + existing_id: Optional[str] = None, + new_geom_type: Optional[DbGeomType] = DbGeomType.POINT, ) -> tuple[str, BytesIO]: """Helper to return the intermediate XLSForm prior to convert.""" log.debug("Appending mandatory FMTM fields to XLSForm") return await append_mandatory_fields( xlsform, - form_category=form_category, + form_name=form_name, additional_entities=additional_entities, existing_id=existing_id, new_geom_type=new_geom_type, @@ -292,15 +292,15 @@ async def append_fields_to_user_xlsform( async def validate_and_update_user_xlsform( xlsform: BytesIO, - form_category: str = "buildings", - additional_entities: list[str] = None, - existing_id: str = None, - new_geom_type: DbGeomType = DbGeomType.POINT, + form_name: str = "buildings", + additional_entities: Optional[list[str]] = None, + existing_id: Optional[str] = None, + new_geom_type: Optional[DbGeomType] = DbGeomType.POINT, ) -> BytesIO: """Wrapper to append mandatory fields and validate user uploaded XLSForm.""" xform_id, updated_file_bytes = await append_fields_to_user_xlsform( xlsform, - form_category=form_category, + form_name=form_name, additional_entities=additional_entities, existing_id=existing_id, new_geom_type=new_geom_type, @@ -315,7 +315,7 @@ async def update_project_xform( xform_id: str, odk_id: int, xlsform: BytesIO, - category: str, + # osm_category: str, odk_credentials: central_schemas.ODKCentralDecrypted, ) -> None: """Update and publish the XForm for a project. @@ -324,7 +324,6 @@ async def update_project_xform( xform_id (str): The UUID of the existing XForm in ODK Central. odk_id (int): ODK Central form ID. xlsform (UploadFile): XForm data. - category (str): Category of the XForm. odk_credentials (central_schemas.ODKCentralDecrypted): ODK Central creds. Returns: None @@ -337,6 +336,7 @@ async def update_project_xform( xform_obj.createForm( odk_id, xform_bytesio, + # NOTE this variable is incorrectly named and should be form_id form_name=xform_id, ) # The draft form must be published after upload diff --git a/src/backend/app/central/central_deps.py b/src/backend/app/central/central_deps.py index ff495fd5db..73d05df6f0 100644 --- a/src/backend/app/central/central_deps.py +++ b/src/backend/app/central/central_deps.py @@ -21,9 +21,8 @@ from contextlib import asynccontextmanager from io import BytesIO from pathlib import Path -from typing import Optional -from fastapi import File, UploadFile +from fastapi import UploadFile from fastapi.exceptions import HTTPException from osm_fieldwork.OdkCentralAsync import OdkDataset, OdkForm @@ -65,8 +64,8 @@ async def get_async_odk_form(odk_creds: ODKCentralDecrypted): async def validate_xlsform_extension(xlsform: UploadFile): """Validate an XLSForm has .xls or .xlsx extension.""" - file = Path(xlsform.filename) - file_ext = file.suffix.lower() + filename = Path(xlsform.filename) + file_ext = filename.suffix.lower() allowed_extensions = [".xls", ".xlsx"] if file_ext not in allowed_extensions: @@ -80,11 +79,3 @@ async def validate_xlsform_extension(xlsform: UploadFile): async def read_xlsform(xlsform: UploadFile) -> BytesIO: """Read an XLSForm, validate extension, return wrapped in BytesIO.""" return await validate_xlsform_extension(xlsform) - - -async def read_optional_xlsform( - xlsform: Optional[UploadFile] = File(None), -) -> Optional[BytesIO]: - """Read an XLSForm, validate extension, return wrapped in BytesIO.""" - if xlsform: - return await validate_xlsform_extension(xlsform) diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index 66afb8de87..fff93caf9a 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -1026,7 +1026,7 @@ class DbProject(BaseModel): custom_tms_url: Optional[str] = None status: Optional[ProjectStatus] = None visibility: Optional[ProjectVisibility] = None - xform_category: Optional[str] = None + osm_category: Optional[str] = None odk_form_id: Optional[str] = None xlsform_content: Optional[bytes] = None mapper_level: Optional[MappingLevel] = None @@ -1036,11 +1036,11 @@ class DbProject(BaseModel): odk_central_user: Optional[str] = None odk_central_password: Optional[str] = None odk_token: Optional[str] = None - data_extract_type: Optional[str] = None data_extract_url: Optional[str] = None task_split_dimension: Optional[int] = None task_num_buildings: Optional[int] = None - new_geom_type: Optional[DbGeomType] = None + primary_geom_type: Optional[DbGeomType] = None # the main geometries surveyed + new_geom_type: Optional[DbGeomType] = None # when new geometries are drawn geo_restrict_distance_meters: Optional[PositiveInt] = None geo_restrict_force_error: Optional[bool] = None hashtags: Optional[list[str]] = None diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index c8e7570382..25dd4030cb 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -774,17 +774,17 @@ def merge_polygons( ) from e -def get_osm_geometries(form_category, geometry): +def get_osm_geometries(osm_category, geometry): """Request a snapshot based on the provided geometry. Args: - form_category(str): feature category type (eg: buildings). + osm_category(str): feature category type (eg: buildings). geometry (str): The geometry data in JSON format. Returns: dict: The JSON response containing the snapshot data. """ - config_filename = XLSFormType(form_category).name + config_filename = XLSFormType(osm_category).name data_model = f"{data_models_path}/{config_filename}.yaml" with open(data_model, "rb") as data_model_yaml: diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index 2c8cb9622f..944d1ad8ed 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -66,10 +66,10 @@ @router.get("/download-template-xlsform") async def download_template( - category: XLSFormType, + form_type: XLSFormType, ): - """Download an XLSForm template to fill out.""" - filename = XLSFormType(category).name + """Download example XLSForm from FMTM.""" + filename = XLSFormType(form_type).name xlsform_path = f"{xlsforms_path}/{filename}.xls" if Path(xlsform_path).exists(): return FileResponse(xlsform_path, filename="form.xls") diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 4e544a0979..cc7eac19ee 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -47,7 +47,6 @@ featcol_keep_single_geom_type, featcol_to_flatgeobuf, flatgeobuf_to_featcol, - get_featcol_dominant_geom_type, parse_geojson_file_to_featcol, split_geojson_by_task_areas, ) @@ -147,7 +146,7 @@ async def generate_data_extract( if not extract_config: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, - detail="To generate a new data extract a form_category must be specified.", + detail="To generate a new data extract a extract_config must be specified.", ) pg = PostgresClient( @@ -203,7 +202,7 @@ async def read_and_insert_xlsforms(db: Connection, directory: str) -> None: # Insert or update new XLSForms from disk for xls_type in XLSFormType: file_name = xls_type.name - category = xls_type.value + form_type = xls_type.value file_path = Path(directory) / f"{file_name}.xls" if not file_path.exists(): @@ -224,12 +223,12 @@ async def read_and_insert_xlsforms(db: Connection, directory: str) -> None: ON CONFLICT (title) DO UPDATE SET xls = EXCLUDED.xls """ - await cur.execute(insert_query, {"title": category, "xls": data}) - log.info(f"XLSForm for '{category}' inserted/updated in the database") + await cur.execute(insert_query, {"title": form_type, "xls": data}) + log.info(f"XLSForm for '{form_type}' inserted/updated in the database") except Exception as e: log.exception( - f"Failed to insert or update {category} in the database. " + f"Failed to insert or update {form_type} in the database. " f"Error: {e}", stack_info=True, ) @@ -278,17 +277,11 @@ async def get_or_set_data_extract_url( raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) return existing_url - # FIXME determine this using get_featcol_dominant_geom_type ? - # FIXME would have to download extract first though - # Perhaps we do this via generate-data-extract URL - extract_type = "polygon" - await DbProject.update( db, project_id, project_schemas.ProjectUpdate( data_extract_url=url, - data_extract_type=extract_type, ), ) @@ -299,7 +292,6 @@ async def upload_custom_extract_to_s3( db: Connection, project_id: int, fgb_content: bytes, - data_extract_type: str, ) -> str: """Uploads custom data extracts to S3. @@ -307,7 +299,6 @@ async def upload_custom_extract_to_s3( db (Connection): The database connection. project_id (int): The ID of the project. fgb_content (bytes): Content of read flatgeobuf file. - data_extract_type (str): centroid/polygon/line for database. Returns: str: URL to fgb file in S3. @@ -336,7 +327,6 @@ async def upload_custom_extract_to_s3( project_id, project_schemas.ProjectUpdate( data_extract_url=s3_fgb_full_url, - data_extract_type=data_extract_type, ), ) @@ -374,36 +364,13 @@ async def upload_custom_fgb_extract( log.error(msg) raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) - data_extract_type = await get_data_extract_type(featcol) - return await upload_custom_extract_to_s3( db, project_id, fgb_content, - data_extract_type, ) -async def get_data_extract_type(featcol: geojson.FeatureCollection) -> str: - """Determine predominant geometry type for extract.""" - geom_type = get_featcol_dominant_geom_type(featcol) - if geom_type not in ["Polygon", "LineString", "Point"]: - msg = ( - "Extract does not contain valid geometry types, from 'Polygon' " - ", 'LineString' and 'Point'." - ) - log.error(msg) - raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) - geom_name_map = { - "Polygon": "polygon", - "Point": "centroid", - "LineString": "line", - } - data_extract_type = geom_name_map.get(geom_type, "polygon") - - return data_extract_type - - async def upload_custom_geojson_extract( db: Connection, project_id: int, @@ -433,8 +400,6 @@ async def upload_custom_geojson_extract( await check_crs(featcol_single_geom_type) - data_extract_type = await get_data_extract_type(featcol_single_geom_type) - log.debug( "Generating fgb object from geojson with " f"{len(featcol_single_geom_type.get('features', []))} features" @@ -447,7 +412,9 @@ async def upload_custom_geojson_extract( raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) return await upload_custom_extract_to_s3( - db, project_id, fgb_data, data_extract_type + db, + project_id, + fgb_data, ) diff --git a/src/backend/app/projects/project_deps.py b/src/backend/app/projects/project_deps.py index 9f3ff5aa33..5e3d41ceba 100644 --- a/src/backend/app/projects/project_deps.py +++ b/src/backend/app/projects/project_deps.py @@ -41,7 +41,7 @@ async def get_project( async def get_project_by_id(db: Connection, project_id: int): """Get a single project by it's ID.""" try: - return await DbProject.one(db, project_id) + return await DbProject.one(db, project_id, warn_on_missing_token=False) except KeyError as e: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) from e diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index cb39a5184f..ca544748af 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -43,7 +43,6 @@ from loguru import logger as log from osm_fieldwork.data_models import data_models_path from osm_fieldwork.make_data_extract import getChoices -from osm_fieldwork.xlsforms import xlsforms_path from pg_nearest_city import AsyncNearestCity from psycopg import Connection @@ -568,7 +567,6 @@ async def validate_form( NOTE this provides a basic sanity check, some fields are omitted so the form is not usable in production: - - form_category - additional_entities - new_geom_type """ @@ -647,7 +645,7 @@ async def get_data_extract( # FIXME once sub project creation implemented, this should be manager only current_user: Annotated[AuthUser, Depends(login_required)], geojson_file: UploadFile = File(...), - form_category: Optional[XLSFormType] = Form(None), + osm_category: Optional[XLSFormType] = Form(None), ): """Get a new data extract for a given project AOI. @@ -658,8 +656,8 @@ async def get_data_extract( clean_boundary_geojson = merge_polygons(boundary_geojson) # Get extract config file from existing data_models - if form_category: - config_filename = XLSFormType(form_category).name + if osm_category: + config_filename = XLSFormType(osm_category).name data_model = f"{data_models_path}/{config_filename}.yaml" with open(data_model, "rb") as data_model_yaml: extract_config = BytesIO(data_model_yaml.read()) @@ -803,30 +801,17 @@ async def update_project_form( db: Annotated[Connection, Depends(db_conn)], project_user_dict: Annotated[ProjectUserDict, Depends(project_manager)], xform_id: str = Form(...), - category: XLSFormType = Form(...), + # FIXME add back in capability to update osm_category + # osm_category: XLSFormType = Form(...), ): - """Update the XForm data in ODK Central. - - Also updates the category and custom XLSForm data in the database. - """ + """Update the XForm data in ODK Central & FMTM DB.""" project = project_user_dict["project"] - # TODO we currently do nothing with the provided category - # TODO allowing for category updates is disabled due to complexity - # TODO as it would mean also updating data extracts, - # TODO so perhaps we just remove this? - # form_filename = XLSFormType(project.xform_category).name - # xlsform_path = Path(f"{xlsforms_path}/{form_filename}.xls") - # file_ext = xlsform_path.suffix.lower() - # with open(xlsform_path, "rb") as f: - # new_xform_data = BytesIO(f.read()) - # Update ODK Central form data await central_crud.update_project_xform( xform_id, project.odkid, xlsform, - category, project.odk_credentials, ) @@ -963,9 +948,7 @@ async def generate_files( db: Annotated[Connection, Depends(db_conn)], project_user_dict: Annotated[ProjectUserDict, Depends(project_manager)], background_tasks: BackgroundTasks, - xlsform_upload: Annotated[ - Optional[BytesIO], Depends(central_deps.read_optional_xlsform) - ] = None, + xlsform_upload: Annotated[BytesIO, Depends(central_deps.read_xlsform)], additional_entities: Annotated[Optional[list[str]], None] = None, combined_features_count: Annotated[int, Form()] = 0, ): @@ -973,7 +956,7 @@ async def generate_files( Boundary, ODK Central forms, QR codes, etc. - Accepts a project ID, category, custom form flag, and an uploaded file as inputs. + Accepts a project ID and an uploaded file as inputs. The generated files are associated with the project ID and stored in the database. This api generates odk appuser tokens, forms. This api also creates an app user for each task and provides the required roles. @@ -982,8 +965,7 @@ async def generate_files( it to the form. Args: - xlsform_upload (UploadFile, optional): A custom XLSForm to use in the project. - A file should be provided if user wants to upload a custom xls form. + xlsform_upload (UploadFile): The XLSForm for the project data collection. additional_entities (list[str]): If additional Entity lists need to be created (i.e. the project form references multiple geometries). combined_features_count (int): Total count of features to be mapped, plus @@ -997,34 +979,24 @@ async def generate_files( """ project = project_user_dict.get("project") project_id = project.id - form_category = project.xform_category new_geom_type = project.new_geom_type log.debug(f"Generating additional files for project: {project.id}") - if xlsform_upload: - log.debug("User provided custom XLSForm") - - # Validate uploaded form - await central_crud.validate_and_update_user_xlsform( - xlsform=xlsform_upload, - form_category=form_category, - additional_entities=additional_entities, - new_geom_type=new_geom_type, - ) - xlsform = xlsform_upload + form_name = f"FMTM_Project_{project.id}" - else: - log.debug(f"Using default XLSForm for category: '{form_category}'") - - form_filename = XLSFormType(form_category).name - xlsform_path = f"{xlsforms_path}/{form_filename}.xls" - with open(xlsform_path, "rb") as f: - xlsform = BytesIO(f.read()) + # Validate uploaded form + await central_crud.validate_and_update_user_xlsform( + xlsform=xlsform_upload, + form_name=form_name, + additional_entities=additional_entities, + new_geom_type=new_geom_type, + ) + xlsform = xlsform_upload xform_id, project_xlsform = await central_crud.append_fields_to_user_xlsform( xlsform=xlsform, - form_category=form_category, + form_name=form_name, additional_entities=additional_entities, new_geom_type=new_geom_type, ) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index c8203abec7..3d6ba023ef 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -173,8 +173,6 @@ def append_fmtm_hashtag_and_slug(self) -> Self: class ProjectIn(ProjectInBase, ODKCentralIn): """Upload new project.""" - # Mandatory - xform_category: str # Ensure geojson_pydantic.Polygon outline: Polygon # Omit new_geom_type as we calculate this automatically diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index 195579c862..cb8c7c053c 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -434,10 +434,10 @@ async def conflate_geojson( submission_geojson = await central_crud.convert_odk_submission_json_to_geojson( task_submission ) - form_category = project.xform_category + osm_category = project.osm_category input_features = submission_geojson["features"] - osm_features = postgis_utils.get_osm_geometries(form_category, task_geojson) + osm_features = postgis_utils.get_osm_geometries(osm_category, task_geojson) conflated_features = postgis_utils.conflate_features( input_features, osm_features.get("features", []), remove_conflated ) diff --git a/src/backend/migrations/004-project-create-refine.sql b/src/backend/migrations/004-project-create-refine.sql index 919625701c..b406d1a6fd 100644 --- a/src/backend/migrations/004-project-create-refine.sql +++ b/src/backend/migrations/004-project-create-refine.sql @@ -1,15 +1,41 @@ -- ## Migration to: -- * Replace projects.xform_category with projects.osm_category +-- * Replace projects.data_extract_type with primary_geom_type +-- (to compliment new_geom_type) -- Start a transaction BEGIN; DO $$ BEGIN - IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'projects' AND column_name = 'xform_category') THEN - ALTER TABLE public.projects RENAME COLUMN xform_category TO osm_category; + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'projects' AND column_name = 'xform_category' + ) THEN + ALTER TABLE public.projects RENAME COLUMN xform_category + TO osm_category; END IF; END $$; +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'projects' AND column_name = 'data_extract_type' + ) THEN + ALTER TABLE public.projects RENAME COLUMN data_extract_type + TO primary_geom_type; + + UPDATE public.projects SET primary_geom_type = NULL; + + ALTER TABLE public.projects + ALTER COLUMN primary_geom_type + TYPE public.geomtype + USING primary_geom_type::public.geomtype, + ALTER COLUMN primary_geom_type SET DEFAULT 'POLYGON'; + END IF; +END $$; + + -- Commit the transaction COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 8de3616c39..96b8e6b021 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -273,7 +273,6 @@ CREATE TABLE public.projects ( odk_central_user character varying, odk_central_password character varying, odk_token character varying, - data_extract_type character varying, data_extract_url character varying, task_split_type public.tasksplittype, task_split_dimension smallint, @@ -284,6 +283,7 @@ CREATE TABLE public.projects ( geo_restrict_distance_meters int2 DEFAULT 50 CHECK ( geo_restrict_distance_meters >= 0 ), + primary_geom_type public.geomtype DEFAULT 'POLYGON', new_geom_type public.geomtype DEFAULT 'POINT', created_at timestamp with time zone NOT NULL DEFAULT now(), updated_at timestamp with time zone DEFAULT now() diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index 1c2f8698af..a33cd6fae7 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -28,7 +28,6 @@ import pytest import pytest_asyncio -import requests from asgi_lifespan import LifespanManager from fastapi import FastAPI from httpx import ASGITransport, AsyncClient @@ -194,7 +193,7 @@ async def project(db, admin_user, organisation): name=project_name, short_description="test", description="test", - xform_category="buildings", + osm_category="buildings", odk_central_url=os.getenv("ODK_CENTRAL_URL"), odk_central_user=os.getenv("ODK_CENTRAL_USER"), odk_central_password=os.getenv("ODK_CENTRAL_PASSWD"), @@ -303,15 +302,19 @@ async def odk_project(db, client, project, tasks): ) internal_s3_url = f"{settings.S3_ENDPOINT}{urlparse(data_extract_s3_path).path}" - response = requests.head(internal_s3_url, allow_redirects=True) - assert response.status_code < 400 + + async with AsyncClient() as client_httpx: + response = await client_httpx.head(internal_s3_url, follow_redirects=True) + assert response.status_code < 400, ( + f"HEAD request failed with status {response.status_code}" + ) xlsform_file = Path(f"{test_data_path}/buildings.xls") with open(xlsform_file, "rb") as xlsform_data: xlsform_obj = BytesIO(xlsform_data.read()) xform_file = { - "xls_form_upload": ( + "xlsform": ( "buildings.xls", xlsform_obj, ) @@ -342,12 +345,15 @@ async def submission(client, odk_project): odk_creds["odk_central_password"], ) - def forms(base_url, auth, pid): + async def forms(base_url, auth, pid): """Fetch a list of forms in a project.""" - url = f"{base_url}/v1/projects/{pid}/forms" - return requests.get(url, auth=auth) + async with AsyncClient(auth=auth) as client_httpx: + url = f"{base_url}/v1/projects/{pid}/forms" + response = await client_httpx.get(url) + response.raise_for_status() + return response - forms_response = forms(base_url, auth, odk_project_id) + forms_response = await forms(base_url, auth, odk_project_id) assert forms_response.status_code == 200, "Failed to fetch forms from ODK Central" forms = forms_response.json() assert forms, "No forms found in ODK Central project" @@ -374,7 +380,6 @@ def forms(base_url, auth, pid): {photo_file_name} 12.750577838121643 -24.776785714285722 0.0 0.0 - building 12.750577838121643 -24.776785714285722 0.0 0.0 @@ -454,7 +459,7 @@ async def project_data(): "name": project_name, "short_description": "test", "description": "test", - "xform_category": "buildings", + "osm_category": "buildings", "hashtags": "testtag", "outline": { "coordinates": [ diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index f9a442cd33..472d9d4a2e 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -26,8 +26,8 @@ from uuid import uuid4 import pytest -import requests from fastapi import HTTPException +from httpx import AsyncClient from loguru import logger as log from app.central.central_crud import create_odk_project @@ -362,8 +362,11 @@ async def test_generate_project_files(db, client, project): ) assert data_extract_s3_path is not None internal_s3_url = f"{settings.S3_ENDPOINT}{urlparse(data_extract_s3_path).path}" - response = requests.head(internal_s3_url, allow_redirects=True) - assert response.status_code < 400 + async with AsyncClient() as client_httpx: + response = await client_httpx.head(internal_s3_url, follow_redirects=True) + assert response.status_code < 400, ( + f"HEAD request failed with status {response.status_code}" + ) # Get custom XLSForm path xlsform_file = Path(f"{test_data_path}/buildings.xls") @@ -371,7 +374,7 @@ async def test_generate_project_files(db, client, project): xlsform_obj = BytesIO(xlsform_data.read()) xform_file = { - "xls_form_upload": ( + "xlsform": ( "buildings.xls", xlsform_obj, ) @@ -395,7 +398,7 @@ async def test_update_project(client, admin_user, project): "name": f"Updated Test Project {uuid4()}", "short_description": "updated short description", "description": "updated description", - "xform_category": "healthcare", + "osm_category": "healthcare", "hashtags": "#FMTM anothertag", } @@ -412,7 +415,7 @@ async def test_update_project(client, admin_user, project): ) assert response_data["description"] == updated_project_data["description"] - assert response_data["xform_category"] == updated_project_data["xform_category"] + assert response_data["osm_category"] == updated_project_data["osm_category"] assert sorted(response_data["hashtags"]) == sorted( [ "#FMTM", @@ -452,7 +455,7 @@ async def test_project_by_id(client, project): assert data["description"] == project.description assert data["per_task_instructions"] == project.per_task_instructions assert data["status"] == project.status - assert data["xform_category"] == project.xform_category + assert data["osm_category"] == project.osm_category assert data["hashtags"] == project.hashtags assert data["organisation_id"] == project.organisation_id assert data["location_str"] == project.location_str diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index 7981435312..98bc567b40 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -200,10 +200,8 @@ const GenerateProjectFilesService = (url: string, projectData: any, formUpload: formData.append('additional_entities', additionalEntities); } - // Append xlsform only if it's a custom form - if (projectData.form_ways === 'custom_form' && formUpload) { - formData.append('xlsform', formUpload); - } + // Append xlsform + formData.append('xlsform', formUpload); // Add combined features count formData.append('combined_features_count', combinedFeaturesCount.toString()); @@ -433,7 +431,8 @@ const PostFormUpdate = (url: string, projectData: Record) => { try { const formFormData = new FormData(); formFormData.append('xform_id', projectData.xformId); - formFormData.append('category', projectData.category); + // FIXME add back in capability to update osm_category + // formFormData.append('category', projectData.osm_category); formFormData.append('xlsform', projectData.upload); const postFormUpdateResponse = await axios.post(url, formFormData); diff --git a/src/frontend/src/api/Project.ts b/src/frontend/src/api/Project.ts index ec74c262ff..c062398efa 100755 --- a/src/frontend/src/api/Project.ts +++ b/src/frontend/src/api/Project.ts @@ -49,7 +49,7 @@ export const ProjectById = (projectId: string) => { short_description: projectResp.short_description, num_contributors: projectResp.num_contributors, total_tasks: projectResp.total_tasks, - xform_category: projectResp.xform_category, + osm_category: projectResp.osm_category, odk_form_id: projectResp?.odk_form_id, data_extract_url: projectResp.data_extract_url, instructions: projectResp?.per_task_instructions, diff --git a/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx b/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx index 30c21fc903..d98e6ec766 100644 --- a/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx +++ b/src/frontend/src/components/ManageProject/EditTab/FormUpdateTab.tsx @@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react'; import { useAppDispatch, useAppSelector } from '@/types/reduxTypes'; import UploadArea from '@/components/common/UploadArea'; import Button from '@/components/common/Button'; -import { CustomSelect } from '@/components/common/Select'; +// import { CustomSelect } from '@/components/common/Select'; import CoreModules from '@/shared/CoreModules'; import { FormCategoryService } from '@/api/CreateProjectService'; import { DownloadProjectForm } from '@/api/Project'; @@ -25,9 +25,9 @@ const FormUpdateTab = ({ projectId }) => { const [formError, setFormError] = useState(false); const xFormId = CoreModules.useAppSelector((state) => state.createproject.editProjectDetails.odk_form_id); - const formCategoryList = useAppSelector((state) => state.createproject.formCategoryList); - const sortedFormCategoryList = formCategoryList.slice().sort((a, b) => a.title.localeCompare(b.title)); - const selectedCategory = useAppSelector((state) => state.createproject.editProjectDetails.xform_category); + // const formExampleList = useAppSelector((state) => state.createproject.formExampleList); + // const sortedFormExampleList = formExampleList.slice().sort((a, b) => a.title.localeCompare(b.title)); + // const selectedCategory = useAppSelector((state) => state.createproject.editProjectDetails.osm_category); const formUpdateLoading = useAppSelector((state) => state.createproject.formUpdateLoading); useEffect(() => { @@ -42,7 +42,7 @@ const FormUpdateTab = ({ projectId }) => { dispatch( PostFormUpdate(`${API_URL}/projects/update-form?project_id=${projectId}`, { xformId: xFormId, - category: selectedCategory, + // osm_category: selectedCategory, upload: uploadForm && uploadForm?.[0]?.file, }), ); @@ -50,11 +50,11 @@ const FormUpdateTab = ({ projectId }) => { return (
-
+ {/*
{ {' '} {`if uploading the final submissions to OSM.`}

-
+
*/}

⚠️ IMPORTANT ⚠️

diff --git a/src/frontend/src/components/createnewproject/DataExtract.tsx b/src/frontend/src/components/createnewproject/DataExtract.tsx index f7636e674f..982064e912 100644 --- a/src/frontend/src/components/createnewproject/DataExtract.tsx +++ b/src/frontend/src/components/createnewproject/DataExtract.tsx @@ -108,7 +108,7 @@ const DataExtract = ({ const dataExtractRequestFormData = new FormData(); const projectAoiGeojsonFile = getFileFromGeojson(projectAoiGeojson); dataExtractRequestFormData.append('geojson_file', projectAoiGeojsonFile); - dataExtractRequestFormData.append('form_category', projectDetails.formCategorySelection); + dataExtractRequestFormData.append('form_category', projectDetails.formExampleSelection); // Set flatgeobuf as loading dispatch(CreateProjectActions.SetFgbFetchingStatus(true)); diff --git a/src/frontend/src/components/createnewproject/Description.tsx b/src/frontend/src/components/createnewproject/Description.tsx index d167138cb7..820c2b85b7 100644 --- a/src/frontend/src/components/createnewproject/Description.tsx +++ b/src/frontend/src/components/createnewproject/Description.tsx @@ -133,7 +133,7 @@ const UploadSurvey = ({ hoveredSection }: hoveredSectionType) => { useEffect(() => { if (!hoveredSection || windowSize.width < 1024) return; - if (hoveredSection === 'selectform-customform') { + if (hoveredSection === 'selectform-osmnote') { customFormRef?.current?.scrollIntoView(scrollOptions); } if (hoveredSection === 'selectform-selectform') { @@ -146,7 +146,7 @@ const UploadSurvey = ({ hoveredSection }: hoveredSectionType) => { className={`${hoveredSection ? 'fmtm-text-gray-400' : 'fmtm-text-gray-500'} lg:fmtm-flex lg:fmtm-flex-col lg:fmtm-gap-3`} > - You may choose a pre-configured form, or upload a custom XLS form. Click{' '} + You may choose to upload a pre-configured XLSForm, or an entirely custom form. Click{' '} {

- For creating a custom XLS form, there are few essential fields that must be present for FMTM to function. You - may either download the sample XLS file and modify all fields that are not hidden, or edit the sample form - interactively in the browser. + Note: Uploading a custom form may make uploading of the final dataset to OSM difficult. { +const SelectForm = ({ flag, geojsonFile, xlsFormFile, setXlsFormFile }) => { useDocumentTitle('Create Project: Upload Survey'); const dispatch = useAppDispatch(); const navigate = useNavigate(); @@ -27,7 +26,7 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) => const submission = () => { dispatch(CreateProjectActions.SetIndividualProjectDetailsData(formValues)); - if (!customFileValidity && formValues.formWays === 'custom_form') { + if (!customFileValidity) { dispatch( CommonActions.SetSnackBar({ message: 'Your file is invalid', @@ -44,8 +43,8 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) => values: formValues, errors, }: any = useForm(projectDetails, submission, SelectFormValidation); - const formCategoryList = useAppSelector((state) => state.createproject.formCategoryList); - const sortedFormCategoryList = formCategoryList.slice().sort((a, b) => a.title.localeCompare(b.title)); + const formExampleList = useAppSelector((state) => state.createproject.formExampleList); + const sortedFormExampleList = formExampleList.slice().sort((a, b) => a.title.localeCompare(b.title)); /** * Function to handle the change event of a file input. @@ -56,14 +55,14 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) => dispatch(CreateProjectActions.SetCustomFileValidity(false)); // Get the selected files from the event target const { files } = event.target; - // Set the selected file as the customFormFile state - setCustomFormFile(files[0]); + // Set the selected file as the xlsFormFile state + setXlsFormFile(files[0]); handleCustomChange('customFormUpload', files[0]); }; const resetFile = (): void => { handleCustomChange('customFormUpload', null); dispatch(CreateProjectActions.SetCustomFileValidity(false)); - setCustomFormFile(null); + setXlsFormFile(null); }; useEffect(() => { @@ -75,10 +74,10 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) => navigate(url); }; useEffect(() => { - if (customFormFile && !customFileValidity) { - dispatch(ValidateCustomForm(`${import.meta.env.VITE_API_URL}/projects/validate-form`, customFormFile)); + if (xlsFormFile && !customFileValidity) { + dispatch(ValidateCustomForm(`${import.meta.env.VITE_API_URL}/projects/validate-form`, xlsFormFile)); } - }, [customFormFile]); + }, [xlsFormFile]); return (

@@ -87,24 +86,59 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) =>
-
+
{ + dispatch(CreateProjectActions.SetDescriptionToFocus('selectform-selectform')); + }} + onMouseLeave={() => dispatch(CreateProjectActions.SetDescriptionToFocus(null))} + > +
+

+ What are you surveying? +

+

*

+
+ +
+ {validateCustomFormLoading && ( +
+ +

Validating form...

+
+ )} +
{ + dispatch(CreateProjectActions.SetDescriptionToFocus('selectform-osmnote')); + }} + onMouseLeave={() => dispatch(CreateProjectActions.SetDescriptionToFocus(null))} + >
+ -
{ - dispatch(CreateProjectActions.SetDescriptionToFocus('selectform-customform')); - }} - onMouseLeave={() => dispatch(CreateProjectActions.SetDescriptionToFocus(null))} - > - { - if (status) { - handleCustomChange('formWays', 'custom_form'); - } else { - handleCustomChange('formWays', 'existing_form'); - } - }} - className="fmtm-text-black" - labelClickable - disabled={!formValues.formCategorySelection} - /> -
- {formValues.formWays === 'custom_form' ? ( -
-
{ - dispatch(CreateProjectActions.SetDescriptionToFocus('selectform-customform')); - }} - onMouseLeave={() => dispatch(CreateProjectActions.SetDescriptionToFocus(null))} +

+ -

- Please extend upon the existing XLSForm for the selected category: -

-

- - Download Form - -

-

- - Edit Interactively - -

-
-
{ - dispatch(CreateProjectActions.SetDescriptionToFocus('selectform-selectform')); - }} - onMouseLeave={() => dispatch(CreateProjectActions.SetDescriptionToFocus(null))} + Download Form + +

+

+ - -

- {validateCustomFormLoading && ( -
- -

Validating form...

-
- )} -
- ) : null} + Edit Interactively + +

+
)} - {extractWays === 'custom_data_extract' && ( + {extractType === 'custom_data_extract' && ( <> { @@ -308,7 +356,7 @@ const DataExtract = ({ {dataExtractGeojson?.features?.length || 0}

- {extractWays && ( + {extractType && (
{ diff --git a/src/frontend/src/components/createnewproject/SelectForm.tsx b/src/frontend/src/components/createnewproject/SelectForm.tsx index 63d829c8cc..75818321c9 100644 --- a/src/frontend/src/components/createnewproject/SelectForm.tsx +++ b/src/frontend/src/components/createnewproject/SelectForm.tsx @@ -14,7 +14,7 @@ import useDocumentTitle from '@/utilfunctions/useDocumentTitle'; import { Loader2 } from 'lucide-react'; import DescriptionSection from '@/components/createnewproject/Description'; -const SelectForm = ({ flag, geojsonFile, xlsFormFile, setXlsFormFile }) => { +const SelectForm = ({ flag, _geojsonFile, xlsFormFile, setXlsFormFile }) => { useDocumentTitle('Create Project: Upload Survey'); const dispatch = useAppDispatch(); const navigate = useNavigate(); @@ -57,10 +57,12 @@ const SelectForm = ({ flag, geojsonFile, xlsFormFile, setXlsFormFile }) => { const { files } = event.target; // Set the selected file as the xlsFormFile state setXlsFormFile(files[0]); - handleCustomChange('customFormUpload', files[0]); + console.log(files[0]); + console.log(xlsFormFile); + handleCustomChange('xlsFormFileUpload', files[0]); }; const resetFile = (): void => { - handleCustomChange('customFormUpload', null); + handleCustomChange('xlsFormFileUpload', null); dispatch(CreateProjectActions.SetCustomFileValidity(false)); setXlsFormFile(null); }; @@ -106,7 +108,7 @@ const SelectForm = ({ flag, geojsonFile, xlsFormFile, setXlsFormFile }) => { btnText="Upload XLSForm" accept=".xls,.xlsx,.xml" fileDescription="*The supported file formats are .xlsx, .xls, .xml" - errorMsg={errors.customFormUpload} + errorMsg={errors.xlsFormFileUpload} />
{validateCustomFormLoading && ( diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index 34a2b6a119..d96f54f19a 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -105,8 +105,8 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF const submission = () => { dispatch(CreateProjectActions.SetIsUnsavedChanges(false)); - dispatch(CreateProjectActions.SetIndividualProjectDetailsData(formValues)); + // Project POST data let projectData = { name: projectDetails.name, @@ -119,6 +119,8 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF odk_central_user: projectDetails.odk_central_user, odk_central_password: projectDetails.odk_central_password, osm_category: projectDetails.formExampleSelection, + primary_geom_type: projectDetails.primaryGeomType, + new_geom_type: projectDetails.newGeomType ? projectDetails.newGeomType : projectDetails.primaryGeomType, task_split_type: taskSplittingMethod, // "uploaded_form": projectDetails.uploaded_form, hashtags: projectDetails.hashtags, @@ -131,6 +133,7 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF } else { projectData = { ...projectData, task_split_dimension: projectDetails.dimension }; } + // Create file object from generated task areas const taskAreaBlob = new Blob([JSON.stringify(dividedTaskGeojson || drawnGeojson)], { type: 'application/json', @@ -145,7 +148,7 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF taskAreaGeojsonFile, xlsFormFile, customDataExtractUpload, - projectDetails.dataExtractWays === 'osm_data_extract', + projectDetails.dataExtractType === 'osm_data_extract', additionalFeature, projectDetails.project_admins as number[], combinedFeaturesCount, @@ -185,7 +188,7 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF dispatch( GetDividedTaskFromGeojson(`${import.meta.env.VITE_API_URL}/projects/preview-split-by-square`, { geojson: drawnGeojsonFile, - extract_geojson: formValues.dataExtractWays === 'osm_data_extract' ? null : dataExtractFile, + extract_geojson: formValues.dataExtractType === 'osm_data_extract' ? null : dataExtractFile, dimension: formValues?.dimension, }), ); @@ -196,7 +199,7 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF drawnGeojsonFile, formValues?.average_buildings_per_task, // Only send dataExtractFile if custom extract - formValues.dataExtractWays === 'osm_data_extract' ? null : dataExtractFile, + formValues.dataExtractType === 'osm_data_extract' ? null : dataExtractFile, ), ); } diff --git a/src/frontend/src/components/createnewproject/validation/DataExtractValidation.tsx b/src/frontend/src/components/createnewproject/validation/DataExtractValidation.tsx index 5bd36c6660..ebb4e765d4 100644 --- a/src/frontend/src/components/createnewproject/validation/DataExtractValidation.tsx +++ b/src/frontend/src/components/createnewproject/validation/DataExtractValidation.tsx @@ -1,5 +1,10 @@ +import { MapGeomTypes } from '@/types/enums'; + interface ProjectValues { - dataExtractWays: string; + primaryGeomType: MapGeomTypes; + useMixedGeomTypes: boolean; + newGeomType: MapGeomTypes; + dataExtractType: string; data_extractFile: object; data_extract_options: string; customDataExtractUpload: string; @@ -7,7 +12,9 @@ interface ProjectValues { additionalFeature: File; } interface ValidationErrors { - dataExtractWays?: string; + primaryGeomType?: string; + newGeomType?: string; + dataExtractType?: string; data_extractFile?: string; data_extract_options?: string; customDataExtractUpload?: string; @@ -17,11 +24,19 @@ interface ValidationErrors { function DataExtractValidation(values: ProjectValues) { const errors: ValidationErrors = {}; - if (!values?.dataExtractWays) { - errors.dataExtractWays = 'Map Features Selection is Required.'; + if (!values?.primaryGeomType) { + errors.primaryGeomType = 'A primary geometry type must be selected.'; + } + + if (values?.useMixedGeomTypes && !values?.newGeomType) { + errors.newGeomType = 'Please select a type for new geometries.'; + } + + if (!values?.dataExtractType) { + errors.dataExtractType = 'Map Features Selection is Required.'; } - if (values.dataExtractWays && values.dataExtractWays === 'custom_data_extract' && !values.customDataExtractUpload) { + if (values.dataExtractType && values.dataExtractType === 'custom_data_extract' && !values.customDataExtractUpload) { errors.customDataExtractUpload = 'A GeoJSON file is required.'; } diff --git a/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx b/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx index 8f7e2083d8..0741fc5d2c 100644 --- a/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx +++ b/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx @@ -1,17 +1,17 @@ interface ProjectValues { formExampleSelection: string; - customFormUpload: File | null; + xlsFormFileUpload: File | null; } interface ValidationErrors { formExampleSelection?: string; - customFormUpload?: any; + xlsFormFileUpload?: any; } function SelectFormValidation(values: ProjectValues) { const errors: ValidationErrors = {}; - if (!values?.customFormUpload) { - errors.customFormUpload = 'Form needs to be Uploaded.'; + if (!values?.xlsFormFileUpload) { + errors.xlsFormFileUpload = 'Form needs to be Uploaded.'; } return errors; diff --git a/src/frontend/src/components/createnewproject/validation/UploadAreaValidation.tsx b/src/frontend/src/components/createnewproject/validation/UploadAreaValidation.tsx index 0621cc0558..ed872a6c69 100644 --- a/src/frontend/src/components/createnewproject/validation/UploadAreaValidation.tsx +++ b/src/frontend/src/components/createnewproject/validation/UploadAreaValidation.tsx @@ -1,6 +1,6 @@ interface ProjectValues { uploadAreaSelection: string; - dataExtractWays: string; + dataExtractType: string; data_extractFile: object; data_extract_options: string; drawnGeojson: string; @@ -8,7 +8,7 @@ interface ProjectValues { } interface ValidationErrors { uploadAreaSelection?: string; - dataExtractWays?: string; + dataExtractType?: string; data_extractFile?: string; data_extract_options?: string; drawnGeojson?: string; diff --git a/src/frontend/src/store/types/ICreateProject.ts b/src/frontend/src/store/types/ICreateProject.ts index 18b5f398c5..aafd129d30 100644 --- a/src/frontend/src/store/types/ICreateProject.ts +++ b/src/frontend/src/store/types/ICreateProject.ts @@ -103,12 +103,15 @@ export type ProjectDetailsTypes = { organisation_id?: number | null; formExampleSelection?: string; average_buildings_per_task?: number; - dataExtractWays?: string; + dataExtractType?: string; per_task_instructions?: string; custom_tms_url: string; hasCustomTMS: boolean; - customFormUpload: any; + xlsFormFileUpload: any; hasAdditionalFeature: boolean; + primaryGeomType: MapGeomTypes; + useMixedGeomTypes: boolean; + newGeomType: MapGeomTypes; project_admins: number[]; }; diff --git a/src/frontend/src/utilfunctions/getTaskStatusStyle.ts b/src/frontend/src/utilfunctions/getTaskStatusStyle.ts index 614e5b7acd..f9e098209a 100644 --- a/src/frontend/src/utilfunctions/getTaskStatusStyle.ts +++ b/src/frontend/src/utilfunctions/getTaskStatusStyle.ts @@ -148,7 +148,7 @@ export const getFeatureStatusStyle = (geomType: string, mapTheme: Record Date: Mon, 24 Feb 2025 22:05:17 +0000 Subject: [PATCH 7/8] fix: correctly handle osm data extracts using enum from form type --- src/backend/app/db/enums.py | 4 ++-- src/backend/app/helpers/helper_routes.py | 6 +++--- src/backend/app/projects/project_routes.py | 3 ++- .../src/components/createnewproject/DataExtract.tsx | 4 +++- src/frontend/src/components/createnewproject/SelectForm.tsx | 5 +++++ src/frontend/src/components/createnewproject/SplitTasks.tsx | 5 ++++- .../createnewproject/validation/SelectFormValidation.tsx | 2 -- src/frontend/src/store/types/ICreateProject.ts | 1 + src/frontend/src/types/enums.ts | 5 +++++ 9 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/backend/app/db/enums.py b/src/backend/app/db/enums.py index 2c78448af5..28330ee54d 100644 --- a/src/backend/app/db/enums.py +++ b/src/backend/app/db/enums.py @@ -247,9 +247,9 @@ class XLSFormType(StrEnum, Enum): The the value is the user facing form name (e.g. healthcare). """ - buildings = "buildings" + buildings = "OSM Buildings" # highways = "highways" - health = "healthcare" + health = "OSM Healthcare" # toilets = "toilets" # religious = "religious" # landusage = "landusage" diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index 944d1ad8ed..631236c340 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -69,10 +69,10 @@ async def download_template( form_type: XLSFormType, ): """Download example XLSForm from FMTM.""" - filename = XLSFormType(form_type).name - xlsform_path = f"{xlsforms_path}/{filename}.xls" + form_filename = XLSFormType(form_type).name + xlsform_path = f"{xlsforms_path}/{form_filename}.xls" if Path(xlsform_path).exists(): - return FileResponse(xlsform_path, filename="form.xls") + return FileResponse(xlsform_path, filename=f"{form_filename}.xls") else: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail="Form not found") diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index ca544748af..ee8c1da933 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -645,7 +645,8 @@ async def get_data_extract( # FIXME once sub project creation implemented, this should be manager only current_user: Annotated[AuthUser, Depends(login_required)], geojson_file: UploadFile = File(...), - osm_category: Optional[XLSFormType] = Form(None), + # FIXME this is currently hardcoded but needs to be user configurable via UI + osm_category: Annotated[Optional[XLSFormType], Form()] = XLSFormType.buildings, ): """Get a new data extract for a given project AOI. diff --git a/src/frontend/src/components/createnewproject/DataExtract.tsx b/src/frontend/src/components/createnewproject/DataExtract.tsx index be063206e7..4496c0d243 100644 --- a/src/frontend/src/components/createnewproject/DataExtract.tsx +++ b/src/frontend/src/components/createnewproject/DataExtract.tsx @@ -117,7 +117,9 @@ const DataExtract = ({ const dataExtractRequestFormData = new FormData(); const projectAoiGeojsonFile = getFileFromGeojson(projectAoiGeojson); dataExtractRequestFormData.append('geojson_file', projectAoiGeojsonFile); - dataExtractRequestFormData.append('form_category', projectDetails.formExampleSelection); + if (projectDetails.osmFormSelectionName) { + dataExtractRequestFormData.append('osm_category', projectDetails.osmFormSelectionName); + } // Set flatgeobuf as loading dispatch(CreateProjectActions.SetFgbFetchingStatus(true)); diff --git a/src/frontend/src/components/createnewproject/SelectForm.tsx b/src/frontend/src/components/createnewproject/SelectForm.tsx index 75818321c9..4780f7a285 100644 --- a/src/frontend/src/components/createnewproject/SelectForm.tsx +++ b/src/frontend/src/components/createnewproject/SelectForm.tsx @@ -13,6 +13,7 @@ import NewDefineAreaMap from '@/views/NewDefineAreaMap'; import useDocumentTitle from '@/utilfunctions/useDocumentTitle'; import { Loader2 } from 'lucide-react'; import DescriptionSection from '@/components/createnewproject/Description'; +import { osm_forms } from '@/types/enums'; const SelectForm = ({ flag, _geojsonFile, xlsFormFile, setXlsFormFile }) => { useDocumentTitle('Create Project: Upload Survey'); @@ -134,6 +135,10 @@ const SelectForm = ({ flag, _geojsonFile, xlsFormFile, setXlsFormFile }) => { value={formValues.formExampleSelection} onValueChange={(value) => { handleCustomChange('formExampleSelection', value); + // Only set osmFormSelectionName for specific 'OSM forms' + if (Object.values(osm_forms).includes(value as osm_forms)) { + handleCustomChange('osmFormSelectionName', value); + } dispatch(CreateProjectActions.setDataExtractGeojson(null)); }} errorMsg={errors.formExampleSelection} diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index d96f54f19a..789a0f4e92 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -118,7 +118,6 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF odk_central_url: projectDetails.odk_central_url, odk_central_user: projectDetails.odk_central_user, odk_central_password: projectDetails.odk_central_password, - osm_category: projectDetails.formExampleSelection, primary_geom_type: projectDetails.primaryGeomType, new_geom_type: projectDetails.newGeomType ? projectDetails.newGeomType : projectDetails.primaryGeomType, task_split_type: taskSplittingMethod, @@ -127,6 +126,10 @@ const SplitTasks = ({ flag, setGeojsonFile, customDataExtractUpload, additionalF data_extract_url: projectDetails.data_extract_url, custom_tms_url: projectDetails.custom_tms_url, }; + // Append osm_category if set + if (projectDetails.osmFormSelectionName) { + projectData = { ...projectData, osm_category: projectDetails.osmFormSelectionName }; + } // Append extra param depending on task split type if (taskSplittingMethod === task_split_type.TASK_SPLITTING_ALGORITHM) { projectData = { ...projectData, task_num_buildings: projectDetails.average_buildings_per_task }; diff --git a/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx b/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx index 0741fc5d2c..8002f96f06 100644 --- a/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx +++ b/src/frontend/src/components/createnewproject/validation/SelectFormValidation.tsx @@ -1,9 +1,7 @@ interface ProjectValues { - formExampleSelection: string; xlsFormFileUpload: File | null; } interface ValidationErrors { - formExampleSelection?: string; xlsFormFileUpload?: any; } diff --git a/src/frontend/src/store/types/ICreateProject.ts b/src/frontend/src/store/types/ICreateProject.ts index aafd129d30..8c552f2da5 100644 --- a/src/frontend/src/store/types/ICreateProject.ts +++ b/src/frontend/src/store/types/ICreateProject.ts @@ -102,6 +102,7 @@ export type ProjectDetailsTypes = { data_extract_options?: string; organisation_id?: number | null; formExampleSelection?: string; + osmFormSelectionName?: string; average_buildings_per_task?: number; dataExtractType?: string; per_task_instructions?: string; diff --git a/src/frontend/src/types/enums.ts b/src/frontend/src/types/enums.ts index bca83d75ba..1ec86a5f05 100644 --- a/src/frontend/src/types/enums.ts +++ b/src/frontend/src/types/enums.ts @@ -74,3 +74,8 @@ export enum submission_status { approved = 'Approved', rejected = 'Rejected', } + +export enum osm_forms { + buildings = 'OSM Buildings', + health = 'OSM Healthcare', +} From 9e2f201880e54096486276cbf1ff1b22741d3e26 Mon Sep 17 00:00:00 2001 From: Nishit Suwal <81785002+NSUWAL123@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:14:19 +0545 Subject: [PATCH 8/8] Feat/rework proj create cont (#2230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate (#2224) updates: - [github.com/commitizen-tools/commitizen: v4.2.1 → v4.2.2](https://github.com/commitizen-tools/commitizen/compare/v4.2.1...v4.2.2) - [github.com/astral-sh/ruff-pre-commit: v0.9.6 → v0.9.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.6...v0.9.7) - [github.com/astral-sh/uv-pre-commit: 0.6.0 → 0.6.2](https://github.com/astral-sh/uv-pre-commit/compare/0.6.0...0.6.2) - [github.com/pycontribs/mirrors-prettier: v3.5.1 → v3.5.2](https://github.com/pycontribs/mirrors-prettier/compare/v3.5.1...v3.5.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * chore(deps): update dependency @types/geojson to v7946.0.16 (#2227) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(dataExtract): if no dataExtract uploaded or fetched, disable next btn * fix(radiobutton): use radio option label as id for radiobutton * fix(description): update uploadSurvey desc section hover * fix(checkbox): update styles * refactor(style): refactor style to maintain consistent gaps in UI * fix(selectForm): disappear validation message after form upload --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 8 +- .../src/components/common/Checkbox.tsx | 4 +- .../components/common/FileInputComponent.tsx | 2 +- .../src/components/common/RadioButton.tsx | 4 +- .../createnewproject/DataExtract.tsx | 115 +- .../createnewproject/Description.tsx | 13 +- .../createnewproject/SelectForm.tsx | 15 +- .../createnewproject/UploadArea.tsx | 1 + src/mapper/pnpm-lock.yaml | 2027 +++++++++-------- 9 files changed, 1176 insertions(+), 1013 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fb3987d75..1130908ac4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: # Versioning: Commit messages & changelog - repo: https://github.com/commitizen-tools/commitizen - rev: v4.2.1 + rev: v4.2.2 hooks: - id: commitizen stages: [commit-msg] @@ -9,7 +9,7 @@ repos: # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.9.6" + rev: "v0.9.7" hooks: # Run the linter - id: ruff @@ -21,7 +21,7 @@ repos: # Deps: ensure Python uv lockfile is up to date - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.6.0 + rev: 0.6.2 hooks: - id: uv-lock files: src/backend/pyproject.toml @@ -50,7 +50,7 @@ repos: # Autoformat: YAML, JSON, Markdown, etc. - repo: https://github.com/pycontribs/mirrors-prettier - rev: v3.5.1 + rev: v3.5.2 hooks: - id: prettier args: diff --git a/src/frontend/src/components/common/Checkbox.tsx b/src/frontend/src/components/common/Checkbox.tsx index f8f117a939..9b2d2ec2dd 100644 --- a/src/frontend/src/components/common/Checkbox.tsx +++ b/src/frontend/src/components/common/Checkbox.tsx @@ -56,12 +56,12 @@ export const CustomCheckbox = ({ ref={null} checked={checked} onCheckedChange={onCheckedChange} - className="fmtm-mt-[2px]" + className="fmtm-mt-1" disabled={disabled} />

{label} diff --git a/src/frontend/src/components/common/FileInputComponent.tsx b/src/frontend/src/components/common/FileInputComponent.tsx index d06243188b..6b2cd0648d 100644 --- a/src/frontend/src/components/common/FileInputComponent.tsx +++ b/src/frontend/src/components/common/FileInputComponent.tsx @@ -22,7 +22,7 @@ const FileInputComponent = ({ }: fileInputComponentType) => { const customFileRef = useRef(null); return ( -

+