Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change the FID field to ID #1

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

Change the FID field to ID #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
30b82d6
Change the FID field to ID
ryanfranz Mar 19, 2019
972c19d
Fix the FID field to be id
ryanfranz Mar 21, 2019
5d09607
Fix crashes for shapefiles without a class-level dbf file
ryanfranz Mar 22, 2019
83bbee9
Convert logical fields as text, since true/false can be many values a…
ryanfranz Mar 27, 2019
3c1a3aa
Update README.md
ryanfranz Apr 5, 2019
7011f70
Merge pull request #1 from Cognitics/master
ryanfranz Apr 8, 2019
34226b6
Cleanup Option3 and Option4 to work in a similar manner to Option1d (…
ryanfranz Apr 17, 2019
ae78d51
Fix the building of the file lists
ryanfranz Apr 17, 2019
1dd7439
Fix the name of the table in the GeoPackage
ryanfranz Apr 24, 2019
d394d6f
Modified to store the dataset, component selectors, lod, row and colu…
ryanfranz Apr 25, 2019
70074b8
Changed to not set the table attributes unless some features exist, s…
ryanfranz Apr 29, 2019
8744fc8
Close the GeoPackage file, so that the spatial indices and extents ar…
ryanfranz Apr 29, 2019
38b74de
Add creation of a search index for file parts that are not in the tab…
ryanfranz Apr 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Converters/Option1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def translateCDB(cDBRoot,ogrPath, removeShapefile):
geoPackageFile = shapefile[0:-3] + "gpkg"
if(os.path.getsize(shapefile)>0):
#'-t_srs', 'EPSG:4326', '-s_srs', 'EPSG:4326',
subprocess.call([ogrPath,'-f', 'GPKG', geoPackageFile,shapefile])
subprocess.call([ogrPath, '-f', 'GPKG', geoPackageFile, "-lco", "FID=id", shapefile])
print(shapefile + ' -> ' + geoPackageFile)
if(removeShapefile):
converter.removeShapeFile(shapefile)
Expand Down
65 changes: 38 additions & 27 deletions Converters/Option1d.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,17 @@ def cleanPath(path):

def getOutputLayerName(shpFilename):
filenameOnly = os.path.basename(shpFilename)
filenameParts = filenameOnly.split("_")
baseName,ext = os.path.splitext(filenameOnly)
filenameParts = baseName.split("_")
datasetCode = filenameParts[1]
datasetName = filenameParts[-4]
componentSelector1 = filenameParts[2]
componentSelector2 = filenameParts[3]
lod = filenameParts[4]
uref = filenameParts[5]
rref = filenameParts[6]
#Create the layer if it doesn't already exist.
outLayerName = datasetName + "_" + lod + "_" + componentSelector1 + "_" + componentSelector2
outLayerName = datasetCode + "_" + componentSelector1 + "_" + componentSelector2 + "_" + lod + "_" + uref + "_" + rref
return outLayerName

def getFilenameComponents(shpFilename):
Expand All @@ -64,17 +66,20 @@ def getFilenameComponents(shpFilename):
baseName,ext = os.path.splitext(filenameOnly)
filenameParts = baseName.split("_")
datasetCode = filenameParts[1]
components['datasetcode'] = datasetCode
components['datasetcode'] = int(datasetCode[1:])
componentSelector1 = filenameParts[2]
components['selector1'] = componentSelector1
components['selector1'] = int(componentSelector1[1:])
componentSelector2 = filenameParts[3]
components['selector2'] = componentSelector2
components['selector2'] = int(componentSelector2[1:])
lod = filenameParts[4]
components['lod'] = lod
if (lod[:2] == "LC"):
components['lod'] = -int(lod[2:])
else:
components['lod'] = int(lod[1:])
uref = filenameParts[5]
components['uref'] = uref
components['uref'] = int(uref[1:])
rref = filenameParts[6]
components['rref'] = rref
components['rref'] = int(rref[1:])

return components

Expand Down Expand Up @@ -105,6 +110,7 @@ def copyFeaturesFromShapeToGeoPackage(shpFilename, gpkgFilename):
outLayerName = getOutputLayerName(shpFilename)

ogrDriver = ogr.GetDriverByName("GPKG")
print(" Creating file " + gpkgFilename)
gpkgFile = ogrDriver.CreateDataSource(gpkgFilename)

if(gpkgFile == None):
Expand All @@ -122,7 +128,7 @@ def copyFeaturesFromShapeToGeoPackage(shpFilename, gpkgFilename):
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1
else:
outLayer = gpkgFile.CreateLayer(outLayerName,srs,geom_type=layerDefinition.GetGeomType())
outLayer = gpkgFile.CreateLayer(outLayerName,srs,geom_type=layerDefinition.GetGeomType(),options=["FID=id"])

# Add fields
for i in range(layerDefinition.GetFieldCount()):
Expand All @@ -139,47 +145,47 @@ def copyFeaturesFromShapeToGeoPackage(shpFilename, gpkgFilename):

# Add the LOD and UXX fields
fieldName = "_DATASET_CODE"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1

fieldName = "_COMPONENT_SELECTOR_1"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1

fieldName = "_COMPONENT_SELECTOR_2"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1

fieldName = "_LOD"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1

fieldName = "_UREF"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
fieldIndexes[fieldName] = fieldIdx
fieldIdx += 1

fieldName = "_RREF"
fieldTypeCode = ogr.OFTString
fieldTypeCode = ogr.OFTInteger
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)
outLayer.CreateField(fieldDef)
convertedFields.append(fieldName)
Expand All @@ -193,11 +199,12 @@ def copyFeaturesFromShapeToGeoPackage(shpFilename, gpkgFilename):
continue
fieldTypeCode = ogr.OFTString
if(isinstance(fieldValue,float)):
fieldTypeCode = ogr.OFSTFloat32
fieldTypeCode = ogr.OFTReal
if(isinstance(fieldValue,int)):
fieldTypeCode = ogr.OFTInteger
if(isinstance(fieldValue,bool)):
fieldTypeCode = ogr.OFSTBoolean
#DBase logical fields can have multiple values for true and false, best converted as text
#if(isinstance(fieldValue,bool)):
# fieldTypeCode = ogr.OFSTBoolean
fieldDef = ogr.FieldDefn(fieldName,fieldTypeCode)

outLayer.CreateField(fieldDef)
Expand All @@ -217,20 +224,24 @@ def copyFeaturesFromShapeToGeoPackage(shpFilename, gpkgFilename):
#Copy the geometry and attributes
outFeature.SetFrom(inFeature)

cnamValue = inFeature.GetField('CNAM')
fclassRecord = fClassRecords[cnamValue]
outFeature.SetField(fieldIndexes["_DATASET_CODE"], layerComponents['datasetcode'])
outFeature.SetField(fieldIndexes["_COMPONENT_SELECTOR_1"], layerComponents['selector1'])
outFeature.SetField(fieldIndexes["_COMPONENT_SELECTOR_2"], layerComponents['selector2'])
outFeature.SetField(fieldIndexes["_LOD"], layerComponents['lod'])
outFeature.SetField(fieldIndexes["_UREF"], layerComponents['uref'])
outFeature.SetField(fieldIndexes["_RREF"], layerComponents['rref'])

#flatten attributes from the feature class attributes table
if(cnamValue in fClassRecords.keys()):
fclassFields = fClassRecords[cnamValue]
for field in fclassFields.keys():
outFeature.SetField(fieldIndexes[field],fclassFields[field])
#flatten attributes from the feature class attributes table, if a CNAM attribute exists
try:
cnamValue = inFeature.GetField('CNAM')
fclassRecord = fClassRecords[cnamValue]
if(cnamValue in fClassRecords.keys()):
fclassFields = fClassRecords[cnamValue]
for field in fclassFields.keys():
outFeature.SetField(fieldIndexes[field],fclassFields[field])
except:
#print(" File does not contain the CNAM attribute")
cnamValue = ""

#write the feature
outLayer.CreateFeature(outFeature)
Expand Down Expand Up @@ -281,13 +292,13 @@ def convertShapeFile(shpFilename, cdbInputDir, cdbOutputDir):

#Read all the feature records from the DBF at once (using GDAL)
#copyFeaturesFromShapeToGeoPackage(shpFilename,outputGeoPackageFile)
fClassRecords = converter.readDBF(fcAttrName)
#fClassRecords = converter.readDBF(fcAttrName)
#Read Featureclass records
featureTableName = converter.getFeatureAttrTableName(shpFilename)
copyFeaturesFromShapeToGeoPackage(shpFilename,outputGeoPackageFile)
#convertSHP(sqliteCon,shpFilename,outputGeoPackageFile, fClassRecords, True)
sqliteCon = sqlite3.connect(outputGeoPackageFile)
if(createExtendedAttributesTable(sqliteCon,shpFilename)):
if(0 and createExtendedAttributesTable(sqliteCon,shpFilename)):
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ryan, could you please explain why you want to disable the above if statement ?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code crashes on Yemen (Tiles\N12\E044\201_RoadNetwork\L00\U0\N12E044_D201_S002_T003_L00_U0_R0.shp). It is trying to get the CNAM attribute, but from an extended attribute DBF file where one doesn't exist. Since it wasn't converting correctly anyway, I disabled it.

dbfTableName = getExtendedAttrTableName(shpFilename)
RelatedTables.createRTESchema(sqliteCon)
relationship = RelatedTables.Relationship()
Expand Down
Loading