From fe9ddb3e1a1f8bdfae7ae42b476462ba1f5eaca7 Mon Sep 17 00:00:00 2001 From: Jarod Lam Date: Thu, 29 Jun 2023 16:04:51 +1000 Subject: [PATCH] [FEAT] Add option to not filter on network_type filters during graph creation --- Project.toml | 2 +- src/graph.jl | 38 ++++++++++++++++++++++++++------------ src/parse.jl | 31 ++++++++++++++++++++++++------- test/download.jl | 20 +++++++++++++++----- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/Project.toml b/Project.toml index 9999f69..40a5c3c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LightOSM" uuid = "d1922b25-af4e-4ba3-84af-fe9bea896051" authors = ["Jack Chan "] -version = "0.2.10" +version = "0.2.11" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" diff --git a/src/graph.jl b/src/graph.jl index a54bf38..8e0c82e 100644 --- a/src/graph.jl +++ b/src/graph.jl @@ -27,6 +27,9 @@ Creates an `OSMGraph` object from download OpenStreetMap network data, use with possible for graphs with large amount of nodes due to memory limits. - `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network. +- `filter_network_type::Bool=true`: Set true to filter out nodes and edges that do not + match the `network_type` filter. Useful if the network data was downloaded with custom + Overpass filters and/or not generated with LightOSM. # Return - `OSMGraph`: Container for storing OpenStreetMap node-, way-, relation- and graph-related @@ -37,9 +40,10 @@ function graph_from_object(osm_data_object::Union{XMLDocument,Dict}; weight_type::Symbol=:time, graph_type::Symbol=:static, precompute_dijkstra_states::Bool=false, - largest_connected_component::Bool=true + largest_connected_component::Bool=true, + filter_network_type::Bool=true )::OSMGraph - g = init_graph_from_object(osm_data_object, network_type) + g = init_graph_from_object(osm_data_object, network_type, filter_network_type=filter_network_type) add_node_and_edge_mappings!(g) add_weights!(g, weight_type) add_graph!(g, graph_type) @@ -61,13 +65,13 @@ function graph_from_object(osm_data_object::Union{XMLDocument,Dict}; end """ - graph_from_object(file_path::String; - network_type::Symbol=:drive, - weight_type::Symbol=:time, - graph_type::Symbol=:static, - precompute_dijkstra_states::Bool=false, - largest_connected_component::Bool=true - )::OSMGraph + graph_from_file(file_path::String; + network_type::Symbol=:drive, + weight_type::Symbol=:time, + graph_type::Symbol=:static, + precompute_dijkstra_states::Bool=false, + largest_connected_component::Bool=true + )::OSMGraph Creates an `OSMGraph` object from a downloaded OpenStreetMap network data file, the extention must be either `.json`, `.osm` or `.xml`. @@ -78,6 +82,9 @@ Creates an `OSMGraph` object from a downloaded OpenStreetMap network data file, - `graph_type::Symbol=:static`: Type of `Graphs.AbstractGraph`, pick from `:static` (StaticDiGraph), `:light` (DiGraph), `:simple_weighted` (SimpleWeightedDiGraph), `:meta` (MetaDiGraph). - `precompute_dijkstra_states::Bool=false`: Set true to precompute dijkstra parent states for every source node in the graph, *NOTE* this may take a while and may not be possible for graphs with large amount of nodes due to memory limits. - `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network. +- `filter_network_type::Bool=true`: Set true to filter out nodes and edges that do not + match the `network_type` filter. Useful if the network data was downloaded with custom + Overpass filters and/or not generated with LightOSM. # Return - `OSMGraph`: Container for storing OpenStreetMap node, way, relation and graph related obejcts. @@ -87,7 +94,8 @@ function graph_from_file(file_path::String; weight_type::Symbol=:time, graph_type::Symbol=:static, precompute_dijkstra_states::Bool=false, - largest_connected_component::Bool=true + largest_connected_component::Bool=true, + filter_network_type::Bool=true )::OSMGraph !isfile(file_path) && throw(ArgumentError("File $file_path does not exist")) @@ -98,7 +106,8 @@ function graph_from_file(file_path::String; weight_type=weight_type, graph_type=graph_type, precompute_dijkstra_states=precompute_dijkstra_states, - largest_connected_component=largest_connected_component) + largest_connected_component=largest_connected_component, + filter_network_type=filter_network_type) end """ @@ -126,6 +135,9 @@ Downloads OpenStreetMap network data and creates an `OSMGraph` object. - `graph_type::Symbol=:static`: Type of `Graphs.AbstractGraph`, pick from `:static` (StaticDiGraph), `:light` (DiGraph), `:simple_weighted` (SimpleWeightedDiGraph), `:meta` (MetaDiGraph). - `precompute_dijkstra_states::Bool=false`: Set true to precompute dijkstra parent states for every source node in the graph, *NOTE* this may take a while and may not be possible for graphs with large amount of nodes due to memory limits. - `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network. +- `filter_network_type::Bool=true`: Set true to filter out nodes and edges that do not + match the `network_type` filter after download. You may want to use this with + `download_method=:custom_filters`. # Required Kwargs for each Download Method @@ -167,6 +179,7 @@ function graph_from_download(download_method::Symbol; graph_type::Symbol=:static, precompute_dijkstra_states::Bool=false, largest_connected_component::Bool=true, + filter_network_type::Bool=true, download_kwargs... )::OSMGraph obj = download_osm_network(download_method, @@ -180,7 +193,8 @@ function graph_from_download(download_method::Symbol; weight_type=weight_type, graph_type=graph_type, precompute_dijkstra_states=precompute_dijkstra_states, - largest_connected_component=largest_connected_component) + largest_connected_component=largest_connected_component, + filter_network_type=filter_network_type) end diff --git a/src/parse.jl b/src/parse.jl index be42d8f..009c01d 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -197,7 +197,10 @@ end """ Parse OpenStreetMap data into `Node`, `Way` and `Restriction` objects. """ -function parse_osm_network_dict(osm_network_dict::AbstractDict, network_type::Symbol=:drive)::OSMGraph +function parse_osm_network_dict(osm_network_dict::AbstractDict, + network_type::Symbol=:drive; + filter_network_type::Bool=true + )::OSMGraph U = DEFAULT_OSM_INDEX_TYPE T = DEFAULT_OSM_ID_TYPE W = DEFAULT_OSM_EDGE_WEIGHT_TYPE @@ -208,7 +211,7 @@ function parse_osm_network_dict(osm_network_dict::AbstractDict, network_type::Sy for way in osm_network_dict["way"] if haskey(way, "tags") && haskey(way, "nodes") tags = way["tags"] - if is_highway(tags) && matches_network_type(tags, network_type) + if is_highway(tags) && (!filter_network_type || matches_network_type(tags, network_type)) tags["maxspeed"] = maxspeed(tags) tags["lanes"] = lanes(tags) tags["oneway"] = is_oneway(tags) @@ -217,7 +220,7 @@ function parse_osm_network_dict(osm_network_dict::AbstractDict, network_type::Sy union!(highway_nodes, nds) id = way["id"] ways[id] = Way(id, nds, tags) - elseif is_railway(tags) && matches_network_type(tags, network_type) + elseif is_railway(tags) && (!filter_network_type || matches_network_type(tags, network_type)) tags["rail_type"] = get(tags, "railway", "unknown") tags["electrified"] = get(tags, "electrified", "unknown") tags["gauge"] = get(tags, "gauge", nothing) @@ -334,15 +337,29 @@ end """ Initialises the OSMGraph object from OpenStreetMap data downloaded in `:xml` or `:osm` format. """ -function init_graph_from_object(osm_xml_object::XMLDocument, network_type::Symbol=:drive)::OSMGraph +function init_graph_from_object(osm_xml_object::XMLDocument, + network_type::Symbol=:drive; + filter_network_type::Bool=true + )::OSMGraph dict_to_parse = osm_dict_from_xml(osm_xml_object) - return parse_osm_network_dict(dict_to_parse, network_type) + return parse_osm_network_dict( + dict_to_parse, + network_type; + filter_network_type=filter_network_type + ) end """ Initialises the OSMGraph object from OpenStreetMap data downloaded in `:json` format. """ -function init_graph_from_object(osm_json_object::AbstractDict, network_type::Symbol=:drive)::OSMGraph +function init_graph_from_object(osm_json_object::AbstractDict, + network_type::Symbol=:drive; + filter_network_type::Bool=true + )::OSMGraph dict_to_parse = osm_dict_from_json(osm_json_object) - return parse_osm_network_dict(dict_to_parse, network_type) + return parse_osm_network_dict( + dict_to_parse, + network_type; + filter_network_type=filter_network_type + ) end diff --git a/test/download.jl b/test/download.jl index 7aecb52..df6390d 100644 --- a/test/download.jl +++ b/test/download.jl @@ -47,12 +47,17 @@ end @testset "Download with custom filters" begin filename = "map.json" format = :json + #= + Compared to the defauilt network_type=:drive, this filter: + - Excludes all ways with highway=tertiary, secondary, primary, living_street + - Includes all ways with highway=service + =# custom_filters = """ way ["highway"] ["motorcar"!~"no"] ["area"!~"yes"] - ["highway"!~"elevator|steps|tertiary|construction|bridleway|proposed|track|pedestrian|secondary|path|living_street|cycleway|primary|footway|platform|abandoned|service|escalator|corridor|raceway"] + ["highway"!~"elevator|steps|tertiary|construction|bridleway|proposed|track|pedestrian|secondary|path|living_street|cycleway|primary|footway|platform|abandoned|escalator|corridor|raceway"] ["motor_vehicle"!~"no"]["access"!~"private"] ["service"!~"parking|parking_aisle|driveway|private|emergency_access"] ; @@ -72,14 +77,19 @@ end ) @test isfile(filename) - g = graph_from_file(filename) + g = graph_from_file(filename, filter_network_type=false) + + # Make sure Overpass included/excluded these tags according to the custom filter + excluded_tags = ["primary", "secondary", "tertiary", "living_street"] + included_tags = ["service"] + included_tags_count = 0 for (_, way) in g.ways - # Make sure Overpass excluded these tags, as they are excluded in the filter - highway_tags_to_check = ["primary", "secondary", "tertiary", "living_street"] if haskey(way.tags, "highway") - @test way.tags["highway"] ∉ highway_tags_to_check + @test way.tags["highway"] ∉ excluded_tags + (way.tags["highway"] ∈ included_tags) && (included_tags_count += 1) end end + @test included_tags_count > 0 catch err # Sometimes gets HTTP.ExceptionRequest.StatusError in tests due to connection to overpass !isa(err, HTTP.ExceptionRequest.StatusError) && rethrow()