From 57672047108c0a41361feb1bce40602ca270a720 Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 29 Sep 2021 12:49:56 +0200 Subject: [PATCH 01/14] Set new version 5.0-SNAPSHOT --- CHANGELOG.md | 2 ++ client-hc/pom.xml | 4 ++-- core/pom.xml | 4 ++-- example/pom.xml | 4 ++-- hmm-lib/pom.xml | 4 ++-- map-matching/pom.xml | 4 ++-- navigation/pom.xml | 4 ++-- pom.xml | 2 +- reader-gtfs/pom.xml | 2 +- tools/pom.xml | 2 +- web-api/pom.xml | 4 ++-- web-bundle/pom.xml | 4 ++-- web-bundle/src/main/resources/com/graphhopper/maps/index.html | 2 +- web/pom.xml | 4 ++-- 14 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eda9dfa8b1..82f72125b27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +### 5.0 [not yet released] + ### 4.0 [29 Sep 2021] - faster node-based CH preparation (~20%), (#2390) diff --git a/client-hc/pom.xml b/client-hc/pom.xml index 6a4d04a9acf..df819da0a4e 100644 --- a/client-hc/pom.xml +++ b/client-hc/pom.xml @@ -22,14 +22,14 @@ 4.0.0 directions-api-client-hc - 4.0 + 5.0-SNAPSHOT jar GraphHopper Directions API hand-crafted Java Client. com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/core/pom.xml b/core/pom.xml index d4d1fdb78e1..afb234df4d4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ graphhopper-core GraphHopper Core - 4.0 + 5.0-SNAPSHOT jar GraphHopper is a fast and memory efficient Java road routing engine @@ -14,7 +14,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/example/pom.xml b/example/pom.xml index cd2546e5dc1..317fe0aac46 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-example - 4.0 + 5.0-SNAPSHOT jar GraphHopper Example com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/hmm-lib/pom.xml b/hmm-lib/pom.xml index c81478cb6ad..0f3f4b919fb 100644 --- a/hmm-lib/pom.xml +++ b/hmm-lib/pom.xml @@ -19,7 +19,7 @@ 4.0.0 hmm-lib-external - 4.0 + 5.0-SNAPSHOT jar hmm-lib @@ -29,7 +29,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/map-matching/pom.xml b/map-matching/pom.xml index f51bb17052a..83eaa17c25c 100644 --- a/map-matching/pom.xml +++ b/map-matching/pom.xml @@ -3,14 +3,14 @@ 4.0.0 com.graphhopper graphhopper-map-matching - 4.0 + 5.0-SNAPSHOT jar GraphHopper Map Matching com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/navigation/pom.xml b/navigation/pom.xml index 1bfb0120b72..655c01bea9c 100644 --- a/navigation/pom.xml +++ b/navigation/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-nav - 4.0 + 5.0-SNAPSHOT jar GraphHopper Navigation com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 02fce502b3d..b8d82eb02c7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.graphhopper graphhopper-parent GraphHopper Parent Project - 4.0 + 5.0-SNAPSHOT pom https://www.graphhopper.com 2012 diff --git a/reader-gtfs/pom.xml b/reader-gtfs/pom.xml index 5028d42a072..b7b611c0fb0 100644 --- a/reader-gtfs/pom.xml +++ b/reader-gtfs/pom.xml @@ -10,7 +10,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/tools/pom.xml b/tools/pom.xml index b425ac19b7c..47099203835 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -10,7 +10,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT package diff --git a/web-api/pom.xml b/web-api/pom.xml index a5e8f1919c9..5350eb5c7c9 100644 --- a/web-api/pom.xml +++ b/web-api/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web-api jar - 4.0 + 5.0-SNAPSHOT GraphHopper Web API JSON Representation of the API classes com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/web-bundle/pom.xml b/web-bundle/pom.xml index a59281060b0..389299dbc41 100644 --- a/web-bundle/pom.xml +++ b/web-bundle/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web-bundle jar - 4.0 + 5.0-SNAPSHOT GraphHopper Dropwizard Bundle Use the GraphHopper routing engine as a web-service com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/web-bundle/src/main/resources/com/graphhopper/maps/index.html b/web-bundle/src/main/resources/com/graphhopper/maps/index.html index 011f6d92920..bc0519f073a 100644 --- a/web-bundle/src/main/resources/com/graphhopper/maps/index.html +++ b/web-bundle/src/main/resources/com/graphhopper/maps/index.html @@ -36,7 +36,7 @@ - + diff --git a/web/pom.xml b/web/pom.xml index f66aa34c0b0..67aa7862306 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web jar - 4.0 + 5.0-SNAPSHOT GraphHopper Web Use the GraphHopper routing engine as a web-service com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT package From ce3a207efd3406cbac064a574a388ce26a392974 Mon Sep 17 00:00:00 2001 From: Andi Date: Wed, 29 Sep 2021 13:24:11 +0200 Subject: [PATCH 02/14] Move nodes and edges storage out of BaseGraph (#2399) --- .../com/graphhopper/storage/BaseGraph.java | 524 +++--------------- .../storage/BaseGraphNodesAndEdges.java | 399 +++++++++++++ .../com/graphhopper/storage/CHStorage.java | 4 +- .../com/graphhopper/storage/GHNodeAccess.java | 58 +- .../storage/GraphHopperStorage.java | 2 +- .../storage/RoutingCHEdgeIteratorImpl.java | 8 +- .../RoutingCHEdgeIteratorStateImpl.java | 8 +- .../graphhopper/storage/TurnCostStorage.java | 2 +- .../storage/AbstractGraphStorageTester.java | 3 +- .../storage/GraphHopperStorageTest.java | 19 +- 10 files changed, 524 insertions(+), 503 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 97273a63757..6eda12fd12c 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -26,7 +26,6 @@ import com.graphhopper.util.shapes.BBox; import java.util.Collections; -import java.util.Locale; import static com.graphhopper.util.Helper.nf; @@ -42,114 +41,47 @@ * loadExisting, (4) usage, (5) flush, (6) close */ class BaseGraph implements Graph { - // Currently distances are stored as 4 byte integers. using a conversion factor of 1000 the minimum distance - // that is not considered zero is 0.0005m (=0.5mm) and the maximum distance per edge is about 2.147.483m=2147km. - // See OSMReader.addEdge and #1871. - private static final double INT_DIST_FACTOR = 1000d; - static double MAX_DIST = Integer.MAX_VALUE / INT_DIST_FACTOR; - - final DataAccess edges; - final DataAccess nodes; - final BBox bounds; - final NodeAccess nodeAccess; private final static String STRING_IDX_NAME_KEY = "name"; + final BaseGraphNodesAndEdges store; + final NodeAccess nodeAccess; final StringIndex stringIndex; // can be null if turn costs are not supported final TurnCostStorage turnCostStorage; final BitUtil bitUtil; - private final int intsForFlags; // length | nodeA | nextNode | ... | nodeB - // as we use integer index in 'egdes' area => 'geometry' area is limited to 4GB (we use pos&neg values!) + // as we use integer index in 'edges' area => 'geometry' area is limited to 4GB (we use pos&neg values!) private final DataAccess wayGeometry; private final Directory dir; - /** - * interval [0,n) - */ - protected int edgeCount; - // node memory layout: - protected int N_EDGE_REF, N_LAT, N_LON, N_ELE, N_TC; - // edge memory layout: - int E_NODEA, E_NODEB, E_LINKA, E_LINKB, E_FLAGS, E_DIST, E_GEO, E_NAME; - /** - * Specifies how many entries (integers) are used per edge. - */ - int edgeEntryBytes; - /** - * Specifies how many entries (integers) are used per node - */ - int nodeEntryBytes; private boolean initialized = false; - /** - * interval [0,n) - */ - private int nodeCount; - private int edgeEntryIndex, nodeEntryIndex; private long maxGeoRef; - private boolean frozen = false; public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; - this.intsForFlags = intsForFlags; this.bitUtil = BitUtil.get(dir.getByteOrder()); this.wayGeometry = dir.find("geometry"); this.stringIndex = new StringIndex(dir); - this.nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); - this.edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); - this.bounds = BBox.createInverse(withElevation); - this.nodeAccess = new GHNodeAccess(this, withElevation); + this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); + this.nodeAccess = new GHNodeAccess(store); if (withTurnCosts) { turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs")); + if (segmentSize >= 0) + turnCostStorage.setSegmentSize(segmentSize); } else { turnCostStorage = null; } if (segmentSize >= 0) { - setSegmentSize(segmentSize); + wayGeometry.setSegmentSize(segmentSize); + stringIndex.setSegmentSize(segmentSize); } } - private void setEdgeRef(int nodeId, int edgeId) { - nodes.setInt(toNodePointer(nodeId) + N_EDGE_REF, edgeId); - } - - int getEdgeRef(int nodeId) { - return nodes.getInt(toNodePointer(nodeId) + N_EDGE_REF); - } - - private int getNodeA(long edgePointer) { - return edges.getInt(edgePointer + E_NODEA); - } - - private int getNodeB(long edgePointer) { - return edges.getInt(edgePointer + E_NODEB); - } - - private int getLinkA(long edgePointer) { - return edges.getInt(edgePointer + E_LINKA); - } - - private int getLinkB(long edgePointer) { - return edges.getInt(edgePointer + E_LINKB); - } - - long toNodePointer(int node) { - if (node < 0 || node >= nodeCount) - throw new IllegalArgumentException("node: " + node + " out of bounds [0," + nodeCount + "["); - return (long) node * nodeEntryBytes; - } - - private long toEdgePointer(int edge) { - if (edge < 0 || edge >= edgeCount) - throw new IllegalArgumentException("edge: " + edge + " out of bounds [0," + edgeCount + "["); - return (long) edge * edgeEntryBytes; - } - private int getOtherNode(int nodeThis, long edgePointer) { - int nodeA = getNodeA(edgePointer); - return nodeThis == nodeA ? getNodeB(edgePointer) : nodeA; + int nodeA = store.getNodeA(edgePointer); + return nodeThis == nodeA ? store.getNodeB(edgePointer) : nodeA; } private boolean isAdjacentToNode(int node, long edgePointer) { - return getNodeA(edgePointer) == node || getNodeB(edgePointer) == node; + return store.getNodeA(edgePointer) == node || store.getNodeB(edgePointer) == node; } private static boolean isTestingEnabled() { @@ -169,90 +101,16 @@ void checkNotInitialized() { + "after calling create or loadExisting. Calling one of the methods twice is also not allowed."); } - void checkInitialized() { - if (!initialized) - throw new IllegalStateException("The graph has not yet been initialized."); - } - - private void loadNodesHeader() { - nodeEntryBytes = nodes.getHeader(0 * 4); - nodeCount = nodes.getHeader(1 * 4); - bounds.minLon = Helper.intToDegree(nodes.getHeader(2 * 4)); - bounds.maxLon = Helper.intToDegree(nodes.getHeader(3 * 4)); - bounds.minLat = Helper.intToDegree(nodes.getHeader(4 * 4)); - bounds.maxLat = Helper.intToDegree(nodes.getHeader(5 * 4)); - - if (bounds.hasElevation()) { - bounds.minEle = Helper.intToEle(nodes.getHeader(6 * 4)); - bounds.maxEle = Helper.intToEle(nodes.getHeader(7 * 4)); - } - - frozen = nodes.getHeader(8 * 4) == 1; - } - - private void setNodesHeader() { - nodes.setHeader(0 * 4, nodeEntryBytes); - nodes.setHeader(1 * 4, nodeCount); - nodes.setHeader(2 * 4, Helper.degreeToInt(bounds.minLon)); - nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.maxLon)); - nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.minLat)); - nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.maxLat)); - if (bounds.hasElevation()) { - nodes.setHeader(6 * 4, Helper.eleToInt(bounds.minEle)); - nodes.setHeader(7 * 4, Helper.eleToInt(bounds.maxEle)); - } - - nodes.setHeader(8 * 4, isFrozen() ? 1 : 0); - } - - protected void loadEdgesHeader() { - edgeEntryBytes = edges.getHeader(0 * 4); - edgeCount = edges.getHeader(1 * 4); - } - - protected void setEdgesHeader() { - edges.setHeader(0, edgeEntryBytes); - edges.setHeader(1 * 4, edgeCount); - } - - protected int loadWayGeometryHeader() { + private void loadWayGeometryHeader() { maxGeoRef = bitUtil.combineIntsToLong(wayGeometry.getHeader(0), wayGeometry.getHeader(4)); - return 1; } - protected int setWayGeometryHeader() { + private void setWayGeometryHeader() { wayGeometry.setHeader(0, bitUtil.getIntLow(maxGeoRef)); wayGeometry.setHeader(4, bitUtil.getIntHigh(maxGeoRef)); - return 1; } - void initStorage() { - edgeEntryIndex = 0; - nodeEntryIndex = 0; - E_NODEA = nextEdgeEntryIndex(4); - E_NODEB = nextEdgeEntryIndex(4); - E_LINKA = nextEdgeEntryIndex(4); - E_LINKB = nextEdgeEntryIndex(4); - E_FLAGS = nextEdgeEntryIndex(intsForFlags * 4); - - E_DIST = nextEdgeEntryIndex(4); - E_GEO = nextEdgeEntryIndex(4); - E_NAME = nextEdgeEntryIndex(4); - - N_EDGE_REF = nextNodeEntryIndex(4); - N_LAT = nextNodeEntryIndex(4); - N_LON = nextNodeEntryIndex(4); - if (nodeAccess.is3D()) - N_ELE = nextNodeEntryIndex(4); - else - N_ELE = -1; - - if (supportsTurnCosts()) - N_TC = nextNodeEntryIndex(4); - else - N_TC = -1; - - initNodeAndEdgeEntrySize(); + private void setInitialized() { initialized = true; } @@ -260,64 +118,14 @@ boolean supportsTurnCosts() { return turnCostStorage != null; } - /** - * Initializes the node storage such that each node has no edge and no turn cost entry - */ - void initNodeRefs(long oldCapacity, long newCapacity) { - for (long pointer = oldCapacity + N_EDGE_REF; pointer < newCapacity; pointer += nodeEntryBytes) { - nodes.setInt(pointer, EdgeIterator.NO_EDGE); - } - if (supportsTurnCosts()) { - for (long pointer = oldCapacity + N_TC; pointer < newCapacity; pointer += nodeEntryBytes) { - nodes.setInt(pointer, TurnCostStorage.NO_TURN_ENTRY); - } - } - } - - protected final int nextEdgeEntryIndex(int sizeInBytes) { - int tmp = edgeEntryIndex; - edgeEntryIndex += sizeInBytes; - return tmp; - } - - protected final int nextNodeEntryIndex(int sizeInBytes) { - int tmp = nodeEntryIndex; - nodeEntryIndex += sizeInBytes; - return tmp; - } - - protected final void initNodeAndEdgeEntrySize() { - nodeEntryBytes = nodeEntryIndex; - edgeEntryBytes = edgeEntryIndex; - } - - /** - * Check if byte capacity of DataAcess nodes object is sufficient to include node index, else - * extend byte capacity - */ - final void ensureNodeIndex(int nodeIndex) { - checkInitialized(); - - if (nodeIndex < nodeCount) - return; - - long oldNodes = nodeCount; - nodeCount = nodeIndex + 1; - boolean capacityIncreased = nodes.ensureCapacity((long) nodeCount * nodeEntryBytes); - if (capacityIncreased) { - long newBytesCapacity = nodes.getCapacity(); - initNodeRefs(oldNodes * nodeEntryBytes, newBytesCapacity); - } - } - @Override public int getNodes() { - return nodeCount; + return store.getNodes(); } @Override public int getEdges() { - return edgeCount; + return store.getEdges(); } @Override @@ -327,34 +135,21 @@ public NodeAccess getNodeAccess() { @Override public BBox getBounds() { - return bounds; - } - - private void setSegmentSize(int bytes) { - checkNotInitialized(); - nodes.setSegmentSize(bytes); - edges.setSegmentSize(bytes); - wayGeometry.setSegmentSize(bytes); - stringIndex.setSegmentSize(bytes); - if (supportsTurnCosts()) { - turnCostStorage.setSegmentSize(bytes); - } + return store.getBounds(); } synchronized void freeze() { if (isFrozen()) throw new IllegalStateException("base graph already frozen"); - - frozen = true; + store.setFrozen(true); } synchronized boolean isFrozen() { - return frozen; + return store.getFrozen(); } void create(long initSize) { - nodes.create(initSize); - edges.create(initSize); + store.create(initSize); initSize = Math.min(initSize, 2000); wayGeometry.create(initSize); @@ -362,51 +157,15 @@ void create(long initSize) { if (supportsTurnCosts()) { turnCostStorage.create(initSize); } - initStorage(); + setInitialized(); // 0 stands for no separate geoRef maxGeoRef = 4; - - initNodeRefs(0, nodes.getCapacity()); } String toDetailsString() { - return "edges:" + nf(edgeCount) + "(" + edges.getCapacity() / Helper.MB + "MB), " - + "nodes:" + nf(getNodes()) + "(" + nodes.getCapacity() / Helper.MB + "MB), " + return store.toDetailsString() + ", " + "name:(" + stringIndex.getCapacity() / Helper.MB + "MB), " - + "geo:" + nf(maxGeoRef) + "(" + wayGeometry.getCapacity() / Helper.MB + "MB), " - + "bounds:" + bounds; - } - - public void debugPrint() { - final int printMax = 100; - System.out.println("nodes:"); - String formatNodes = "%12s | %12s | %12s | %12s \n"; - System.out.format(Locale.ROOT, formatNodes, "#", "N_EDGE_REF", "N_LAT", "N_LON"); - NodeAccess nodeAccess = getNodeAccess(); - for (int i = 0; i < Math.min(nodeCount, printMax); ++i) { - System.out.format(Locale.ROOT, formatNodes, i, getEdgeRef(i), nodeAccess.getLat(i), nodeAccess.getLon(i)); - } - if (nodeCount > printMax) { - System.out.format(Locale.ROOT, " ... %d more nodes\n", nodeCount - printMax); - } - System.out.println("edges:"); - String formatEdges = "%12s | %12s | %12s | %12s | %12s | %12s | %12s \n"; - System.out.format(Locale.ROOT, formatEdges, "#", "E_NODEA", "E_NODEB", "E_LINKA", "E_LINKB", "E_FLAGS", "E_DIST"); - IntsRef intsRef = new IntsRef(intsForFlags); - for (int i = 0; i < Math.min(edgeCount, printMax); ++i) { - long edgePointer = toEdgePointer(i); - readFlags(edgePointer, intsRef); - System.out.format(Locale.ROOT, formatEdges, i, - getNodeA(edgePointer), - getNodeB(edgePointer), - getLinkA(edgePointer), - getLinkB(edgePointer), - intsRef, - getDist(edgePointer)); - } - if (edgeCount > printMax) { - System.out.printf(Locale.ROOT, " ... %d more edges", edgeCount - printMax); - } + + "geo:" + nf(maxGeoRef) + "(" + wayGeometry.getCapacity() / Helper.MB + "MB)"; } /** @@ -431,10 +190,7 @@ public void flush() { if (!stringIndex.isClosed()) stringIndex.flush(); - setNodesHeader(); - setEdgesHeader(); - edges.flush(); - nodes.flush(); + store.flush(); if (supportsTurnCosts()) { turnCostStorage.flush(); } @@ -445,15 +201,14 @@ public void close() { wayGeometry.close(); if (!stringIndex.isClosed()) stringIndex.close(); - edges.close(); - nodes.close(); + store.close(); if (supportsTurnCosts()) { turnCostStorage.close(); } } long getCapacity() { - return edges.getCapacity() + nodes.getCapacity() + stringIndex.getCapacity() + return store.getCapacity() + stringIndex.getCapacity() + wayGeometry.getCapacity() + (supportsTurnCosts() ? turnCostStorage.getCapacity() : 0); } @@ -462,16 +217,13 @@ long getMaxGeoRef() { } void loadExisting(String dim) { - if (!nodes.loadExisting()) - throw new IllegalStateException("Cannot load nodes. corrupt file or directory? " + dir); + if (!store.loadExisting()) + throw new IllegalStateException("Cannot load edges or nodes. corrupt file or directory? " + dir); if (!dim.equalsIgnoreCase("" + nodeAccess.getDimension())) throw new IllegalStateException("Configured dimension (" + nodeAccess.getDimension() + ") is not equal " + "to dimension of loaded graph (" + dim + ")"); - if (!edges.loadExisting()) - throw new IllegalStateException("Cannot load edges. corrupt file or directory? " + dir); - if (!wayGeometry.loadExisting()) throw new IllegalStateException("Cannot load geometry. corrupt file or directory? " + dir); @@ -481,12 +233,7 @@ void loadExisting(String dim) { if (supportsTurnCosts() && !turnCostStorage.loadExisting()) throw new IllegalStateException("Cannot load turn cost storage. corrupt file or directory? " + dir); - // first define header indices of this storage - initStorage(); - - // now load some properties from stored data - loadNodesHeader(); - loadEdgesHeader(); + setInitialized(); loadWayGeometryHeader(); } @@ -496,8 +243,8 @@ void loadExisting(String dim) { * @return the updated iterator the properties where copied to. */ EdgeIteratorState copyProperties(EdgeIteratorState from, EdgeIteratorStateImpl to) { - long edgePointer = toEdgePointer(to.getEdge()); - writeFlags(edgePointer, from.getFlags()); + long edgePointer = store.toEdgePointer(to.getEdge()); + store.writeFlags(edgePointer, from.getFlags()); // copy the rest with higher level API to.setDistance(from.getDistance()). @@ -516,71 +263,13 @@ EdgeIteratorState copyProperties(EdgeIteratorState from, EdgeIteratorStateImpl t public EdgeIteratorState edge(int nodeA, int nodeB) { if (isFrozen()) throw new IllegalStateException("Cannot create edge if graph is already frozen"); - - ensureNodeIndex(Math.max(nodeA, nodeB)); - int edgeId = internalEdgeAdd(nextEdgeId(), nodeA, nodeB); + int edgeId = store.edge(nodeA, nodeB); EdgeIteratorStateImpl edge = new EdgeIteratorStateImpl(this); boolean valid = edge.init(edgeId, nodeB); assert valid; return edge; } - /** - * Writes a new edge to the array of edges and adds it to the linked list of edges at nodeA and nodeB - */ - final int internalEdgeAdd(int newEdgeId, int nodeA, int nodeB) { - writeEdge(newEdgeId, nodeA, nodeB); - long edgePointer = toEdgePointer(newEdgeId); - - int edge = getEdgeRef(nodeA); - if (edge > EdgeIterator.NO_EDGE) - edges.setInt(E_LINKA + edgePointer, edge); - setEdgeRef(nodeA, newEdgeId); - - if (nodeA != nodeB) { - edge = getEdgeRef(nodeB); - if (edge > EdgeIterator.NO_EDGE) - edges.setInt(E_LINKB + edgePointer, edge); - setEdgeRef(nodeB, newEdgeId); - } - return newEdgeId; - } - - /** - * Writes plain edge information to the edges index - */ - private long writeEdge(int edgeId, int nodeA, int nodeB) { - if (!EdgeIterator.Edge.isValid(edgeId)) - throw new IllegalStateException("Cannot write edge with illegal ID:" + edgeId + "; nodeA:" + nodeA + ", nodeB:" + nodeB); - - long edgePointer = toEdgePointer(edgeId); - edges.setInt(edgePointer + E_NODEA, nodeA); - edges.setInt(edgePointer + E_NODEB, nodeB); - edges.setInt(edgePointer + E_LINKA, EdgeIterator.NO_EDGE); - edges.setInt(edgePointer + E_LINKB, EdgeIterator.NO_EDGE); - return edgePointer; - } - - // for test only - void setEdgeCount(int cnt) { - edgeCount = cnt; - } - - /** - * Determine next free edgeId and ensure byte capacity to store edge - * - * @return next free edgeId - */ - protected int nextEdgeId() { - int nextEdge = edgeCount; - edgeCount++; - if (edgeCount < 0) - throw new IllegalStateException("too many edges. new edge id would be negative. " + toString()); - - edges.ensureCapacity(((long) edgeCount + 1) * edgeEntryBytes); - return nextEdge; - } - @Override public EdgeIteratorState getEdgeIteratorState(int edgeId, int adjNode) { EdgeIteratorStateImpl edge = new EdgeIteratorStateImpl(this); @@ -597,11 +286,6 @@ public EdgeIteratorState getEdgeIteratorStateForKey(int edgeKey) { return edge; } - final void checkAdjNodeBounds(int adjNode) { - if (adjNode < 0 && adjNode != Integer.MIN_VALUE || adjNode >= nodeCount) - throw new IllegalStateException("adjNode " + adjNode + " out of bounds [0," + nf(nodeCount) + ")"); - } - @Override public EdgeExplorer createEdgeExplorer(EdgeFilter filter) { return new EdgeIteratorImpl(this, filter); @@ -629,64 +313,23 @@ public Weighting wrapWeighting(Weighting weighting) { @Override public int getOtherNode(int edge, int node) { - long edgePointer = toEdgePointer(edge); + long edgePointer = store.toEdgePointer(edge); return getOtherNode(node, edgePointer); } @Override public boolean isAdjacentToNode(int edge, int node) { - long edgePointer = toEdgePointer(edge); + long edgePointer = store.toEdgePointer(edge); return isAdjacentToNode(node, edgePointer); } - private void readFlags(long edgePointer, IntsRef edgeFlags) { - int size = edgeFlags.ints.length; - for (int i = 0; i < size; i++) { - edgeFlags.ints[i] = edges.getInt(edgePointer + E_FLAGS + i * 4); - } - } - - private void writeFlags(long edgePointer, IntsRef edgeFlags) { - int size = edgeFlags.ints.length; - for (int i = 0; i < size; i++) { - edges.setInt(edgePointer + E_FLAGS + i * 4, edgeFlags.ints[i]); - } - } - - private void setDist(long edgePointer, double distance) { - edges.setInt(edgePointer + E_DIST, distToInt(distance)); - } - - /** - * Translates double distance to integer in order to save it in a DataAccess object - */ - private int distToInt(double distance) { - if (distance < 0) - throw new IllegalArgumentException("Distance cannot be negative: " + distance); - if (distance > MAX_DIST) { - distance = MAX_DIST; - } - int integ = (int) Math.round(distance * INT_DIST_FACTOR); - assert integ >= 0 : "distance out of range"; - return integ; - } - - /** - * returns distance (already translated from integer to double) - */ - private double getDist(long pointer) { - int val = edges.getInt(pointer + E_DIST); - // do never return infinity even if INT MAX, see #435 - return val / INT_DIST_FACTOR; - } - private void setWayGeometry_(PointList pillarNodes, long edgePointer, boolean reverse) { if (pillarNodes != null && !pillarNodes.isEmpty()) { if (pillarNodes.getDimension() != nodeAccess.getDimension()) throw new IllegalArgumentException("Cannot use pointlist which is " + pillarNodes.getDimension() + "D for graph which is " + nodeAccess.getDimension() + "D"); - long existingGeoRef = Helper.toUnsignedLong(edges.getInt(edgePointer + E_GEO)); + long existingGeoRef = Helper.toUnsignedLong(store.getGeoRef(edgePointer)); int len = pillarNodes.size(); int dim = nodeAccess.getDimension(); @@ -701,7 +344,7 @@ private void setWayGeometry_(PointList pillarNodes, long edgePointer, boolean re long nextGeoRef = nextGeoRef(len * dim); setWayGeometryAtGeoRef(pillarNodes, edgePointer, reverse, nextGeoRef); } else { - edges.setInt(edgePointer + E_GEO, 0); + store.setGeoRef(edgePointer, 0); } } @@ -713,7 +356,7 @@ private void setWayGeometryAtGeoRef(PointList pillarNodes, long edgePointer, boo ensureGeometry(geoRefPosition, totalLen); byte[] wayGeometryBytes = createWayGeometryBytes(pillarNodes, reverse); wayGeometry.setBytes(geoRefPosition, wayGeometryBytes, wayGeometryBytes.length); - edges.setInt(edgePointer + E_GEO, Helper.toSignedInt(geoRef)); + store.setGeoRef(edgePointer, Helper.toSignedInt(geoRef)); } private byte[] createWayGeometryBytes(PointList pillarNodes, boolean reverse) { @@ -750,7 +393,7 @@ private PointList fetchWayGeometry_(long edgePointer, boolean reverse, FetchMode pillarNodes.add(nodeAccess, adjNode); return pillarNodes; } - long geoRef = Helper.toUnsignedLong(edges.getInt(edgePointer + E_GEO)); + long geoRef = Helper.toUnsignedLong(store.getGeoRef(edgePointer)); int count = 0; byte[] bytes = null; if (geoRef > 0) { @@ -814,8 +457,7 @@ private void setName(long edgePointer, String name) { int stringIndexRef = (int) stringIndex.add(Collections.singletonMap(STRING_IDX_NAME_KEY, name)); if (stringIndexRef < 0) throw new IllegalStateException("Too many names are stored, currently limited to int pointer"); - - edges.setInt(edgePointer + E_NAME, stringIndexRef); + store.setNameRef(edgePointer, stringIndexRef); } private void ensureGeometry(long bytePos, int byteLength) { @@ -831,6 +473,10 @@ private long nextGeoRef(int arrayLength) { return tmp; } + public boolean isClosed() { + return store.isClosed(); + } + protected static class EdgeIteratorImpl extends EdgeIteratorStateImpl implements EdgeExplorer, EdgeIterator { final EdgeFilter filter; int nextEdgeId; @@ -845,7 +491,7 @@ public EdgeIteratorImpl(BaseGraph baseGraph, EdgeFilter filter) { @Override public EdgeIterator setBaseNode(int baseNode) { - nextEdgeId = edgeId = baseGraph.getEdgeRef(baseNode); + nextEdgeId = edgeId = store.getEdgeRef(store.toNodePointer(baseNode)); this.baseNode = baseNode; return this; } @@ -861,16 +507,16 @@ public final boolean next() { } void goToNext() { - edgePointer = baseGraph.toEdgePointer(nextEdgeId); + edgePointer = store.toEdgePointer(nextEdgeId); edgeId = nextEdgeId; - int nodeA = baseGraph.getNodeA(edgePointer); + int nodeA = store.getNodeA(edgePointer); boolean baseNodeIsNodeA = baseNode == nodeA; - adjNode = baseNodeIsNodeA ? baseGraph.getNodeB(edgePointer) : nodeA; + adjNode = baseNodeIsNodeA ? store.getNodeB(edgePointer) : nodeA; reverse = !baseNodeIsNodeA; freshFlags = false; // position to next edge - nextEdgeId = baseNodeIsNodeA ? baseGraph.getLinkA(edgePointer) : baseGraph.getLinkB(edgePointer); + nextEdgeId = baseNodeIsNodeA ? store.getLinkA(edgePointer) : store.getLinkB(edgePointer); assert nextEdgeId != edgeId : ("endless loop detected for base node: " + baseNode + ", adj node: " + adjNode + ", edge pointer: " + edgePointer + ", edge: " + edgeId); } @@ -893,17 +539,17 @@ public AllEdgeIterator(BaseGraph baseGraph) { @Override public int length() { - return baseGraph.edgeCount; + return store.getEdges(); } @Override public boolean next() { edgeId++; - if (edgeId >= baseGraph.edgeCount) + if (edgeId >= store.getEdges()) return false; - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; reverse = false; return true; @@ -932,6 +578,7 @@ public final EdgeIteratorState detach(boolean reverseArg) { static class EdgeIteratorStateImpl implements EdgeIteratorState { final BaseGraph baseGraph; + final BaseGraphNodesAndEdges store; long edgePointer = -1; int baseNode; int adjNode; @@ -943,19 +590,20 @@ static class EdgeIteratorStateImpl implements EdgeIteratorState { public EdgeIteratorStateImpl(BaseGraph baseGraph) { this.baseGraph = baseGraph; - this.edgeFlags = new IntsRef(baseGraph.intsForFlags); + this.edgeFlags = new IntsRef(baseGraph.store.getIntsForFlags()); + store = baseGraph.store; } /** * @return false if the edge has not a node equal to expectedAdjNode */ final boolean init(int edgeId, int expectedAdjNode) { - if (edgeId < 0 || edgeId >= baseGraph.edgeCount) - throw new IllegalArgumentException("edge: " + edgeId + " out of bounds: [0," + baseGraph.edgeCount + "["); + if (edgeId < 0 || edgeId >= store.getEdges()) + throw new IllegalArgumentException("edge: " + edgeId + " out of bounds: [0," + store.getEdges() + "["); this.edgeId = edgeId; - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; if (expectedAdjNode == adjNode || expectedAdjNode == Integer.MIN_VALUE) { @@ -978,9 +626,9 @@ final void init(int edgeKey) { if (edgeKey < 0) throw new IllegalArgumentException("edge keys must not be negative, given: " + edgeKey); this.edgeId = GHUtility.getEdgeFromEdgeKey(edgeKey); - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; if (edgeKey % 2 == 0 || baseNode == adjNode) { @@ -1005,19 +653,19 @@ public final int getAdjNode() { @Override public double getDistance() { - return baseGraph.getDist(edgePointer); + return store.getDist(edgePointer); } @Override public EdgeIteratorState setDistance(double dist) { - baseGraph.setDist(edgePointer, dist); + store.setDist(edgePointer, dist); return this; } @Override public IntsRef getFlags() { if (!freshFlags) { - baseGraph.readFlags(edgePointer, edgeFlags); + store.readFlags(edgePointer, edgeFlags); freshFlags = true; } return edgeFlags; @@ -1025,8 +673,8 @@ public IntsRef getFlags() { @Override public final EdgeIteratorState setFlags(IntsRef edgeFlags) { - assert edgeId < baseGraph.edgeCount : "must be edge but was shortcut: " + edgeId + " >= " + baseGraph.edgeCount + ". Use setFlagsAndWeight"; - baseGraph.writeFlags(edgePointer, edgeFlags); + assert edgeId < store.getEdges() : "must be edge but was shortcut: " + edgeId + " >= " + store.getEdges() + ". Use setFlagsAndWeight"; + store.writeFlags(edgePointer, edgeFlags); for (int i = 0; i < edgeFlags.ints.length; i++) { this.edgeFlags.ints[i] = edgeFlags.ints[i]; } @@ -1042,7 +690,7 @@ public boolean get(BooleanEncodedValue property) { @Override public EdgeIteratorState set(BooleanEncodedValue property, boolean value) { property.setBool(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1054,7 +702,7 @@ public boolean getReverse(BooleanEncodedValue property) { @Override public EdgeIteratorState setReverse(BooleanEncodedValue property, boolean value) { property.setBool(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1064,7 +712,7 @@ public EdgeIteratorState set(BooleanEncodedValue property, boolean fwd, boolean throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setBool(reverse, getFlags(), fwd); property.setBool(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1076,7 +724,7 @@ public int get(IntEncodedValue property) { @Override public EdgeIteratorState set(IntEncodedValue property, int value) { property.setInt(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1088,7 +736,7 @@ public int getReverse(IntEncodedValue property) { @Override public EdgeIteratorState setReverse(IntEncodedValue property, int value) { property.setInt(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1098,7 +746,7 @@ public EdgeIteratorState set(IntEncodedValue property, int fwd, int bwd) { throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setInt(reverse, getFlags(), fwd); property.setInt(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1110,7 +758,7 @@ public double get(DecimalEncodedValue property) { @Override public EdgeIteratorState set(DecimalEncodedValue property, double value) { property.setDecimal(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1122,7 +770,7 @@ public double getReverse(DecimalEncodedValue property) { @Override public EdgeIteratorState setReverse(DecimalEncodedValue property, double value) { property.setDecimal(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1132,7 +780,7 @@ public EdgeIteratorState set(DecimalEncodedValue property, double fwd, double bw throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setDecimal(reverse, getFlags(), fwd); property.setDecimal(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1144,7 +792,7 @@ public > T get(EnumEncodedValue property) { @Override public > EdgeIteratorState set(EnumEncodedValue property, T value) { property.setEnum(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1156,7 +804,7 @@ public > T getReverse(EnumEncodedValue property) { @Override public > EdgeIteratorState setReverse(EnumEncodedValue property, T value) { property.setEnum(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1166,7 +814,7 @@ public > EdgeIteratorState set(EnumEncodedValue property, T throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setEnum(reverse, getFlags(), fwd); property.setEnum(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1178,7 +826,7 @@ public String get(StringEncodedValue property) { @Override public EdgeIteratorState set(StringEncodedValue property, String value) { property.setString(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1190,7 +838,7 @@ public String getReverse(StringEncodedValue property) { @Override public EdgeIteratorState setReverse(StringEncodedValue property, String value) { property.setString(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1200,7 +848,7 @@ public EdgeIteratorState set(StringEncodedValue property, String fwd, String bwd throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setString(reverse, getFlags(), fwd); property.setString(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1242,7 +890,7 @@ public int getOrigEdgeLast() { @Override public String getName() { - int stringIndexRef = baseGraph.edges.getInt(edgePointer + baseGraph.E_NAME); + int stringIndexRef = store.getNameRef(edgePointer); String name = baseGraph.stringIndex.get(stringIndexRef, STRING_IDX_NAME_KEY); // preserve backward compatibility (returns null if not explicitly set) return name == null ? "" : name; diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java new file mode 100644 index 00000000000..f2e647c5921 --- /dev/null +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -0,0 +1,399 @@ +/* + * Licensed to GraphHopper GmbH under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + * + * GraphHopper GmbH licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphhopper.storage; + +import com.graphhopper.util.EdgeIterator; +import com.graphhopper.util.Helper; +import com.graphhopper.util.shapes.BBox; + +import java.util.Locale; + +import static com.graphhopper.util.EdgeIterator.NO_EDGE; +import static com.graphhopper.util.Helper.nf; + +/** + * Underlying storage for nodes and edges of {@link BaseGraph}. Nodes and edges are stored using two {@link DataAccess} + * instances. Nodes and edges are simply stored sequentially, see the memory layout in the constructor. + */ +class BaseGraphNodesAndEdges { + // Currently distances are stored as 4 byte integers. using a conversion factor of 1000 the minimum distance + // that is not considered zero is 0.0005m (=0.5mm) and the maximum distance per edge is about 2.147.483m=2147km. + // See OSMReader.addEdge and #1871. + private static final double INT_DIST_FACTOR = 1000d; + static double MAX_DIST = Integer.MAX_VALUE / INT_DIST_FACTOR; + + // nodes + private final DataAccess nodes; + private final int N_EDGE_REF, N_LAT, N_LON, N_ELE, N_TC; + private int nodeEntryBytes; + private int nodeCount; + + // edges + private final DataAccess edges; + private final int E_NODEA, E_NODEB, E_LINKA, E_LINKB, E_FLAGS, E_DIST, E_GEO, E_NAME; + private final int intsForFlags; + private int edgeEntryBytes; + private int edgeCount; + + private final boolean withTurnCosts; + private final boolean withElevation; + + // we do not write the bounding box directly to storage, but rather to this bbox object. we only write to storage + // when flushing. why? just because we did it like this in the past, and otherwise we run into rounding errors, + // because of: #2393 + public final BBox bounds; + private boolean frozen; + + public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { + nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); + edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); + if (segmentSize >= 0) { + nodes.setSegmentSize(segmentSize); + edges.setSegmentSize(segmentSize); + } + this.intsForFlags = intsForFlags; + this.withTurnCosts = withTurnCosts; + this.withElevation = withElevation; + bounds = BBox.createInverse(withElevation); + + // memory layout for nodes + N_EDGE_REF = 0; + N_LAT = 4; + N_LON = 8; + N_ELE = N_LON + (withElevation ? 4 : 0); + N_TC = N_ELE + (withTurnCosts ? 4 : 0); + nodeEntryBytes = N_TC + 4; + + // memory layout for edges + E_NODEA = 0; + E_NODEB = 4; + E_LINKA = 8; + E_LINKB = 12; + E_FLAGS = 16; + E_DIST = E_FLAGS + intsForFlags * 4; + E_GEO = E_DIST + 4; + E_NAME = E_GEO + 4; + edgeEntryBytes = E_NAME + 4; + } + + public void create(long initSize) { + nodes.create(initSize); + edges.create(initSize); + } + + public boolean loadExisting() { + if (!nodes.loadExisting() || !edges.loadExisting()) + return false; + + // now load some properties from stored data + nodeEntryBytes = nodes.getHeader(0 * 4); + nodeCount = nodes.getHeader(1 * 4); + bounds.minLon = Helper.intToDegree(nodes.getHeader(2 * 4)); + bounds.maxLon = Helper.intToDegree(nodes.getHeader(3 * 4)); + bounds.minLat = Helper.intToDegree(nodes.getHeader(4 * 4)); + bounds.maxLat = Helper.intToDegree(nodes.getHeader(5 * 4)); + if (withElevation) { + bounds.minEle = Helper.intToEle(nodes.getHeader(6 * 4)); + bounds.maxEle = Helper.intToEle(nodes.getHeader(7 * 4)); + } + frozen = nodes.getHeader(8 * 4) == 1; + + edgeEntryBytes = edges.getHeader(0 * 4); + edgeCount = edges.getHeader(1 * 4); + return true; + } + + public void flush() { + nodes.setHeader(0 * 4, nodeEntryBytes); + nodes.setHeader(1 * 4, nodeCount); + nodes.setHeader(2 * 4, Helper.degreeToInt(bounds.minLon)); + nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.maxLon)); + nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.minLat)); + nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.maxLat)); + if (withElevation) { + nodes.setHeader(6 * 4, Helper.eleToInt(bounds.minEle)); + nodes.setHeader(7 * 4, Helper.eleToInt(bounds.maxEle)); + } + nodes.setHeader(8 * 4, frozen ? 1 : 0); + + edges.setHeader(0, edgeEntryBytes); + edges.setHeader(1 * 4, edgeCount); + + edges.flush(); + nodes.flush(); + } + + public void close() { + edges.close(); + nodes.close(); + } + + public int getNodes() { + return nodeCount; + } + + public int getEdges() { + return edgeCount; + } + + public int getIntsForFlags() { + return intsForFlags; + } + + public boolean withElevation() { + return withElevation; + } + + public boolean withTurnCosts() { + return withTurnCosts; + } + + public BBox getBounds() { + return bounds; + } + + public long getCapacity() { + return nodes.getCapacity() + edges.getCapacity(); + } + + public boolean isClosed() { + assert nodes.isClosed() == edges.isClosed(); + return nodes.isClosed(); + } + + public int edge(int nodeA, int nodeB) { + if (edgeCount == Integer.MAX_VALUE) + throw new IllegalStateException("Maximum edge count exceeded: " + edgeCount); + ensureNodeCapacity(Math.max(nodeA, nodeB)); + final int edge = edgeCount; + final long edgePointer = (long) edgeCount * edgeEntryBytes; + edgeCount++; + edges.ensureCapacity((long) edgeCount * edgeEntryBytes); + + setNodeA(edgePointer, nodeA); + setNodeB(edgePointer, nodeB); + // we keep a linked list of edges at each node. here we prepend the new edge at the already existing linked + // list of edges. + long nodePointerA = toNodePointer(nodeA); + int edgeRefA = getEdgeRef(nodePointerA); + setLinkA(edgePointer, EdgeIterator.Edge.isValid(edgeRefA) ? edgeRefA : NO_EDGE); + setEdgeRef(nodePointerA, edge); + + if (nodeA != nodeB) { + long nodePointerB = toNodePointer(nodeB); + int edgeRefB = getEdgeRef(nodePointerB); + setLinkB(edgePointer, EdgeIterator.Edge.isValid(edgeRefB) ? edgeRefB : NO_EDGE); + setEdgeRef(nodePointerB, edge); + } + return edge; + } + + public void ensureNodeCapacity(int node) { + if (node < nodeCount) + return; + + int oldNodes = nodeCount; + nodeCount = node + 1; + nodes.ensureCapacity((long) nodeCount * nodeEntryBytes); + for (int n = oldNodes; n < nodeCount; ++n) { + setEdgeRef(toNodePointer(n), NO_EDGE); + if (withTurnCosts) + setTurnCostRef(toNodePointer(n), TurnCostStorage.NO_TURN_ENTRY); + } + } + + public long toNodePointer(int node) { + if (node < 0 || node >= nodeCount) + throw new IllegalArgumentException("node: " + node + " out of bounds [0," + nodeCount + "["); + return (long) node * nodeEntryBytes; + } + + public long toEdgePointer(int edge) { + if (edge < 0 || edge >= edgeCount) + throw new IllegalArgumentException("edge: " + edge + " out of bounds [0," + edgeCount + "["); + return (long) edge * edgeEntryBytes; + } + + public void readFlags(long edgePointer, IntsRef edgeFlags) { + int size = edgeFlags.ints.length; + for (int i = 0; i < size; ++i) + edgeFlags.ints[i] = edges.getInt(edgePointer + E_FLAGS + i * 4); + } + + public void writeFlags(long edgePointer, IntsRef edgeFlags) { + int size = edgeFlags.ints.length; + for (int i = 0; i < size; ++i) + edges.setInt(edgePointer + E_FLAGS + i * 4, edgeFlags.ints[i]); + } + + public void setNodeA(long edgePointer, int nodeA) { + edges.setInt(edgePointer + E_NODEA, nodeA); + } + + public void setNodeB(long edgePointer, int nodeB) { + edges.setInt(edgePointer + E_NODEB, nodeB); + } + + public void setLinkA(long edgePointer, int linkA) { + edges.setInt(edgePointer + E_LINKA, linkA); + } + + public void setLinkB(long edgePointer, int linkB) { + edges.setInt(edgePointer + E_LINKB, linkB); + } + + public void setDist(long edgePointer, double distance) { + edges.setInt(edgePointer + E_DIST, distToInt(distance)); + } + + public void setGeoRef(long edgePointer, int geoRef) { + edges.setInt(edgePointer + E_GEO, geoRef); + } + + public void setNameRef(long edgePointer, int nameRef) { + edges.setInt(edgePointer + E_NAME, nameRef); + } + + public int getNodeA(long edgePointer) { + return edges.getInt(edgePointer + E_NODEA); + } + + public int getNodeB(long edgePointer) { + return edges.getInt(edgePointer + E_NODEB); + } + + public int getLinkA(long edgePointer) { + return edges.getInt(edgePointer + E_LINKA); + } + + public int getLinkB(long edgePointer) { + return edges.getInt(edgePointer + E_LINKB); + } + + public double getDist(long pointer) { + int val = edges.getInt(pointer + E_DIST); + // do never return infinity even if INT MAX, see #435 + return val / INT_DIST_FACTOR; + } + + public int getGeoRef(long edgePointer) { + return edges.getInt(edgePointer + E_GEO); + } + + public int getNameRef(long edgePointer) { + return edges.getInt(edgePointer + E_NAME); + } + + public void setEdgeRef(long nodePointer, int edgeRef) { + nodes.setInt(nodePointer + N_EDGE_REF, edgeRef); + } + + public void setLat(long nodePointer, double lat) { + nodes.setInt(nodePointer + N_LAT, Helper.degreeToInt(lat)); + } + + public void setLon(long nodePointer, double lon) { + nodes.setInt(nodePointer + N_LON, Helper.degreeToInt(lon)); + } + + public void setEle(long elePointer, double ele) { + nodes.setInt(elePointer + N_ELE, Helper.eleToInt(ele)); + } + + public void setTurnCostRef(long nodePointer, int tcRef) { + nodes.setInt(nodePointer + N_TC, tcRef); + } + + public int getEdgeRef(long nodePointer) { + return nodes.getInt(nodePointer + N_EDGE_REF); + } + + public double getLat(long nodePointer) { + return Helper.intToDegree(nodes.getInt(nodePointer + N_LAT)); + } + + public double getLon(long nodePointer) { + return Helper.intToDegree(nodes.getInt(nodePointer + N_LON)); + } + + public double getEle(long nodePointer) { + return Helper.intToEle(nodes.getInt(nodePointer + N_ELE)); + } + + public int getTurnCostRef(long nodePointer) { + return nodes.getInt(nodePointer + N_TC); + } + + public void setFrozen(boolean frozen) { + this.frozen = frozen; + } + + public boolean getFrozen() { + return frozen; + } + + public void debugPrint() { + final int printMax = 100; + System.out.println("nodes:"); + String formatNodes = "%12s | %12s | %12s | %12s \n"; + System.out.format(Locale.ROOT, formatNodes, "#", "N_EDGE_REF", "N_LAT", "N_LON"); + for (int i = 0; i < Math.min(nodeCount, printMax); ++i) { + long nodePointer = toNodePointer(i); + System.out.format(Locale.ROOT, formatNodes, i, getEdgeRef(nodePointer), getLat(nodePointer), getLon(nodePointer)); + } + if (nodeCount > printMax) { + System.out.format(Locale.ROOT, " ... %d more nodes\n", nodeCount - printMax); + } + System.out.println("edges:"); + String formatEdges = "%12s | %12s | %12s | %12s | %12s | %12s | %12s \n"; + System.out.format(Locale.ROOT, formatEdges, "#", "E_NODEA", "E_NODEB", "E_LINKA", "E_LINKB", "E_FLAGS", "E_DIST"); + IntsRef intsRef = new IntsRef(intsForFlags); + for (int i = 0; i < Math.min(edgeCount, printMax); ++i) { + long edgePointer = toEdgePointer(i); + readFlags(edgePointer, intsRef); + System.out.format(Locale.ROOT, formatEdges, i, + getNodeA(edgePointer), + getNodeB(edgePointer), + getLinkA(edgePointer), + getLinkB(edgePointer), + intsRef, + getDist(edgePointer)); + } + if (edgeCount > printMax) { + System.out.printf(Locale.ROOT, " ... %d more edges", edgeCount - printMax); + } + } + + private int distToInt(double distance) { + if (distance < 0) + throw new IllegalArgumentException("Distance cannot be negative: " + distance); + if (distance > MAX_DIST) { + distance = MAX_DIST; + } + int intDist = (int) Math.round(distance * INT_DIST_FACTOR); + assert intDist >= 0 : "distance out of range"; + return intDist; + } + + public String toDetailsString() { + return "edges: " + nf(edgeCount) + "(" + edges.getCapacity() / Helper.MB + "MB), " + + "nodes: " + nf(nodeCount) + "(" + nodes.getCapacity() / Helper.MB + "MB), " + + "bounds: " + bounds; + } +} diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index 74324f90f67..b59ad6849f1 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -31,7 +31,7 @@ * DataAccess-based storage for CH shortcuts. Stores shortcuts and CH levels sequentially using two DataAccess objects * and gives read/write access to the different shortcut and node fields. *

- * This can be seen as an extension to a base graph: We assign a CH level to each nodes and add additional edges to + * This can be seen as an extension to a base graph: We assign a CH level to each node and add additional edges to * the graph ('shortcuts'). The shortcuts need to be ordered in a certain way, but this is not enforced here. * * @see CHStorageBuilder to build a valid storage that can be used for routing @@ -58,7 +58,7 @@ public class CHStorage { private int nodeCount = -1; private boolean edgeBased; - // some shortcuts exceed the maximum storable weight and we count them here + // some shortcuts exceed the maximum storable weight, and we count them here private int numShortcutsExceedingWeight; // use this to report shortcuts with too small weights diff --git a/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java b/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java index 6ad8cdaa3e9..9d0d396e4a0 100644 --- a/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java +++ b/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java @@ -17,8 +17,6 @@ */ package com.graphhopper.storage; -import com.graphhopper.util.Helper; - /** * A helper class for GraphHopperStorage for its node access. *

@@ -26,59 +24,55 @@ * @author Peter Karich */ class GHNodeAccess implements NodeAccess { - private final BaseGraph baseGraph; - private final boolean elevation; + private final BaseGraphNodesAndEdges store; - public GHNodeAccess(BaseGraph baseGraph, boolean withElevation) { - this.baseGraph = baseGraph; - this.elevation = withElevation; + public GHNodeAccess(BaseGraphNodesAndEdges store) { + this.store = store; } @Override public void ensureNode(int nodeId) { - baseGraph.ensureNodeIndex(nodeId); + store.ensureNodeCapacity(nodeId); } @Override public final void setNode(int nodeId, double lat, double lon, double ele) { - baseGraph.ensureNodeIndex(nodeId); - long tmp = baseGraph.toNodePointer(nodeId); - baseGraph.nodes.setInt(tmp + baseGraph.N_LAT, Helper.degreeToInt(lat)); - baseGraph.nodes.setInt(tmp + baseGraph.N_LON, Helper.degreeToInt(lon)); + store.ensureNodeCapacity(nodeId); + store.setLat(store.toNodePointer(nodeId), lat); + store.setLon(store.toNodePointer(nodeId), lon); - if (is3D()) { + if (store.withElevation()) { // meter precision is sufficient for now - baseGraph.nodes.setInt(tmp + baseGraph.N_ELE, Helper.eleToInt(ele)); - baseGraph.bounds.update(lat, lon, ele); - + store.setEle(store.toNodePointer(nodeId), ele); + store.bounds.update(lat, lon, ele); } else { - baseGraph.bounds.update(lat, lon); + store.bounds.update(lat, lon); } } @Override public final double getLat(int nodeId) { - return Helper.intToDegree(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_LAT)); + return store.getLat(store.toNodePointer(nodeId)); } @Override public final double getLon(int nodeId) { - return Helper.intToDegree(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_LON)); + return store.getLon(store.toNodePointer(nodeId)); } @Override public final double getEle(int nodeId) { - if (!elevation) - throw new IllegalStateException("Cannot access elevation - 3D is not enabled"); - - return Helper.intToEle(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_ELE)); + if (!store.withElevation()) + throw new IllegalStateException("elevation is disabled"); + return store.getEle(store.toNodePointer(nodeId)); } + @Override public final void setTurnCostIndex(int index, int turnCostIndex) { - if (baseGraph.supportsTurnCosts()) { - baseGraph.ensureNodeIndex(index); - long tmp = baseGraph.toNodePointer(index); - baseGraph.nodes.setInt(tmp + baseGraph.N_TC, turnCostIndex); + if (store.withTurnCosts()) { + // todo: remove ensure? + store.ensureNodeCapacity(index); + store.setTurnCostRef(store.toNodePointer(index), turnCostIndex); } else { throw new AssertionError("This graph does not support turn costs"); } @@ -86,21 +80,19 @@ public final void setTurnCostIndex(int index, int turnCostIndex) { @Override public final int getTurnCostIndex(int index) { - if (baseGraph.supportsTurnCosts()) - return baseGraph.nodes.getInt(baseGraph.toNodePointer(index) + baseGraph.N_TC); + if (store.withTurnCosts()) + return store.getTurnCostRef(store.toNodePointer(index)); else throw new AssertionError("This graph does not support turn costs"); } @Override public final boolean is3D() { - return elevation; + return store.withElevation(); } @Override public int getDimension() { - if (elevation) - return 3; - return 2; + return store.withElevation() ? 3 : 2; } } diff --git a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java index ba38078c6c8..bd03a44d47e 100644 --- a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java +++ b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java @@ -314,7 +314,7 @@ public void close() { } public boolean isClosed() { - return baseGraph.nodes.isClosed(); + return baseGraph.isClosed(); } public long getCapacity() { diff --git a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java index 0180f96a0b8..f0245508d97 100644 --- a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java +++ b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java @@ -54,7 +54,7 @@ public RoutingCHEdgeIterator setBaseNode(int baseNode) { assert baseGraph.isFrozen(); baseIterator.setBaseNode(baseNode); int lastShortcut = store.getLastShortcut(store.toNodePointer(baseNode)); - nextEdgeId = edgeId = lastShortcut < 0 ? baseIterator.edgeId : baseGraph.edgeCount + lastShortcut; + nextEdgeId = edgeId = lastShortcut < 0 ? baseIterator.edgeId : baseGraph.getEdges() + lastShortcut; return this; } @@ -63,13 +63,13 @@ public boolean next() { // we first traverse shortcuts (in decreasing order) and when we are done we use the base iterator to traverse // the base edges as well. shortcuts are filtered using shortcutFilter, but base edges are only filtered by // access/finite weight. - while (nextEdgeId >= baseGraph.edgeCount) { - shortcutPointer = store.toShortcutPointer(nextEdgeId - baseGraph.edgeCount); + while (nextEdgeId >= baseGraph.getEdges()) { + shortcutPointer = store.toShortcutPointer(nextEdgeId - baseGraph.getEdges()); baseNode = store.getNodeA(shortcutPointer); adjNode = store.getNodeB(shortcutPointer); edgeId = nextEdgeId; nextEdgeId--; - if (nextEdgeId < baseGraph.edgeCount || store.getNodeA(store.toShortcutPointer(nextEdgeId - baseGraph.edgeCount)) != baseNode) + if (nextEdgeId < baseGraph.getEdges() || store.getNodeA(store.toShortcutPointer(nextEdgeId - baseGraph.getEdges())) != baseNode) nextEdgeId = baseIterator.edgeId; // todo: note that it would be more efficient (but cost more memory) to separate in/out edges, // especially for edge-based where we do not use bidirectional shortcuts diff --git a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java index b5a45ac7e1c..54626fc0d24 100644 --- a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java +++ b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java @@ -45,11 +45,11 @@ public RoutingCHEdgeIteratorStateImpl(CHStorage store, BaseGraph baseGraph, Base } boolean init(int edge, int expectedAdjNode) { - if (edge < 0 || edge >= baseGraph.edgeCount + store.getShortcuts()) - throw new IllegalArgumentException("edge must be in bounds: [0," + (baseGraph.edgeCount + store.getShortcuts()) + "["); + if (edge < 0 || edge >= baseGraph.getEdges() + store.getShortcuts()) + throw new IllegalArgumentException("edge must be in bounds: [0," + (baseGraph.getEdges() + store.getShortcuts()) + "["); edgeId = edge; if (isShortcut()) { - shortcutPointer = store.toShortcutPointer(edge - baseGraph.edgeCount); + shortcutPointer = store.toShortcutPointer(edge - baseGraph.getEdges()); baseNode = store.getNodeA(shortcutPointer); adjNode = store.getNodeB(shortcutPointer); @@ -103,7 +103,7 @@ public int getAdjNode() { @Override public boolean isShortcut() { - return edgeId >= baseGraph.edgeCount; + return edgeId >= baseGraph.getEdges(); } @Override diff --git a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java index a362a02f120..8347d116663 100644 --- a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java +++ b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java @@ -243,7 +243,7 @@ public interface TurnRelationIterator { private class Itr implements TurnRelationIterator { private int viaNode = -1; private int turnCostIndex = -1; - private IntsRef intsRef = TurnCost.createFlags(); + private final IntsRef intsRef = TurnCost.createFlags(); private long turnCostPtr() { return (long) turnCostIndex * BYTES_PER_ENTRY; diff --git a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java index ab039dedd96..257fd25ad98 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.Closeable; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -117,7 +116,7 @@ public void tearDown() { public void testSetTooBigDistance_435() { graph = createGHStorage(); - double maxDist = BaseGraph.MAX_DIST; + double maxDist = BaseGraphNodesAndEdges.MAX_DIST; EdgeIteratorState edge1 = GHUtility.setSpeed(60, true, true, carEncoder, graph.edge(0, 1).setDistance(maxDist)); assertEquals(maxDist, edge1.getDistance(), 1); diff --git a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java index 13956f7b8d1..d2dab7fbd48 100644 --- a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java @@ -57,14 +57,7 @@ protected GraphHopperStorage newGHStorage(Directory dir, boolean enabled3D, int @Test public void testNoCreateCalled() { - try (GraphHopperStorage gs = GraphBuilder.start(encodingManager).build()) { - ((BaseGraph) gs.getBaseGraph()).ensureNodeIndex(123); - fail("IllegalStateException should be raised"); - } catch (IllegalStateException err) { - // ok - } catch (Exception ex) { - fail("IllegalStateException should be raised, but was " + ex.toString()); - } + assertThrows(Throwable.class, () -> GraphBuilder.start(encodingManager).build().edge(0, 1)); } @Test @@ -160,16 +153,6 @@ protected void checkGraph(Graph g) { assertEquals(GHUtility.asSet(0), GHUtility.getNeighbors(explorer.setBaseNode(2))); } - @Test - public void testBigDataEdge() { - Directory dir = new RAMDirectory(); - GraphHopperStorage graph = new GraphHopperStorage(dir, encodingManager, false); - graph.create(defaultSize); - ((BaseGraph) graph.getBaseGraph()).setEdgeCount(Integer.MAX_VALUE / 2); - assertTrue(graph.getAllEdges().next()); - graph.close(); - } - @Test public void testDoThrowExceptionIfDimDoesNotMatch() { graph = newGHStorage(new RAMDirectory(defaultGraphLoc, true), false); From e98be3b800e932654e2e02f55adec6058fd7c610 Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 29 Sep 2021 13:32:55 +0200 Subject: [PATCH 03/14] Mention turn restrictions in profiles.md --- docs/core/profiles.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/core/profiles.md b/docs/core/profiles.md index d5410409493..780f844dcb1 100644 --- a/docs/core/profiles.md +++ b/docs/core/profiles.md @@ -54,6 +54,9 @@ weightings: - curvature (prefers routes with lots of curves for enjoyable motorcycle rides) - custom (enables custom profiles, see the next section) +Another important profile setting is `turn_costs: true/false`. Use this to enable turn restrictions for each profile. +You can learn more about this setting [here](./turn-restrictions.md) + The profile name is used to select the profile when executing routing queries. To do this use the `profile` request parameter, for example `/route?point=49.5,11.1&profile=car` or `/route?point=49.5,11.1&profile=some_other_profile`. From 023378185018eb842a1947edc3afc884b2c3f228 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 30 Sep 2021 19:39:30 +0200 Subject: [PATCH 04/14] country_rules: fix documentation in example config (#2414) --- config-example.yml | 6 +++--- core/src/main/java/com/graphhopper/GraphHopper.java | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/config-example.yml b/config-example.yml index 0432d559945..a933e485550 100644 --- a/config-example.yml +++ b/config-example.yml @@ -164,9 +164,9 @@ graphhopper: # custom_areas.directory: path/to/custom_areas ##### Country Rules ##### - # GraphHopper applies country-specific routing rules (enabled by default) during import. - # Use this flag to disable these rules (you need to repeat the import for changes to take effect) - # country_rules.enable: false + # GraphHopper applies country-specific routing rules during import (not enabled by default). + # You need to redo the import for changes to take effect. + # country_rules.enabled: true # Dropwizard server configuration server: diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index 9e368495078..ff5e0ab4f1d 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -460,10 +460,7 @@ public GraphHopper init(GraphHopperConfig ghConfig) { graphHopperFolder = pruneFileEnd(osmFile) + "-gh"; } - if (ghConfig.has("country_rules.enabled")) { - boolean countryRulesEnabled = ghConfig.getBool("country_rules.enabled", false); - countryRuleFactory = countryRulesEnabled ? new CountryRuleFactory() : null; - } + countryRuleFactory = ghConfig.getBool("country_rules.enabled", false) ? new CountryRuleFactory() : null; customAreasDirectory = ghConfig.getString("custom_areas.directory", customAreasDirectory); // graph From fb59390c927f94999bc38593a039dbc2867c1647 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 1 Oct 2021 07:13:12 +0200 Subject: [PATCH 05/14] HMM-Lib: Bump Java version to 8 (#2413) --- hmm-lib/pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hmm-lib/pom.xml b/hmm-lib/pom.xml index 0f3f4b919fb..cb420216004 100644 --- a/hmm-lib/pom.xml +++ b/hmm-lib/pom.xml @@ -57,17 +57,4 @@ UTF-8 UTF-8 - - - - - maven-compiler-plugin - 3.1 - - 1.7 - 1.7 - - - - From ff8d658c6537251d42ad9d7827067863b5a43484 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 1 Oct 2021 11:19:07 +0200 Subject: [PATCH 06/14] Consider heading when snapping (#2411) * current tests pass * minor clarifications * fix test and heading for only one point is fine * changed comment * make test stable * no need to check edge twice as it is the same segment for both directions; do not forget to check end pillar node in distance calc * only calc heading of geometry first (#2415) * only calc heading of geometry first * Fix test (?!) * minor * Flip heading edge filter condition * move method and add comments * minor Co-authored-by: Andi --- .../graphhopper/routing/HeadingResolver.java | 11 +- .../java/com/graphhopper/routing/Router.java | 23 ++- .../com/graphhopper/routing/ViaRouting.java | 23 ++- .../routing/util/HeadingEdgeFilter.java | 68 +++++++++ .../java/com/graphhopper/GraphHopperTest.java | 2 +- .../routing/HeadingRoutingTest.java | 139 +++++++++++++++++- 6 files changed, 238 insertions(+), 28 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java diff --git a/core/src/main/java/com/graphhopper/routing/HeadingResolver.java b/core/src/main/java/com/graphhopper/routing/HeadingResolver.java index 1548ffabba5..db22126a898 100644 --- a/core/src/main/java/com/graphhopper/routing/HeadingResolver.java +++ b/core/src/main/java/com/graphhopper/routing/HeadingResolver.java @@ -19,12 +19,14 @@ package com.graphhopper.routing; import com.carrotsearch.hppc.IntArrayList; +import com.graphhopper.routing.util.EdgeFilter; import com.graphhopper.storage.Graph; import com.graphhopper.util.*; +import com.graphhopper.util.shapes.GHPoint; public class HeadingResolver { private final EdgeExplorer edgeExplorer; - private double toleranceRad = deg2Rad(100); + private double toleranceRad = Math.toRadians(100); public HeadingResolver(Graph graph) { this.edgeExplorer = graph.createEdgeExplorer(); @@ -63,12 +65,7 @@ public IntArrayList getEdgesWithDifferentHeading(int baseNode, double heading) { * Sets the tolerance for {@link #getEdgesWithDifferentHeading} in degrees. */ public HeadingResolver setTolerance(double tolerance) { - this.toleranceRad = deg2Rad(tolerance); + this.toleranceRad = Math.toRadians(tolerance); return this; } - - private static double deg2Rad(double deg) { - return Math.toRadians(deg); - } - } diff --git a/core/src/main/java/com/graphhopper/routing/Router.java b/core/src/main/java/com/graphhopper/routing/Router.java index 26d9ce07b0d..26b23a746b5 100644 --- a/core/src/main/java/com/graphhopper/routing/Router.java +++ b/core/src/main/java/com/graphhopper/routing/Router.java @@ -24,6 +24,7 @@ import com.graphhopper.ResponsePath; import com.graphhopper.config.Profile; import com.graphhopper.routing.ch.CHRoutingAlgorithmFactory; +import com.graphhopper.routing.ev.BooleanEncodedValue; import com.graphhopper.routing.ev.EncodedValueLookup; import com.graphhopper.routing.ev.Subnetwork; import com.graphhopper.routing.lm.LMRoutingAlgorithmFactory; @@ -165,12 +166,12 @@ private void checkHeadings(GHRequest request) { } private void checkPointHints(GHRequest request) { - if (request.getPointHints().size() > 0 && request.getPointHints().size() != request.getPoints().size()) + if (!request.getPointHints().isEmpty() && request.getPointHints().size() != request.getPoints().size()) throw new IllegalArgumentException("If you pass " + POINT_HINT + ", you need to pass exactly one hint for every point, empty hints will be ignored"); } private void checkCurbsides(GHRequest request) { - if (request.getCurbsides().size() > 0 && request.getCurbsides().size() != request.getPoints().size()) + if (!request.getCurbsides().isEmpty() && request.getCurbsides().size() != request.getPoints().size()) throw new IllegalArgumentException("If you pass " + CURBSIDE + ", you need to pass exactly one curbside for every point, empty curbsides will be ignored"); } @@ -196,7 +197,7 @@ protected GHResponse routeRoundTrip(GHRequest request, FlexSolver solver) { StopWatch sw = new StopWatch().start(); double startHeading = request.getHeadings().isEmpty() ? Double.NaN : request.getHeadings().get(0); RoundTripRouting.Params params = new RoundTripRouting.Params(request.getHints(), startHeading, routerConfig.getMaxRoundTripRetries()); - List snaps = RoundTripRouting.lookup(request.getPoints(), solver.getSnapFilter(), locationIndex, params); + List snaps = RoundTripRouting.lookup(request.getPoints(), solver.createSnapFilter(), locationIndex, params); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); QueryGraph queryGraph = QueryGraph.create(ghStorage, snaps); @@ -216,7 +217,8 @@ protected GHResponse routeAlt(GHRequest request, Solver solver) { throw new IllegalArgumentException("Currently alternative routes work only with start and end point. You tried to use: " + request.getPoints().size() + " points"); GHResponse ghRsp = new GHResponse(); StopWatch sw = new StopWatch().start(); - List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.getSnapFilter(), locationIndex, request.getSnapPreventions(), request.getPointHints()); + List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.createSnapFilter(), locationIndex, + request.getSnapPreventions(), request.getPointHints(), solver.createDirectedSnapFilter(), request.getHeadings()); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); QueryGraph queryGraph = QueryGraph.create(ghStorage, snaps); PathCalculator pathCalculator = solver.createPathCalculator(queryGraph); @@ -246,7 +248,8 @@ protected GHResponse routeAlt(GHRequest request, Solver solver) { protected GHResponse routeVia(GHRequest request, Solver solver) { GHResponse ghRsp = new GHResponse(); StopWatch sw = new StopWatch().start(); - List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.getSnapFilter(), locationIndex, request.getSnapPreventions(), request.getPointHints()); + List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.createSnapFilter(), locationIndex, + request.getSnapPreventions(), request.getPointHints(), solver.createDirectedSnapFilter(), request.getHeadings()); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); // (base) query graph used to resolve headings, curbsides etc. this is not necessarily the same thing as // the (possibly implementation specific) query graph used by PathCalculator @@ -254,7 +257,8 @@ protected GHResponse routeVia(GHRequest request, Solver solver) { PathCalculator pathCalculator = solver.createPathCalculator(queryGraph); boolean passThrough = getPassThrough(request.getHints()); boolean forceCurbsides = getForceCurbsides(request.getHints()); - ViaRouting.Result result = ViaRouting.calcPaths(request.getPoints(), queryGraph, snaps, solver.weighting, pathCalculator, request.getCurbsides(), forceCurbsides, request.getHeadings(), passThrough); + ViaRouting.Result result = ViaRouting.calcPaths(request.getPoints(), queryGraph, snaps, solver.weighting, + pathCalculator, request.getCurbsides(), forceCurbsides, request.getHeadings(), passThrough); if (request.getPoints().size() != result.paths.size() + 1) throw new RuntimeException("There should be exactly one more point than paths. points:" + request.getPoints().size() + ", paths:" + result.paths.size()); @@ -377,10 +381,15 @@ protected void checkProfileCompatibility() { protected abstract Weighting createWeighting(); - protected EdgeFilter getSnapFilter() { + protected EdgeFilter createSnapFilter() { return new DefaultSnapFilter(weighting, lookup.getBooleanEncodedValue(Subnetwork.key(profile.getName()))); } + protected EdgeFilter createDirectedSnapFilter() { + BooleanEncodedValue inSubnetworkEnc = lookup.getBooleanEncodedValue(Subnetwork.key(profile.getName())); + return edgeState -> !edgeState.get(inSubnetworkEnc) && Double.isFinite(weighting.calcEdgeWeightWithAccess(edgeState, false)); + } + protected abstract PathCalculator createPathCalculator(QueryGraph queryGraph); private List getTurnCostProfiles() { diff --git a/core/src/main/java/com/graphhopper/routing/ViaRouting.java b/core/src/main/java/com/graphhopper/routing/ViaRouting.java index 6966dd1e5b8..2cc67a77e14 100644 --- a/core/src/main/java/com/graphhopper/routing/ViaRouting.java +++ b/core/src/main/java/com/graphhopper/routing/ViaRouting.java @@ -24,7 +24,7 @@ import com.graphhopper.routing.ev.RoadEnvironment; import com.graphhopper.routing.querygraph.QueryGraph; import com.graphhopper.routing.util.EdgeFilter; -import com.graphhopper.routing.util.FiniteWeightFilter; +import com.graphhopper.routing.util.HeadingEdgeFilter; import com.graphhopper.routing.util.NameSimilarityEdgeFilter; import com.graphhopper.routing.util.SnapPreventionEdgeFilter; import com.graphhopper.routing.weighting.Weighting; @@ -54,27 +54,36 @@ public class ViaRouting { /** * @throws MultiplePointsNotFoundException in case one or more points could not be resolved */ - public static List lookup(EncodedValueLookup lookup, List points, EdgeFilter edgeFilter, LocationIndex locationIndex, List snapPreventions, List pointHints) { + public static List lookup(EncodedValueLookup lookup, List points, EdgeFilter snapFilter, + LocationIndex locationIndex, List snapPreventions, List pointHints, + EdgeFilter directedSnapFilter, List headings) { if (points.size() < 2) throw new IllegalArgumentException("At least 2 points have to be specified, but was:" + points.size()); final EnumEncodedValue roadClassEnc = lookup.getEnumEncodedValue(RoadClass.KEY, RoadClass.class); final EnumEncodedValue roadEnvEnc = lookup.getEnumEncodedValue(RoadEnvironment.KEY, RoadEnvironment.class); EdgeFilter strictEdgeFilter = snapPreventions.isEmpty() - ? edgeFilter - : new SnapPreventionEdgeFilter(edgeFilter, roadClassEnc, roadEnvEnc, snapPreventions); + ? snapFilter + : new SnapPreventionEdgeFilter(snapFilter, roadClassEnc, roadEnvEnc, snapPreventions); List snaps = new ArrayList<>(points.size()); IntArrayList pointsNotFound = new IntArrayList(); for (int placeIndex = 0; placeIndex < points.size(); placeIndex++) { GHPoint point = points.get(placeIndex); Snap snap = null; - if (!pointHints.isEmpty()) + if (placeIndex < headings.size() && !Double.isNaN(headings.get(placeIndex))) { + if (!pointHints.isEmpty() && !Helper.isEmpty(pointHints.get(placeIndex))) + throw new IllegalArgumentException("Cannot specify heading and point_hint at the same time. " + + "Make sure you specify either an empty point_hint (String) or a NaN heading (double) for point " + placeIndex); + snap = locationIndex.findClosest(point.lat, point.lon, new HeadingEdgeFilter(directedSnapFilter, headings.get(placeIndex), point)); + } else if (!pointHints.isEmpty()) { snap = locationIndex.findClosest(point.lat, point.lon, new NameSimilarityEdgeFilter(strictEdgeFilter, pointHints.get(placeIndex), point, 100)); - else if (!snapPreventions.isEmpty()) + } else if (!snapPreventions.isEmpty()) { snap = locationIndex.findClosest(point.lat, point.lon, strictEdgeFilter); + } + if (snap == null || !snap.isValid()) - snap = locationIndex.findClosest(point.lat, point.lon, edgeFilter); + snap = locationIndex.findClosest(point.lat, point.lon, snapFilter); if (!snap.isValid()) pointsNotFound.add(placeIndex); diff --git a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java new file mode 100644 index 00000000000..a6b986fe5ad --- /dev/null +++ b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java @@ -0,0 +1,68 @@ +package com.graphhopper.routing.util; + +import com.graphhopper.util.*; +import com.graphhopper.util.shapes.GHPoint; + +public class HeadingEdgeFilter implements EdgeFilter { + + private final double heading; + private final EdgeFilter directedEdgeFilter; + private final GHPoint pointNearHeading; + + public HeadingEdgeFilter(EdgeFilter directedEdgeFilter, double heading, GHPoint pointNearHeading) { + this.directedEdgeFilter = directedEdgeFilter; + this.heading = heading; + this.pointNearHeading = pointNearHeading; + } + + @Override + public boolean accept(EdgeIteratorState edgeState) { + final double tolerance = 30; + // we only accept edges that are not too far away. It might happen that only far away edges match the heading + // in which case we rather rely on the fallback snapping than return a match here. + final double maxDistance = 20; + double headingOfEdge = getHeadingOfGeometryNearPoint(edgeState, pointNearHeading, maxDistance); + if (Double.isNaN(headingOfEdge)) + // this edge is too far away. we do not accept it. + return false; + // we accept the edge if either of the two directions roughly has the right heading + return Math.abs(headingOfEdge - heading) < tolerance && directedEdgeFilter.accept(edgeState) || + Math.abs((headingOfEdge + 180) % 360 - heading) < tolerance && directedEdgeFilter.accept(edgeState.detach(true)); + } + + /** + * Calculates the heading (in degrees) of the given edge in fwd direction near the given point. If the point is + * too far away from the edge (according to the maxDistance parameter) it returns Double.NaN. + */ + static double getHeadingOfGeometryNearPoint(EdgeIteratorState edgeState, GHPoint point, double maxDistance) { + final DistanceCalc calcDist = DistanceCalcEarth.DIST_EARTH; + double closestDistance = Double.POSITIVE_INFINITY; + PointList points = edgeState.fetchWayGeometry(FetchMode.ALL); + int closestPoint = -1; + for (int i = 1; i < points.size(); i++) { + double fromLat = points.getLat(i - 1), fromLon = points.getLon(i - 1); + double toLat = points.getLat(i), toLon = points.getLon(i); + // the 'distance' between the point and an edge segment is either the vertical distance to the segment or + // the distance to the closer one of the two endpoints. here we save one call to calcDist per segment, + // because each endpoint appears in two segments (except the first and last). + double distance = calcDist.validEdgeDistance(point.lat, point.lon, fromLat, fromLon, toLat, toLon) + ? calcDist.calcDenormalizedDist(calcDist.calcNormalizedEdgeDistance(point.lat, point.lon, fromLat, fromLon, toLat, toLon)) + : calcDist.calcDist(fromLat, fromLon, point.lat, point.lon); + if (i == points.size() - 1) + distance = Math.min(distance, calcDist.calcDist(toLat, toLon, point.lat, point.lon)); + if (distance > maxDistance) + continue; + if (distance < closestDistance) { + closestDistance = distance; + closestPoint = i; + } + } + if (closestPoint < 0) + return Double.NaN; + + double fromLat = points.getLat(closestPoint - 1), fromLon = points.getLon(closestPoint - 1); + double toLat = points.getLat(closestPoint), toLon = points.getLon(closestPoint); + // calcOrientation returns value relative to East, but heading is relative to North + return (Math.toDegrees(AngleCalc.ANGLE_CALC.calcOrientation(fromLat, fromLon, toLat, toLon)) + 90) % 360; + } +} diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index 6bcc4759762..f68fe4cc2c6 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -230,7 +230,7 @@ public void testUTurn() { request.addPoint(new GHPoint(43.743887, 7.431151)); request.addPoint(new GHPoint(43.744007, 7.431076)); //Force initial U-Turn - request.setHeadings(Arrays.asList(200., Double.NaN)); + request.setHeadings(Arrays.asList(200.)); request.setAlgorithm(ASTAR).setProfile(profile); GHResponse rsp = hopper.route(request); diff --git a/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java b/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java index 37b037a10c5..23d911f108a 100644 --- a/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java +++ b/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java @@ -155,7 +155,7 @@ public void headingTest5() { GHPoint via = new GHPoint(0.000, 0.0015); GHRequest req = new GHRequest(). setPoints(Arrays.asList(start, via, end)). - setHeadings(Arrays.asList(0., 3.14 / 2, Double.NaN)). + setHeadings(Arrays.asList(0., 90., Double.NaN)). setProfile("profile"). setPathDetails(Collections.singletonList("edge_key")); req.putHint(Parameters.Routing.PASS_THROUGH, true); @@ -164,6 +164,96 @@ public void headingTest5() { assertArrayEquals(new int[]{5, 4, 3, 8, 7, 6, 5, 4, 3, 2}, calcNodes(graph, response.getAll().get(0))); } + @Test + public void testHeadingWithSnapFilter() { + GraphHopperStorage graph = createSquareGraphWithTunnel(); + Router router = createRouter(graph); + // Start at 8 (slightly north to make it independent on some edge ordering and always use 8-3 or 3-8 as fallback) + GHPoint start = new GHPoint(0.0011, 0.001); + // End at middle of edge 2-3 + GHPoint end = new GHPoint(0.002, 0.0005); + + // no heading + GHRequest req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + GHResponse response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + // same start + end but heading=0, parallel to 3-8-7 + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(0.)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + // heading=90 parallel to 1->5 + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(90., Double.NaN)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{1, 5, 4, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + for (double angle = 0; angle < 360; angle += 10) { + // Ignore angles nearly parallel to 1->5. I.e. it should fallback to results with 8-3.. or 3-8.. + if (angle >= 60 && angle <= 120) continue; + + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(angle, Double.NaN)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + + int[] expectedNodes = (angle >= 130 && angle <= 250) ? new int[]{3, 8, 7, 0, 1, 2, 3} : new int[]{8, 3, 2}; + // System.out.println(Arrays.toString(calcNodes(graph, response.getAll().get(0))) + " angle:" + angle); + assertArrayEquals(expectedNodes, calcNodes(graph, response.getAll().get(0)), "angle: " + angle); + } + } + + @Test + public void testHeadingWithSnapFilter2() { + GraphHopperStorage graph = createSquareGraphWithTunnel(); + Router router = createRouter(graph); + // Start at 8 (slightly east to snap to edge 1->5 per default) + GHPoint start = new GHPoint(0.001, 0.0011); + // End at middle of edge 2-3 + GHPoint end = new GHPoint(0.002, 0.0005); + + GHRequest req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setHeadings(Arrays.asList(0.)). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + GHResponse response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setHeadings(Arrays.asList(180.)). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + } + @Test public void headingTest6() { // Test if snaps at tower nodes are ignored @@ -199,11 +289,11 @@ private GraphHopperStorage createSquareGraph() { EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).add(Subnetwork.create("profile")).build(); GraphHopperStorage g = new GraphBuilder(encodingManager).create(); - // 2---3---4 - // / | \ - // 1----8----5 - // / | / - // 0----7---6 + // 2---3---4 + // | | | + // 1---8---5 + // | | | + // 0---7---6 NodeAccess na = g.getNodeAccess(); na.setNode(0, 0.000, 0.000); na.setNode(1, 0.001, 0.000); @@ -232,6 +322,43 @@ private GraphHopperStorage createSquareGraph() { return g; } + private GraphHopperStorage createSquareGraphWithTunnel() { + CarFlagEncoder carEncoder = new CarFlagEncoder(); + EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).add(Subnetwork.create("profile")).build(); + GraphHopperStorage g = new GraphBuilder(encodingManager).create(); + + // 2----3---4 + // | | | + // 1->- 8 >-5 (edge 1->5 is not connected to 8) + // | | | + // 0----7---6 + NodeAccess na = g.getNodeAccess(); + na.setNode(0, 0.000, 0.000); + na.setNode(1, 0.001, 0.000); + na.setNode(2, 0.002, 0.000); + na.setNode(3, 0.002, 0.001); + na.setNode(4, 0.002, 0.002); + na.setNode(5, 0.001, 0.002); + na.setNode(6, 0.000, 0.002); + na.setNode(7, 0.000, 0.001); + na.setNode(8, 0.001, 0.001); + + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(0, 1).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(1, 2).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(2, 3).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(3, 4).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(4, 5).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(5, 6).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(6, 7).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(7, 0).setDistance(100)); + + GHUtility.setSpeed(60, true, false, carEncoder, g.edge(1, 5).setDistance(110)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(3, 8).setDistance(110)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(7, 8).setDistance(110)); + + return g; + } + private int[] calcNodes(Graph graph, ResponsePath responsePath) { List edgeKeys = responsePath.getPathDetails().get("edge_key"); int[] result = new int[edgeKeys.size() + 1]; From 2c4351ce07c821ce688516e6ebbfa81975e0b6de Mon Sep 17 00:00:00 2001 From: Andi Date: Fri, 1 Oct 2021 12:10:13 +0200 Subject: [PATCH 07/14] Convert `Directory#find` to `Directory#create` (#2412) * Make sure GHDirectory#find only uses each name once * Rename Directory#find -> Directory#create --- .../java/com/graphhopper/coll/OSMIDMap.java | 4 +- .../com/graphhopper/reader/PillarInfo.java | 2 +- .../dem/AbstractSRTMElevationProvider.java | 2 +- .../dem/AbstractTiffElevationProvider.java | 2 +- .../routing/lm/LandmarkStorage.java | 9 ++++- .../routing/subnetwork/SubnetworkStorage.java | 2 +- .../com/graphhopper/search/StringIndex.java | 4 +- .../com/graphhopper/storage/BaseGraph.java | 4 +- .../storage/BaseGraphNodesAndEdges.java | 4 +- .../com/graphhopper/storage/CHStorage.java | 4 +- .../com/graphhopper/storage/Directory.java | 9 +---- .../com/graphhopper/storage/GHDirectory.java | 37 ++++--------------- .../storage/StorableProperties.java | 2 +- .../storage/index/LineIntIndex.java | 2 +- .../storage/index/LocationIndexTree.java | 25 ++++++------- .../com/graphhopper/coll/OSMIDMapTest.java | 2 +- .../reader/dem/HeightTileTest.java | 6 +-- .../routing/lm/LandmarkStorageTest.java | 6 +-- .../storage/AbstractDirectoryTester.java | 10 ++--- .../storage/AbstractGraphStorageTester.java | 1 + .../com/graphhopper/gtfs/GraphHopperGtfs.java | 24 +++++++----- 21 files changed, 71 insertions(+), 90 deletions(-) diff --git a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java b/core/src/main/java/com/graphhopper/coll/OSMIDMap.java index 079bf2c99a5..08f8868def5 100644 --- a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java +++ b/core/src/main/java/com/graphhopper/coll/OSMIDMap.java @@ -45,9 +45,9 @@ public OSMIDMap(Directory dir) { public OSMIDMap(Directory dir, int noNumber) { this.dir = dir; this.noEntryValue = noNumber; - keys = dir.find("osmid_map_keys"); + keys = dir.create("osmid_map_keys"); keys.create(2000); - values = dir.find("osmid_map_values"); + values = dir.create("osmid_map_values"); values.create(1000); } diff --git a/core/src/main/java/com/graphhopper/reader/PillarInfo.java b/core/src/main/java/com/graphhopper/reader/PillarInfo.java index 81ec1f8c893..fd2e3eb221d 100644 --- a/core/src/main/java/com/graphhopper/reader/PillarInfo.java +++ b/core/src/main/java/com/graphhopper/reader/PillarInfo.java @@ -38,7 +38,7 @@ public class PillarInfo implements PointAccess { public PillarInfo(boolean enabled3D, Directory dir) { this.enabled3D = enabled3D; this.dir = dir; - this.da = dir.find("tmp_pillar_info").create(100); + this.da = dir.create("tmp_pillar_info").create(100); this.rowSizeInBytes = getDimension() * 4; } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 3a5e1075755..106d586893f 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -102,7 +102,7 @@ public double getEle(double lat, double lon) { if (fileName == null) return 0; - DataAccess heights = getDirectory().find("dem" + intKey); + DataAccess heights = getDirectory().create("dem" + intKey); boolean loadExisting = false; try { loadExisting = heights.loadExisting(); diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index bf0a491a71b..2d16b527c11 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -117,7 +117,7 @@ public double getEle(double lat, double lon) { demProvider.setInterpolate(interpolate); cacheData.put(name, demProvider); - DataAccess heights = getDirectory().find(name + ".gh"); + DataAccess heights = getDirectory().create(name + ".gh"); demProvider.setHeights(heights); boolean loadExisting = false; try { diff --git a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java index 17f4e1c0fa3..b0ef05e67af 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java @@ -132,7 +132,7 @@ public String toString() { // use the node based traversal as this is a smaller weight approximation and will still produce correct results // In this sense its even 'better' to use node-based. this.traversalMode = TraversalMode.NODE_BASED; - this.landmarkWeightDA = dir.find("landmarks_" + lmConfig.getName()); + this.landmarkWeightDA = dir.create("landmarks_" + lmConfig.getName()); this.landmarks = landmarks; // one short per landmark and two directions => 2*2 byte @@ -759,6 +759,13 @@ private LandmarkExplorer findLandmarks(int[] landmarkNodeIdsToReturn, int startN return explorer; } + /** + * For testing only + */ + DataAccess _getInternalDA() { + return landmarkWeightDA; + } + /** * This class is used to calculate landmark location (equally distributed). * It derives from DijkstraBidirectionRef, but is only used as forward or backward search. diff --git a/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java b/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java index 18e107f6e6a..703bba5d36e 100644 --- a/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java +++ b/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java @@ -32,7 +32,7 @@ public class SubnetworkStorage { public SubnetworkStorage(Directory dir, String postfix) { DAType type = dir.getDefaultType(); - da = dir.find("subnetwork_" + postfix, type.isMMap() ? DAType.MMAP : (type.isStoring() ? DAType.RAM_STORE : DAType.RAM)); + da = dir.create("subnetwork_" + postfix, type.isMMap() ? DAType.MMAP : (type.isStoring() ? DAType.RAM_STORE : DAType.RAM)); } /** diff --git a/core/src/main/java/com/graphhopper/search/StringIndex.java b/core/src/main/java/com/graphhopper/search/StringIndex.java index 50fab782462..381082bd07b 100644 --- a/core/src/main/java/com/graphhopper/search/StringIndex.java +++ b/core/src/main/java/com/graphhopper/search/StringIndex.java @@ -58,9 +58,9 @@ public StringIndex(Directory dir) { * Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object. */ public StringIndex(Directory dir, final int cacheSize) { - keys = dir.find("string_index_keys"); + keys = dir.create("string_index_keys"); keys.setSegmentSize(10 * 1024); - vals = dir.find("string_index_vals"); + vals = dir.create("string_index_vals"); smallCache = new LinkedHashMap(cacheSize, 0.75f, true) { @Override protected boolean removeEldestEntry(Map.Entry entry) { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 6eda12fd12c..0456f4181e4 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -58,12 +58,12 @@ class BaseGraph implements Graph { public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; this.bitUtil = BitUtil.get(dir.getByteOrder()); - this.wayGeometry = dir.find("geometry"); + this.wayGeometry = dir.create("geometry"); this.stringIndex = new StringIndex(dir); this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); this.nodeAccess = new GHNodeAccess(store); if (withTurnCosts) { - turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs")); + turnCostStorage = new TurnCostStorage(this, dir.create("turn_costs")); if (segmentSize >= 0) turnCostStorage.setSegmentSize(segmentSize); } else { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java index f2e647c5921..178bfbcaa83 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -61,8 +61,8 @@ class BaseGraphNodesAndEdges { private boolean frozen; public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { - nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); - edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); + nodes = dir.create("nodes", DAType.getPreferredInt(dir.getDefaultType())); + edges = dir.create("edges", DAType.getPreferredInt(dir.getDefaultType())); if (segmentSize >= 0) { nodes.setSegmentSize(segmentSize); edges.setSegmentSize(segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index b59ad6849f1..7f30c18480f 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -66,8 +66,8 @@ public class CHStorage { public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased) { this.edgeBased = edgeBased; - this.nodesCH = dir.find("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType())); - this.shortcuts = dir.find("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType())); + this.nodesCH = dir.create("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType())); + this.shortcuts = dir.create("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType())); if (segmentSize >= 0) { nodesCH.setSegmentSize(segmentSize); shortcuts.setSegmentSize(segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index 6ae6d633755..8811378ede0 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -43,9 +43,9 @@ public interface Directory { * Tries to find the object with that name if not existent it creates one and associates the * location with it. A name is unique in one Directory. */ - DataAccess find(String name); + DataAccess create(String name); - DataAccess find(String name, DAType type); + DataAccess create(String name, DAType type); /** * Removes the specified object from the directory. @@ -67,10 +67,5 @@ public interface Directory { */ void close(); - /** - * Returns all created directories. - */ - Collection getAll(); - Directory create(); } diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java index 7934a629bfe..9c0c051f486 100644 --- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java +++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java @@ -19,7 +19,6 @@ import java.io.File; import java.nio.ByteOrder; -import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -35,7 +34,6 @@ public class GHDirectory implements Directory { private final DAType defaultType; private final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; protected Map map = new HashMap<>(); - protected Map types = new HashMap<>(); public GHDirectory(String _location, DAType defaultType) { this.defaultType = defaultType; @@ -56,36 +54,22 @@ public ByteOrder getByteOrder() { return byteOrder; } - public Directory put(String name, DAType type) { - if (!name.equals(toLowerCase(name))) - throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); - - types.put(name, type); - return this; - } - @Override - public DataAccess find(String name) { - DAType type = types.get(name); - if (type == null) - type = defaultType; - - return find(name, type); + public DataAccess create(String name) { + return create(name, defaultType); } @Override - public DataAccess find(String name, DAType type) { + public DataAccess create(String name, DAType type) { if (!name.equals(toLowerCase(name))) throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); - DataAccess da = map.get(name); - if (da != null) { - if (!type.equals(da.getType())) - throw new IllegalStateException("Found existing DataAccess object '" + name - + "' but types did not match. Requested:" + type + ", was:" + da.getType()); - return da; - } + if (map.containsKey(name)) + // we do not allow creating two DataAccess with the same name, because on disk there can only be one DA + // per file name + throw new IllegalStateException("DataAccess " + name + " has already been created"); + DataAccess da; if (type.isInMemory()) { if (type.isInteg()) { if (type.isStoring()) @@ -154,11 +138,6 @@ public Directory create() { return this; } - @Override - public Collection getAll() { - return map.values(); - } - @Override public String toString() { return getLocation(); diff --git a/core/src/main/java/com/graphhopper/storage/StorableProperties.java b/core/src/main/java/com/graphhopper/storage/StorableProperties.java index e798ca75b49..39c3a93c291 100644 --- a/core/src/main/java/com/graphhopper/storage/StorableProperties.java +++ b/core/src/main/java/com/graphhopper/storage/StorableProperties.java @@ -41,7 +41,7 @@ public class StorableProperties { private final DataAccess da; public StorableProperties(Directory dir) { - this.da = dir.find("properties"); + this.da = dir.create("properties"); // reduce size da.setSegmentSize(1 << 15); } diff --git a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java index 4237dc38fe6..6e0e3a8657f 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java +++ b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java @@ -49,7 +49,7 @@ public class LineIntIndex { public LineIntIndex(BBox bBox, Directory dir, String name) { this.bounds = bBox; - this.dataAccess = dir.find(name, DAType.getPreferredInt(dir.getDefaultType())); + this.dataAccess = dir.create(name, DAType.getPreferredInt(dir.getDefaultType())); } public boolean loadExisting() { diff --git a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java index ef1fe4f7f4d..7a19325d715 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java +++ b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java @@ -76,6 +76,18 @@ public LocationIndexTree(Graph g, Directory dir) { this.graph = g; this.nodeAccess = g.getNodeAccess(); this.directory = dir; + + // Clone this defensively -- In case something funny happens and things get added to the Graph after + // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we + // need it to be immutable. + BBox bounds = graph.getBounds().clone(); + + // I want to be able to create a location index for the empty graph without error, but for that + // I need valid bounds so that the initialization logic works. + if (!bounds.isValid()) + bounds = new BBox(-10.0, 10.0, -10.0, 10.0); + + lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); } public int getMinResolutionInMeter() { @@ -116,18 +128,6 @@ public LocationIndex setResolution(int minResolutionInMeter) { } public boolean loadExisting() { - // Clone this defensively -- In case something funny happens and things get added to the Graph after - // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we - // need it to be immutable. - BBox bounds = graph.getBounds().clone(); - - // I want to be able to create a location index for the empty graph without error, but for that - // I need valid bounds so that the initialization logic works. - if (!bounds.isValid()) - bounds = new BBox(-10.0,10.0,-10.0,10.0); - - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); - if (!lineIntIndex.loadExisting()) return false; @@ -166,7 +166,6 @@ public LocationIndex prepareIndex(EdgeFilter edgeFilter) { InMemConstructionIndex inMemConstructionIndex = prepareInMemConstructionIndex(bounds, edgeFilter); - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); lineIntIndex.setMinResolutionInMeter(minResolutionInMeter); lineIntIndex.store(inMemConstructionIndex); lineIntIndex.setChecksum(checksum()); diff --git a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java b/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java index fa7128f4f8f..d035a0de0bc 100644 --- a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java +++ b/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java @@ -59,7 +59,7 @@ public void testGet() { @Test public void testBinSearch() { - DataAccess da = new RAMDirectory().find(""); + DataAccess da = new RAMDirectory().create(""); da.create(100); da.setInt(0 * 4, 1); diff --git a/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java b/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java index a6bb09b5bd7..02a70c42f99 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java @@ -34,7 +34,7 @@ public void testGetHeight() { int width = 10; int height = 20; HeightTile instance = new HeightTile(0, 0, width, height, 1e-6, 10, 20); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * width * height); instance.setHeights(heights); init(heights, width, height, 1); @@ -77,7 +77,7 @@ public void testGetHeight() { public void testGetHeightForNegativeTile() { int width = 10; HeightTile instance = new HeightTile(-20, -20, width, width, 1e-6, 10, 10); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * 10 * 10); instance.setHeights(heights); init(heights, width, width, 1); @@ -98,7 +98,7 @@ public void testGetHeightForNegativeTile() { @Test public void testInterpolate() { HeightTile instance = new HeightTile(0, 0, 2, 2, 1e-6, 10, 10).setInterpolate(true); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * 2 * 2); instance.setHeights(heights); double topLeft = 0; diff --git a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java index bcdf8fb544c..89fbc7130da 100644 --- a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java +++ b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java @@ -95,11 +95,9 @@ public double calcEdgeWeight(EdgeIteratorState edgeState, boolean reverse) { public void testSetGetWeight() { GHUtility.setSpeed(60, true, true, encoder, graph.edge(0, 1).setDistance(40.1)); Directory dir = new RAMDirectory(); - DataAccess da = dir.find("landmarks_c1"); - da.create(2000); - LandmarkStorage lms = new LandmarkStorage(graph, dir, new LMConfig("c1", new FastestWeighting(encoder)), 4). setMaximumWeight(LandmarkStorage.PRECISION); + lms._getInternalDA().create(2000); // 2^16=65536, use -1 for infinity and -2 for maximum lms.setWeight(0, 65536); // reached maximum value but do not reset to 0 instead use 2^16-2 @@ -109,7 +107,7 @@ public void testSetGetWeight() { lms.setWeight(0, 79999); assertEquals(65534, lms.getFromWeight(0, 0)); - da.setInt(0, Integer.MAX_VALUE); + lms._getInternalDA().setInt(0, Integer.MAX_VALUE); assertTrue(lms.isInfinity(0)); // for infinity return much bigger value // assertEquals(Integer.MAX_VALUE, lms.getFromWeight(0, 0)); diff --git a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java index 345225df6b9..cc2d6eda0dd 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java @@ -24,7 +24,7 @@ import java.io.File; -import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author Peter Karich @@ -50,17 +50,15 @@ public void setUp() { @Test public void testNoDuplicates() { Directory dir = createDir(); - DataAccess da1 = dir.find("testing"); - DataAccess da2 = dir.find("testing"); - assertSame(da1, da2); + DataAccess da1 = dir.create("testing"); + assertThrows(IllegalStateException.class, () -> dir.create("testing")); da1.close(); - da2.close(); } @Test public void testNoErrorForDACreate() { Directory dir = createDir(); - da = dir.find("testing"); + da = dir.create("testing"); da.create(100); da.flush(); } diff --git a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java index 257fd25ad98..60b1f6ad127 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java @@ -677,6 +677,7 @@ public String toString() { assertEquals(Integer.MAX_VALUE / 3, edge.getFlags().ints[0]); graph.close(); + dir = new RAMDirectory(); graph = new GraphHopperStorage(dir, manager, false).create(defaultSize); DecimalEncodedValue avSpeed0Enc = manager.getDecimalEncodedValue(getKey("car0", "average_speed")); diff --git a/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java b/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java index 0b108b21cb6..4bce6490c92 100644 --- a/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java +++ b/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java @@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.*; @@ -66,16 +68,15 @@ protected void importOSM() { @Override protected LocationIndex createLocationIndex(Directory dir) { - LocationIndexTree tmpIndex = new LocationIndexTree(getGraphHopperStorage(), dir); - if (tmpIndex.loadExisting()) { - return tmpIndex; - } else { - LocationIndexTree locationIndexTree = new LocationIndexTree(getGraphHopperStorage(), new RAMDirectory()); - if (!locationIndexTree.loadExisting()) { - locationIndexTree.prepareIndex(); - } - return locationIndexTree; + // if the location index was already created (we are 'loading') we use it. but we must not create the location + // index object in case the index does not exist yet, because we only can create it once. we are not ready yet, + // because first we need to import PT. + if (Files.exists(Paths.get(getGraphHopperLocation()).resolve("location_index"))) { + LocationIndexTree index = new LocationIndexTree(getGraphHopperStorage(), dir); + index.loadExisting(); + return index; } + return null; } static class TransferWithTime { @@ -91,7 +92,9 @@ protected void importPublicTransit() { ensureWriteAccess(); getGtfsStorage().create(); GraphHopperStorage graphHopperStorage = getGraphHopperStorage(); - LocationIndex streetNetworkIndex = getLocationIndex(); + // temporary location index for the street network that we only use during import + LocationIndexTree streetNetworkIndex = new LocationIndexTree(getGraphHopperStorage(), new RAMDirectory()); + streetNetworkIndex.prepareIndex(); try { int idx = 0; List gtfsFiles = ghConfig.has("gtfs.file") ? Arrays.asList(ghConfig.getString("gtfs.file", "").split(",")) : Collections.emptyList(); @@ -125,6 +128,7 @@ protected void importPublicTransit() { throw new RuntimeException("Error while constructing transit network. Is your GTFS file valid? Please check log for possible causes.", e); } streetNetworkIndex.close(); + // now we build the final location index LocationIndexTree locationIndex = new LocationIndexTree(getGraphHopperStorage(), getGraphHopperStorage().getDirectory()); PtEncodedValues ptEncodedValues = PtEncodedValues.fromEncodingManager(getEncodingManager()); EnumEncodedValue typeEnc = ptEncodedValues.getTypeEnc(); From 9c184bb42fc15ccdfe202100e9d21167393948b9 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 12:24:38 +0200 Subject: [PATCH 08/14] Remove small segment size optimization for sea level elevation tiles --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 106d586893f..68f46b0552d 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10) + .flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 2d16b527c11..8bdc8c84973 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,8 +135,7 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10).flush(); return 0; } From 0b703cc75ebaa6a5269fd69b061f697cf7510715 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 12:25:17 +0200 Subject: [PATCH 09/14] Revert "Remove small segment size optimization for sea level elevation tiles" This reverts commit 9c184bb42fc15ccdfe202100e9d21167393948b9. --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 68f46b0552d..106d586893f 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.create(10) - .flush(); + heights.setSegmentSize(100).create(10). + flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 8bdc8c84973..2d16b527c11 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,7 +135,8 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.create(10).flush(); + heights.setSegmentSize(100).create(10). + flush(); return 0; } From 44ff428e100e441ac3fea83c18ea2c3d0d69c378 Mon Sep 17 00:00:00 2001 From: otbutz Date: Fri, 1 Oct 2021 12:50:03 +0200 Subject: [PATCH 10/14] Register country rules in a map (#2418) Co-authored-by: Thomas Butz --- .../util/countryrules/AustriaCountryRule.java | 1 - .../util/countryrules/CountryRuleFactory.java | 26 +++++++++++++------ .../util/countryrules/GermanyCountryRule.java | 1 - 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java index ae51cfb1970..e214ac0f8ad 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java @@ -24,7 +24,6 @@ import com.graphhopper.routing.util.TransportationMode; public class AustriaCountryRule implements CountryRule { - public final static AustriaCountryRule RULE = new AustriaCountryRule(); @Override public double getMaxSpeed(ReaderWay readerWay, TransportationMode transportationMode, double currentMaxSpeed) { diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java index 2ccd8a2ccbc..7a68a6dbfd1 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java @@ -18,18 +18,28 @@ package com.graphhopper.routing.util.countryrules; + +import static com.graphhopper.routing.ev.Country.*; + +import java.util.EnumMap; +import java.util.Map; + import com.graphhopper.routing.ev.Country; public class CountryRuleFactory { + + private final Map rules = new EnumMap<>(Country.class); + + public CountryRuleFactory() { + rules.put(AUT, new AustriaCountryRule()); + rules.put(DEU, new GermanyCountryRule()); + } public CountryRule getCountryRule(Country country) { - switch (country) { - case DEU: - return GermanyCountryRule.RULE; - case AUT: - return AustriaCountryRule.RULE; - default: - return null; - } + return rules.get(country); + } + + public Map getCountryToRuleMap() { + return rules; } } diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java index 8c4adcee1e4..d78c800da9d 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java @@ -28,7 +28,6 @@ * @author Robin Boldt */ public class GermanyCountryRule implements CountryRule { - public final static GermanyCountryRule RULE = new GermanyCountryRule(); /** * In Germany there are roads without a speed limit. For these roads, this method From db214409e72857b7ee6a72e98f1b085483269ef3 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 14:04:50 +0200 Subject: [PATCH 11/14] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f72125b27..97005f16e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ### 5.0 [not yet released] +- consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) + ### 4.0 [29 Sep 2021] - faster node-based CH preparation (~20%), (#2390) From f7397fdceb82bdfa679318e2447c7cd6032d074e Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 4 Oct 2021 10:40:36 +0200 Subject: [PATCH 12/14] use jdk17 ga and jdk18 ea (#2419) * use jdk17 ga and jdk18 ea * adoptopenjdk is adoptium and I cannot find ea versions, so use the default ones (sormuras/bach -> install-jdk.properties) * since jdk17 locale IDs changed --- .travis.yml | 6 +++--- .../com/graphhopper/util/TranslationMap.java | 20 +++++++++++-------- .../graphhopper/util/TranslationMapTest.java | 8 ++++++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98fe5bdd04d..afa1b626fff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,10 @@ env: matrix: include: - jdk: openjdk8 - - env: JDK='OpenJDK 16' - install: . ./install-jdk.sh -F 16 -C --url 'https://api.adoptopenjdk.net/v3/binary/latest/16/ga/linux/x64/jdk/hotspot/normal/adoptopenjdk' - env: JDK='OpenJDK 17' - install: . ./install-jdk.sh -F 17 -C --url 'https://api.adoptopenjdk.net/v3/binary/latest/17/ea/linux/x64/jdk/hotspot/normal/adoptopenjdk' + install: . ./install-jdk.sh -F 17 -C + - env: JDK='OpenJDK 18' + install: . ./install-jdk.sh -F ea -C # avoid default dependency command for maven, 'true' means 'return true' and continue install: true diff --git a/core/src/main/java/com/graphhopper/util/TranslationMap.java b/core/src/main/java/com/graphhopper/util/TranslationMap.java index bd5052ae9e9..8fb172603d6 100644 --- a/core/src/main/java/com/graphhopper/util/TranslationMap.java +++ b/core/src/main/java/com/graphhopper/util/TranslationMap.java @@ -85,14 +85,18 @@ public void add(Translation tr) { if (!locale.getCountry().isEmpty() && !translations.containsKey(tr.getLanguage())) translations.put(tr.getLanguage(), tr); - // Map old Java 'standard' to latest, Java is a bit ugly here: http://stackoverflow.com/q/13974169/194609 - // Hebrew - if ("iw".equals(locale.getLanguage())) - translations.put("he", tr); - - // Indonesia - if ("in".equals(locale.getLanguage())) - translations.put("id", tr); + // Hebrew locale was "iw" in old JDKs but is now he + // required in old JDKs: + if ("iw".equals(locale.getLanguage())) translations.put("he", tr); + // required since jdk17 to still provide translation for "iw": + if ("he".equals(locale.getLanguage())) translations.put("iw", tr); + + // Indonesia locale was "in_ID" in old JDKs but is now id_ID + // required in old JDKs: + if ("in".equals(locale.getLanguage())) translations.put("id", tr); + // required since jdk17 to still provide translation for "in": + if ("id".equals(locale.getLanguage())) translations.put("in", tr); + // Indian locales are: en-IN and hi-IN and are not overwritten by that } /** diff --git a/core/src/test/java/com/graphhopper/util/TranslationMapTest.java b/core/src/test/java/com/graphhopper/util/TranslationMapTest.java index 4ce2d78b6f2..1922c08f878 100644 --- a/core/src/test/java/com/graphhopper/util/TranslationMapTest.java +++ b/core/src/test/java/com/graphhopper/util/TranslationMapTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Locale; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -60,8 +61,11 @@ public void testToString() { assertEquals("רגל", trMap.tr("web.FOOT")); // Indonesian - assertEquals("in", SINGLETON.get("in").getLanguage()); - assertEquals("in", SINGLETON.get("in_ID").getLanguage()); + // for jdk17 and later "id" is returned, before "in" was returned + String lang = SINGLETON.get("id").getLanguage(); + assertTrue(Arrays.asList("id", "in").contains(lang)); + assertEquals(lang, SINGLETON.get("in").getLanguage()); + assertEquals(lang, SINGLETON.get("in_ID").getLanguage()); // Vietnamese assertEquals("vi", SINGLETON.get("vi").getLanguage()); From 3073c0adf1e194f12b29f5b6336af14fae24ada9 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 4 Oct 2021 10:32:05 +0200 Subject: [PATCH 13/14] fix orientation calculation for #2411 #1463 --- .../routing/util/HeadingEdgeFilter.java | 3 +-- .../java/com/graphhopper/GraphHopperTest.java | 6 ++--- .../routing/util/HeadingEdgeFilterTest.java | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java diff --git a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java index a6b986fe5ad..74ff0aed256 100644 --- a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java +++ b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java @@ -62,7 +62,6 @@ static double getHeadingOfGeometryNearPoint(EdgeIteratorState edgeState, GHPoint double fromLat = points.getLat(closestPoint - 1), fromLon = points.getLon(closestPoint - 1); double toLat = points.getLat(closestPoint), toLon = points.getLon(closestPoint); - // calcOrientation returns value relative to East, but heading is relative to North - return (Math.toDegrees(AngleCalc.ANGLE_CALC.calcOrientation(fromLat, fromLon, toLat, toLon)) + 90) % 360; + return AngleCalc.ANGLE_CALC.calcAzimuth(fromLat, fromLon, toLat, toLon); } } diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index f68fe4cc2c6..214109c2877 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -238,12 +238,12 @@ public void testUTurn() { assertFalse(rsp.hasErrors()); ResponsePath res = rsp.getBest(); InstructionList il = res.getInstructions(); - assertEquals(3, il.size()); + assertEquals(4, il.size()); // Initial U-turn - assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(0).getTurnDescription(tr)); - // Second U-turn to get to destination assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(1).getTurnDescription(tr)); + // Second U-turn to get to destination + assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(2).getTurnDescription(tr)); } private void testImportCloseAndLoad(boolean ch, boolean lm, boolean sort, boolean custom) { diff --git a/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java b/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java new file mode 100644 index 00000000000..95045e55f8f --- /dev/null +++ b/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java @@ -0,0 +1,26 @@ +package com.graphhopper.routing.util; + +import com.graphhopper.storage.GraphBuilder; +import com.graphhopper.storage.GraphHopperStorage; +import com.graphhopper.util.EdgeIteratorState; +import com.graphhopper.util.shapes.GHPoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class HeadingEdgeFilterTest { + + @Test + public void getHeading() { + GHPoint point = new GHPoint(55.67093, 12.577294); + CarFlagEncoder carEncoder = new CarFlagEncoder(); + EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).build(); + GraphHopperStorage g = new GraphBuilder(encodingManager).create(); + EdgeIteratorState edge = g.edge(0, 1); + g.getNodeAccess().setNode(0, 55.671044, 12.5771583); + g.getNodeAccess().setNode(1, 55.6704136, 12.5784324); + // GHUtility.setSpeed(50, 0, carEncoder, edge.getFlags()); + + assertEquals(131.2, HeadingEdgeFilter.getHeadingOfGeometryNearPoint(edge, point, 20), .1); + } +} \ No newline at end of file From 3be6bdd873b2ac7bef436780326751b8700527df Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 4 Oct 2021 12:22:01 +0200 Subject: [PATCH 14/14] Remove small segment size optimization for sea level elevation tiles (#2417) --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 +-- .../java/com/graphhopper/reader/dem/CGIARProviderTest.java | 4 ++-- .../java/com/graphhopper/reader/dem/GMTEDProviderTest.java | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 106d586893f..68f46b0552d 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10) + .flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 2d16b527c11..8bdc8c84973 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,8 +135,7 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10).flush(); return 0; } diff --git a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java index 6e161418b56..e62213bf86b 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java @@ -89,9 +89,9 @@ public void downloadFile(String url, String toFile) throws IOException { }); assertEquals(0, instance.getEle(46, -20), 1); - // file not found => small! + // file not found assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override diff --git a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java index 1bc732a0d07..64eb6c1c005 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java @@ -96,9 +96,9 @@ public void downloadFile(String url, String toFile) throws IOException { }); assertEquals(0, instance.getEle(46, -20), 1); - // file not found => small! + // file not found assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override