From defd8785fce8255ad0ca5c329bc0ed745fa1b4e1 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Tue, 23 Jul 2019 10:43:05 +0100 Subject: [PATCH 1/8] renderer2d/tiles: Change openstreetmap url and reduce sleep for testing --- src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java | 2 +- src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index 3a8c7f23e4..3f6f0b90cb 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -464,7 +464,7 @@ public void run() { switch (tileLoadingState) { case START: { - double sleepT = 2000 + 15000 * rnd.nextDouble(); + double sleepT = 100;//2000 + 15000 * rnd.nextDouble(); if (isInStateForbidden && retries == 0) sleepT = 60000 + 30000 * rnd.nextDouble(); // NeptusLog.pub().info("<###> "+sleepT + "ms"); diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java index f92d26e3d7..01f84ceffb 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java @@ -121,8 +121,8 @@ protected String createTileRequestURL() { // urlGet = "http://" + sv + "." + "tile.opencyclemap.org/cycle/" + levelOfDetail + "/" // + tileX + "/" + tileY + ".png"; // else - String urlGet = "http://" + sv + "." + "tile.openstreetmap.org/" + levelOfDetail + "/" - + tileX + "/" + tileY + ".png"; + String urlGet = "http://localhost:8080/openstreetmap/" + levelOfDetail + "/" + + tileX + "/" + tileY; return urlGet; From 990b117b84c8a5dab10c3b9119920a722a1cd735 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Tue, 23 Jul 2019 11:17:25 +0100 Subject: [PATCH 2/8] renderer2d/tiles: Add parsing of cache-control header to tile expiration --- src/pt/lsts/neptus/renderer2d/tiles/Tile.java | 3 +++ .../neptus/renderer2d/tiles/TileHttpFetcher.java | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index c0417c2b4e..6c935ba49f 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -103,6 +103,7 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING public final int levelOfDetail; public final int tileX, tileY; public final int worldX, worldY; + public long expiration; protected BufferedImage image = null; protected boolean temporaryTransparencyDetectedOnImageOnDisk = false; //only for base layers private boolean showTileId = false; @@ -392,6 +393,8 @@ public boolean saveTile() { try { File outFile = new File(getTileFilePath()); outFile.mkdirs(); + System.out.println("Saving expiration date for tile: " + getId()); + System.out.println("expiration = " + expiration); return ImageIO.write(image, TILE_FX_EXTENSION.toUpperCase(), outFile); } catch (Exception e) { diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index 3f6f0b90cb..f897e18dc5 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -44,6 +44,7 @@ import javax.imageio.ImageIO; import javax.imageio.stream.ImageInputStream; +import org.apache.http.HeaderElement; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpGet; @@ -287,7 +288,13 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { } retries = 0; isInStateForbidden = false; - + + for (HeaderElement cacheInfo : resp.getFirstHeader("cache-control").getElements()){ + if(cacheInfo.getName().equalsIgnoreCase("max-age")){ + expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); + } + } + InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); @@ -636,6 +643,12 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; + for (HeaderElement cacheInfo : resp.getFirstHeader("cache-control").getElements()){ + if(cacheInfo.getName().equalsIgnoreCase("max-age")){ + expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); + } + } + InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); From e70402c85c81c0e8600da48d147dcfa3ae492e17 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Wed, 24 Jul 2019 16:40:05 +0100 Subject: [PATCH 3/8] renderer2d/tiles: Add expiration to file metadata and dynamic build of expiration map --- .../neptus/renderer2d/WorldRenderPainter.java | 7 +- src/pt/lsts/neptus/renderer2d/tiles/Tile.java | 107 ++++++++++++++++-- 2 files changed, 105 insertions(+), 9 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java index d4aef50489..90e3de84ae 100644 --- a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java +++ b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java @@ -302,8 +302,10 @@ else if (i == lst.size() - 1) { for (String mapDefTag : list) { String[] tags = mapDefTag.split(":"); String mapDef = tags[0]; - if (mapActiveHolderList.containsKey(mapDef)) + if (mapActiveHolderList.containsKey(mapDef)) { mapActiveHolderList.put(mapDef, true); + Tile.setCache(mapDef,true); + } if (mapLayerPrioriryHolderList.containsKey(mapDef) && tags.length > 1) { try { short prio = Short.parseShort(tags[1]); @@ -488,6 +490,7 @@ public void onSelectedChange(boolean selected) { for (String key : mapStyle) { if (mapActiveHolderList.containsKey(key)) { + Tile.setCache(key, true); mapActiveHolderList.put(key, true); } } @@ -664,10 +667,12 @@ public void setMapStyle(boolean exclusive, boolean activate, String... mapStyleN for (String mapKey : mapActiveHolderList.keySet()) { if (mapKey.equalsIgnoreCase(mapStyle)) { mapActiveHolderList.put(mapKey, activate); + Tile.setCache(mapKey, activate); } else { if (exclusive && !mapStyleList.contains(mapKey)) { mapActiveHolderList.put(mapKey, !activate); + Tile.setCache(mapKey, !activate); } } } diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index 6c935ba49f..8866361c15 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -39,16 +39,27 @@ import java.awt.Image; import java.awt.geom.Point2D; import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileFilter; import java.io.Serializable; +import java.util.HashMap; import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.Vector; import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.imageio.IIOImage; import javax.imageio.ImageIO; +import javax.imageio.ImageTypeSpecifier; +import javax.imageio.ImageWriteParam; +import javax.imageio.ImageWriter; +import javax.imageio.metadata.IIOMetadata; +import javax.imageio.metadata.IIOMetadataNode; +import javax.imageio.stream.ImageOutputStream; + +import org.apache.commons.io.FileUtils; import pt.lsts.neptus.NeptusLog; import pt.lsts.neptus.gui.PropertiesEditor; @@ -86,6 +97,8 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable { } protected static final String TILE_FX_EXTENSION = "png"; + + static HashMap> cacheExpiration = new HashMap<>(); public static final long MILISECONDS_TO_TILE_MEM_REMOVAL = 20000; private static final int MILLIS_TO_NOT_TRY_LOAD_LOW_LEVEL_IMAGE = 30000; @@ -393,9 +406,51 @@ public boolean saveTile() { try { File outFile = new File(getTileFilePath()); outFile.mkdirs(); - System.out.println("Saving expiration date for tile: " + getId()); + outFile.delete(); + outFile.createNewFile(); + System.out.println("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName()); System.out.println("expiration = " + expiration); - return ImageIO.write(image, TILE_FX_EXTENSION.toUpperCase(), outFile); + + + ImageWriter writer = ImageIO.getImageWritersByFormatName("png").next(); + + ImageWriteParam writeParam = writer.getDefaultWriteParam(); + ImageTypeSpecifier typeSpecifier = ImageTypeSpecifier.createFromBufferedImageType(BufferedImage.TYPE_INT_RGB); + + //adding metadata + IIOMetadata metadata = writer.getDefaultImageMetadata(typeSpecifier, writeParam); + + IIOMetadataNode textEntry = new IIOMetadataNode("tEXtEntry"); + textEntry.setAttribute("keyword", "expiration"); + textEntry.setAttribute("value", Long.toString(expiration)); + + IIOMetadataNode text = new IIOMetadataNode("tEXt"); + text.appendChild(textEntry); + + IIOMetadataNode root = new IIOMetadataNode("javax_imageio_png_1.0"); + root.appendChild(text); + + metadata.mergeTree("javax_imageio_png_1.0", root); + + //writing the data + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageOutputStream stream = ImageIO.createImageOutputStream(baos); + writer.setOutput(stream); + writer.write(metadata, new IIOImage(image, null, metadata), writeParam); + stream.close(); + + FileUtils.writeByteArrayToFile(outFile, baos.toByteArray()); + + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + if(currMapStyleCache != null){ + currMapStyleCache.put(id, expiration); + } else { + HashMap newMap = new HashMap<>(); + newMap.put(id, expiration); + cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); + } + + return true; } catch (Exception e) { e.printStackTrace(); @@ -418,12 +473,20 @@ public boolean loadTile() { if (image == null) state = TileState.LOADING; File inFile = new File(getTileFilePath()); - if (!inFile.exists()) { - lasErrorMessage = "Error loading tile from file not existing!"; - if (image == null) - state = TileState.ERROR; -// scheduleLoadImageFromLowerLevelOfDetail(); - return false; + + if(hasExpired()){ + if (!inFile.exists()) { + lasErrorMessage = "Error loading tile from file not existing!"; + if (image == null) + state = TileState.ERROR; + // scheduleLoadImageFromLowerLevelOfDetail(); + return false; + } else { + if(hasExpired(inFile)){ + state = TileState.ERROR; + return false; + } + } } BufferedImage img; @@ -452,6 +515,34 @@ public boolean loadTile() { } } + private boolean hasExpired() { + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + Long expiration = currMapStyleCache.get(id); + if(expiration != null) { + return expiration <= System.currentTimeMillis(); + } else { + return true; + } + } + + private boolean hasExpired(File inFile) { + // TODO: 24/07/2019 read file metadata field with name expiration + return true; + } + + public static void setCache(String mapKey, boolean state) { + if(state) { + cacheExpiration.put(mapKey,loadCacheExpiration(mapKey)); + } else { + cacheExpiration.remove(mapKey); + } + } + + private static HashMap loadCacheExpiration(String mapKey) { + // TODO: 24/07/2019 load previously saved cache from disk + return new HashMap<>(); + } + /** * @param img */ From b335075adb85f3a6bcdee0c2db0f36fcd3fc248e Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Thu, 25 Jul 2019 11:10:30 +0100 Subject: [PATCH 4/8] renderer2d/tiles: Add expiration reading from file --- src/pt/lsts/neptus/renderer2d/tiles/Tile.java | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index 8866361c15..c9a3a85e32 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -32,6 +32,7 @@ */ package pt.lsts.neptus.renderer2d.tiles; +import com.sun.imageio.plugins.png.PNGMetadata; import java.awt.AlphaComposite; import java.awt.Color; import java.awt.Font; @@ -42,6 +43,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileFilter; +import java.io.IOException; import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -52,6 +54,7 @@ import javax.imageio.IIOImage; import javax.imageio.ImageIO; +import javax.imageio.ImageReader; import javax.imageio.ImageTypeSpecifier; import javax.imageio.ImageWriteParam; import javax.imageio.ImageWriter; @@ -61,6 +64,8 @@ import org.apache.commons.io.FileUtils; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import pt.lsts.neptus.NeptusLog; import pt.lsts.neptus.gui.PropertiesEditor; import pt.lsts.neptus.i18n.I18n; @@ -412,6 +417,7 @@ public boolean saveTile() { System.out.println("expiration = " + expiration); + // https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html ImageWriter writer = ImageIO.getImageWritersByFormatName("png").next(); ImageWriteParam writeParam = writer.getDefaultWriteParam(); @@ -525,8 +531,46 @@ private boolean hasExpired() { } } - private boolean hasExpired(File inFile) { - // TODO: 24/07/2019 read file metadata field with name expiration + private boolean hasExpired(File inFile) throws IOException { + // https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html + ImageReader imageReader = ImageIO.getImageReadersByFormatName("png").next(); + + imageReader.setInput(ImageIO.createImageInputStream(inFile), true); + + // read metadata of first image + IIOMetadata metadata = imageReader.getImageMetadata(0); + + // the PNG image reader already create a PNGMetadata Object + PNGMetadata pngmeta = (PNGMetadata) metadata; + NodeList childNodes = pngmeta.getStandardTextNode().getChildNodes(); + + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + String keyword = node.getAttributes().getNamedItem("keyword").getNodeValue(); + String value = node.getAttributes().getNamedItem("value").getNodeValue(); + if("expiration".equals(keyword)){ + try { + long expirationValue = Long.valueOf(value); + + // add new entry to cache map + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + if(currMapStyleCache != null){ + currMapStyleCache.put(id, expiration); + } else { + HashMap newMap = new HashMap<>(); + newMap.put(id, expiration); + cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); + } + + return expirationValue <= System.currentTimeMillis(); + } catch (NumberFormatException e) { + NeptusLog.pub().info(String.format("Could not load expiration metadata for map tile %s of style '%s'", + id, + getClass().getSimpleName())); + return true; + } + } + } return true; } From 64658bc7d2bb456085a0b154b065af0b87a8bbb3 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Thu, 25 Jul 2019 15:32:29 +0100 Subject: [PATCH 5/8] renderer2d/tiles: Add parsing for "Expires" header --- .../renderer2d/tiles/TileHttpFetcher.java | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index f897e18dc5..f92ab7ba43 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -35,6 +35,8 @@ import java.awt.image.BufferedImage; import java.io.InputStream; import java.net.UnknownHostException; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Random; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -44,6 +46,7 @@ import javax.imageio.ImageIO; import javax.imageio.stream.ImageInputStream; +import org.apache.http.Header; import org.apache.http.HeaderElement; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -289,11 +292,8 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; - for (HeaderElement cacheInfo : resp.getFirstHeader("cache-control").getElements()){ - if(cacheInfo.getName().equalsIgnoreCase("max-age")){ - expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); - } - } + System.out.println("Fetched tile from: " + urlGet); + parseExpirationHeader(resp, urlGet); InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); @@ -347,7 +347,7 @@ public void run() { } finally { get.abort(); - + // Mostly for GoogleMaps wait time between connections if (waitTime > 0) { if (getWaitTimeLock() != null) { @@ -415,6 +415,33 @@ public void run() { } } + private void parseExpirationHeader(HttpResponse resp, String urlGet) { + Header cacheControlHeader = resp.getFirstHeader("cache-control"); + if(cacheControlHeader != null) { + for (HeaderElement cacheInfo : cacheControlHeader.getElements()){ + if(cacheInfo.getName().equalsIgnoreCase("max-age")){ + try { + expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); + return; + } catch (NumberFormatException e) { + NeptusLog.pub().info("An error occurred while parsing cache control:max-age from request url: " + urlGet); + } + } + } + } + + Header expiresHeader = resp.getFirstHeader("Expires"); + if(expiration < System.currentTimeMillis() && expiresHeader != null) { + try { + expiration = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz").parse(expiresHeader.getValue()).getTime(); + return; + } catch (ParseException e){ + NeptusLog.pub().info("An error occurred while parsing expires header from request url: " + urlGet); + } + } + expiration = System.currentTimeMillis() + 86400000;// one day + } + // --------------- Less threads for loading tiles code (test) ------------------------ private static final boolean DEBUG = false; private static boolean useThreadPool = true; @@ -643,11 +670,8 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; - for (HeaderElement cacheInfo : resp.getFirstHeader("cache-control").getElements()){ - if(cacheInfo.getName().equalsIgnoreCase("max-age")){ - expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); - } - } + System.out.println("Fetched tile from: " + urlGet); + parseExpirationHeader(resp, urlGet); InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); @@ -722,6 +746,8 @@ public void run() { finally { get.abort(); + System.out.println(lasErrorMessage); + // Mostly for GoogleMaps wait time between connections long waitTime = getWaitTimeMillisToSeparateConnections(); if (waitTime > MAX_WAIT_TIME_MILLIS) From 7b0abcd4a3995d215dd543305f205f97f67a6482 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Thu, 25 Jul 2019 18:23:24 +0100 Subject: [PATCH 6/8] renderer2d/tiles: Add permanent storage and loading of cache data --- .../neptus/renderer2d/WorldRenderPainter.java | 1 + src/pt/lsts/neptus/renderer2d/tiles/Tile.java | 78 +++++++++++++++++-- .../renderer2d/tiles/TileHttpFetcher.java | 2 - 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java index 90e3de84ae..f2e67c6b5b 100644 --- a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java +++ b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java @@ -411,6 +411,7 @@ public void run() { public void run() { clearMemCache(); + Tile.cleanup(); ttask.cancel(); ttask1.cancel(); timer.cancel(); diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index c9a3a85e32..f8f9e6b567 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -43,13 +43,18 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.Vector; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.imageio.IIOImage; @@ -94,7 +99,7 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable { protected static String TILE_BASE_CACHE_DIR; - { + static { if (new File("../" + ".cache/wmcache").exists()) TILE_BASE_CACHE_DIR = "../" + ".cache/wmcache"; else @@ -133,7 +138,11 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING private Timer timer = null; // new Timer(this.getClass().getSimpleName() + " [" + Integer.toHexString(this.hashCode()) + "]"); private TimerTask timerTask = null; - + + private static Timer saveTimer = new Timer("TileExpirationMapSaveTimer"); + private static AtomicBoolean hasSaveTimer = new AtomicBoolean(false); + private static final long SAVE_INTERVAL = 120000; //2 minutes + /** * @param levelOfDetail * @param tileX @@ -410,8 +419,7 @@ public boolean saveTile() { tileCacheDiskClearOrTileSaveLock.readLock().lock(); try { File outFile = new File(getTileFilePath()); - outFile.mkdirs(); - outFile.delete(); + outFile.getParentFile().mkdirs(); outFile.createNewFile(); System.out.println("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName()); System.out.println("expiration = " + expiration); @@ -455,6 +463,16 @@ public boolean saveTile() { newMap.put(id, expiration); cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); } + if(!hasSaveTimer.get()) { + saveTimer.schedule(new TimerTask() { + @Override + public void run() { + saveCacheExpiration(); + hasSaveTimer.set(false); + } + }, SAVE_INTERVAL); + hasSaveTimer.set(true); + } return true; } @@ -488,6 +506,7 @@ public boolean loadTile() { // scheduleLoadImageFromLowerLevelOfDetail(); return false; } else { + System.out.println("Checking file expiration"); if(hasExpired(inFile)){ state = TileState.ERROR; return false; @@ -550,7 +569,7 @@ private boolean hasExpired(File inFile) throws IOException { String value = node.getAttributes().getNamedItem("value").getNodeValue(); if("expiration".equals(keyword)){ try { - long expirationValue = Long.valueOf(value); + expiration = Long.valueOf(value); // add new entry to cache map HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); @@ -562,7 +581,7 @@ private boolean hasExpired(File inFile) throws IOException { cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); } - return expirationValue <= System.currentTimeMillis(); + return expiration <= System.currentTimeMillis(); } catch (NumberFormatException e) { NeptusLog.pub().info(String.format("Could not load expiration metadata for map tile %s of style '%s'", id, @@ -583,8 +602,51 @@ public static void setCache(String mapKey, boolean state) { } private static HashMap loadCacheExpiration(String mapKey) { - // TODO: 24/07/2019 load previously saved cache from disk - return new HashMap<>(); + System.out.println("Loading cache file for: " + mapKey); + File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapKey); + if(!serFile.exists()) { + System.out.println(String.format("No cache expiration found at '%s'", serFile.getPath())); + return new HashMap<>(); + } + + try (FileInputStream fis = new FileInputStream(serFile); + ObjectInputStream ois = new ObjectInputStream(fis)) { + Object savedObject = ois.readObject(); + if (savedObject instanceof HashMap){ + return ((HashMap) savedObject); + } else { + throw new Exception("Saved Object is not instance of HashMap"); + } + } catch(Exception e) { + System.out.println("An error occurred while saving cache expiration data"); + e.printStackTrace(); + return new HashMap<>(); + } + } + + private static void saveCacheExpiration() { + System.out.println("Saving cache"); + for (Map.Entry> mapEntry : cacheExpiration.entrySet()) { + System.out.println("Key = " + mapEntry.getKey()); + try { + File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapEntry.getKey()); + serFile.getParentFile().mkdirs(); + serFile.createNewFile(); + FileOutputStream fos = new FileOutputStream(serFile); + ObjectOutputStream oos = new ObjectOutputStream(fos); + oos.writeObject(mapEntry.getValue()); + oos.close(); + fos.close(); + } catch(IOException ioe) { + System.out.println("An error occurred while saving cache expiration data"); + ioe.printStackTrace(); + } + } + } + + public static void cleanup() { + saveTimer.cancel(); + saveCacheExpiration(); } /** diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index f92ab7ba43..816dbb4bdf 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -746,8 +746,6 @@ public void run() { finally { get.abort(); - System.out.println(lasErrorMessage); - // Mostly for GoogleMaps wait time between connections long waitTime = getWaitTimeMillisToSeparateConnections(); if (waitTime > MAX_WAIT_TIME_MILLIS) From 7ad5cee5299aad63fce04d511c5e83af52d95cc7 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Sun, 28 Jul 2019 21:02:41 +0100 Subject: [PATCH 7/8] renderer2d/tiles: Improve debug information format to write to NeptusLog --- src/pt/lsts/neptus/renderer2d/tiles/Tile.java | 19 ++++++++++--------- .../renderer2d/tiles/TileHttpFetcher.java | 4 ++-- .../renderer2d/tiles/TileOpenStreetMap.java | 4 ++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index f8f9e6b567..b1a5e4743e 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -49,6 +49,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Timer; @@ -421,8 +422,8 @@ public boolean saveTile() { File outFile = new File(getTileFilePath()); outFile.getParentFile().mkdirs(); outFile.createNewFile(); - System.out.println("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName()); - System.out.println("expiration = " + expiration); + NeptusLog.pub().debug("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName()); + NeptusLog.pub().debug("expiration = " + new Date(expiration)); // https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html @@ -506,7 +507,7 @@ public boolean loadTile() { // scheduleLoadImageFromLowerLevelOfDetail(); return false; } else { - System.out.println("Checking file expiration"); + NeptusLog.pub().debug("Checking file expiration"); if(hasExpired(inFile)){ state = TileState.ERROR; return false; @@ -602,10 +603,10 @@ public static void setCache(String mapKey, boolean state) { } private static HashMap loadCacheExpiration(String mapKey) { - System.out.println("Loading cache file for: " + mapKey); + NeptusLog.pub().info("Loading cache file for: " + mapKey); File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapKey); if(!serFile.exists()) { - System.out.println(String.format("No cache expiration found at '%s'", serFile.getPath())); + NeptusLog.pub().error(String.format("No cache expiration found at '%s'", serFile.getPath())); return new HashMap<>(); } @@ -618,16 +619,16 @@ private static HashMap loadCacheExpiration(String mapKey) { throw new Exception("Saved Object is not instance of HashMap"); } } catch(Exception e) { - System.out.println("An error occurred while saving cache expiration data"); + NeptusLog.pub().error("An error occurred while saving cache expiration data"); e.printStackTrace(); return new HashMap<>(); } } private static void saveCacheExpiration() { - System.out.println("Saving cache"); + NeptusLog.pub().debug("Saving tile cache"); for (Map.Entry> mapEntry : cacheExpiration.entrySet()) { - System.out.println("Key = " + mapEntry.getKey()); + NeptusLog.pub().debug("Saving Map Style: " + mapEntry.getKey()); try { File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapEntry.getKey()); serFile.getParentFile().mkdirs(); @@ -638,7 +639,7 @@ private static void saveCacheExpiration() { oos.close(); fos.close(); } catch(IOException ioe) { - System.out.println("An error occurred while saving cache expiration data"); + NeptusLog.pub().error("An error occurred while saving cache expiration data for map style: " + mapEntry.getKey()); ioe.printStackTrace(); } } diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index 816dbb4bdf..41704c9ce7 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -292,7 +292,7 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; - System.out.println("Fetched tile from: " + urlGet); + NeptusLog.pub().debug("Fetched tile from: " + urlGet); parseExpirationHeader(resp, urlGet); InputStream is = resp.getEntity().getContent(); @@ -670,7 +670,7 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; - System.out.println("Fetched tile from: " + urlGet); + NeptusLog.pub().debug("Fetched tile from: " + urlGet); parseExpirationHeader(resp, urlGet); InputStream is = resp.getEntity().getContent(); diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java index 01f84ceffb..b1b1f782a7 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java @@ -121,8 +121,8 @@ protected String createTileRequestURL() { // urlGet = "http://" + sv + "." + "tile.opencyclemap.org/cycle/" + levelOfDetail + "/" // + tileX + "/" + tileY + ".png"; // else - String urlGet = "http://localhost:8080/openstreetmap/" + levelOfDetail + "/" - + tileX + "/" + tileY; + String urlGet = "http://" + sv + ".tile.openstreetmap.org/" + levelOfDetail + "/" + + tileX + "/" + tileY + ".png"; return urlGet; From 55a014a6b76bc51c8a9d1da18f410b3038d6a933 Mon Sep 17 00:00:00 2001 From: edramos-97 Date: Sun, 28 Jul 2019 21:05:32 +0100 Subject: [PATCH 8/8] renderer2d/tiles: Restore original sleep time in TileHttpFetcher --- src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index 41704c9ce7..04210cf93c 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -498,7 +498,7 @@ public void run() { switch (tileLoadingState) { case START: { - double sleepT = 100;//2000 + 15000 * rnd.nextDouble(); + double sleepT = 2000 + 15000 * rnd.nextDouble(); if (isInStateForbidden && retries == 0) sleepT = 60000 + 30000 * rnd.nextDouble(); // NeptusLog.pub().info("<###> "+sleepT + "ms");