diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java b/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java index cbb6a09..748f8ab 100755 --- a/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java +++ b/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java @@ -222,16 +222,16 @@ public void setCurrentImage(BufferedImage currentImage) { public static final String CURRENT_RADAR_URL_POSTFIX = ".GIF"; public static final String CURRENT_SAT_URL_POSTFIX = "_1070_100.jpg"; public static final String CURRENT_LIGHTNING_URL_POSTFIX = ".png"; - public static final String HISTORICAL_RADAR_URL_PREFIX_PRE_20110920 = "http://www.climate.weatheroffice.gc.ca/radar/get_image.cfm?img="; + //public static final String HISTORICAL_RADAR_URL_PREFIX_PRE_20110920 = "http://www.climate.weatheroffice.gc.ca/radar/get_image.cfm?img="; public static final String HISTORICAL_RADAR_URL_PREFIX = "http://www.climate.weatheroffice.gc.ca/radar/image.php?time="; public static final String HISTORICAL_RADAR_URL_SEARCH_PAGE_PREFIX = "http://climate.weather.gc.ca/radar/index_e.html?site=";//http://www.climate.weatheroffice.gc.ca/radar/index_e.html?RadarSite="; - public static final String HISTORICAL_RADAR_URL_POSTFIX_PRE_20110920 = ":RADAR:GIF"; + //public static final String HISTORICAL_RADAR_URL_POSTFIX_PRE_20110920 = ":RADAR:GIF"; public static final String[] HISTORICAL_RADAR_URL_FRAGMENTS = { "_PRECIP_RAIN_WEATHEROFFICE_ARC~PRECIP,125,18,MPRATE", "_CAPPI_RAIN_WEATHEROFFICE_ARC~CAPPI,1.5,AGL,MPRATE" }; public static final int[] HISTORICAL_RADAR_URL_FRAGMENTS_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; //public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_" }; - public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_", "_PRECIP_SNOW_"}; + public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_", "_PRECIP_SNOW_", "_COMP_PRECIP_RAIN", "_COMP_PRECIP_SNOW_"}; //public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; // post 201320709 XFT TO PRECIP,XSI,XBE,XTI TO CAPPI, - all down //public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0 };// 20130709 prior @@ -244,7 +244,9 @@ public void setCurrentImage(BufferedImage currentImage) { // 20150131 PRECIP_SNOW //public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1 };// 201340102 prior // 20190323 (wmb=cappi) - public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 };// 20130709 prior + //public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 };// 20130709 prior + // 20190720 + public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1 };// 20130709 prior public static final String[] SITE_GPSS = { "", diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java b/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java index 2061117..f522e09 100644 --- a/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java +++ b/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java @@ -36,6 +36,7 @@ public class BlockingCallable implements Callable { private String unprocessedImagePath; //private FutureTask task; + public BlockingCallable(//FutureTask task, ApplicationService service, Site site, diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/ResourceManager.java b/radar.se/src/main/java/org/obrienscience/radar/integration/ResourceManager.java index c63b0f5..d679197 100755 --- a/radar.se/src/main/java/org/obrienscience/radar/integration/ResourceManager.java +++ b/radar.se/src/main/java/org/obrienscience/radar/integration/ResourceManager.java @@ -57,6 +57,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.obrienscience.radar.RadarView; import org.obrienscience.radar.model.RadarSite; import org.obrienscience.radar.model.Site; import org.obrienscience.radar.model.Sweep; @@ -67,6 +68,9 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.PutObjectRequest; +import javax.imageio.*; +import java.nio.channels.*; + /** * Images are stored in the file system and referenced from the database * @@ -171,7 +175,9 @@ public String captureImage(String unprocessedImagePath, ApplicationService servi public String captureImage(String unprocessedImagePath, ApplicationService service, Site site, String urlAppend, String fullURL, String postfix, String subdir) throws Exception { // use a timed callable - return blockingCallable(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir); + //return blockingCallable(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir); + // 20190324 + return blockingCallable0(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir); } @@ -289,11 +295,173 @@ public String call() throws Exception { //} return fileName; } + /*private String blockingCallable1(final String unprocessedImagePath, final ApplicationService service, final Site site, + final String urlAppend, final String fullURL, final String postfix, final String subdir) { + boolean _found = false; + String fileName = null; + //Image image = null; + String filenamePath = ResourceManager.getFilename(unprocessedImagePath, site, urlAppend, postfix, subdir); + System.out.println("_Writing to: " + filenamePath); + + + //image = ImageIO.read(url); + ReadableByteChannel readableByteChannel = Channels.newChannel(fullURL.openStream()); + URL aURL = null; + try { + aURL = new URL(fullURL); + FileOutputStream fileOutputStream = new FileOutputStream(FILE_NAME); + FileChannel fileChannel = fileOutputStream.getChannel(); + fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE); + } catch () { + + } + return fileName; + }*/ + + + + private String blockingCallable0(final String unprocessedImagePath, final ApplicationService service, final Site site, final String urlAppend, final String fullURL, final String postfix, final String subdir) { boolean _found = false; String fileName = null; + FutureTask aFutureTask = null; + + try { + aFutureTask = new FutureTask(new Callable() { + public String call() throws Exception { + FileOutputStream fileOutputStream = null; + FileChannel fileChannel = null; + + String imageName = null; + + /** this stream is used to get the BufferedInputStream below */ + // InputStream abstractInputStream = null; + // /** stream to read from the FTP server */ + // BufferedInputStream aBufferedInputStream = null; + /** stream to file system */ + String filenamePath = ResourceManager.getFilename(unprocessedImagePath, site, urlAppend, postfix, subdir); +// FileOutputStream aFileWriter = new FileOutputStream(filenamePath); + System.out.println("_Writing to: " + filenamePath); +// HttpURLConnection aURLConnection = null; + URL aURL = null; + long byteCount = 0; +// int bytesRead; + long tsstart = System.currentTimeMillis(); + try { + aURL = new URL(fullURL); + ReadableByteChannel readableByteChannel = Channels.newChannel(aURL.openStream()); + fileOutputStream = new FileOutputStream(filenamePath); + fileChannel = fileOutputStream.getChannel(); + fileOutputStream.getChannel() + .transferFrom(readableByteChannel, 0, Long.MAX_VALUE); + +// aURLConnection = (HttpURLConnection)aURL.openConnection(); + // fake the agent +// HttpURLConnection.setDefaultAllowUserInteraction(true); +// aURLConnection.setRequestProperty("Upgrade-Insecure-Requests", "1"); +// aURLConnection.setRequestProperty("User-Agent", //"Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US"); +// "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"); + //aURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"); +// abstractInputStream = aURLConnection.getInputStream(); +// aBufferedInputStream = new BufferedInputStream(abstractInputStream); +// byteCount = 0; + System.out.println("Downloading from: " + fullURL); +// byte b[] = new byte[ResourceManager.INPUT_BUFFER_SIZE]; +// while ((bytesRead = aBufferedInputStream.read( +// b, // name of buffer +// 0, // start of buffer to start reading into +// b.length // save actual bytes read, not default max buffer size +// )) >= 0 && ((System.currentTimeMillis() - tsstart) < 30000)) { + byteCount = fileChannel.size();//+= bytesRead; +// aFileWriter.write(b, 0, b.length); + //aFileWriter.write(b, 0, bytesRead);//b.length); + + Thread.sleep(1); + if(Thread.currentThread().isInterrupted() || Thread.currentThread().isDaemon()) {//aFutureTask.isCancelled()) { + throw new InterruptedException(); + } +// } // while + System.out.println((System.currentTimeMillis() - tsstart) + "ms:HTML capture/processing complete: bytes: " + byteCount); +// aBufferedInputStream.close(); +// if(byteCount > 6144) { + imageName = urlAppend; +// service.setCurrentImage(filenamePath); +// } else { +// System.out.println("Truncated download of " + byteCount + " bytes: " + urlAppend); +// } +// if(null != aFileWriter) { +// aFileWriter.flush(); +// aFileWriter.close(); +// } + } catch (InterruptedException ie) { + System.out.println("_Interrupted: " + fullURL); + } catch (IllegalStateException e) { + System.out.println(e.getMessage()); + } catch (UnknownServiceException e) { + System.out.println(e.getMessage()); + } catch (MalformedURLException e) { + System.out.println(e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + System.out.println(e.getMessage()); + } catch (Exception e) { + System.out.println(e.getMessage()); + } finally { + try { + if(fileOutputStream != null) { + fileOutputStream.close(); + } + } catch (IOException fe) { + fe.printStackTrace(); + } + +/* + if(aBufferedInputStream != null) { + aBufferedInputStream.close(); + } // if + if(aFileWriter != null) { + //aFileWriter.flush(); + aFileWriter.close(); + }*/ + } // finally + + // only return a non-null image name if the image is invalid + return imageName; + }}); + executorService.submit(aFutureTask); + fileName = aFutureTask.get(120, TimeUnit.SECONDS); // returns an immediate result or throws TimeoutException + _found = true; + } catch (TimeoutException te) { + System.out.println("Timeout on: " + fullURL); + te.printStackTrace(); + _found = false; + aFutureTask.cancel(true); + } catch (InterruptedException ie) { + ie.printStackTrace(); + _found = false; + } catch (ExecutionException ee) { + ee.printStackTrace(); + _found = false; + } finally { + + } + // verify that we returned normally or had a TimeoutException + if(_found) { + } + //} catch ( Exception e) { + // e.printStackTrace(); + //} finally { + //} + return fileName; + } + + /* data corruption saving under http pre 20190720 */ + private String blockingCallable0old(final String unprocessedImagePath, final ApplicationService service, final Site site, + final String urlAppend, final String fullURL, final String postfix, final String subdir) { + boolean _found = false; + String fileName = null; //BlockingCallable aBlockingCallable; FutureTask aFutureTask = null; @@ -331,7 +499,10 @@ public String call() throws Exception { aURLConnection = (HttpURLConnection)aURL.openConnection(); // fake the agent HttpURLConnection.setDefaultAllowUserInteraction(true); - aURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US"); + aURLConnection.setRequestProperty("Upgrade-Insecure-Requests", "1"); + aURLConnection.setRequestProperty("User-Agent", //"Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US"); + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"); + //aURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"); abstractInputStream = aURLConnection.getInputStream(); aBufferedInputStream = new BufferedInputStream(abstractInputStream); byteCount = 0; @@ -343,8 +514,10 @@ public String call() throws Exception { b.length // save actual bytes read, not default max buffer size )) >= 0 && ((System.currentTimeMillis() - tsstart) < 30000)) { byteCount += bytesRead; - aFileWriter.write(b, 0, bytesRead);//b.length); - Thread.sleep(10); + aFileWriter.write(b, 0, b.length); + //aFileWriter.write(b, 0, bytesRead);//b.length); + + Thread.sleep(1); if(Thread.currentThread().isInterrupted() || Thread.currentThread().isDaemon()) {//aFutureTask.isCancelled()) { throw new InterruptedException(); } @@ -631,7 +804,8 @@ public String getTimestampFileFormat(Calendar universalTime, String delimeter) { // 20110928_000000 public String getTimestampFileFormat(Calendar universalTime, String delimeter, String midfix, String postfix, boolean dropMin) {//, int extraMinOffset) { // subtract - likely 20 min first (for satellite) - //universalTime.add(Calendar.MINUTE);//, -extraMinOffset); + // 20190720 subtract 10 for some sites like CASBV + universalTime.add(Calendar.MINUTE, -10);//-extraMinOffset); StringBuilder buffer = new StringBuilder(); buffer.append(universalTime.get(Calendar.YEAR)); diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/SatService.java b/radar.se/src/main/java/org/obrienscience/radar/integration/SatService.java index 6671531..70d5aea 100755 --- a/radar.se/src/main/java/org/obrienscience/radar/integration/SatService.java +++ b/radar.se/src/main/java/org/obrienscience/radar/integration/SatService.java @@ -1,6 +1,12 @@ package org.obrienscience.radar.integration; import java.awt.image.BufferedImage; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; import java.util.ArrayList; import java.util.Calendar; import java.util.List; @@ -10,7 +16,9 @@ import org.obrienscience.radar.model.Site; public class SatService extends ApplicationService { - public static final String CURRENT_URL_SAT_PREFIX = "http://www.weatheroffice.gc.ca/data/satellite/goes_"; + //public static final String CURRENT_URL_SAT_PREFIX = "https://www.weatheroffice.gc.ca/data/satellite/goes_"; + // 20190720 + public static final String CURRENT_URL_SAT_PREFIX = "https://weather.gc.ca/data/satellite/goes_"; public static final String CURRENT_URL_SAT_POSTFIX = "_1070_100.jpg"; /* @@ -36,6 +44,32 @@ public void persistImage(Site site, BufferedImage image, String name) { public void performCapture(Site site, boolean persist, long minDelayMS, long maxDelayMS) throws Exception { } + public void performCaptureNIO(RadarView view) { + URL url; + String FILE_NAME = "/Users/michaelobrien/geos_ecan_1070_100.jpg"; + FileOutputStream fileOutputStream = null; + FileChannel fileChannel = null; + + try { + url = new URL("https://weather.gc.ca/data/satellite/goes_ecan_1070_100.jpg"); + //url = new URL("http://wiki.obrienlabs.cloud/download/attachments/983215/Screenshot%202019-07-02%2016.21.27.png?version=1&modificationDate=1562098913258&api=v2"); + ReadableByteChannel readableByteChannel = Channels.newChannel(url.openStream()); + fileOutputStream = new FileOutputStream(FILE_NAME); + fileChannel = fileOutputStream.getChannel(); + fileOutputStream.getChannel() + .transferFrom(readableByteChannel, 0, Long.MAX_VALUE); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + fileOutputStream.close(); + } catch (IOException fe) { + fe.printStackTrace(); + } + } + } + public void performCapture(RadarView view) { List sites = new ArrayList<>(); for(int i=0;i