Skip to content

Commit

Permalink
#7 ec changes and prep for 6 min new site radar resolution
Browse files Browse the repository at this point in the history
  • Loading branch information
@obriensystems
obriensystems committed Jul 29, 2019
1 parent 7645aaf commit bce2aa7
Show file tree
Hide file tree
Showing 4 changed files with 221 additions and 10 deletions.
10 changes: 6 additions & 4 deletions radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,16 +222,16 @@ public void setCurrentImage(BufferedImage currentImage) {
public static final String CURRENT_RADAR_URL_POSTFIX = ".GIF";
public static final String CURRENT_SAT_URL_POSTFIX = "_1070_100.jpg";
public static final String CURRENT_LIGHTNING_URL_POSTFIX = ".png";
public static final String HISTORICAL_RADAR_URL_PREFIX_PRE_20110920 = "http://www.climate.weatheroffice.gc.ca/radar/get_image.cfm?img=";
//public static final String HISTORICAL_RADAR_URL_PREFIX_PRE_20110920 = "http://www.climate.weatheroffice.gc.ca/radar/get_image.cfm?img=";
public static final String HISTORICAL_RADAR_URL_PREFIX = "http://www.climate.weatheroffice.gc.ca/radar/image.php?time=";
public static final String HISTORICAL_RADAR_URL_SEARCH_PAGE_PREFIX = "http://climate.weather.gc.ca/radar/index_e.html?site=";//http://www.climate.weatheroffice.gc.ca/radar/index_e.html?RadarSite=";
public static final String HISTORICAL_RADAR_URL_POSTFIX_PRE_20110920 = ":RADAR:GIF";
//public static final String HISTORICAL_RADAR_URL_POSTFIX_PRE_20110920 = ":RADAR:GIF";
public static final String[] HISTORICAL_RADAR_URL_FRAGMENTS = {
"_PRECIP_RAIN_WEATHEROFFICE_ARC~PRECIP,125,18,MPRATE",
"_CAPPI_RAIN_WEATHEROFFICE_ARC~CAPPI,1.5,AGL,MPRATE" };
public static final int[] HISTORICAL_RADAR_URL_FRAGMENTS_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
//public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_" };
public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_", "_PRECIP_SNOW_"};
public static final String[] CURRENT_RADAR_URL_MIDFIX = { "_PRECIP_RAIN_", "_CAPPI_RAIN_", "_PRECIP_SNOW_", "_COMP_PRECIP_RAIN", "_COMP_PRECIP_SNOW_"};
//public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
// post 201320709 XFT TO PRECIP,XSI,XBE,XTI TO CAPPI, - all down
//public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0 };// 20130709 prior
Expand All @@ -244,7 +244,9 @@ public void setCurrentImage(BufferedImage currentImage) {
// 20150131 PRECIP_SNOW
//public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1 };// 201340102 prior
// 20190323 (wmb=cappi)
public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 };// 20130709 prior
//public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 };// 20130709 prior
// 20190720
public static final int[] CURRENT_RADAR_URL_MIDFIX_INDEX = { 0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1 };// 20130709 prior

public static final String[] SITE_GPSS = {
"",
Expand Down
1 change: 1 addition & 0 deletions radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public class BlockingCallable<T> implements Callable<T> {
private String unprocessedImagePath;
//private FutureTask<T> task;


public BlockingCallable(//FutureTask<T> task,
ApplicationService service,
Site site,
Expand Down
184 changes: 179 additions & 5 deletions radar.se/src/main/java/org/obrienscience/radar/integration/ResourceManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.obrienscience.radar.RadarView;
import org.obrienscience.radar.model.RadarSite;
import org.obrienscience.radar.model.Site;
import org.obrienscience.radar.model.Sweep;
Expand All @@ -67,6 +68,9 @@
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.PutObjectRequest;

import javax.imageio.*;
import java.nio.channels.*;

/**
* Images are stored in the file system and referenced from the database
*
Expand Down Expand Up @@ -171,7 +175,9 @@ public String captureImage(String unprocessedImagePath, ApplicationService servi
public String captureImage(String unprocessedImagePath, ApplicationService service, Site site,
String urlAppend, String fullURL, String postfix, String subdir) throws Exception {
// use a timed callable
return blockingCallable(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir);
//return blockingCallable(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir);
// 20190324
return blockingCallable0(unprocessedImagePath, service, site, urlAppend, fullURL, postfix, subdir);
}


Expand Down Expand Up @@ -289,11 +295,173 @@ public String call() throws Exception {
//}
return fileName;
}
/*private String blockingCallable1(final String unprocessedImagePath, final ApplicationService service, final Site site,
final String urlAppend, final String fullURL, final String postfix, final String subdir) {
boolean _found = false;
String fileName = null;
//Image image = null;
String filenamePath = ResourceManager.getFilename(unprocessedImagePath, site, urlAppend, postfix, subdir);
System.out.println("_Writing to: " + filenamePath);
//image = ImageIO.read(url);
ReadableByteChannel readableByteChannel = Channels.newChannel(fullURL.openStream());
URL aURL = null;
try {
aURL = new URL(fullURL);
FileOutputStream fileOutputStream = new FileOutputStream(FILE_NAME);
FileChannel fileChannel = fileOutputStream.getChannel();
fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
} catch () {
}
return fileName;
}*/




private String blockingCallable0(final String unprocessedImagePath, final ApplicationService service, final Site site,
final String urlAppend, final String fullURL, final String postfix, final String subdir) {
boolean _found = false;
String fileName = null;
FutureTask<String> aFutureTask = null;

try {
aFutureTask = new FutureTask<String>(new Callable<String>() {
public String call() throws Exception {
FileOutputStream fileOutputStream = null;
FileChannel fileChannel = null;

String imageName = null;

/** this stream is used to get the BufferedInputStream below */
// InputStream abstractInputStream = null;
// /** stream to read from the FTP server */
// BufferedInputStream aBufferedInputStream = null;
/** stream to file system */
String filenamePath = ResourceManager.getFilename(unprocessedImagePath, site, urlAppend, postfix, subdir);
// FileOutputStream aFileWriter = new FileOutputStream(filenamePath);
System.out.println("_Writing to: " + filenamePath);
// HttpURLConnection aURLConnection = null;
URL aURL = null;
long byteCount = 0;
// int bytesRead;
long tsstart = System.currentTimeMillis();
try {
aURL = new URL(fullURL);
ReadableByteChannel readableByteChannel = Channels.newChannel(aURL.openStream());
fileOutputStream = new FileOutputStream(filenamePath);
fileChannel = fileOutputStream.getChannel();
fileOutputStream.getChannel()
.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);

// aURLConnection = (HttpURLConnection)aURL.openConnection();
// fake the agent
// HttpURLConnection.setDefaultAllowUserInteraction(true);
// aURLConnection.setRequestProperty("Upgrade-Insecure-Requests", "1");
// aURLConnection.setRequestProperty("User-Agent", //"Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US");
// "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36");
//aURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
// abstractInputStream = aURLConnection.getInputStream();
// aBufferedInputStream = new BufferedInputStream(abstractInputStream);
// byteCount = 0;
System.out.println("Downloading from: " + fullURL);
// byte b[] = new byte[ResourceManager.INPUT_BUFFER_SIZE];
// while ((bytesRead = aBufferedInputStream.read(
// b, // name of buffer
// 0, // start of buffer to start reading into
// b.length // save actual bytes read, not default max buffer size
// )) >= 0 && ((System.currentTimeMillis() - tsstart) < 30000)) {
byteCount = fileChannel.size();//+= bytesRead;
// aFileWriter.write(b, 0, b.length);
//aFileWriter.write(b, 0, bytesRead);//b.length);

Thread.sleep(1);
if(Thread.currentThread().isInterrupted() || Thread.currentThread().isDaemon()) {//aFutureTask.isCancelled()) {
throw new InterruptedException();
}
// } // while
System.out.println((System.currentTimeMillis() - tsstart) + "ms:HTML capture/processing complete: bytes: " + byteCount);
// aBufferedInputStream.close();
// if(byteCount > 6144) {
imageName = urlAppend;
// service.setCurrentImage(filenamePath);
// } else {
// System.out.println("Truncated download of " + byteCount + " bytes: " + urlAppend);
// }
// if(null != aFileWriter) {
// aFileWriter.flush();
// aFileWriter.close();
// }
} catch (InterruptedException ie) {
System.out.println("_Interrupted: " + fullURL);
} catch (IllegalStateException e) {
System.out.println(e.getMessage());
} catch (UnknownServiceException e) {
System.out.println(e.getMessage());
} catch (MalformedURLException e) {
System.out.println(e.getMessage());
} catch (IOException e) {
e.printStackTrace();
System.out.println(e.getMessage());
} catch (Exception e) {
System.out.println(e.getMessage());
} finally {
try {
if(fileOutputStream != null) {
fileOutputStream.close();
}
} catch (IOException fe) {
fe.printStackTrace();
}

/*
if(aBufferedInputStream != null) {
aBufferedInputStream.close();
} // if
if(aFileWriter != null) {
//aFileWriter.flush();
aFileWriter.close();
}*/
} // finally

// only return a non-null image name if the image is invalid
return imageName;
}});
executorService.submit(aFutureTask);
fileName = aFutureTask.get(120, TimeUnit.SECONDS); // returns an immediate result or throws TimeoutException
_found = true;
} catch (TimeoutException te) {
System.out.println("Timeout on: " + fullURL);
te.printStackTrace();
_found = false;
aFutureTask.cancel(true);
} catch (InterruptedException ie) {
ie.printStackTrace();
_found = false;
} catch (ExecutionException ee) {
ee.printStackTrace();
_found = false;
} finally {

}
// verify that we returned normally or had a TimeoutException
if(_found) {
}
//} catch ( Exception e) {
// e.printStackTrace();
//} finally {
//}
return fileName;
}

/* data corruption saving under http pre 20190720 */
private String blockingCallable0old(final String unprocessedImagePath, final ApplicationService service, final Site site,
final String urlAppend, final String fullURL, final String postfix, final String subdir) {
boolean _found = false;
String fileName = null;
//BlockingCallable aBlockingCallable;
FutureTask<String> aFutureTask = null;

Expand Down Expand Up @@ -331,7 +499,10 @@ public String call() throws Exception {
aURLConnection = (HttpURLConnection)aURL.openConnection();
// fake the agent
HttpURLConnection.setDefaultAllowUserInteraction(true);
aURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US");
aURLConnection.setRequestProperty("Upgrade-Insecure-Requests", "1");
aURLConnection.setRequestProperty("User-Agent", //"Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US");
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36");
//aURLConnection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
abstractInputStream = aURLConnection.getInputStream();
aBufferedInputStream = new BufferedInputStream(abstractInputStream);
byteCount = 0;
Expand All @@ -343,8 +514,10 @@ public String call() throws Exception {
b.length // save actual bytes read, not default max buffer size
)) >= 0 && ((System.currentTimeMillis() - tsstart) < 30000)) {
byteCount += bytesRead;
aFileWriter.write(b, 0, bytesRead);//b.length);
Thread.sleep(10);
aFileWriter.write(b, 0, b.length);
//aFileWriter.write(b, 0, bytesRead);//b.length);

Thread.sleep(1);
if(Thread.currentThread().isInterrupted() || Thread.currentThread().isDaemon()) {//aFutureTask.isCancelled()) {
throw new InterruptedException();
}
Expand Down Expand Up @@ -631,7 +804,8 @@ public String getTimestampFileFormat(Calendar universalTime, String delimeter) {
// 20110928_000000
public String getTimestampFileFormat(Calendar universalTime, String delimeter, String midfix, String postfix, boolean dropMin) {//, int extraMinOffset) {
// subtract - likely 20 min first (for satellite)
//universalTime.add(Calendar.MINUTE);//, -extraMinOffset);
// 20190720 subtract 10 for some sites like CASBV
universalTime.add(Calendar.MINUTE, -10);//-extraMinOffset);

StringBuilder buffer = new StringBuilder();
buffer.append(universalTime.get(Calendar.YEAR));
Expand Down
36 changes: 35 additions & 1 deletion radar.se/src/main/java/org/obrienscience/radar/integration/SatService.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
package org.obrienscience.radar.integration;

import java.awt.image.BufferedImage;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
Expand All @@ -10,7 +16,9 @@
import org.obrienscience.radar.model.Site;

public class SatService extends ApplicationService {
public static final String CURRENT_URL_SAT_PREFIX = "http://www.weatheroffice.gc.ca/data/satellite/goes_";
//public static final String CURRENT_URL_SAT_PREFIX = "https://www.weatheroffice.gc.ca/data/satellite/goes_";
// 20190720
public static final String CURRENT_URL_SAT_PREFIX = "https://weather.gc.ca/data/satellite/goes_";
public static final String CURRENT_URL_SAT_POSTFIX = "_1070_100.jpg";

/*
Expand All @@ -36,6 +44,32 @@ public void persistImage(Site site, BufferedImage image, String name) {
public void performCapture(Site site, boolean persist, long minDelayMS, long maxDelayMS) throws Exception {
}

public void performCaptureNIO(RadarView view) {
URL url;
String FILE_NAME = "/Users/michaelobrien/geos_ecan_1070_100.jpg";
FileOutputStream fileOutputStream = null;
FileChannel fileChannel = null;

try {
url = new URL("https://weather.gc.ca/data/satellite/goes_ecan_1070_100.jpg");
//url = new URL("http://wiki.obrienlabs.cloud/download/attachments/983215/Screenshot%202019-07-02%2016.21.27.png?version=1&modificationDate=1562098913258&api=v2");
ReadableByteChannel readableByteChannel = Channels.newChannel(url.openStream());
fileOutputStream = new FileOutputStream(FILE_NAME);
fileChannel = fileOutputStream.getChannel();
fileOutputStream.getChannel()
.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);

} catch (Exception e) {
e.printStackTrace();
} finally {
try {
fileOutputStream.close();
} catch (IOException fe) {
fe.printStackTrace();
}
}
}

public void performCapture(RadarView view) {
List<Site> sites = new ArrayList<>();
for(int i=0;i<NUMBER_SAT_SITES;i++) {
Expand Down

0 comments on commit bce2aa7

Please sign in to comment.