#6 start 2018 GC refactor for Env Canada changes

obrienlabs · Sep 30, 2018 · d4e1545 · d4e1545
1 parent 3921814
commit d4e1545
Show file tree

Hide file tree

Showing 14 changed files with 104 additions and 66 deletions.
diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/radar.model/.classpath b/radar.model/.classpath
@@ -10,6 +10,7 @@
 		<attributes>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
@@ -18,16 +19,10 @@
 			<attribute name="org.eclipse.jst.component.nondependency" value=""/>
 		</attributes>
 	</classpathentry>
-	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
-		<attributes>
-			<attribute name="maven.pomderived" value="true"/>
-		</attributes>
-	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>
diff --git a/radar.model/.settings/org.eclipse.jdt.core.prefs b/radar.model/.settings/org.eclipse.jdt.core.prefs
@@ -1,8 +1,9 @@
 eclipse.preferences.version=1
 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
 org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.7
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/radar.model/.settings/org.eclipse.wst.common.component b/radar.model/.settings/org.eclipse.wst.common.component
@@ -1,8 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?><project-modules id="moduleCoreId" project-version="1.5.0">
     <wb-module deploy-name="radar.model">
         <wb-resource deploy-path="/" source-path="/src/main/resources"/>
-        <wb-resource deploy-path="/" source-path="/src/test/resources"/>
         <wb-resource deploy-path="/" source-path="/src/main/java"/>
-        <wb-resource deploy-path="/" source-path="/src/test/java"/>
     </wb-module>
 </project-modules>
diff --git a/radar.model/.settings/org.eclipse.wst.common.project.facet.core.xml b/radar.model/.settings/org.eclipse.wst.common.project.facet.core.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <faceted-project>
-  <installed facet="java" version="1.7"/>
   <installed facet="jst.utility" version="1.0"/>
+  <installed facet="java" version="1.8"/>
 </faceted-project>
diff --git a/radar.se/.classpath b/radar.se/.classpath
@@ -16,18 +16,18 @@
 		<attributes>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>
diff --git a/radar.se/.settings/org.eclipse.jdt.core.prefs b/radar.se/.settings/org.eclipse.jdt.core.prefs
@@ -1,5 +1,6 @@
 eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.7
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/radar.se/src/main/java/org/obrienscience/radar/PreProcessor.java b/radar.se/src/main/java/org/obrienscience/radar/PreProcessor.java
@@ -78,8 +78,8 @@ private void initialize(RadarView aView) {
     	// set static variables
     	//RING_FILTER_IMAGE = loadImage("xft/XFT_ring_filter", "gif");
 	    //statImage = new BufferedImage (RING_FILTER_IMAGE.getWidth(), RING_FILTER_IMAGE.getHeight(), BufferedImage.TYPE_INT_RGB);
-        //statView = aView;
-        //statView.setBufferedImage(statImage);     
+        statView = aView;
+        statView.setBufferedImage(statImage);     
         statView.applicationInit();
         statView.setFlash(false);
     }

diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java b/radar.se/src/main/java/org/obrienscience/radar/integration/ApplicationService.java
@@ -98,8 +98,8 @@ public void setCurrentImage(BufferedImage currentImage) {
 	public static final int DEC_MIN_INTERVAL_ITERATIONS_FOR_SAT = 3;
     public static final int DEC_MIN_INTERVAL_ITERATIONS_FOR_LIGHTNING = 6;
     public static final int LIGHTNING_REFRESH_MINUTE = 31;//27;
-	public static final long MIN_DELAY_BETWEEN_URL_CAPTURE_MS = 800;//9667 - 258; // 5 min / 31 sites ( 8 sec or .258 processing time)
-    public static final long MAX_DELAY_BETWEEN_URL_CAPTURE_MS = 2900;
+	public static final long MIN_DELAY_BETWEEN_URL_CAPTURE_MS = 4500;//800;//9667 - 258; // 5 min / 31 sites ( 8 sec or .258 processing time)
+    public static final long MAX_DELAY_BETWEEN_URL_CAPTURE_MS = 8900;//2900;
     public static final int SWEEP_INTERVAL_MIN = 10;
     public static final int NUMBER_RADAR_SITES = 31;
     public static final int NUMBER_SAT_SITES = 9;
@@ -218,7 +218,7 @@ public void setCurrentImage(BufferedImage currentImage) {
     public static final String CURRENT_LIGHTNING_URL_POSTFIX = ".png";   
 	public static final String HISTORICAL_RADAR_URL_PREFIX_PRE_20110920 = "http://www.climate.weatheroffice.gc.ca/radar/get_image.cfm?img=";
     public static final String HISTORICAL_RADAR_URL_PREFIX = "http://www.climate.weatheroffice.gc.ca/radar/image.php?time=";
-    public static final String HISTORICAL_RADAR_URL_SEARCH_PAGE_PREFIX = "http://www.climate.weatheroffice.gc.ca/radar/index_e.html?RadarSite=";
+    public static final String HISTORICAL_RADAR_URL_SEARCH_PAGE_PREFIX = "http://climate.weather.gc.ca/radar/index_e.html?site=";//http://www.climate.weatheroffice.gc.ca/radar/index_e.html?RadarSite=";
 	public static final String HISTORICAL_RADAR_URL_POSTFIX_PRE_20110920 = ":RADAR:GIF";
 	public static final String[] HISTORICAL_RADAR_URL_FRAGMENTS = {
 		"_PRECIP_RAIN_WEATHEROFFICE_ARC~PRECIP,125,18,MPRATE",

diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java b/radar.se/src/main/java/org/obrienscience/radar/integration/BlockingCallable.java
@@ -82,6 +82,8 @@ public T captureImage(String unprocessedImagePath,
 		BufferedInputStream aBufferedInputStream = null;
 		/** stream to file system */
 		String filenamePath = ResourceManager.getFilename(unprocessedImagePath, site, urlAppend, postfix, subdir);
+		// 2018 historical add GIF
+		filenamePath += ".GIF";
 		FileOutputStream aFileWriter = new FileOutputStream(filenamePath);
 		System.out.println("_Writing to: " + filenamePath);
 		/** connection based on the aURL */

diff --git a/radar.se/src/main/java/org/obrienscience/radar/integration/HistoricalRadarService.java b/radar.se/src/main/java/org/obrienscience/radar/integration/HistoricalRadarService.java
@@ -18,6 +18,7 @@ public class HistoricalRadarService extends RadarService {
     //private int offsetMonth;
     public static final boolean IS_HISTORICAL = true;
     private Calendar universalTime;
+    private Calendar universalTimeToPersist;
     private int siteIndex = -1;
 
     public HistoricalRadarService(int year, int month, int day, String site) {
@@ -27,48 +28,32 @@ public HistoricalRadarService(int year, int month, int day, String site) {
     		universalTime.set(Calendar.YEAR, year);
     		universalTime.set(Calendar.MONTH, month);
     		universalTime.set(Calendar.DAY_OF_MONTH, day);
+    		universalTime.add(Calendar.HOUR, -14);
+    	}
+    	universalTimeToPersist = GregorianCalendar.getInstance();
+    	if(year > 0 && month > 0 && day > 0) {
+    		universalTimeToPersist.set(Calendar.YEAR, year);
+    		universalTimeToPersist.set(Calendar.MONTH, month);
+    		universalTimeToPersist.set(Calendar.DAY_OF_MONTH, day);
+    		universalTimeToPersist.add(Calendar.HOUR, -14);
+    		// first image is always 10-16 min ahead
+    		universalTimeToPersist.add(Calendar.MINUTE, SWEEP_INTERVAL_MIN);
     	}
     	if(null != site) {
     		siteIndex = Integer.parseInt(site);
     		//ApplicationService.RADAR_SITE_IDENTIFIERS[siteIndex];
 
     	}
 		System.out.println("_Time set to " + universalTime.toString());
+		System.out.println("_Time set to " + universalTimeToPersist.toString() + " for writes");
     }
-
-    /*public void setOffsetDate(int year, int month, int day) {
-        offsetDay = day;
-        offsetMonth = month;
-        offsetYear = year;
-    }*/
-
-    //@Override
-    public void persistImage(Site site, BufferedImage image, String name) {
-    }
-
-
-	/**
-     * http://www.climate.weatheroffice.gc.ca/radar/index_e.html
-     * ?RadarSite=XFT&sYear=2011&sMonth=8&sDay=7&sHour=19&sMin=40&sec=00&Duration=2&ImageType=Default
-	 */
-    @Override
-	public void performCapture(Site site, boolean persist, long minDelayMS, long maxDelayMS) throws Exception {
-
-    }
-
+
     public void performCapture(List<Site> sites, boolean persist, long minDelayMS, long maxDelayMS, RadarView view) throws Exception {
         PreProcessor preProcessor = new PreProcessor();
         preProcessor.setApplicationServiceOnView(this);
         BufferedImage reducedImage = null;
         String outputPath = null;
         String filenameRoot = null;
-	    // search for missing images from current date - down
-	    //Calendar universalTime = GregorianCalendar.getInstance();
-	    //universalTime.add(Calendar.HOUR, 4); // get UTC
-	    //universalTime.add(Calendar.DAY_OF_MONTH, -day); // start with a day less
-	    //universalTime.add(Calendar.DAY_OF_MONTH, -35);
-	    //universalTime.add(Calendar.MONTH, -4);
-		// start at 20110513
 	    String imageName = null;
 	    List<Site> siteFilterList = null;
 	    if(siteIndex > -1) {
@@ -80,9 +65,12 @@ public void performCapture(List<Site> sites, boolean persist, long minDelayMS, l
 		for(;;) {
 		    // remember to subtract 10 min. or sleep 10 min
 		    universalTime.add(Calendar.MINUTE, -SWEEP_INTERVAL_MIN);
+		    universalTimeToPersist.add(Calendar.MINUTE, -SWEEP_INTERVAL_MIN);
+
         	for(Site site : siteFilterList) {
         	    // look for image on hd, get it if missing, wait random time
-        	    String filename = getResourceManager().getTimestampFileFormat(universalTime, "_");
+        	    String filename = getResourceManager().getTimestampFileFormat(universalTimeToPersist, "_");
+        	    String filenameCapture = getResourceManager().getTimestampFileFormat(universalTime, "_");
         	    String filenamePath = getResourceManager().getFilename(site, filename, CURRENT_RADAR_URL_POSTFIX);
         	    BufferedImage image = loadImage(filenamePath);
         	    if(null == image) {
@@ -91,24 +79,24 @@ public void performCapture(List<Site> sites, boolean persist, long minDelayMS, l
         	            // as of 20110920 : we cannot directly get the image name - it is encoded - get the page first
         	            StringBuffer historicalURL  = new StringBuffer(HISTORICAL_RADAR_URL_SEARCH_PAGE_PREFIX);
         	            historicalURL.append(site.getName());
-        	            historicalURL.append("&sYear=");
+        	            historicalURL.append("&year=");
         	            historicalURL.append(universalTime.get(Calendar.YEAR));//"2011");
-        	            historicalURL.append("&sMonth=");
+        	            historicalURL.append("&month=");
         	            historicalURL.append(1 + universalTime.get(Calendar.MONTH));
-        	            historicalURL.append("&sDay=");
+        	            historicalURL.append("&day=");
         	            historicalURL.append(universalTime.get(Calendar.DAY_OF_MONTH));
-        	            historicalURL.append("&sHour=");
+        	            historicalURL.append("&hour=");
         	            if(universalTime.get(Calendar.AM_PM) > 0) {
         	                historicalURL.append((universalTime.get(Calendar.HOUR) + 12));
         	            } else {
         	                historicalURL.append((universalTime.get(Calendar.HOUR)));
         	            }
-        	            historicalURL.append("&sMin=");
+        	            historicalURL.append("&min=");
         	            historicalURL.append((((int)(universalTime.get(Calendar.MINUTE)) / 10)));
         	            historicalURL.append("0");
         	            historicalURL.append("&sec=");
         	            historicalURL.append("00");
-        	            historicalURL.append("&Duration=2&ImageType=Default");
+        	            historicalURL.append("&duration=2&image_type=PRECIPET_RAIN_WEATHEROFFICE");//Default");
         	            String historicalHTML = getResourceManager().captureURL(
                                 this,
                                 site, 
@@ -122,22 +110,38 @@ public void performCapture(List<Site> sites, boolean persist, long minDelayMS, l
         	            //"<img src='./image.php?time=07-AUG-11%2008.07.52.603743%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2008.28.04.148385%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2008.28.01.505846%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2008.37.49.863105%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2008.47.47.521850%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2008.57.51.041539%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2009.07.56.197892%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2009.17.49.274688%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2009.27.51.785989%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2009.37.52.981699%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","<img src='./image.php?time=07-AUG-11%2009.47.52.122318%20PM&amp;site=XFT' class='noBorder' alt='Radar Image' />","");
         	            //System.out.println(historicalHTML);
         	            // get first image
-        	            String searchMatch = "image.php?time=";
+
+        	            // 2018
+        	            // http://climate.weather.gc.ca/radar/image_e.html?time=28-SEP-18+09.56.42.467757+PM&site=NAT
+        	            // from
+        	            // blobArray = [
+        	            //'/radar/image_e.html?time=28-SEP-18+09.56.42.467757+PM&site=NAT',
+        	            String searchMatch = "image_e.html?time="; // image.php?time=
         	            int searchPosition1 = historicalHTML.indexOf(searchMatch);
         	            if(searchPosition1 < 1) {
                             //System.out.println(historicalHTML);
         	            	imageName = null;
         	            	System.out.println(filename + " not available - skipping");
         	            } else {
-        	            	int searchPosition2 = historicalHTML.indexOf("&amp;site=", searchPosition1);
+        	            	int searchPosition2 = historicalHTML.indexOf("&site=", searchPosition1);
         	            	String codedFragment = historicalHTML.substring(searchPosition1 + searchMatch.length(), searchPosition2); 
-        	            	//System.out.println(codedFragment);
+        	            	System.out.println(codedFragment);
         	            	imageName = getResourceManager().captureImage(
+        	            			this,
+        	            			site,
+        	            			filename,
+        	            			//getURL(site, universalTime, 0, codedFragment),
+        	            			"http://climate.weather.gc.ca/radar/image_e.html?time=" 
+        	            					+ codedFragment + "&site=" + site.getName(),
+        	            			""
+        	            			);
+
+        	            	/*imageName = getResourceManager().captureImage(
         	            			this,
         	            			site, 
         	            			filename, 
         	            			getURL(site, universalTime, 0, codedFragment),
-        	            			CURRENT_RADAR_URL_POSTFIX);				
+        	            			CURRENT_RADAR_URL_POSTFIX);	*/			
         	            }
         	        } catch (Exception e) {
         	            e.printStackTrace();
@@ -153,7 +157,8 @@ public void performCapture(List<Site> sites, boolean persist, long minDelayMS, l
         	                setCurrentImage(image);//reducedImage);
         	                reducedImage = preProcessor.doFilter(0, image, RadarSite.PRECIP_INTENSITY_COLOR_CODES_SIZE - 1);        	            
         	                filenameRoot = filename.substring(0, filename.length());
-        	                outputPath = PreProcessor.FILTERED_DATA_DIR + site.getName() + "/" + site.getName() + "_" + filenameRoot + "_f";
+        	                outputPath = PreProcessor.FILTERED_DATA_DIR + site.getName() 
+        	                	+ "/" + site.getName() + "_" + filenameRoot + "_f";
         	                preProcessor.writeImage(reducedImage, outputPath, "gif");
         	                if(null != view) {
         	                	view.setBufferedImage(reducedImage);
@@ -173,6 +178,20 @@ public void performCapture(List<Site> sites, boolean persist, long minDelayMS, l
 	    }
 	}
 
+
+    //@Override
+    public void persistImage(Site site, BufferedImage image, String name) {
+    }
+
+
+	/**
+     * http://www.climate.weatheroffice.gc.ca/radar/index_e.html
+     * ?RadarSite=XFT&sYear=2011&sMonth=8&sDay=7&sHour=19&sMin=40&sec=00&Duration=2&ImageType=Default
+	 */
+    @Override
+	public void performCapture(Site site, boolean persist, long minDelayMS, long maxDelayMS) throws Exception {
+
+    }
 	/**
 	 * 
 	 * @param siteName
@@ -261,9 +280,12 @@ public static void main(String[] args) {
             RadarView aRadarView = new RadarView();
             aRadarView.applicationInit();
             //aRadarView.setApplicationService(aService);
-            aRadarView.setFlash(true);
+            //aRadarView.setFlash(true);
         	//aService.set
         	aService.performCapture(aRadarView);
+        	// 2018
+        	// http://climate.weather.gc.ca/radar/index_e.html?site=NAT&year=2018&month=9&day=28&hour=21&minute=40&duration=2&image_type=PRECIPET_RAIN_WEATHEROFFICE
+        	// http://www.climate.weatheroffice.gc.ca/radar/index_e.html?RadarSite=WMB&sYear=2018&sMonth=9&sDay=1&sHour=22&sMin=30&sec=00&Duration=2&ImageType=PRECIPET_RAIN_WEATHEROFFICE
         } catch (Exception e) {
         	e.printStackTrace();
         }

diff --git a/radar.web/.classpath b/radar.web/.classpath
@@ -12,6 +12,17 @@
 			<attribute name="org.eclipse.jst.component.dependency" value="/WEB-INF/lib"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>
diff --git a/radar.web/.settings/org.eclipse.jdt.core.prefs b/radar.web/.settings/org.eclipse.jdt.core.prefs
@@ -10,4 +10,5 @@ org.eclipse.jdt.core.compiler.debug.sourceFile=generate
 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
 org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.release=disabled
 org.eclipse.jdt.core.compiler.source=1.8