Strabon

changeset 504:b408cde6beca

Added scripts according to George's suggestions. Now conforming to URI representation of hotspots.
Also fixed minor bug regarding date values (month was printed instead of minutes in the corresponding slot) in both v2.2 and v2.3
author Manos Karpathiotakis <mk@di.uoa.gr>
date Wed Jul 25 01:59:00 2012 +0300 (2012-07-25)
parents ae32e022c593
children bd55cc99cea2
files scripts/v2.2/runNoaRefinementChain.sh scripts/v2.3/count.sparql scripts/v2.3/deleteSeaHotspots.sparql scripts/v2.3/discover.sparql scripts/v2.3/insertMunicipalities.sparql scripts/v2.3/landUseInvalidForFires.sparql scripts/v2.3/refinePartialSeaHotspots.sparql scripts/v2.3/refineTimePersistence.sparql scripts/v2.3/runNoaRefinementChain-papos.sh scripts/v2.3/runNoaRefinementChain.sh
line diff
     1.1 --- a/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 25 01:17:59 2012 +0300
     1.2 +++ b/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 25 01:59:00 2012 +0300
     1.3 @@ -222,7 +222,7 @@
     1.4  
     1.5  		# refineTimePersistence
     1.6  		echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
     1.7 -		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
     1.8 +		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
     1.9  		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
    1.10  		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
    1.11  		sed "s/SENSOR/${SENSOR}/g" | \
    1.12 @@ -243,8 +243,8 @@
    1.13  
    1.14  		# discover
    1.15  		echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
    1.16 -		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
    1.17 -		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
    1.18 +		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
    1.19 +		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
    1.20  		query=`echo "${discover}" | \
    1.21  			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
    1.22  			sed "s/SENSOR/${SENSOR}/g" | \
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/scripts/v2.3/count.sparql	Wed Jul 25 01:59:00 2012 +0300
     2.3 @@ -0,0 +1,2 @@
     2.4 +SELECT (count(*) AS ?count)
     2.5 +WHERE {?s ?p ?o}
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/scripts/v2.3/deleteSeaHotspots.sparql	Wed Jul 25 01:59:00 2012 +0300
     3.3 @@ -0,0 +1,17 @@
     3.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#>
     3.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
     3.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#>
     3.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
     3.8 +PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
     3.9 +
    3.10 +
    3.11 +INSERT {?h noa:isDiscarded "1"^^xsd:int} 
    3.12 +WHERE {
    3.13 +?h   noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ;
    3.14 +     noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
    3.15 +     noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    3.16 +       OPTIONAL {
    3.17 +    ?h	gag:hasMunicipality ?muni .
    3.18 +  } 
    3.19 +  FILTER(!bound(?muni)) .
    3.20 +}
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/scripts/v2.3/discover.sparql	Wed Jul 25 01:59:00 2012 +0300
     4.3 @@ -0,0 +1,19 @@
     4.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 
     4.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     4.6 +PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
     4.7 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     4.8 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     4.9 +PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
    4.10 +
    4.11 +SELECT ?h  (strdf:transform(?hGeo, <http://www.opengis.net/def/crs/EPSG/0/4326>) AS ?geo) ?conf ?muni
    4.12 +WHERE { 
    4.13 +?h   noa:hasGeometry ?hGeo ;
    4.14 +     noa:hasAcquisitionTime ?hAcquisitionTime ; 
    4.15 +     noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    4.16 +     noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    4.17 +     noa:hasConfidence ?conf ;
    4.18 +     gag:hasMunicipality ?muni .
    4.19 +     FILTER("MIN_ACQUISITION_TIME"^^xsd:dateTime <= ?hAcquisitionTime && ?hAcquisitionTime < "MAX_ACQUISITION_TIME"^^xsd:dateTime ) .
    4.20 +     OPTIONAL {?h noa:isDiscarded ?disc }.
    4.21 +     FILTER (!bound(?disc)) .
    4.22 +}
     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/scripts/v2.3/insertMunicipalities.sparql	Wed Jul 25 01:59:00 2012 +0300
     5.3 @@ -0,0 +1,23 @@
     5.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#>
     5.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
     5.6 +PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
     5.7 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#>
     5.8 +PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
     5.9 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
    5.10 +
    5.11 +INSERT { ?h gag:hasMunicipality ?muni }
    5.12 +WHERE { 
    5.13 +SELECT ?h (SAMPLE(?mLabel) AS ?muni)
    5.14 +  WHERE {
    5.15 +	?h  rdf:type noa:Hotspot ;
    5.16 +		noa:hasGeometry ?hGeo ;
    5.17 +		noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    5.18 +		noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
    5.19 +        noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime.
    5.20 +	?m	rdf:type gag:Dhmos ;
    5.21 +		rdfs:label ?mLabel ;
    5.22 +		strdf:hasGeometry ?mGeo .
    5.23 +	FILTER(strdf:mbbIntersects(?hGeo, ?mGeo)) .
    5.24 +  }  
    5.25 +  GROUP BY ?h
    5.26 +}
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/scripts/v2.3/landUseInvalidForFires.sparql	Wed Jul 25 01:59:00 2012 +0300
     6.3 @@ -0,0 +1,23 @@
     6.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 
     6.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     6.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     6.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     6.8 +PREFIX clc: <http://geo.linkedopendata.gr/corine/ontology#>
     6.9 +
    6.10 +
    6.11 +
    6.12 +INSERT {?h noa:isDiscarded "1"^^xsd:int} 
    6.13 +WHERE {
    6.14 +SELECT ?h WHERE 
    6.15 +{
    6.16 +?h   noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ;
    6.17 +     noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
    6.18 +     noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    6.19 +     noa:hasGeometry ?hGeo.
    6.20 +?a   rdf:type   clc:ExcludeArea;
    6.21 +  clc:hasGeometry ?aGeo.
    6.22 +FILTER(strdf:mbbIntersects(?hGeo,?aGeo)).
    6.23 +} 
    6.24 +GROUP BY ?h ?hGeo
    6.25 +HAVING strdf:contains(strdf:union(?aGeo),?hGeo)
    6.26 +}
     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/scripts/v2.3/refinePartialSeaHotspots.sparql	Wed Jul 25 01:59:00 2012 +0300
     7.3 @@ -0,0 +1,35 @@
     7.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 
     7.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     7.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     7.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     7.8 +PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
     7.9 +
    7.10 +
    7.11 +INSERT { ?h noa:isDiscarded "1"^^xsd:int . 
    7.12 +	 ?valid rdf:type noa:Hotspot ;
    7.13 +		noa:hasConfidence ?conf ;
    7.14 +	        noa:hasGeometry ?dif ;
    7.15 +	        gag:hasMunicipality ?muni ;
    7.16 +		noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 
    7.17 +	        noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    7.18 +		noa:hasConfirmation noa:unknown ;
    7.19 +	        noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    7.20 +		noa:isProducedBy noa:noa ;
    7.21 +		noa:isDerivedFromSatellite "SAT"^^xsd:string .
    7.22 +}
    7.23 +WHERE { 
    7.24 +  SELECT ?h  (strdf:intersection(?hGeo, strdf:union(?cGeo)) AS ?dif) (URI(CONCAT(STR(?h),"/refined")) AS ?valid) ?conf ?muni
    7.25 +  WHERE { 
    7.26 +?h     noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime; 
    7.27 +       noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    7.28 +       noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    7.29 +       noa:hasGeometry ?hGeo ;
    7.30 +       gag:hasMunicipality ?muni ;
    7.31 +       noa:hasConfidence ?conf .
    7.32 +?c     rdf:type noa:Coastline ;
    7.33 +       noa:hasGeometry ?cGeo .
    7.34 +	   FILTER(strdf:mbbIntersects(?hGeo, ?cGeo)) . 
    7.35 +  }
    7.36 +  GROUP BY ?h ?hGeo ?conf ?muni
    7.37 +  HAVING strdf:overlap(?hGeo, strdf:union(?cGeo))
    7.38 +}
     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/scripts/v2.3/refineTimePersistence.sparql	Wed Jul 25 01:59:00 2012 +0300
     8.3 @@ -0,0 +1,47 @@
     8.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 
     8.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     8.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     8.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     8.8 +PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
     8.9 +
    8.10 +
    8.11 +INSERT {
    8.12 + ?newHotspot rdf:type noa:Hotspot ;
    8.13 +		noa:hasConfidence ?hConfidence ;
    8.14 +	        noa:hasGeometry ?hGeometry1 ;
    8.15 +		noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 
    8.16 +	        noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    8.17 +		noa:hasConfirmation noa:unknown ;
    8.18 +	        noa:producedFromProcessingChain "PROCESSING_CHAIN-TimePersistence"^^xsd:string ; 
    8.19 +		noa:isProducedBy noa:noa ;
    8.20 +		 gag:hasMunicipality ?muni ;
    8.21 +		noa:isDerivedFromSatellite "SAT"^^xsd:string .
    8.22 +
    8.23 +}
    8.24 +WHERE {
    8.25 +  SELECT (URI(CONCAT(STR(MAX(?H1)),"virtual/NEW_HOTSPOT")) AS ?newHotspot)
    8.26 +  (SUM(?hConfidence1)/ACQUISITIONS_IN_HALF_AN_HOUR AS ?hConfidence)
    8.27 +  ?hGeometry1 ?muni
    8.28 +  WHERE {
    8.29 +    ?H1 noa:hasConfidence ?hConfidence1 .
    8.30 +    ?H1 noa:hasGeometry ?hGeometry1 .
    8.31 +    ?H1 gag:hasMunicipality ?muni .
    8.32 +    ?H1 noa:hasAcquisitionTime ?hAcquisitionTime1 .
    8.33 +    ?H1 noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    8.34 +    ?H1 noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string .
    8.35 +    OPTIONAL { ?H1 noa:isDiscarded ?z } .
    8.36 +    FILTER (!BOUND(?z)) .
    8.37 +    FILTER( "MIN_ACQUISITION_TIME"^^xsd:dateTime <= ?hAcquisitionTime1 && ?hAcquisitionTime1 < "TIMESTAMP"^^xsd:dateTime ) .
    8.38 +    OPTIONAL {
    8.39 +      ?H2 noa:hasGeometry ?HGEO2 .
    8.40 +      ?H2 noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime .
    8.41 +      ?H2 noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    8.42 +      ?H2 noa:producedFromProcessingChain ?hProcessingChain2 .
    8.43 +      FILTER(("PROCESSING_CHAIN"^^xsd:string = ?hProcessingChain2)||("PROCESSING_CHAIN-TimePersistence"^^xsd:string = ?hProcessingChain2)).
    8.44 +      FILTER( strdf:mbbEquals(?hGeometry1, ?HGEO2) ) .
    8.45 +    }
    8.46 +    FILTER( !BOUND(?H2) ) .
    8.47 +  }
    8.48 +  GROUP BY ?hGeometry1 ?muni
    8.49 +  HAVING(SUM(?hConfidence1)>0.0)
    8.50 +}
     9.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     9.2 +++ b/scripts/v2.3/runNoaRefinementChain-papos.sh	Wed Jul 25 01:59:00 2012 +0300
     9.3 @@ -0,0 +1,255 @@
     9.4 +#!/bin/bash
     9.5 +LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
     9.6 +
     9.7 +############################ CONFIGURATION #############################
     9.8 +ENDPOINT="http://localhost:8080/endpoint"
     9.9 +DB="endpoint"
    9.10 +
    9.11 +HOTSPOTS_URL="http://localhost/rdf"
    9.12 +       URLDIR=( "msg1_rss"              "msg2")
    9.13 +    URLPREFIX=( "HMSG1_RSS_IR_039_s7_"  "HMSG2_IR_039_s7_")
    9.14 +       SENSOR=( "MSG1_RSS"              "MSG2")
    9.15 +     SATELITE=( "METEOSAT8"             "METEOSAT9")
    9.16 +ACQ_HALF_HOUR=( "7.0"                   "3.0")
    9.17 +SUFFIX=".hotspots.n3"
    9.18 +PROCESSING_CHAIN="DynamicThresholds"
    9.19 +
    9.20 +
    9.21 +# log files
    9.22 +logFile="chain.log"
    9.23 +timings="chain-times.log"
    9.24 +timingsDiscover="discover.txt"
    9.25 +
    9.26 +# stSPARQL statements
    9.27 +insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql` 
    9.28 +deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` 
    9.29 +invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
    9.30 +refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql`
    9.31 +refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql`
    9.32 +discover=`cat ${LOC}/discover.sparql`
    9.33 +# | sed 's/\"/\\\"/g'`
    9.34 +########################################################################
    9.35 +
    9.36 +function timer()
    9.37 +{
    9.38 +   if [[ $# -eq 0 ]]; then
    9.39 +       t=$(date '+%s%N')
    9.40 +       t=$((t/1000000))
    9.41 +       echo $t
    9.42 +   else
    9.43 +       local  stime=$1
    9.44 +       etime=$(date '+%s%N')
    9.45 +       etime=$((etime/1000000))
    9.46 +
    9.47 +       if [[ -z "$stime" ]]; then stime=$etime; fi
    9.48 +       dt=$((etime - stime)) #dt in milliseconds
    9.49 +       dM=$((dt%1000))
    9.50 +       Dt=$((dt/1000)) #delta t in seconds
    9.51 +       ds=$((Dt % 60))
    9.52 +       dm=$(((Dt / 60) % 60))
    9.53 +       dh=$((Dt / 3600))
    9.54 +       printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
    9.55 +   fi
    9.56 +}
    9.57 +
    9.58 +# find out the postgres service to use
    9.59 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
    9.60 +
    9.61 +tomcat=
    9.62 +function chooseTomcat()
    9.63 +{
    9.64 +	if test -s /etc/fedora-release ; then
    9.65 +		tomcat="tomcat"
    9.66 +	#elif test -s /etc/centos-release ; then
    9.67 +	#elif test -s /etc/yellowdog-release ; then
    9.68 +	#elif test -s /etc/redhat-release ; then
    9.69 +	#elif test -s /etc/SuSE-release ; then
    9.70 +	#elif test -s /etc/gentoo-release ; then
    9.71 +	elif test -s /etc/lsb-release ; then # Ubuntu
    9.72 +			if test -s /etc/init.d/tomcat6 ; then
    9.73 +				tomcat="tomcat6"
    9.74 +			elif test -s /etc/init.d/tomcat7 ; then
    9.75 +				tomcat="tomcat7"
    9.76 +			fi
    9.77 +	elif test -s /etc/debian_version ; then
    9.78 +			tomcat="tomcat"
    9.79 +	fi
    9.80 +
    9.81 +	# check for service availability
    9.82 +	if ! test -e "/etc/init.d/${tomcat}"; then
    9.83 +		tomcat=
    9.84 +	fi
    9.85 +}
    9.86 +
    9.87 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
    9.88 +chooseTomcat
    9.89 +echo "stopping tomcat"
    9.90 +if test -z "${tomcat}"; then
    9.91 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
    9.92 +	stop_tomcat.sh
    9.93 +else
    9.94 +	sudo service ${tomcat} stop
    9.95 +fi
    9.96 +
    9.97 +sudo service ${postgres} restart
    9.98 +
    9.99 +# get the main version of postgres
   9.100 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
   9.101 +
   9.102 +echo "Dropping endpoint database";
   9.103 +dropdb -U postgres ${DB}
   9.104 +
   9.105 +echo "Creating endpoint database"
   9.106 +createdb -U postgres ${DB} 
   9.107 +
   9.108 +# load background data
   9.109 +echo "initializing database"
   9.110 +curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB}
   9.111 +psql ${DB} -U postgres -c 'VACUUM ANALYZE '
   9.112 +
   9.113 +echo "starting tomcat"
   9.114 +if test -z "${tomcat}"; then
   9.115 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   9.116 +	start_tomcat.sh
   9.117 +else
   9.118 +	sudo service ${tomcat} start
   9.119 +fi
   9.120 +
   9.121 +# the chain :)
   9.122 +echo "Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > ${timings}
   9.123 +for (( i = 0 ; i < ${#URLDIR[@]} ; i++ )) do
   9.124 +	dir=${URLDIR[$i]}
   9.125 +	prefix=${URLPREFIX[$i]}
   9.126 +	sensor=${SENSOR[$i]}
   9.127 +	satelite=${SATELITE[$i]}
   9.128 +    acquisitions=${ACQ_HALF_HOUR[$i]}
   9.129 +	# get hotpost URLS
   9.130 +	for hot in $(curl -s ${HOTSPOTS_URL}/${dir}/ | grep -o ">${prefix}.*\.n3" | colrm 1 1); do
   9.131 +		echo $hot
   9.132 +		file="${HOTSPOTS_URL}/${dir}/${hot}"
   9.133 +		echo $file
   9.134 +
   9.135 +		# get time information for acquisition
   9.136 +		offset=$(( ${#prefix} + 1 ))
   9.137 +		year=$(expr substr ${hot} ${offset} 2)
   9.138 +		month=$(expr substr ${hot} $(( ${offset} + 2 )) 2)
   9.139 +		day=$(expr substr ${hot} $(( ${offset} + 4 )) 2)
   9.140 +		t1=$(expr substr ${hot} $(( ${offset} + 7 )) 2)
   9.141 +		t2=$(expr substr ${hot} $(( ${offset} + 9 )) 2)
   9.142 +		time2="${t1}:${t2}"
   9.143 +
   9.144 +		printf "$hot " >> ${timings}
   9.145 +
   9.146 +		# store file
   9.147 +		echo -n "storing " $file; echo; echo; 
   9.148 +		# ${countTime} ./strabon -db endpoint store $file
   9.149 +
   9.150 +		tmr1=$(timer)
   9.151 +		../endpoint store ${ENDPOINT} N-Triples -u ${file}
   9.152 +		tmr2=$(timer)
   9.153 +		printf '%s ' $((tmr2-tmr1)) >>  ${timings}
   9.154 +
   9.155 +		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
   9.156 +
   9.157 +		echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   9.158 +
   9.159 +		# insertMunicipalities
   9.160 +		echo -n "inserting Municipalities " ;echo; echo; echo;
   9.161 +		# query=`echo "${insertMunicipalities}" `
   9.162 +		# ${countTime} ./strabon -db endpoint update "${query}"
   9.163 +
   9.164 +		tmr1=$(timer)
   9.165 +
   9.166 +		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   9.167 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.168 +		sed "s/SENSOR/${sensor}/g"`
   9.169 +
   9.170 +		../endpoint update ${ENDPOINT} "${query}"
   9.171 +		
   9.172 +		tmr2=$(timer)
   9.173 +printf '%s ' $((tmr2-tmr1)) >> ${timings}
   9.174 +		echo;echo;echo;echo "File ${file} inserted Municipalities!"
   9.175 +		
   9.176 +		# deleteSeaHotspots
   9.177 +		echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
   9.178 +		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   9.179 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.180 +		sed "s/SENSOR/${sensor}/g"`
   9.181 +		# ${countTime} ./strabon -db endpoint update "${query}"
   9.182 +
   9.183 +		tmr1=$(timer)
   9.184 +		../endpoint update ${ENDPOINT} "${query}"
   9.185 +
   9.186 +		tmr2=$(timer)
   9.187 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   9.188 +		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   9.189 +
   9.190 +		# echo "Continue?"
   9.191 +		# read a
   9.192 +			# invalidForFires
   9.193 +		echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   9.194 +		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   9.195 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.196 +		sed "s/SENSOR/${sensor}/g" |\
   9.197 +		sed "s/SAT/${satelite}/g"`
   9.198 +		# ${countTime} ./strabon -db endpoint update "${query}"
   9.199 +		tmr1=$(timer)
   9.200 +		../endpoint update ${ENDPOINT} "${query}"
   9.201 +		tmr2=$(timer)
   9.202 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   9.203 +		echo "File ${file} invalidForFires done!"
   9.204 + 
   9.205 +		# refinePartialSeaHotspots
   9.206 +		echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   9.207 +		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   9.208 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.209 +		sed "s/SENSOR/${sensor}/g" |\
   9.210 +		sed "s/SAT/${satelite}/g"`
   9.211 +		# ${countTime} ./strabon -db endpoint update "${query}"
   9.212 +		tmr1=$(timer)
   9.213 +		../endpoint update ${ENDPOINT} "${query}"
   9.214 +		tmr2=$(timer)
   9.215 +		printf '%s ' $((tmr2-tmr1)) >> ${timings}
   9.216 +
   9.217 +		echo "File ${file} refinePartialSeaHotspots done!"
   9.218 +		# echo "Continue?"
   9.219 +		# read a
   9.220 +
   9.221 +		# refineTimePersistence
   9.222 +		echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   9.223 +		min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
   9.224 +		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   9.225 +		sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.226 +		sed "s/SENSOR/${sensor}/g" | \
   9.227 +		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/${acquisitions}/g" | \
   9.228 +		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   9.229 +		sed "s/SAT/${satelite}/g"`
   9.230 +
   9.231 +		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   9.232 +
   9.233 +		tmr1=$(timer)
   9.234 +		../endpoint update ${ENDPOINT} "${query}"
   9.235 +		 tmr2=$(timer)
   9.236 +		printf '%s \n' $((tmr2-tmr1)) >> ${timings}
   9.237 +		echo;echo;echo;echo "File ${file} timePersistence done!"
   9.238 +		# echo "Continue?"
   9.239 +		# read a
   9.240 +
   9.241 +
   9.242 +		# discover
   9.243 +		echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   9.244 +		min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
   9.245 +		max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
   9.246 +		query=`echo "${discover}" | \
   9.247 +			sed "s/PROCESSING_CHAIN/${PROCESSING_CHAIN}/g" | \
   9.248 +			sed "s/SENSOR/${sensor}/g" | \
   9.249 +			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   9.250 +			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   9.251 +			
   9.252 +		tmr1=$(timer)
   9.253 +		../endpoint query ${ENDPOINT} "${query}"
   9.254 +		tmr2=$(timer)
   9.255 +		printf '%s \n' $((tmr2-tmr1)) >> ${timingsDiscover}
   9.256 +		echo;echo;echo;echo "Discovered hotspots done!"
   9.257 +	done
   9.258 +done
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/scripts/v2.3/runNoaRefinementChain.sh	Wed Jul 25 01:59:00 2012 +0300
    10.3 @@ -0,0 +1,265 @@
    10.4 +#!/bin/bash
    10.5 +LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    10.6 +
    10.7 +ENDPOINT="http://localhost:8080/endpoint"
    10.8 +DB="endpoint"
    10.9 +
   10.10 +#HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG2"
   10.11 +HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
   10.12 +
   10.13 +logFile="chain.log"
   10.14 +
   10.15 +function timer()
   10.16 +{
   10.17 +   if [[ $# -eq 0 ]]; then
   10.18 +       t=$(date '+%s%N')
   10.19 +       t=$((t/1000000))
   10.20 +       echo $t
   10.21 +   else
   10.22 +       local  stime=$1
   10.23 +       etime=$(date '+%s%N')
   10.24 +       etime=$((etime/1000000))
   10.25 +
   10.26 +       if [[ -z "$stime" ]]; then stime=$etime; fi
   10.27 +       dt=$((etime - stime)) #dt in milliseconds
   10.28 +       dM=$((dt%1000))
   10.29 +       Dt=$((dt/1000)) #delta t in seconds
   10.30 +       ds=$((Dt % 60))
   10.31 +       dm=$(((Dt / 60) % 60))
   10.32 +       dh=$((Dt / 3600))
   10.33 +       printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
   10.34 +   fi
   10.35 +}
   10.36 +
   10.37 +# find out the postgres service to use
   10.38 +postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
   10.39 +
   10.40 +tomcat=
   10.41 +function chooseTomcat()
   10.42 +{
   10.43 +	if test -s /etc/fedora-release ; then
   10.44 +		tomcat="tomcat"
   10.45 +	#elif test -s /etc/centos-release ; then
   10.46 +	#elif test -s /etc/yellowdog-release ; then
   10.47 +	#elif test -s /etc/redhat-release ; then
   10.48 +	#elif test -s /etc/SuSE-release ; then
   10.49 +	#elif test -s /etc/gentoo-release ; then
   10.50 +	elif test -s /etc/lsb-release ; then # Ubuntu
   10.51 +			tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
   10.52 +	elif test -s /etc/debian_version ; then
   10.53 +		tomcat="tomcat"
   10.54 +	fi
   10.55 +
   10.56 +	# check for service availability
   10.57 +	if ! test -e "/etc/init.d/${tomcat}"; then
   10.58 +		tomcat=
   10.59 +	fi
   10.60 +}
   10.61 +
   10.62 +insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql` 
   10.63 +deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
   10.64 +refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
   10.65 +refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
   10.66 +invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
   10.67 +discover=`cat ${LOC}/discover.sparql`
   10.68 +#InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
   10.69 +
   10.70 +# Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
   10.71 +chooseTomcat
   10.72 +echo "stopping tomcat"
   10.73 +if test -z "${tomcat}"; then
   10.74 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   10.75 +	stop_tomcat.sh
   10.76 +else
   10.77 +	sudo service ${tomcat} stop
   10.78 +fi
   10.79 +
   10.80 +sudo service ${postgres} restart
   10.81 +
   10.82 +# get the main version of postgres
   10.83 +POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
   10.84 +
   10.85 +echo "Dropping endpoint database";
   10.86 +dropdb  ${DB}
   10.87 +
   10.88 +echo "Creating endpoint database"
   10.89 +createdb  ${DB} 
   10.90 +
   10.91 +# load data
   10.92 +curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
   10.93 +psql ${DB} -c 'VACUUM ANALYZE' 
   10.94 +
   10.95 +echo "starting tomcat"
   10.96 +if test -z "${tomcat}"; then
   10.97 +	# work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
   10.98 +	start_tomcat.sh
   10.99 +	sleep 2
  10.100 +else
  10.101 +	sudo service ${tomcat} start
  10.102 +fi
  10.103 +
  10.104 +echo "initializing database"
  10.105 +echo "Timestamp	Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence" > stderr.txt
  10.106 +
  10.107 +
  10.108 +#./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
  10.109 +#sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
  10.110 +#sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
  10.111 +
  10.112 +#for y in 2007 2008 2010 2011 ;do
  10.113 +for y in 2012; do
  10.114 +	# get hotpost URLS
  10.115 +	for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
  10.116 +		file="${HOTSPOTS_URL}/${y}/${hot}"
  10.117 +
  10.118 +		time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
  10.119 +		
  10.120 +		# get sensor
  10.121 +		SENSOR=$(echo ${hot} | grep -o 'MSG.')
  10.122 +
  10.123 +		# get satellite
  10.124 +		if test "${SENSOR}" = "MSG2"; then
  10.125 +			SAT="METEOSAT9"
  10.126 +		else
  10.127 +			SAT="METEOSAT8"
  10.128 +			SENSOR="MSG1_RSS"
  10.129 +		fi
  10.130 +
  10.131 +		# get time information for acquisition and construct timestamp
  10.132 +		year="20$(expr substr ${time_status} 1 2)"
  10.133 +		month=$(expr substr ${time_status} 3 2)
  10.134 +		day=$(expr substr ${time_status} 5 2)
  10.135 +		time2=$(expr substr ${time_status} 8 2)
  10.136 +		time2="${time2}:$(expr substr ${time_status} 10 2)"
  10.137 +
  10.138 +		# construct timestamp
  10.139 +		TIMESTAMP="${year}-${month}-${day}T${time2}:00"
  10.140 +
  10.141 +		# store file
  10.142 +		echo -n "storing " $file; echo; echo; 
  10.143 +		# ${countTime} ./strabon -db endpoint store $file
  10.144 +
  10.145 +		# print timestamp
  10.146 +		echo -n "${TIMESTAMP} " >> stderr.txt
  10.147 +
  10.148 +		tmr1=$(timer)
  10.149 +		../endpoint store ${ENDPOINT} N-Triples -u ${file}
  10.150 +		tmr2=$(timer)
  10.151 +		printf '%s ' $((tmr2-tmr1)) >> stderr.txt
  10.152 +
  10.153 +		# sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
  10.154 +
  10.155 +		echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
  10.156 +
  10.157 +		# insertMunicipalities
  10.158 +		echo -n "inserting Municipalities " ;echo; echo; echo;
  10.159 +		# query=`echo "${insertMunicipalities}" `
  10.160 +		# ${countTime} ./strabon -db endpoint update "${query}"
  10.161 +
  10.162 +		tmr1=$(timer)
  10.163 +
  10.164 +		query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
  10.165 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.166 +		sed "s/SENSOR/${SENSOR}/g"`
  10.167 +
  10.168 +		../endpoint update ${ENDPOINT} "${query}"
  10.169 +		
  10.170 +		tmr2=$(timer)
  10.171 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
  10.172 +		echo;echo;echo;echo "File ${file} inserted Municipalities!"
  10.173 +
  10.174 +		# execute an explicit VACUUM ANALYZE when a query takes longer than it should
  10.175 +		duration=$((tmr2-tmr1))
  10.176 +		if test ${duration} -ge 30000; then
  10.177 +			psql ${DB} -c 'VACUUM ANALYZE' 
  10.178 +			echo "Explicit VACUUM ANALYZE"
  10.179 +		fi
  10.180 +		
  10.181 +		# deleteSeaHotspots
  10.182 +		echo -n "Going to deleteSeaHotspots ${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
  10.183 +		query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
  10.184 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.185 +		sed "s/SENSOR/${SENSOR}/g"`
  10.186 +		# ${countTime} ./strabon -db endpoint update "${query}"
  10.187 +
  10.188 +		tmr1=$(timer)
  10.189 +		../endpoint update ${ENDPOINT} "${query}"
  10.190 +
  10.191 +		tmr2=$(timer)
  10.192 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
  10.193 +		echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
  10.194 +
  10.195 +		# echo "Continue?"
  10.196 +		# read a
  10.197 +			# invalidForFires
  10.198 +		echo -n "invalidForFires ${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
  10.199 +		query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
  10.200 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.201 +		sed "s/SENSOR/${SENSOR}/g" |\
  10.202 +		sed "s/SAT/${SAT}/g"`
  10.203 +		# ${countTime} ./strabon -db endpoint update "${query}"
  10.204 +		tmr1=$(timer)
  10.205 +		../endpoint update ${ENDPOINT} "${query}"
  10.206 +		tmr2=$(timer)
  10.207 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
  10.208 +		echo "File ${file} invalidForFires done!"
  10.209 + 
  10.210 +		# refinePartialSeaHotspots
  10.211 +		echo -n "refinePartialSeaHotspots ${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
  10.212 +		query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
  10.213 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.214 +		sed "s/SENSOR/${SENSOR}/g" |\
  10.215 +		sed "s/SAT/${SAT}/g"`
  10.216 +		# ${countTime} ./strabon -db endpoint update "${query}"
  10.217 +		tmr1=$(timer)
  10.218 +		../endpoint update ${ENDPOINT} "${query}"
  10.219 +		tmr2=$(timer)
  10.220 +		printf '%s ' $((tmr2-tmr1)) >>stderr.txt
  10.221 +
  10.222 +		echo "File ${file} refinePartialSeaHotspots done!"
  10.223 +		# echo "Continue?"
  10.224 +		# read a
  10.225 +
  10.226 +		# refineTimePersistence
  10.227 +		echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
  10.228 +		min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
  10.229 +		newHotspotTimestamp=`date --date="${year}-${month}-${day} ${time2}:00" +%y%m%d_%H%M`
  10.230 +		
  10.231 +		query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
  10.232 +		sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.233 +		sed "s/SENSOR/${SENSOR}/g" | \
  10.234 +		sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
  10.235 +		sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
  10.236 +		sed "s/SAT/${SAT}/g" | \
  10.237 +		sed "s/NEW_HOTSPOT/${newHotspotTimestamp}/g"`
  10.238 +
  10.239 +		#sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
  10.240 +
  10.241 +		tmr1=$(timer)
  10.242 +		../endpoint update ${ENDPOINT} "${query}"
  10.243 +		 tmr2=$(timer)
  10.244 +		printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
  10.245 +		echo;echo;echo;echo "File ${file} timePersistence done!"
  10.246 +		# echo "Continue?"
  10.247 +		# read a
  10.248 +
  10.249 +
  10.250 +		# discover
  10.251 +		echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
  10.252 +		min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
  10.253 +		max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
  10.254 +		query=`echo "${discover}" | \
  10.255 +			sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
  10.256 +			sed "s/SENSOR/${SENSOR}/g" | \
  10.257 +			sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
  10.258 +			sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
  10.259 +			
  10.260 +		tmr1=$(timer)
  10.261 +		../endpoint query ${ENDPOINT} "${query}"
  10.262 +		tmr2=$(timer)
  10.263 +		printf '%s \n' $((tmr2-tmr1)) >>discover.txt
  10.264 +		echo;echo;echo;echo "Discovered hotspots done!"
  10.265 +
  10.266 +	done
  10.267 +done
  10.268 +