Strabon

changeset 433:8416f5bb7999

added invalidfor fires and modified chain script
author Konstantina Mpereta <grad1110@di.uoa.gr>
date Wed Jul 18 12:35:16 2012 +0300 (2012-07-18)
parents 3b01363b9d58
children e4c4b31ef4b6
files scripts/v2.2/DeleteInSea.sparql scripts/v2.2/InsertMunicipalities.sparql scripts/v2.2/InvalidForFires.sparql scripts/v2.2/Refine.sparql scripts/v2.2/TimePersistence.sparql scripts/v2.2/discover.sparql scripts/v2.2/runNoaRefinementChain.sh
line diff
     1.1 --- a/scripts/v2.2/DeleteInSea.sparql	Wed Jul 18 02:47:07 2012 +0300
     1.2 +++ b/scripts/v2.2/DeleteInSea.sparql	Wed Jul 18 12:35:16 2012 +0300
     1.3 @@ -8,9 +8,8 @@
     1.4  WHERE {
     1.5  ?h   noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ;
     1.6       noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
     1.7 -     noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
     1.8 -     noa:hasGeometry ?hGeo.
     1.9 -  OPTIONAL {
    1.10 +     noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    1.11 +       OPTIONAL {
    1.12      ?h	gag:hasMunicipality ?muni .
    1.13    } 
    1.14    FILTER(!bound(?muni)) .
     2.1 --- a/scripts/v2.2/InsertMunicipalities.sparql	Wed Jul 18 02:47:07 2012 +0300
     2.2 +++ b/scripts/v2.2/InsertMunicipalities.sparql	Wed Jul 18 12:35:16 2012 +0300
     2.3 @@ -11,6 +11,8 @@
     2.4    WHERE {
     2.5  	?h  rdf:type noa:Hotspot ;
     2.6  		noa:hasGeometry ?hGeo ;
     2.7 +		noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
     2.8 +		noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
     2.9          noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime.
    2.10  	?m	rdf:type gag:Dhmos ;
    2.11  		rdfs:label ?mLabel ;
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/scripts/v2.2/InvalidForFires.sparql	Wed Jul 18 12:35:16 2012 +0300
     3.3 @@ -0,0 +1,19 @@
     3.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 
     3.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     3.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     3.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     3.8 +PREFIX clc: <http://geo.linkedopendata.gr/corine/ontology#>
     3.9 +
    3.10 +
    3.11 +
    3.12 +INSERT {?h noa:isDiscarded "1"^^xsd:int} 
    3.13 +WHERE {
    3.14 +?h   noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ;
    3.15 +     noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ;
    3.16 +     noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    3.17 +     noa:hasGeometry ?hGeo.
    3.18 +?a   rdf:type   clc:ExcludeArea;
    3.19 +  clc:hasGeometry ?aGeo.
    3.20 +FILTER(strdf:mbbIntersects(?hGeo,?aGeo)).
    3.21 +} GROUP BY ?h ?hGeo
    3.22 +HAVING strdf:contains(strdf:union(?aGeo),?hGeo)
     4.1 --- a/scripts/v2.2/Refine.sparql	Wed Jul 18 02:47:07 2012 +0300
     4.2 +++ b/scripts/v2.2/Refine.sparql	Wed Jul 18 12:35:16 2012 +0300
     4.3 @@ -2,11 +2,14 @@
     4.4  PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     4.5  PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     4.6  PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     4.7 +PREFIX gag: <http://www.semanticweb.org/ontologies/2011/gagKallikratis.rdf#>
     4.8 +
     4.9  
    4.10  INSERT { ?h noa:isDiscarded "1"^^xsd:int . 
    4.11  	 ?valid rdf:type noa:Hotspot ;
    4.12  		noa:hasConfidence ?conf ;
    4.13  	        noa:hasGeometry ?dif ;
    4.14 +	        gag:hasMunicipality ?muni ;
    4.15  		noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 
    4.16  	        noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    4.17  		noa:hasConfirmation noa:unknown ;
    4.18 @@ -15,17 +18,18 @@
    4.19  		noa:isDerivedFromSatellite "SAT"^^xsd:string .
    4.20  }
    4.21  WHERE { 
    4.22 -  SELECT ?h  (strdf:intersection(?hGeo, strdf:union(?cGeo)) AS ?dif) (BNODE() AS ?valid) ?conf 
    4.23 +  SELECT ?h  (strdf:intersection(?hGeo, strdf:union(?cGeo)) AS ?dif) (BNODE() AS ?valid) ?conf ?muni
    4.24    WHERE { 
    4.25  ?h     noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime; 
    4.26         noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    4.27         noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    4.28         noa:hasGeometry ?hGeo ;
    4.29 +       gag:hasMunicipality ?muni ;
    4.30         noa:hasConfidence ?conf .
    4.31  ?c     rdf:type noa:Coastline ;
    4.32         noa:hasGeometry ?cGeo .
    4.33 -    FILTER(strdf:anyInteract(?hGeo, ?cGeo)) . 
    4.34 +	   FILTER(strdf:mbbIntersects(?hGeo, ?cGeo)) . 
    4.35    }
    4.36 -  GROUP BY ?h ?hGeo ?conf 
    4.37 +  GROUP BY ?h ?hGeo ?conf ?muni
    4.38    HAVING strdf:overlap(?hGeo, strdf:union(?cGeo))
    4.39  }
     5.1 --- a/scripts/v2.2/TimePersistence.sparql	Wed Jul 18 02:47:07 2012 +0300
     5.2 +++ b/scripts/v2.2/TimePersistence.sparql	Wed Jul 18 12:35:16 2012 +0300
     5.3 @@ -2,6 +2,8 @@
     5.4  PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
     5.5  PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 
     5.6  PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     5.7 +PREFIX gag: <http://www.semanticweb.org/ontologies/2011/gagKallikratis.rdf#>
     5.8 +
     5.9  
    5.10  INSERT {
    5.11   ?newHotspot rdf:type noa:Hotspot ;
    5.12 @@ -12,16 +14,18 @@
    5.13  		noa:hasConfirmation noa:unknown ;
    5.14  	        noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    5.15  		noa:isProducedBy noa:noa ;
    5.16 +		 gag:hasMunicipality ?muni ;
    5.17  		noa:isDerivedFromSatellite "SAT"^^xsd:string .
    5.18  
    5.19  }
    5.20  WHERE {
    5.21    SELECT (BNODE() AS ?newHotspot)
    5.22    (SUM(?hConfidence1)/ACQUISITIONS_IN_HALF_AN_HOUR AS ?hConfidence)
    5.23 -  ?hGeometry1
    5.24 +  ?hGeometry1 ?muni
    5.25    WHERE {
    5.26      ?H1 noa:hasConfidence ?hConfidence1 .
    5.27      ?H1 noa:hasGeometry ?hGeometry1 .
    5.28 +    ?H1 gag:hasMunicipality ?muni .
    5.29      ?H1 noa:hasAcquisitionTime ?hAcquisitionTime1 .
    5.30      ?H1 noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    5.31      ?H1 noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string .
    5.32 @@ -34,10 +38,10 @@
    5.33        ?H2 noa:isDerivedFromSensor "SENSOR"^^xsd:string .
    5.34        ?H2 noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string .
    5.35        FILTER(("PROCESSING_CHAIN"^^xsd:string = ?hProcessingChain2)||("PROCESSING_CHAIN-TimePersistence"^^xsd:string = ?hProcessingChain2)).
    5.36 -      FILTER( strdf:equals(?hGeometry1, ?HGEO2) ) .
    5.37 +      FILTER( strdf:mbbEquals(?hGeometry1, ?HGEO2) ) .
    5.38      }
    5.39      FILTER( !BOUND(?H2) ) .
    5.40    }
    5.41 -  GROUP BY ?hGeometry1
    5.42 +  GROUP BY ?hGeometry1 ?muni
    5.43    HAVING(SUM(?hConfidence1)>0.0)
    5.44  }
     6.1 --- a/scripts/v2.2/discover.sparql	Wed Jul 18 02:47:07 2012 +0300
     6.2 +++ b/scripts/v2.2/discover.sparql	Wed Jul 18 12:35:16 2012 +0300
     6.3 @@ -9,13 +9,15 @@
     6.4  PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 
     6.5  PREFIX gag: <http://teleios.di.uoa.gr/ontologies/gagKallikratis.rdf#>
     6.6  
     6.7 -SELECT ?h ?geo ?conf (GROUP_CONCAT(?dLabel ; SEPARATOR=', ') AS ?dimoi )
     6.8 +SELECT ?h  (strdf:transform(?hGeo, <http://www.opengis.net/def/crs/EPSG/0/4326>) AS ?geo) ?conf ?muni
     6.9  WHERE { 
    6.10  ?h   noa:hasGeometry ?hGeo ;
    6.11 -     noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 
    6.12 +     noa:hasAcquisitionTime ?hAcquisitionTime ; 
    6.13       noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 
    6.14       noa:isDerivedFromSensor "SENSOR"^^xsd:string ;
    6.15       noa:hasConfidence ?conf ;
    6.16  	 gag:hasMunicipality ?muni .
    6.17 +	 FILTER( "MIN_ACQUISITION_TIME"^^xsd:dateTime <= ?hAcquisitionTime && ?hAcquisitionTime < "MAX_ACQUISITION_TIME"^^xsd:dateTime ) .
    6.18 +
    6.19 +	 
    6.20  }
    6.21 -GROUP BY ?h (strdf:transform(?hGeo, <http://www.opengis.net/def/crs/EPSG/0/4326>) AS ?geo) ?conf 
     7.1 --- a/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 18 02:47:07 2012 +0300
     7.2 +++ b/scripts/v2.2/runNoaRefinementChain.sh	Wed Jul 18 12:35:16 2012 +0300
     7.3 @@ -27,20 +27,24 @@
     7.4  
     7.5  function timer()
     7.6  {
     7.7 -    if [[ $# -eq 0 ]]; then
     7.8 -        echo $(date '+%s')
     7.9 -    else
    7.10 -        local  stime=$1
    7.11 -        etime=$(date '+%s')
    7.12 +   if [[ $# -eq 0 ]]; then
    7.13 +       t=$(date '+%s%N')
    7.14 +       t=$((t/1000000))
    7.15 +       echo $t
    7.16 +   else
    7.17 +       local  stime=$1
    7.18 +       etime=$(date '+%s%N')
    7.19 +       etime=$((etime/1000000))
    7.20  
    7.21 -        if [[ -z "$stime" ]]; then stime=$etime; fi
    7.22 -
    7.23 -        dt=$((etime - stime))
    7.24 -        ds=$((dt % 60))
    7.25 -        dm=$(((dt / 60) % 60))
    7.26 -        dh=$((dt / 3600))
    7.27 -        printf '%d:%02d:%02d' $dh $dm $ds
    7.28 -    fi
    7.29 +       if [[ -z "$stime" ]]; then stime=$etime; fi
    7.30 +       dt=$((etime - stime)) #dt in milliseconds
    7.31 +       dM=$((dt%1000))
    7.32 +       Dt=$((dt/1000)) #delta t in seconds
    7.33 +       ds=$((Dt % 60))
    7.34 +       dm=$(((Dt / 60) % 60))
    7.35 +       dh=$((Dt / 3600))
    7.36 +       printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
    7.37 +   fi
    7.38  }
    7.39  
    7.40  function chooseTomcat()
    7.41 @@ -59,10 +63,12 @@
    7.42  	fi
    7.43  }
    7.44  
    7.45 -insertMunicipalities=`cat ${LOC}/InsertMunicipalities.sparql`
    7.46 +insertMunicipalities=`cat ${LOC}/InsertMunicipalities.sparql` 
    7.47  deleteSeaHotspots=`cat ${LOC}/DeleteInSea.sparql` # | sed 's/\"/\\\"/g'`
    7.48  refinePartialSeaHotspots=`cat ${LOC}/Refine.sparql` # | sed 's/\"/\\\"/g'`
    7.49  refineTimePersistence=`cat ${LOC}/TimePersistence.sparql` # | sed 's/\"/\\\"/g'`
    7.50 +invalidForFires=`cat ${LOC}/InvalidForFires.sparql`
    7.51 +discover=`cat ${LOC}/discover.sparql`
    7.52  #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
    7.53  
    7.54  
    7.55 @@ -73,16 +79,15 @@
    7.56  sudo -u postgres dropdb ${DB}
    7.57  echo "Creating endpoint database"
    7.58  sudo -u postgres createdb ${DB} 
    7.59 +sudo -u postgres sh -c "curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump.tgz|tar xz -O|psql -d ${DB}"
    7.60 +sudo -u postgres psql ${DB} -c 'VACUUM ANALYZE '
    7.61  echo "restarting tomcat"
    7.62  sudo service ${tomcat} restart
    7.63  
    7.64  echo "initializing database"
    7.65  echo "IM S D R TP" >>stderr.txt
    7.66  
    7.67 - ../endpoint store ${ENDPOINT} N-Triples -u ${INIT}
    7.68  
    7.69 -
    7.70 -sudo -u postgres sh -c "curl -s  http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump.tgz|tar xz -O|psql -d ${DB}"
    7.71  #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
    7.72  #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
    7.73  #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
    7.74 @@ -90,17 +95,7 @@
    7.75  #echo "Continue?"
    7.76  #read a
    7.77  
    7.78 -            # insertMunicipalities
    7.79 -            echo -n "inserting Municipalities " ;echo; echo; echo;
    7.80 -           # query=`echo "${insertMunicipalities}" `
    7.81 -#            ${countTime} ./strabon -db endpoint update "${query}"
    7.82  
    7.83 -tmr1=$(timer)
    7.84 -  ../endpoint update ${ENDPOINT} "${query}"
    7.85 -
    7.86 -tmr2=$(timer)
    7.87 -
    7.88 -            echo;echo;echo;echo "File ${file} inserted Municipalities!"
    7.89  
    7.90  for y in 7 8 10 11 ;do
    7.91  for mon in `seq 7 10`; do
    7.92 @@ -122,6 +117,7 @@
    7.93                then echo "FILE" $check "NOT EXISTS" ; continue
    7.94  	      fi
    7.95  
    7.96 +
    7.97              # store file
    7.98              echo -n "storing " $file; echo; echo; 
    7.99  	  # echo "Hotspot : " $h:$m >> stderr.txt
   7.100 @@ -136,6 +132,23 @@
   7.101  
   7.102              echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
   7.103  
   7.104 +            # insertMunicipalities
   7.105 +            echo -n "inserting Municipalities " ;echo; echo; echo;
   7.106 +           # query=`echo "${insertMunicipalities}" `
   7.107 +#            ${countTime} ./strabon -db endpoint update "${query}"
   7.108 +
   7.109 +tmr1=$(timer)
   7.110 +
   7.111 +                 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   7.112 +                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   7.113 +                sed "s/SENSOR/MSG2/g"`
   7.114 +
   7.115 +  ../endpoint update ${ENDPOINT} "${query}"
   7.116 +                
   7.117 +tmr2=$(timer)
   7.118 +
   7.119 +            echo;echo;echo;echo "File ${file} inserted Municipalities!"
   7.120 +            
   7.121              # deleteSeaHotspots
   7.122              echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
   7.123              query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   7.124 @@ -152,7 +165,19 @@
   7.125              echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
   7.126  #            echo "Continue?"
   7.127  #            read a
   7.128 -
   7.129 +            # invalidForFires
   7.130 +            echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   7.131 +            query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   7.132 +                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   7.133 +                sed "s/SENSOR/MSG2/g" |\
   7.134 +		sed "s/SAT/METEOSAT9/g"`
   7.135 +#            ${countTime} ./strabon -db endpoint update "${query}"
   7.136 +tmr1=$(timer)
   7.137 +              ../endpoint update ${ENDPOINT} "${query}"
   7.138 +tmr2=$(timer)
   7.139 +printf '%s ' $((tmr2-tmr1)) >>stderr.txt
   7.140 + echo "File ${file} invalidForFires done!"
   7.141 + 
   7.142              # refinePartialSeaHotspots
   7.143              echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 "  ; echo; echo ; echo;
   7.144              query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
   7.145 @@ -179,7 +204,7 @@
   7.146                  sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   7.147  		sed "s/SAT/METEOSAT9/g"`
   7.148  
   7.149 - sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   7.150 + #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
   7.151  
   7.152  tmr1=$(timer)
   7.153                ../endpoint update ${ENDPOINT} "${query}"
   7.154 @@ -188,8 +213,17 @@
   7.155              echo;echo;echo;echo "File ${file} timePersistence done!"
   7.156  #            echo "Continue?"
   7.157  #            read a
   7.158 -    done
   7.159 -done
   7.160 -done
   7.161 -done
   7.162 -done
   7.163 +    done #minutes
   7.164 +done #hours
   7.165 +            # discover
   7.166 +            echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo; 
   7.167 +            min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
   7.168 +            max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
   7.169 +            query=`echo "${discover}" | \
   7.170 +                sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
   7.171 +                sed "s/SENSOR/MSG2/g" | \
   7.172 +                sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
   7.173 +                sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
   7.174 +done #days
   7.175 +done #months
   7.176 +done #years