Strabon
changeset 288:095644d86e05 noa
Adding v2 queries and a modified version of the script that simulates the NOA workload
author | Kostis Kyzirakos <kkyzir@di.uoa.gr> |
---|---|
date | Mon Jun 18 21:44:27 2012 +0300 (2012-06-18) |
parents | 84fffede6df8 |
children | e648ef180923 |
files | scripts/v2/DeleteInSea.sparql scripts/v2/Refine.sparql scripts/v2/TimePersistence.sparql scripts/v2/count.sparql scripts/v2/runNoaRefinementChain.sh |
line diff
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 1.2 +++ b/scripts/v2/DeleteInSea.sparql Mon Jun 18 21:44:27 2012 +0300 1.3 @@ -0,0 +1,20 @@ 1.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 1.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 1.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 1.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 1.8 + 1.9 +INSERT {?h noa:isDiscarded "1"^^xsd:int} 1.10 +WHERE { 1.11 + ?h rdf:type noa:Hotspot; 1.12 + noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime; 1.13 + noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string; 1.14 + noa:isDerivedFromSensor "SENSOR"^^xsd:string; 1.15 + noa:hasGeometry ?hGeo; 1.16 + ?property ?object . 1.17 + OPTIONAL { 1.18 + ?c rdf:type noa:Coastline; 1.19 + noa:hasGeometry ?cGeo . 1.20 + FILTER(strdf:anyInteract(?hGeo, ?cGeo)) . 1.21 + } 1.22 + FILTER(!bound(?c)) . 1.23 +}
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 2.2 +++ b/scripts/v2/Refine.sparql Mon Jun 18 21:44:27 2012 +0300 2.3 @@ -0,0 +1,35 @@ 2.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 2.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 2.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 2.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 2.8 + 2.9 + 2.10 +INSERT { ?h noa:isDiscarded "1"^^xsd:int . 2.11 + ?valid rdf:type noa:Hotspot ; 2.12 + noa:hasConfidence ?conf ; 2.13 + noa:hasGeometry ?dif ; 2.14 + noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 2.15 + noa:isDerivedFromSensor "SENSOR"^^xsd:string ; 2.16 + noa:hasConfirmation noa:unknown ; 2.17 + noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 2.18 + noa:isProducedBy noa:noa ; 2.19 + noa:isDerivedFromSatellite ?sat . 2.20 +} 2.21 +WHERE { 2.22 + SELECT DISTINCT ?h ?hGeo (strdf:intersection(?hGeo, strdf:union(?cGeo)) AS ?dif) (BNODE() AS ?valid) ?conf ?sat 2.23 + WHERE { 2.24 + ?h rdf:type noa:Hotspot ; 2.25 + noa:hasAcquisitionTime ?hAcqTime ; 2.26 + noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string ; 2.27 + noa:isDerivedFromSensor "SENSOR"^^xsd:string ; 2.28 + noa:hasGeometry ?hGeo ; 2.29 + noa:hasConfidence ?conf ; 2.30 + noa:isDerivedFromSatellite ?sat . 2.31 + FILTER("TIMESTAMP"^^xsd:dateTime = ?hAcqTime) . 2.32 + ?c rdf:type noa:Coastline ; 2.33 + noa:hasGeometry ?cGeo . 2.34 + FILTER(strdf:anyInteract(?hGeo, ?cGeo)) . 2.35 + } 2.36 + GROUP BY ?h ?hGeo 2.37 + HAVING strdf:overlap(?hGeo, strdf:union(?cGeo)) 2.38 +}
3.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 3.2 +++ b/scripts/v2/TimePersistence.sparql Mon Jun 18 21:44:27 2012 +0300 3.3 @@ -0,0 +1,47 @@ 3.4 +PREFIX noa: <http://teleios.di.uoa.gr/ontologies/noaOntology.owl#> 3.5 +PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 3.6 +PREFIX strdf: <http://strdf.di.uoa.gr/ontology#> 3.7 +PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> 3.8 + 3.9 +INSERT { 3.10 + ?newHotspot rdf:type noa:Hotspot ; 3.11 + noa:hasConfidence ?hConfidence ; 3.12 + noa:hasGeometry ?hGeometry1 ; 3.13 + noa:hasAcquisitionTime "TIMESTAMP"^^xsd:dateTime ; 3.14 + noa:isDerivedFromSensor "SENSOR"^^xsd:string ; 3.15 + noa:hasConfirmation noa:unknown ; 3.16 + noa:producedFromProcessingChain ?hProcessingChain ; 3.17 + noa:isProducedBy noa:noa . 3.18 +} 3.19 +WHERE { 3.20 + SELECT (BNODE() AS ?newHotspot) 3.21 + (SUM(?hConfidence1)/ACQUISITIONS_IN_HALF_AN_HOUR AS ?hConfidence) 3.22 + ?hGeometry1 3.23 + (CONCAT("PROCESSING_CHAIN"^^xsd:string,"-TimePersistence") AS ?hProcessingChain) 3.24 + WHERE { 3.25 + ?H1 rdf:type noa:Hotspot . 3.26 + ?H1 noa:hasConfidence ?hConfidence1 . 3.27 + ?H1 noa:hasGeometry ?hGeometry1 . 3.28 + ?H1 noa:hasAcquisitionTime ?hAcquisitionTime1 . 3.29 + ?H1 noa:isDerivedFromSensor "SENSOR"^^xsd:string . 3.30 + ?H1 noa:hasConfirmation noa:unknown . 3.31 + ?H1 noa:producedFromProcessingChain "PROCESSING_CHAIN"^^xsd:string . 3.32 + OPTIONAL { ?H1 noa:isDiscarded ?z } . 3.33 + FILTER (!BOUND(?z)) . 3.34 + ?H1 noa:isProducedBy noa:noa . 3.35 + FILTER( "MIN_ACQUISITION_TIME"^^xsd:dateTime <= ?hAcquisitionTime1 && ?hAcquisitionTime1 < "TIMESTAMP"^^xsd:dateTime ) . 3.36 + OPTIONAL { 3.37 + ?H2 rdf:type noa:Hotspot . 3.38 + ?H2 noa:hasGeometry ?HGEO2 . 3.39 + ?H2 noa:hasAcquisitionTime ?hAcquisitionTime2 . 3.40 + ?H2 noa:isDerivedFromSensor "SENSOR"^^xsd:string . 3.41 + ?H2 noa:producedFromProcessingChain ?hProcessingChain2 . 3.42 + FILTER(regex("PROCESSING_CHAIN"^^xsd:string, ?hProcessingChain2)). 3.43 + FILTER( strdf:equals(?hGeometry1, ?HGEO2) ) . 3.44 + FILTER(?hAcquisitionTime2 = "TIMESTAMP"^^xsd:dateTime) . 3.45 + } 3.46 + FILTER( !BOUND(?H2) ) . 3.47 + } 3.48 + GROUP BY ?hGeometry1 3.49 + HAVING(SUM(?hConfidence1)>0.0) 3.50 +}
4.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 4.2 +++ b/scripts/v2/count.sparql Mon Jun 18 21:44:27 2012 +0300 4.3 @@ -0,0 +1,2 @@ 4.4 +SELECT (count(*) AS ?count) 4.5 +WHERE {?s ?p ?o}
5.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 5.2 +++ b/scripts/v2/runNoaRefinementChain.sh Mon Jun 18 21:44:27 2012 +0300 5.3 @@ -0,0 +1,168 @@ 5.4 +#!/bin/bash 5.5 +LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 5.6 + 5.7 +ENDPOINT="http://localhost:8080/strabonendpoint" 5.8 +DB="explicit" 5.9 + 5.10 +#dataDir="http://localhost/noa-teleios/out_triples/" 5.11 +dataUrl="http://kk.di.uoa.gr/out_triples/" 5.12 +name="HMSG2_IR_039_s7_070825" 5.13 +suffix=".hotspots.n3" 5.14 + 5.15 +logFile="chain.log" 5.16 +#countWTime="/usr/bin/time -p %e" 5.17 +#echo > ${logFile} 5.18 + 5.19 +function timer() 5.20 +{ 5.21 + if [[ $# -eq 0 ]]; then 5.22 + echo $(date '+%s') 5.23 + else 5.24 + local stime=$1 5.25 + etime=$(date '+%s') 5.26 + 5.27 + if [[ -z "$stime" ]]; then stime=$etime; fi 5.28 + 5.29 + dt=$((etime - stime)) 5.30 + ds=$((dt % 60)) 5.31 + dm=$(((dt / 60) % 60)) 5.32 + dh=$((dt / 3600)) 5.33 + printf '%d:%02d:%02d' $dh $dm $ds 5.34 + fi 5.35 +} 5.36 + 5.37 +#prin 5.38 +#tmr1=$(timer) 5.39 +#meta 5.40 +#tmr2=$(timer) 5.41 +#ektypwsi 5.42 +#printf 'LALA %s %s\n' $((tmr2-tmr1)) $(timer $tmr1) 5.43 + 5.44 + 5.45 +deleteSeaHotspots=`cat ${LOC}/DeleteInSea.sparql | sed 's/\"/\\\"/g'` 5.46 +refinePartialSeaHotspots=`cat ${LOC}/Refine.sparql | sed 's/\"/\\\"/g'` 5.47 +refineTimePersistence=`cat ${LOC}/TimePersistence.sparql | sed 's/\"/\\\"/g'` 5.48 + 5.49 + 5.50 +# Initialize 5.51 +sudo service postgresql restart 5.52 +echo "Dropping endpoint database"; 5.53 +sudo -u postgres dropdb ${DB} 5.54 +echo "Creating endpoint database" 5.55 +sudo -u postgres createdb ${DB} -T template_postgis 5.56 +echo "restarting tomcat" 5.57 +sudo service tomcat7 restart 5.58 + 5.59 + 5.60 +echo "initializing database" 5.61 +echo "S D R TP" >>stderr.txt 5.62 + 5.63 + ../endpoint store ${ENDPOINT} N-Triples -u http://localhost/noa-teleios/grid_4.nt 5.64 + 5.65 +#./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}" 5.66 +#sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)'; 5.67 +#sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 5.68 + 5.69 + 5.70 +#echo "Continue?" 5.71 +#read a 5.72 + 5.73 + 5.74 + 5.75 + 5.76 +for h in `seq 0 23 `; do 5.77 + for m in `seq 0 15 45`; do 5.78 + time=`printf "%02d%02d\n" $h $m` 5.79 + time2=`printf "%02d:%02d\n" $h $m` 5.80 + file=${dataDir}${name}_${time}$suffix 5.81 +# file=${dataUrl}${name}_${time}$suffix 5.82 + 5.83 + # store file 5.84 + echo -n "storing " $file; echo; echo; 5.85 + # echo "Hotspot : " $h:$m >> stderr.txt 5.86 +# ${countTime} ./strabon -db endpoint store $file 5.87 + 5.88 + tmr1=$(timer) 5.89 + ../endpoint store ${ENDPOINT} N-Triples -u ${file} 5.90 + tmr2=$(timer) 5.91 +printf '%s ' $((tmr2-tmr1)) >>stderr.txt 5.92 + 5.93 + # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;'; 5.94 + 5.95 + echo;echo;echo;echo "File ${file} stored!" >> ${logFile} 5.96 +# echo "Continue?" 5.97 +# read a 5.98 + 5.99 + # deleteSeaHotspots 5.100 + echo -n "Going to deleteSeaHotspots 2007-08-25T${time2}:00 " ;echo; echo; echo; 5.101 + query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/2007-08-25T${time2}:00/g" | \ 5.102 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 5.103 + sed "s/SENSOR/MSG2/g"` 5.104 +# ${countTime} ./strabon -db endpoint update "${query}" 5.105 + 5.106 +tmr1=$(timer) 5.107 + ../endpoint update ${ENDPOINT} "${query}" 5.108 + 5.109 +tmr2=$(timer) 5.110 +printf '%s ' $((tmr2-tmr1)) >>stderr.txt 5.111 + 5.112 + echo;echo;echo;echo "File ${file} deleteSeaHotspots done!" 5.113 +# echo "Continue?" 5.114 +# read a 5.115 + 5.116 + # refinePartialSeaHotspots 5.117 + echo -n "refinePartialSeaHotspots 2007-08-25T${time2}:00 " ; echo; echo ; echo; 5.118 + query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/2007-08-25T${time2}:00/g" | \ 5.119 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 5.120 + sed "s/SENSOR/MSG2/g"` 5.121 +# ${countTime} ./strabon -db endpoint update "${query}" 5.122 +tmr1=$(timer) 5.123 + ../endpoint update ${ENDPOINT} "${query}" 5.124 + 5.125 +tmr2=$(timer) 5.126 +printf '%s ' $((tmr2-tmr1)) >>stderr.txt 5.127 + 5.128 + echo "File ${file} refinePartialSeaHotspots done!" 5.129 +# echo "Continue?" 5.130 +# read a 5.131 + 5.132 + # refineTimePersistence 5.133 + echo -n "Going to refineTimePersistence 2007-08-25T${time2}:00 ";echo;echo;echo; 5.134 + min_acquisition_time=`date --date="2007-08-25 ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00` 5.135 + query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/2007-08-25T${time2}:00/g" | \ 5.136 + sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \ 5.137 + sed "s/SENSOR/MSG2/g" | \ 5.138 + sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \ 5.139 + sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g"` 5.140 + 5.141 +# echo "Query:" 5.142 +# echo "${query}" 5.143 +# echo "Continue?" 5.144 +# read a 5.145 +# ${countTime} ./strabon -db endpoint update "${query}" 5.146 +# ${countTime} ../endpoint update ${ENDPOINT} "${query}" 5.147 + 5.148 + sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;'; 5.149 + 5.150 + 5.151 +tmr1=$(timer) 5.152 + ../endpoint update ${ENDPOINT} "${query}" 5.153 + tmr2=$(timer) 5.154 +printf '%s \n' $((tmr2-tmr1)) >>stderr.txt 5.155 + 5.156 + echo;echo;echo;echo "File ${file} timePersistence done!" 5.157 +# echo "Continue?" 5.158 +# read a 5.159 + done 5.160 +done 5.161 + 5.162 + 5.163 +#for f in `ls /home/konstantina/noa-teleios/out_triples/HMSG2_IR_039_s7_070825_*.hotspots.n3` 5.164 +#do 5.165 + 5.166 +# echo "Store $f" 5.167 +# ${countTime} ./scripts/strabon -db endpoint store $f 5.168 +# 5.169 +# 5.170 +#done 5.171 +