Strabon

view examples/teleios/runChain.sh @ 1426:014379fdddf2

fixed the tests of geof:distance and geof:buffer, that had wrong results due to the
conversion of the resulting geometry to the srid 3857.
The results are validated using PostGIS, where in the case of buffer there are
small differences in the precision.
Moreover, in the case of BufferMetresTest, there was no point in having as distance
1m, because the geometry did not change, so i added 55000 metres instead (about 0.5 degrees).
author Stella Giannakopoulou <sgian@di.uoa.gr>
date Wed Sep 24 17:43:53 2014 +0300 (2014-09-24)
parents e9521141694b
children
line source
1 #! /bin/bash
2 #
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this file, you
5 # can obtain one at http://mozilla.org/MPL/2.0/.
6 #
7 # Copyright (C) 2010, 2011, 2012, 2013 Pyravlos Team
8 #
9 # http://www.strabon.di.uoa.gr/
10 #
11 # Author: George Garbis <ggarbis@di.uoa.gr>
12 # Author: Charalampos (Babis) Nikolaou <charnik@di.uoa.gr>
13 # Author: Manos Karpathiotakis <mk@di.uoa.gr>
14 # Author: Konstantina Bereta <Konstantina.Bereta@di.uoa.gr>
15 #
17 # If tomcat is standalone then environment variable TOMCATPATH should be set
19 # Example run command: examples/teleios/runChain.sh -b http://dev.strabon.di.uoa.gr/rdf/data-dump-postgres-9.tgz -l ${HOME}/runChain.log -e http://pathway.di.uoa.gr:8080/endpoint
21 # Command name
22 cmd="$(basename ${0})"
23 # Get the directory where the script resides
24 loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
26 function help() {
27 echo "Usage: ${cmd} [OPTIONS] "
28 echo
29 echo "Execute NOA chain with refinements and measure time."
30 echo
31 echo "OPTIONS can be any of the following"
32 echo " -d,--db : PostGIS database"
33 echo " -e,--endpoint : Strabon Endpoint"
34 echo " -h,--hotposts : URL where hotspots are stored"
35 echo " -b,--background : Background data"
36 echo " -l,--log : Log file"
37 echo " -c,--chain : Processing chain of hotspots"
38 echo " -p,--persistence : Value of persistence of discoverFires query"
39 echo " -r,--repeat_in_persistence : Value of repeat_in_persistence of discoverFires query"
40 }
42 # If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC
43 # Else if a time (miliseconds form ...) is given it returns the delta between
44 # given time and current time
45 function timer()
46 {
47 if [[ $# -eq 0 ]]; then
48 t=$(date '+%s%N')
49 t=$((t/1000000))
50 echo $t
51 else
52 local stime=$1
53 etime=$(date '+%s%N')
54 etime=$((etime/1000000))
56 if [[ -z "$stime" ]]; then stime=$etime; fi
57 dt=$((etime - stime)) #dt in milliseconds
58 dM=$((dt%1000))
59 Dt=$((dt/1000)) #delta t in seconds
60 ds=$((Dt % 60))
61 dm=$(((Dt / 60) % 60))
62 dh=$((Dt / 3600))
63 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
64 fi
65 }
67 # Handle the postgres service
68 # -$1: Command for the service
69 function handlePostgresService()
70 {
71 # find out the postgres service to use
72 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
74 echo "Service ${postgres} received command: $1"
75 sudo service ${postgres} $1
76 }
78 # Handled a postgres database
79 # -$1: Command (create/drop/store)
80 # -$2: Dump file to store (if runscript is given as command)
81 # or 'spatial' to create a spatial database (if create is given as command)
82 function handlePostgresDatabase() {
83 local command=$1
84 local db=$2
85 shift; shift
86 local options="$*"
87 case "${command}" in
88 create)
89 if test "${options}" = "spatial"; then
90 options="-T template_postgis"
91 elif test ! -z "${options}"; then
92 echo "ERROR: only spatial is allowed for create option"
93 echo "options: ${options}"
94 exit -1
95 fi
96 echo "Creating database ${db}... with options ${options}"
97 createdb -U postgres ${db} ${options}
98 ;;
99 drop)
100 if test ! -z "${options}"; then
101 echo "ERROR: dropdb takes no extra options"
102 echo "options: ${options}"
103 exit -1
104 fi
105 echo "Dropping database ${db}..."
106 dropdb -U postgres ${db}
107 ;;
108 vacuum)
109 if test "${options}" = "analyze"; then
110 psql -U postgres ${db} -c 'VACUUM ANALYZE'
111 echo "VACUUM ANALYZE ${db}"
112 else
113 psql -U postgres ${db} -c 'VACUUM'
114 echo "VACUUM ${db}"
115 fi
116 ;;
117 runscript)
118 if test ! -f "${options}"; then
119 echo "ERROR: No dump file to run"
120 exit -1
121 fi
122 echo "Storing dump file ${options} in database ${db}..."
123 psql -U postgres ${db} -f ${options}
124 ;;
125 esac
126 }
128 # Handle the tomcat service
129 # -$1: Command for the service
130 function handleTomcatService()
131 {
132 if test ! -z "${TOMCATPATH}" ; then
133 case "${1}" in
134 start)
135 ${TOMCATPATH}/bin/startup.sh
136 ;;
137 stop)
138 ${TOMCATPATH}/bin/shutdown.sh
139 ;;
140 restart)
141 ${TOMCATPATH}/bin/startup.sh
142 ${TOMCATPATH}/bin/shutdown.sh
143 ;;
144 esac
145 return
146 # find out the tomcat service to use
147 elif test -s /etc/fedora-release ; then
148 tomcat="tomcat"
149 #elif test -s /etc/centos-release ; then
150 #elif test -s /etc/yellowdog-release ; then
151 #elif test -s /etc/redhat-release ; then
152 #elif test -s /etc/SuSE-release ; then
153 #elif test -s /etc/gentoo-release ; then
154 elif test -s /etc/lsb-release ; then # Ubuntu
155 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
156 elif test -s /etc/debian_version ; then
157 tomcat="tomcat"
158 fi
160 # check for service availability
161 if ! test -e "/etc/init.d/${tomcat}"; then
162 echo "ERROR: No tomcat service found"
163 exit -1
164 fi
166 echo "Service ${tomcat} received command: $1"
167 sudo service ${tomcat} $1
168 }
171 # get the main version of postgres
172 function getPostgresMainVersion() {
173 echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
174 }
176 # It stores the backgroud data
177 # - $1: database
178 # - $2: backgound data file
179 function storeBackgroundData() {
180 local db=$1
181 local bgFile=$2
183 if test -f ${bgFile}; then
184 handlePostgresDatabase runscript ${db} ${bgFile}
185 elif test "${bgFile:0:7}" = "http://"; then
186 curl -s ${bgFile} | tar xzf - -O > /tmp/bgFiles$$.sql
187 # wget ${bgFile} -O /tmp/bgFile$$.tar.gz
188 # tar xzf /tmp/bgFile$$.tar.gz
189 handlePostgresDatabase runscript ${db} /tmp/bgFiles$$.sql
190 # rm /tmp/bgFile$$.tar.gz
191 rm /tmp/bgFiles$$.sql
192 else
193 echo "Backgound file not found"
194 exit -1
195 fi
196 handlePostgresDatabase vacuum ${db} analyze
197 }
199 # Handle Stabon Endpoint
200 # - $1: endpoint
201 # - $2: command (store/query)
202 # - $2: file/query
203 function handleStrabonEndpoint(){
204 endpoint=$1
205 command=$2
206 options=$3
208 endpointScript=${loc}/../../scripts/endpoint
209 case ${command} in
210 store)
211 url=${options}
213 tmr1=$(timer)
214 #${endpointScript} store ${endpoint} N-Triples -u ${url}
215 #read t
216 ${endpointScript} store ${endpoint} N-Triples -u ${url}
217 tmr2=$(timer)
219 # execute an explicit VACUUM ANALYZE when a query takes longer than it should
220 duration=$((tmr2-tmr1))
221 if test ${duration} -ge 30000; then
222 handlePostgresDatabase vacuum ${db} analyze
223 # psql -U postgres ${DB} -c 'VACUUM ANALYZE'
224 # echo "Explicit VACUUM ANALYZE"
225 tmr2=$(timer)
226 fi
227 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
228 ;;
229 query)
230 query=${options}
231 tmr1=$(timer)
232 #${endpointScript} query ${endpoint} "${query}"
233 #read t
234 ${endpointScript} query ${endpoint} "${query}"
235 tmr2=$(timer)
236 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
237 ;;
238 update)
239 update=${options}
240 tmr1=$(timer)
241 #${endpointScript} update ${endpoint} "${update}"
242 #read t
243 ${endpointScript} update ${endpoint} "${update}"
244 tmr2=$(timer)
245 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
246 ;;
247 *)
248 echo "ERROR: Unknown endpoint command"
249 exit -1
250 ;;
251 esac
252 }
254 # default values
255 endpoint="http://teleios3.di.uoa.gr:8080/endpoint"
256 db="NOA2012"
257 hotspotsURL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
258 # ./examples/teleios/data/data-dump-9.sql
259 bgFile="http://dev.strabon.di.uoa.gr/rdf/data-dump-9.sql"
260 logFile="${HOME}/runChain.log"
262 chain="DynamicThresholds"
263 persistence=10
264 repeatInPers=3
266 # read script options
267 while test $# -gt 0 -a "X${1:0:1}" == "X-"; do
268 case "${1}" in
269 --help)
270 help
271 exit 0
272 ;;
273 -e|--endpoint)
274 shift
275 endpoint=${1}
276 shift
277 ;;
278 -d|--db)
279 shift
280 db=${1}
281 shift
282 ;;
283 -h|--hotspots)
284 shift
285 hotspots_url=${1}
286 shift
287 ;;
288 -b|--background)
289 shift
290 bgFile=${1}
291 shift
292 ;;
293 -l|--log)
294 shift
295 logFile=${1}
296 shift
297 ;;
298 -c|--chain)
299 shift
300 chain=${1}
301 shift
302 ;;
303 -p|--persistence)
304 shift
305 persistence=${1}
306 shift
307 ;;
308 -r|--repeat_in_persistence)
309 shift
310 repeat_in_persistence=${1}
311 shift
312 ;;
313 *)
314 echo "unknown argument ${1}"
315 help
316 exit -1
317 ;;
318 esac
319 done
321 echo "endpoint: ${endpoint}"
322 echo "db: ${db}"
323 echo "hotspots: ${hotspotsURL}"
324 echo "background: ${bgFile}"
325 echo "logFile: ${logFile}"
327 echo > ${logFile}
328 instantiate=${loc}/instantiate.sh
330 #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
331 handleTomcatService stop
332 handlePostgresService restart
334 handlePostgresDatabase drop ${db}
335 handlePostgresDatabase create ${db}
337 storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql
339 handleTomcatService start
340 # Wait until tomcat server is up
341 until [ "`curl --silent --show-error --connect-timeout 1 -I ${endpoint} | grep 'Coyote'`" != "" ]; do sleep 1; done;
344 #${loc}/../../scripts/endpoint query ${endpoint} size
345 #exit -1
346 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence DiscoverHotspots" > ${logFile}
347 echo > ${HOME}/discoverFires.log
348 echo > ${HOME}/discover.log
350 years="2012" #"2007 2008 2010 2011"
351 for y in ${years}; do
352 # hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"
353 # get hotpost URLS
354 for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
355 # for hot in ${hotspots}; do
356 file="${hotspotsURL}/${y}/${hot}"
358 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
360 # get sensor
361 sensor=$(echo ${hot} | grep -o 'MSG.')
363 # get satellite and set number of acquisitions per hour
364 if test "${sensor}" = "MSG1"; then
365 sensor="MSG1_RSS"
366 fi
368 # get time information for acquisition and construct timestamp
369 year="20$(expr substr ${time_status} 1 2)"
370 month=$(expr substr ${time_status} 3 2)
371 day=$(expr substr ${time_status} 5 2)
372 time2=$(expr substr ${time_status} 8 2)
373 time2="${time2}:$(expr substr ${time_status} 10 2)"
375 # construct timestamp
376 timestamp="${year}-${month}-${day}T${time2}:00"
377 # print timestamp
378 echo -n "${timestamp} " >> ${logFile}
380 handleStrabonEndpoint ${endpoint} store ${file}
381 echo "Processing File ${file}" ; # read t
383 # Insert Municipalities
384 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"
385 # echo "Insert Municipalities: ${update}" ; read t
386 handleStrabonEndpoint ${endpoint} update "${update}"
389 # Delete Sea Hotspots
390 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"
391 # echo "Delete Sea Hotspots: ${update}" ; read t
392 handleStrabonEndpoint ${endpoint} update "${update}"
394 # Invalid For Fires
395 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"
396 # echo "Invalid For Fires: ${update}" ; read t
397 handleStrabonEndpoint ${endpoint} update "${update}"
399 # # Delete Reflections
400 # minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`
401 # update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"
402 ## echo "Delete Reflections: ${update}" ;
403 # handleStrabonEndpoint ${endpoint} update "${update}"
405 # Refine Partial Sea Hotspots
406 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"
407 # echo "Refine Partial Sea Hotspots: ${update}" ; read t
408 handleStrabonEndpoint ${endpoint} update "${update}"
410 # Refine Time Persistence
411 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
412 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"
413 # echo "Refine Time Persistence: ${update}" ; read t
414 handleStrabonEndpoint ${endpoint} update "${update}" #2>&1 | tee ${HOME}/timePersistence.log
416 #psql -U postgres -d ${DB} -c 'VACUUM ANALYZE;';
418 # Discover
419 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
420 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
421 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"
422 # echo "Discover: ${query}" ; #read t
423 handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discover.log
424 #
425 # # Discover Fires
426 # minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
427 # maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
428 # query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"
429 ## echo "Discover Fires: ${query}" ; #read t
430 # handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discoverFires.log
432 # Add a new line
433 echo >> ${logFile}
434 done
435 done