Strabon

view examples/teleios/runChain.sh @ 933:e9521141694b

corrected identation in runChain.sh script
author Babis Nikolaou <charnik@di.uoa.gr>
date Tue Mar 26 20:49:12 2013 +0200 (2013-03-26)
parents 1c1b26b37c41
children 802e52d68629
line source
1 #! /bin/bash
2 #
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this file, you
5 # can obtain one at http://mozilla.org/MPL/2.0/.
6 #
7 # Copyright (C) 2010, 2011, 2012, 2013 Pyravlos Team
8 #
9 # http://www.strabon.di.uoa.gr/
10 #
11 # Author: George Garbis <ggarbis@di.uoa.gr>
12 # Author: Charalampos (Babis) Nikolaou <charnik@di.uoa.gr>
13 # Author: Manos Karpathiotakis <mk@di.uoa.gr>
14 # Author: Konstantina Bereta <Konstantina.Bereta@di.uoa.gr>
15 #
17 # If tomcat is standalone then environment variable TOMCATPATH should be set
19 # Example run command: examples/teleios/runChain.sh -b http://dev.strabon.di.uoa.gr/rdf/data-dump-postgres-9.tgz -l /home/ggarbis/runChain.log -e http://pathway.di.uoa.gr:8080/endpoint
21 # Command name
22 cmd="$(basename ${0})"
23 # Get the directory where the script resides
24 loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
26 function help() {
27 echo "Usage: ${cmd} [OPTIONS] "
28 echo
29 echo "Execute NOA chain with refinements and measure time."
30 echo
31 echo "OPTIONS can be any of the following"
32 echo " -d,--db : PostGIS database"
33 echo " -e,--endpoint : Strabon Endpoint"
34 echo " -h,--hotposts : URL where hotspots are stored"
35 echo " -b,--background : Background data"
36 echo " -l,--log : Log file"
37 echo " -c,--chain : Processing chain of hotspots"
38 echo " -p,--persistence : Value of persistence of discoverFires query"
39 echo " -r,--repeat_in_persistence : Value of repeat_in_persistence of discoverFires query"
40 }
42 # If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC
43 # Else if a time (miliseconds form ...) is given it returns the delta between
44 # given time and current time
45 function timer()
46 {
47 if [[ $# -eq 0 ]]; then
48 t=$(date '+%s%N')
49 t=$((t/1000000))
50 echo $t
51 else
52 local stime=$1
53 etime=$(date '+%s%N')
54 etime=$((etime/1000000))
56 if [[ -z "$stime" ]]; then stime=$etime; fi
57 dt=$((etime - stime)) #dt in milliseconds
58 dM=$((dt%1000))
59 Dt=$((dt/1000)) #delta t in seconds
60 ds=$((Dt % 60))
61 dm=$(((Dt / 60) % 60))
62 dh=$((Dt / 3600))
63 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
64 fi
65 }
67 # Handle the postgres service
68 # -$1: Command for the service
69 function handlePostgresService()
70 {
71 # find out the postgres service to use
72 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
74 echo "Service ${postgres} received command: $1"
75 sudo service ${postgres} $1
76 }
78 # Handled a postgres database
79 # -$1: Command (create/drop/store)
80 # -$2: Dump file to store (if runscript is given as command)
81 # or 'spatial' to create a spatial database (if create is given as command)
82 function handlePostgresDatabase() {
83 local command=$1
84 local db=$2
85 shift; shift
86 local options="$*"
87 case "${command}" in
88 create)
89 if test "${options}" = "spatial"; then
90 options="-T template_postgis"
91 elif test ! -z "${options}"; then
92 echo "ERROR: only spatial is allowed for create option"
93 echo "options: ${options}"
94 exit -1
95 fi
96 echo "Creating database ${db}... with options ${options}"
97 createdb -U postgres ${db} ${options}
98 ;;
99 drop)
100 if test ! -z "${options}"; then
101 echo "ERROR: dropdb takes no extra options"
102 echo "options: ${options}"
103 exit -1
104 fi
105 echo "Dropping database ${db}..."
106 dropdb -U postgres ${db}
107 ;;
108 vacuum)
109 if test "${options}" = "analyze"; then
110 psql -U postgres ${db} -c 'VACUUM ANALYZE'
111 echo "VACUUM ANALYZE ${db}"
112 else
113 psql -U postgres ${db} -c 'VACUUM'
114 echo "VACUUM ${db}"
115 fi
116 ;;
117 runscript)
118 if test ! -f "${options}"; then
119 echo "ERROR: No dump file to run"
120 exit -1
121 fi
122 echo "Storing dump file ${options} in database ${db}..."
123 psql -U postgres ${db} -f ${options}
124 ;;
125 esac
126 }
128 # Handle the tomcat service
129 # -$1: Command for the service
130 function handleTomcatService()
131 {
132 if test ! -z "${TOMCATPATH}" ; then
133 case "${1}" in
134 start)
135 ${TOMCATPATH}/bin/startup.sh
136 ;;
137 stop)
138 ${TOMCATPATH}/bin/shutdown.sh
139 ;;
140 restart)
141 ${TOMCATPATH}/bin/startup.sh
142 ${TOMCATPATH}/bin/shutdown.sh
143 ;;
144 esac
145 return
146 # find out the tomcat service to use
147 elif test -s /etc/fedora-release ; then
148 tomcat="tomcat"
149 #elif test -s /etc/centos-release ; then
150 #elif test -s /etc/yellowdog-release ; then
151 #elif test -s /etc/redhat-release ; then
152 #elif test -s /etc/SuSE-release ; then
153 #elif test -s /etc/gentoo-release ; then
154 elif test -s /etc/lsb-release ; then # Ubuntu
155 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)
156 elif test -s /etc/debian_version ; then
157 tomcat="tomcat"
158 fi
160 # check for service availability
161 if ! test -e "/etc/init.d/${tomcat}"; then
162 echo "ERROR: No tomcat service found"
163 exit -1
164 fi
166 echo "Service ${tomcat} received command: $1"
167 sudo service ${tomcat} $1
168 }
171 # get the main version of postgres
172 function getPostgresMainVersion() {
173 echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
174 }
176 # It stores the backgroud data
177 # - $1: database
178 # - $2: backgound data file
179 function storeBackgroundData() {
180 local db=$1
181 local bgFile=$2
183 if test -f ${bgFile}; then
184 handlePostgresDatabase runscript ${db} ${bgFile}
185 elif test "${bgFile:0:7}" = "http://"; then
186 curl -s ${bgFile} | tar xzf - -O > /tmp/bgFiles$$.sql
187 # wget ${bgFile} -O /tmp/bgFile$$.tar.gz
188 # tar xzf /tmp/bgFile$$.tar.gz
189 handlePostgresDatabase runscript ${db} /tmp/bgFiles$$.sql
190 # rm /tmp/bgFile$$.tar.gz
191 rm /tmp/bgFiles$$.sql
192 else
193 echo "Backgound file not foung"
194 exit -1
195 fi
196 handlePostgresDatabase vacuum ${db} analyze
197 }
199 # Handle Stabon Endpoint
200 # - $1: endpoint
201 # - $2: command (store/query)
202 # - $2: file/query
203 function handleStrabonEndpoint(){
204 endpoint=$1
205 command=$2
206 options=$3
208 endpointScript=${loc}/../../scripts/endpoint
209 case ${command} in
210 store)
211 url=${options}
213 tmr1=$(timer)
214 #${endpointScript} store ${endpoint} N-Triples -u ${url}
215 #read t
216 ${endpointScript} store ${endpoint} N-Triples -u ${url}
217 tmr2=$(timer)
219 # execute an explicit VACUUM ANALYZE when a query takes longer than it should
220 duration=$((tmr2-tmr1))
221 if test ${duration} -ge 30000; then
222 handlePostgresDatabase vacuum ${db} analyze
223 # psql -U postgres ${DB} -c 'VACUUM ANALYZE'
224 # echo "Explicit VACUUM ANALYZE"
225 tmr2=$(timer)
226 fi
227 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
228 ;;
229 query)
230 query=${options}
231 tmr1=$(timer)
232 #${endpointScript} query ${endpoint} "${query}"
233 #read t
234 ${endpointScript} query ${endpoint} "${query}"
235 tmr2=$(timer)
236 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
237 ;;
238 update)
239 update=${options}
240 tmr1=$(timer)
241 #${endpointScript} update ${endpoint} "${update}"
242 #read t
243 ${endpointScript} update ${endpoint} "${update}"
244 tmr2=$(timer)
245 printf '%s ' $((tmr2-tmr1)) >> ${logFile}
246 ;;
247 *)
248 echo "ERROR: Unknown endpoint command"
249 exit -1
250 ;;
251 esac
252 }
254 # default values
255 endpoint="http://teleios3.di.uoa.gr:8080/endpoint"
256 db="NOA2012"
257 hotspotsURL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"
258 # ./examples/teleios/data/data-dump-9.sql
259 bgFile="http://dev.strabon.di.uoa.gr/rdf/data-dump-9.sql"
260 logFile="/home/ggarbis/runChain.log"
262 chain="DynamicThresholds"
263 persistence=10
264 repeatInPers=3
266 # read script options
267 while test $# -gt 0 -a "X${1:0:1}" == "X-"; do
268 case "${1}" in
269 --help)
270 help
271 exit 0
272 ;;
273 -e|--endpoint)
274 shift
275 endpoint=${1}
276 shift
277 ;;
278 -d|--db)
279 shift
280 db=${1}
281 shift
282 ;;
283 -h|--hotspots)
284 shift
285 hotspots_url=${1}
286 shift
287 ;;
288 -b|--background)
289 shift
290 bgFile=${1}
291 shift
292 ;;
293 -l|--log)
294 shift
295 logFile=${1}
296 shift
297 ;;
298 -c|--chain)
299 shift
300 chain=${1}
301 shift
302 ;;
303 -p|--persistence)
304 shift
305 persistence=${1}
306 shift
307 ;;
308 -r|--repeat_in_persistence)
309 shift
310 repeat_in_persistence=${1}
311 shift
312 ;;
313 *)
314 echo "unknown argument ${1}"
315 help
316 exit -1
317 ;;
318 esac
319 done
321 echo "endpoint: ${endpoint}"
322 echo "db: ${db}"
323 echo "hotspots: ${hotspotsURL}"
324 echo "background: ${bgFile}"
325 echo "logFile: ${logFile}"
327 echo > ${logFile}
328 echo 2222
329 instantiate=${loc}/instantiate.sh
331 #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
332 echo 111
333 handleTomcatService stop
334 handlePostgresService restart
336 handlePostgresDatabase drop ${db}
337 handlePostgresDatabase create ${db}
339 storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql
341 handleTomcatService start
342 # Wait until tomcat server is up
343 until [ "`curl --silent --show-error --connect-timeout 1 -I ${endpoint} | grep 'Coyote'`" != "" ]; do sleep 1; done;
346 #${loc}/../../scripts/endpoint query ${endpoint} size
347 #exit -1
348 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence DiscoverHotspots" > ${logFile}
349 echo > /home/ggarbis/discoverFires.log
350 echo > /home/ggarbis/discover.log
352 years="2012" #"2007 2008 2010 2011"
353 for y in ${years}; do
354 # hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"
355 # get hotpost URLS
356 for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
357 # for hot in ${hotspots}; do
358 file="${hotspotsURL}/${y}/${hot}"
360 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
362 # get sensor
363 sensor=$(echo ${hot} | grep -o 'MSG.')
365 # get satellite and set number of acquisitions per hour
366 if test "${sensor}" = "MSG1"; then
367 sensor="MSG1_RSS"
368 fi
370 # get time information for acquisition and construct timestamp
371 year="20$(expr substr ${time_status} 1 2)"
372 month=$(expr substr ${time_status} 3 2)
373 day=$(expr substr ${time_status} 5 2)
374 time2=$(expr substr ${time_status} 8 2)
375 time2="${time2}:$(expr substr ${time_status} 10 2)"
377 # construct timestamp
378 timestamp="${year}-${month}-${day}T${time2}:00"
379 # print timestamp
380 echo -n "${timestamp} " >> ${logFile}
382 handleStrabonEndpoint ${endpoint} store ${file}
383 echo "Processing File ${file}" ; # read t
385 # Insert Municipalities
386 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"
387 # echo "Insert Municipalities: ${update}" ; read t
388 handleStrabonEndpoint ${endpoint} update "${update}"
391 # Delete Sea Hotspots
392 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"
393 # echo "Delete Sea Hotspots: ${update}" ; read t
394 handleStrabonEndpoint ${endpoint} update "${update}"
396 # Invalid For Fires
397 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"
398 # echo "Invalid For Fires: ${update}" ; read t
399 handleStrabonEndpoint ${endpoint} update "${update}"
401 # # Delete Reflections
402 # minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`
403 # update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"
404 ## echo "Delete Reflections: ${update}" ;
405 # handleStrabonEndpoint ${endpoint} update "${update}"
407 # Refine Partial Sea Hotspots
408 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"
409 # echo "Refine Partial Sea Hotspots: ${update}" ; read t
410 handleStrabonEndpoint ${endpoint} update "${update}"
412 # Refine Time Persistence
413 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`
414 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"
415 # echo "Refine Time Persistence: ${update}" ; read t
416 handleStrabonEndpoint ${endpoint} update "${update}" #2>&1 | tee /home/ggarbis/timePersistence.log
418 #psql -U postgres -d ${DB} -c 'VACUUM ANALYZE;';
420 # Discover
421 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
422 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
423 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"
424 # echo "Discover: ${query}" ; #read t
425 handleStrabonEndpoint ${endpoint} query "${query}" &>> /home/ggarbis/discover.log
426 #
427 # # Discover Fires
428 # minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`
429 # maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`
430 # query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"
431 ## echo "Discover Fires: ${query}" ; #read t
432 # handleStrabonEndpoint ${endpoint} query "${query}" &>> /home/ggarbis/discoverFires.log
434 # Add a new line
435 echo >> ${logFile}
436 done
437 done