Strabon: 8e6437ba6ba2 examples/teleios/runChain.sh

19 # Example run command: examples/teleios/runChain.sh -b http://dev.strabon.di.uoa.gr/rdf/data-dump-postgres-9.tgz -l ${HOME}/runChain.log -e http://pathway.di.uoa.gr:8080/endpoint

21 # Command name

22 cmd="$(basename ${0})"

23 # Get the directory where the script resides

24 loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

26 function help() {

27 echo "Usage: ${cmd} [OPTIONS] "

28 echo

29 echo "Execute NOA chain with refinements and measure time."

30 echo

31 echo "OPTIONS can be any of the following"

32 echo " -d,--db : PostGIS database"

33 echo " -e,--endpoint : Strabon Endpoint"

34 echo " -h,--hotposts : URL where hotspots are stored"

35 echo " -b,--background : Background data"

36 echo " -l,--log : Log file"

37 echo " -c,--chain : Processing chain of hotspots"

38 echo " -p,--persistence : Value of persistence of discoverFires query"

39 echo " -r,--repeat_in_persistence : Value of repeat_in_persistence of discoverFires query"

40 }

42 # If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC

43 # Else if a time (miliseconds form ...) is given it returns the delta between

44 # given time and current time

45 function timer()

46 {

47 if [[ $# -eq 0 ]]; then

48 t=$(date '+%s%N')

49 t=$((t/1000000))

50 echo $t

51 else

52 local stime=$1

53 etime=$(date '+%s%N')

54 etime=$((etime/1000000))

56 if [[ -z "$stime" ]]; then stime=$etime; fi

57 dt=$((etime - stime)) #dt in milliseconds

58 dM=$((dt%1000))

59 Dt=$((dt/1000)) #delta t in seconds

60 ds=$((Dt % 60))

61 dm=$(((Dt / 60) % 60))

62 dh=$((Dt / 3600))

63 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM

64 fi

65 }

67 # Handle the postgres service

68 # -$1: Command for the service

69 function handlePostgresService()

70 {

71 # find out the postgres service to use

72 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)

74 echo "Service ${postgres} received command: $1"

75 sudo service ${postgres} $1

76 }

78 # Handled a postgres database

79 # -$1: Command (create/drop/store)

80 # -$2: Dump file to store (if runscript is given as command)

81 # or 'spatial' to create a spatial database (if create is given as command)

82 function handlePostgresDatabase() {

83 local command=$1

84 local db=$2

85 shift; shift

86 local options="$*"

87 case "${command}" in

88 create)

89 if test "${options}" = "spatial"; then

90 options="-T template_postgis"

91 elif test ! -z "${options}"; then

92 echo "ERROR: only spatial is allowed for create option"

93 echo "options: ${options}"

94 exit -1

95 fi

96 echo "Creating database ${db}... with options ${options}"

97 createdb -U postgres ${db} ${options}

98 ;;

99 drop)

100 if test ! -z "${options}"; then

101 echo "ERROR: dropdb takes no extra options"

102 echo "options: ${options}"

103 exit -1

104 fi

105 echo "Dropping database ${db}..."

106 dropdb -U postgres ${db}

107 ;;

108 vacuum)

109 if test "${options}" = "analyze"; then

110 psql -U postgres ${db} -c 'VACUUM ANALYZE'

111 echo "VACUUM ANALYZE ${db}"

112 else

113 psql -U postgres ${db} -c 'VACUUM'

114 echo "VACUUM ${db}"

115 fi

116 ;;

117 runscript)

118 if test ! -f "${options}"; then

119 echo "ERROR: No dump file to run"

120 exit -1

121 fi

122 echo "Storing dump file ${options} in database ${db}..."

123 psql -U postgres ${db} -f ${options}

124 ;;

125 esac

126 }

127

128 # Handle the tomcat service

129 # -$1: Command for the service

130 function handleTomcatService()

131 {

132 if test ! -z "${TOMCATPATH}" ; then

133 case "${1}" in

134 start)

135 ${TOMCATPATH}/bin/startup.sh

136 ;;

137 stop)

138 ${TOMCATPATH}/bin/shutdown.sh

139 ;;

140 restart)

141 ${TOMCATPATH}/bin/startup.sh

142 ${TOMCATPATH}/bin/shutdown.sh

143 ;;

144 esac

145 return

146 # find out the tomcat service to use

147 elif test -s /etc/fedora-release ; then

148 tomcat="tomcat"

149 #elif test -s /etc/centos-release ; then

150 #elif test -s /etc/yellowdog-release ; then

151 #elif test -s /etc/redhat-release ; then

152 #elif test -s /etc/SuSE-release ; then

153 #elif test -s /etc/gentoo-release ; then

154 elif test -s /etc/lsb-release ; then # Ubuntu

155 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)

156 elif test -s /etc/debian_version ; then

157 tomcat="tomcat"

158 fi

159

160 # check for service availability

161 if ! test -e "/etc/init.d/${tomcat}"; then

162 echo "ERROR: No tomcat service found"

163 exit -1

164 fi

165

166 echo "Service ${tomcat} received command: $1"

167 sudo service ${tomcat} $1

168 }

169

170

171 # get the main version of postgres

172 function getPostgresMainVersion() {

173 echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)

174 }

175

176 # It stores the backgroud data

177 # - $1: database

178 # - $2: backgound data file

179 function storeBackgroundData() {

180 local db=$1

181 local bgFile=$2

182

183 if test -f ${bgFile}; then

184 handlePostgresDatabase runscript ${db} ${bgFile}

185 elif test "${bgFile:0:7}" = "http://"; then

186 curl -s ${bgFile} | tar xzf - -O > /tmp/bgFiles$$.sql

187 # wget ${bgFile} -O /tmp/bgFile$$.tar.gz

188 # tar xzf /tmp/bgFile$$.tar.gz

189 handlePostgresDatabase runscript ${db} /tmp/bgFiles$$.sql

190 # rm /tmp/bgFile$$.tar.gz

191 rm /tmp/bgFiles$$.sql

192 else

193 echo "Backgound file not found"

194 exit -1

195 fi

196 handlePostgresDatabase vacuum ${db} analyze

197 }

198

199 # Handle Stabon Endpoint

200 # - $1: endpoint

201 # - $2: command (store/query)

202 # - $2: file/query

203 function handleStrabonEndpoint(){

204 endpoint=$1

205 command=$2

206 options=$3

207

208 endpointScript=${loc}/../../scripts/endpoint

209 case ${command} in

210 store)

211 url=${options}

212

213 tmr1=$(timer)

214 #${endpointScript} store ${endpoint} N-Triples -u ${url}

215 #read t

216 ${endpointScript} store ${endpoint} N-Triples -u ${url}

217 tmr2=$(timer)

218

219 # execute an explicit VACUUM ANALYZE when a query takes longer than it should

220 duration=$((tmr2-tmr1))

221 if test ${duration} -ge 30000; then

222 handlePostgresDatabase vacuum ${db} analyze

223 # psql -U postgres ${DB} -c 'VACUUM ANALYZE'

224 # echo "Explicit VACUUM ANALYZE"

225 tmr2=$(timer)

226 fi

227 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

228 ;;

229 query)

230 query=${options}

231 tmr1=$(timer)

232 #${endpointScript} query ${endpoint} "${query}"

233 #read t

234 ${endpointScript} query ${endpoint} "${query}"

235 tmr2=$(timer)

236 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

237 ;;

238 update)

239 update=${options}

240 tmr1=$(timer)

241 #${endpointScript} update ${endpoint} "${update}"

242 #read t

243 ${endpointScript} update ${endpoint} "${update}"

244 tmr2=$(timer)

245 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

246 ;;

247 *)

248 echo "ERROR: Unknown endpoint command"

249 exit -1

250 ;;

251 esac

252 }

253

254 # default values

255 endpoint="http://teleios3.di.uoa.gr:8080/endpoint"

256 db="NOA2012"

257 hotspotsURL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"

258 # ./examples/teleios/data/data-dump-9.sql

259 bgFile="http://dev.strabon.di.uoa.gr/rdf/data-dump-9.sql"

260 logFile="${HOME}/runChain.log"

261

262 chain="DynamicThresholds"

263 persistence=10

264 repeatInPers=3

265

266 # read script options

267 while test $# -gt 0 -a "X${1:0:1}" == "X-"; do

268 case "${1}" in

269 --help)

270 help

271 exit 0

272 ;;

273 -e|--endpoint)

274 shift

275 endpoint=${1}

276 shift

277 ;;

278 -d|--db)

279 shift

280 db=${1}

281 shift

282 ;;

283 -h|--hotspots)

284 shift

285 hotspots_url=${1}

286 shift

287 ;;

288 -b|--background)

289 shift

290 bgFile=${1}

291 shift

292 ;;

293 -l|--log)

294 shift

295 logFile=${1}

296 shift

297 ;;

298 -c|--chain)

299 shift

300 chain=${1}

301 shift

302 ;;

303 -p|--persistence)

304 shift

305 persistence=${1}

306 shift

307 ;;

308 -r|--repeat_in_persistence)

309 shift

310 repeat_in_persistence=${1}

311 shift

312 ;;

313 *)

314 echo "unknown argument ${1}"

315 help

316 exit -1

317 ;;

318 esac

319 done

320

321 echo "endpoint: ${endpoint}"

322 echo "db: ${db}"

323 echo "hotspots: ${hotspotsURL}"

324 echo "background: ${bgFile}"

325 echo "logFile: ${logFile}"

326

327 echo > ${logFile}

328 instantiate=${loc}/instantiate.sh

329

330 #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)

331 handleTomcatService stop

332 handlePostgresService restart

333

334 handlePostgresDatabase drop ${db}

335 handlePostgresDatabase create ${db}

336

337 storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql

338

339 handleTomcatService start

340 # Wait until tomcat server is up

341 until [ "`curl --silent --show-error --connect-timeout 1 -I ${endpoint} | grep 'Coyote'`" != "" ]; do sleep 1; done;

342

343

344 #${loc}/../../scripts/endpoint query ${endpoint} size

345 #exit -1

346 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires RefineInCoast TimePersistence DiscoverHotspots" > ${logFile}

347 echo > ${HOME}/discoverFires.log

348 echo > ${HOME}/discover.log

349

350 years="2012" #"2007 2008 2010 2011"

351 for y in ${years}; do

352 # hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"

353 # get hotpost URLS

354 for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do

355 # for hot in ${hotspots}; do

356 file="${hotspotsURL}/${y}/${hot}"

357

358 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')

359

360 # get sensor

361 sensor=$(echo ${hot} | grep -o 'MSG.')

362

363 # get satellite and set number of acquisitions per hour

364 if test "${sensor}" = "MSG1"; then

365 sensor="MSG1_RSS"

366 fi

367

368 # get time information for acquisition and construct timestamp

369 year="20$(expr substr ${time_status} 1 2)"

370 month=$(expr substr ${time_status} 3 2)

371 day=$(expr substr ${time_status} 5 2)

372 time2=$(expr substr ${time_status} 8 2)

373 time2="${time2}:$(expr substr ${time_status} 10 2)"

374

375 # construct timestamp

376 timestamp="${year}-${month}-${day}T${time2}:00"

377 # print timestamp

378 echo -n "${timestamp} " >> ${logFile}

379

380 handleStrabonEndpoint ${endpoint} store ${file}

381 echo "Processing File ${file}" ; # read t

382

383 # Insert Municipalities

384 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"

385 # echo "Insert Municipalities: ${update}" ; read t

386 handleStrabonEndpoint ${endpoint} update "${update}"

387

388

389 # Delete Sea Hotspots

390 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"

391 # echo "Delete Sea Hotspots: ${update}" ; read t

392 handleStrabonEndpoint ${endpoint} update "${update}"

393

394 # Invalid For Fires

395 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"

396 # echo "Invalid For Fires: ${update}" ; read t

397 handleStrabonEndpoint ${endpoint} update "${update}"

398

399 # # Delete Reflections

400 # minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`

401 # update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"

402 ## echo "Delete Reflections: ${update}" ;

403 # handleStrabonEndpoint ${endpoint} update "${update}"

404

405 # Refine Partial Sea Hotspots

406 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"

407 # echo "Refine Partial Sea Hotspots: ${update}" ; read t

408 handleStrabonEndpoint ${endpoint} update "${update}"

409

410 # Refine Time Persistence

411 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`

412 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"

413 # echo "Refine Time Persistence: ${update}" ; read t

414 handleStrabonEndpoint ${endpoint} update "${update}" #2>&1 | tee ${HOME}/timePersistence.log

415

416 #psql -U postgres -d ${DB} -c 'VACUUM ANALYZE;';

417

418 # Discover

419 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`

420 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`

421 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"

422 # echo "Discover: ${query}" ; #read t

423 handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discover.log

424 #

425 # # Discover Fires

426 # minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`

427 # maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`

428 # query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"

429 ## echo "Discover Fires: ${query}" ; #read t

430 # handleStrabonEndpoint ${endpoint} query "${query}" &>> ${HOME}/discoverFires.log

431

432 # Add a new line

433 echo >> ${logFile}

434 done

435 done