Strabon: 5e4a561d0a11 examples/teleios/runChain.sh

17 # Example run command: examples/teleios/runChain.sh -b http://dev.strabon.di.uoa.gr/rdf/data-dump-postgres-9.tgz -l /home/ggarbis/runChain.log -e http://pathway.di.uoa.gr:8080/endpoint

19 # Command name

20 cmd="$(basename ${0})"

21 # Get the directory where the script resides

22 loc="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

24 function help() {

25 echo "Usage: ${cmd} [OPTIONS] "

26 echo

27 echo "Execute NOA chain with refinements and measure time."

28 echo

29 echo "OPTIONS can be any of the following"

30 echo " -d,--db : PostGIS database"

31 echo " -e,--endpoint : Strabon Endpoint"

32 echo " -h,--hotposts : URL where hotspots are stored"

33 echo " -b,--background : Background data"

34 echo " -l,--log : Log file"

35 echo " -c,--chain : Processing chain of hotspots"

36 echo " -p,--persistence : Value of persistence of discoverFires query"

37 echo " -r,--repeat_in_persistence : Value of repeat_in_persistence of discoverFires query"

38 }

40 # If no arguments are given it returns miliseconds from 1970-01-01 00:00:00 UTC

41 # Else if a time (miliseconds form ...) is given it returns the delta between

42 # given time and current time

43 function timer()

44 {

45 if [[ $# -eq 0 ]]; then

46 t=$(date '+%s%N')

47 t=$((t/1000000))

48 echo $t

49 else

50 local stime=$1

51 etime=$(date '+%s%N')

52 etime=$((etime/1000000))

54 if [[ -z "$stime" ]]; then stime=$etime; fi

55 dt=$((etime - stime)) #dt in milliseconds

56 dM=$((dt%1000))

57 Dt=$((dt/1000)) #delta t in seconds

58 ds=$((Dt % 60))

59 dm=$(((Dt / 60) % 60))

60 dh=$((Dt / 3600))

61 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM

62 fi

63 }

65 # Handle the postgres service

66 # -$1: Command for the service

67 function handlePostgresService()

68 {

69 # find out the postgres service to use

70 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)

72 echo "Service ${postgres} received command: $1"

73 sudo service ${postgres} $1

74 }

76 # Handled a postgres database

77 # -$1: Command (create/drop/store)

78 # -$2: Dump file to store (if runscript is given as command)

79 # or 'spatial' to create a spatial database (if create is given as command)

80 function handlePostgresDatabase() {

81 local command=$1

82 local db=$2

83 shift; shift

84 local options="$*"

85 case "${command}" in

86 create)

87 if test "${options}" = "spatial"; then

88 options="-T template_postgis"

89 elif test ! -z "${options}"; then

90 echo "ERROR: only spatial is allowed for create option"

91 echo "options: ${options}"

92 exit -1

93 fi

94 echo "Creating database ${db}... with options ${options}"

95 createdb -U postgres ${db} ${options}

96 ;;

97 drop)

98 if test ! -z "${options}"; then

99 echo "ERROR: dropdb takes no extra options"

100 echo "options: ${options}"

101 exit -1

102 fi

103 echo "Dropping database ${db}..."

104 dropdb -U postgres ${db}

105 ;;

106 vacuum)

107 if test "${options}" = "analyze"; then

108 psql -U postgres ${db} -c 'VACUUM ANALYZE'

109 echo "VACUUM ANALYZE ${db}"

110 else

111 psql -U postgres ${db} -c 'VACUUM'

112 echo "VACUUM ${db}"

113 fi

114 ;;

115 runscript)

116 if test ! -f "${options}"; then

117 echo "ERROR: No dump file to run"

118 exit -1

119 fi

120 echo "Storing dump file ${options} in database ${db}..."

121 psql ${db} -f ${options}

122 ;;

123 esac

124 }

125

126 # Handle the tomcat service

127 # -$1: Command for the service

128 function handleTomcatService()

129 {

130 # find out the tomcat service to use

131 if test -s /etc/fedora-release ; then

132 tomcat="tomcat"

133 #elif test -s /etc/centos-release ; then

134 #elif test -s /etc/yellowdog-release ; then

135 #elif test -s /etc/redhat-release ; then

136 #elif test -s /etc/SuSE-release ; then

137 #elif test -s /etc/gentoo-release ; then

138 elif test -s /etc/lsb-release ; then # Ubuntu

139 tomcat=$(ls -1 /etc/init.d/| grep tomcat | head -1)

140 elif test -s /etc/debian_version ; then

141 tomcat="tomcat"

142 fi

143

144 # check for service availability

145 if ! test -e "/etc/init.d/${tomcat}"; then

146 echo "ERROR: No tomcat service found"

147 exit -1

148 fi

149

150 echo "Service ${tomcat} received command: $1"

151 sudo service ${tomcat} $1

152 }

153

154 # get the main version of postgres

155 function getPostgresMainVersion() {

156 echo $(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)

157 }

158

159 # It stores the backgroud data

160 # - $1: database

161 # - $2: backgound data file

162 function storeBackgroundData() {

163 local db=$1

164 local bgFile=$2

165

166 if test -f ${bgFile}; then

167 handlePostgresDatabase runscript ${db} ${bgFile}

168 elif test "${bgFile:0:7}" = "http://"; then

169 curl -s ${bgFile} | tar xzf - -O > /tmp/bgFiles$$.sql

170 # wget ${bgFile} -O /tmp/bgFile$$.tar.gz

171 # tar xzf /tmp/bgFile$$.tar.gz

172 handlePostgresDatabase runscript ${db} /tmp/bgFiles$$.sql

173 # rm /tmp/bgFile$$.tar.gz

174 rm /tmp/bgFiles$$.sql

175 else

176 echo "Backgound file not foung"

177 exit -1

178 fi

179 handlePostgresDatabase vacuum ${db} analyze

180 }

181

182 # Handle Stabon Endpoint

183 # - $1: endpoint

184 # - $2: command (store/query)

185 # - $2: file/query

186 function handleStrabonEndpoint(){

187 endpoint=$1

188 command=$2

189 options=$3

190

191 endpointScript=${loc}/../../scripts/endpoint

192 case ${command} in

193 store)

194 url=${options}

195

196 tmr1=$(timer)

197 ${endpointScript} store ${endpoint} N-Triples -u ${url}

198 tmr2=$(timer)

199

200 # execute an explicit VACUUM ANALYZE when a query takes longer than it should

201 duration=$((tmr2-tmr1))

202 if test ${duration} -ge 30000; then

203 handlePostgresDatabase vacuum ${db} analyze

204 # psql -U postgres ${DB} -c 'VACUUM ANALYZE'

205 # echo "Explicit VACUUM ANALYZE"

206 tmr2=$(timer)

207 fi

208 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

209 ;;

210 query)

211 query=${options}

212 tmr1=$(timer)

213 ${endpointScript} query ${endpoint} "${query}"

214 tmr2=$(timer)

215 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

216 ;;

217 update)

218 update=${options}

219 tmr1=$(timer)

220 ${endpointScript} update ${endpoint} "${update}"

221 tmr2=$(timer)

222 printf '%s ' $((tmr2-tmr1)) >> ${logFile}

223 ;;

224 *)

225 echo "ERROR: Unknown endpoint command"

226 exit -1

227 ;;

228 esac

229 }

230

231 # default values

232 endpoint="http://pathway.di.uoa.gr:8080/endpoint"

233 db="NOA2012"

234 hotspotsURL="http://jose.di.uoa.gr/rdf/hotspots/MSG1"

235 # ./examples/teleios/data/data-dump-9.sql

236 bgFile="http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-ExcludeArea-dump.tgz"

237 logFile="runChain.log"

238

239 chain="DynamicThresholds"

240 persistence=10

241 repeatInPers=3

242

243 # read script options

244 while test $# -gt 0 -a "X${1:0:1}" == "X-"; do

245 case "${1}" in

246 --help)

247 help

248 exit 0

249 ;;

250 -e|--endpoint)

251 shift

252 endpoint=${1}

253 shift

254 ;;

255 -d|--db)

256 shift

257 db=${1}

258 shift

259 ;;

260 -h|--hotspots)

261 shift

262 hotspots_url=${1}

263 shift

264 ;;

265 -b|--background)

266 shift

267 bgFile=${1}

268 shift

269 ;;

270 -l|--log)

271 shift

272 logFile=${1}

273 shift

274 ;;

275 -c|--chain)

276 shift

277 chain=${1}

278 shift

279 ;;

280 -p|--persistence)

281 shift

282 persistence=${1}

283 shift

284 ;;

285 -r|--repeat_in_persistence)

286 shift

287 repeat_in_persistence=${1}

288 shift

289 ;;

290 *)

291 echo "unknown argument ${1}"

292 help

293 exit -1

294 ;;

295 esac

296 done

297

298 echo "endpoint: ${endpoint}"

299 echo "db: ${db}"

300 echo "hotspots: ${hotspotsURL}"

301 echo "background: ${bgFile}"

302 echo "logFile: ${logFile}"

303

304 instantiate=${loc}/instantiate.sh

305

306 #Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)

307 handleTomcatService stop

308 handlePostgresService restart

309

310 handlePostgresDatabase drop ${db}

311 handlePostgresDatabase create ${db}

312

313 storeBackgroundData ${db} ${bgFile} # ~/Temp/Kallikratis-Coastline-Corine-postgres-9.sql

314

315 handleTomcatService start

316

317 #${loc}/../../scripts/endpoint query ${endpoint} size

318 #exit -1

319 echo "Timestamp Store Municipalities DeleteInSea InvalidForFires DeleteReflections RefineInCoast TimePersistence DiscoverHotspots DiscoverFires" > ${logFile}

320 echo > /home/ggarbis/discoverFires.log

321 echo > /home/ggarbis/discover.log

322

323 years="2012" #"2007 2008 2010 2011"

324 for y in ${years}; do

325 # hotspots="`ls /var/www/hotspots/${y} | sort | grep -o 'HMSG.*\.nt'`"

326 # get hotpost URLS

327 for hot in $(curl -s ${hotspotsURL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do

328 # for hot in ${hotspots}; do

329 file="${hotspotsURL}/${y}/${hot}"

330

331 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')

332

333 # get sensor

334 sensor=$(echo ${hot} | grep -o 'MSG.')

335

336 # get satellite and set number of acquisitions per hour

337 if test "${sensor}" = "MSG1"; then

338 sensor="MSG1_RSS"

339 fi

340

341 # get time information for acquisition and construct timestamp

342 year="20$(expr substr ${time_status} 1 2)"

343 month=$(expr substr ${time_status} 3 2)

344 day=$(expr substr ${time_status} 5 2)

345 time2=$(expr substr ${time_status} 8 2)

346 time2="${time2}:$(expr substr ${time_status} 10 2)"

347

348 # construct timestamp

349 timestamp="${year}-${month}-${day}T${time2}:00"

350 # print timestamp

351 echo -n "${timestamp} " >> ${logFile}

352

353 handleStrabonEndpoint ${endpoint} store ${file}

354 echo "Processing File ${file}" ; # read t

355

356 # Insert Municipalities

357 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/insertMunicipalities.rq`"

358 # echo "Insert Municipalities: ${update}" ; read t

359 handleStrabonEndpoint ${endpoint} update "${update}"

360

361

362 # Delete Sea Hotspots

363 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/deleteSeaHotspots.rq`"

364 # echo "Delete Sea Hotspots: ${update}" ; read t

365 handleStrabonEndpoint ${endpoint} update "${update}"

366

367 # Invalid For Fires

368 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/landUseInvalidForFires.rq`"

369 # echo "Invalid For Fires: ${update}" ; read t

370 handleStrabonEndpoint ${endpoint} update "${update}"

371

372 # Delete Reflections

373 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -60 minutes" +%Y-%m-%dT%H:%M:00`

374 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/deleteReflections.rq`"

375 # echo "Delete Reflections: ${update}" ;

376 handleStrabonEndpoint ${endpoint} update "${update}"

377

378 # Refine Partial Sea Hotspots

379 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} ${loc}/refinePartialSeaHotspots.rq`"

380 # echo "Refine Partial Sea Hotspots: ${update}" ; read t

381 handleStrabonEndpoint ${endpoint} update "${update}"

382

383 # Refine Time Persistence

384 minTime=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%M:00`

385 update="`${instantiate} -t ${timestamp} -c ${chain} -s ${sensor} -m ${minTime} ${loc}/refineTimePersistence.rq`"

386 # echo "Refine Time Persistence: ${update}" ; read t

387 handleStrabonEndpoint ${endpoint} update "${update}" #2>&1 | tee /home/ggarbis/timePersistence.log

388

389 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';

390

391 # Discover

392 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`

393 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`

394 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} ${loc}/discover.rq`"

395 # echo "Discover: ${query}" ; #read t

396 handleStrabonEndpoint ${endpoint} query "${query}" &>> /home/ggarbis/discover.log

397

398 # Discover Fires

399 minTime=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%M:00`

400 maxTime=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%M:00`

401 query="`${instantiate} -c ${chain} -s ${sensor} -m ${minTime} -M ${maxTime} -p 10 -r 3 ${loc}/discoverFires.rq`"

402 # echo "Discover Fires: ${query}" ; #read t

403 handleStrabonEndpoint ${endpoint} query "${query}" &>> /home/ggarbis/discoverFires.log

404

405 echo >> ${logFile}

406 done

407 done