Strabon

view scripts/v2.2/runNoaRefinementChain.sh @ 463:e81a36154bb9

made runNoaRefinementChain.sh agnostic to MSG type
author Babis Nikolaou <charnik@di.uoa.gr>
date Thu Jul 19 18:28:53 2012 +0300 (2012-07-19)
parents 206040860011
children ebb1e9fb543b
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
6 #GRIDURL="http://kk.di.uoa.gr/grid_4.nt"
7 GRIDURL="http://jose.di.uoa.gr/rdf/coastline/grid_4.nt"
8 #INIT="http://jose.di.uoa.gr/rdf/Kallikratis-Coastline.ntriples"
9 INIT="../Kalli_coast.sql"
11 #CHECKDIR="/home/konstantina/allhot/"
12 #CHECKDIR="${HOME}/teleios/nkua/Hotspots/"
14 POSTGISTEMPLATE="postgistemplate"
15 #POSTGISTEMPLATE="template_postgis"
17 #dataDir="http://localhost/noa-teleios/out_triples/"
18 #dataDir="http://kk.di.uoa.gr/out_triples/"
19 #dataDir="http://godel.di.uoa.gr/allhot/"
20 dataDir="http://jose.di.uoa.gr/rdf/hotspots/20"
22 #HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots"
23 HOTSPOTS_URL="http://jose.di.uoa.gr/rdf/hotspots/msg1"
25 logFile="chain.log"
26 #countWTime="/usr/bin/time -p %e"
27 #echo > ${logFile}
29 function timer()
30 {
31 if [[ $# -eq 0 ]]; then
32 t=$(date '+%s%N')
33 t=$((t/1000000))
34 echo $t
35 else
36 local stime=$1
37 etime=$(date '+%s%N')
38 etime=$((etime/1000000))
40 if [[ -z "$stime" ]]; then stime=$etime; fi
41 dt=$((etime - stime)) #dt in milliseconds
42 dM=$((dt%1000))
43 Dt=$((dt/1000)) #delta t in seconds
44 ds=$((Dt % 60))
45 dm=$(((Dt / 60) % 60))
46 dh=$((Dt / 3600))
47 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
48 fi
49 }
51 # find out the postgres service to use
52 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
54 tomcat=
55 function chooseTomcat()
56 {
57 if test -s /etc/fedora-release ; then
58 tomcat="tomcat"
59 #elif test -s /etc/centos-release ; then
60 #elif test -s /etc/yellowdog-release ; then
61 #elif test -s /etc/redhat-release ; then
62 #elif test -s /etc/SuSE-release ; then
63 #elif test -s /etc/gentoo-release ; then
64 elif test -s /etc/lsb-release ; then # Ubuntu
65 tomcat="tomcat7"
66 elif test -s /etc/debian_version ; then
67 tomcat="tomcat"
68 fi
70 # check for service availability
71 if ! test -e "/etc/init.d/${tomcat}"; then
72 tomcat=
73 fi
74 }
76 insertMunicipalities=`cat ${LOC}/insertMunicipalities.sparql`
77 deleteSeaHotspots=`cat ${LOC}/deleteSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
78 refinePartialSeaHotspots=`cat ${LOC}/refinePartialSeaHotspots.sparql` # | sed 's/\"/\\\"/g'`
79 refineTimePersistence=`cat ${LOC}/refineTimePersistence.sparql` # | sed 's/\"/\\\"/g'`
80 invalidForFires=`cat ${LOC}/landUseInvalidForFires.sparql`
81 discover=`cat ${LOC}/discover.sparql`
82 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
84 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
85 chooseTomcat
86 echo "stopping tomcat"
87 if test -z "${tomcat}"; then
88 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
89 stop_tomcat.sh
90 else
91 sudo service ${tomcat} stop
92 fi
94 sudo service ${postgres} restart
96 # get the main version of postgres
97 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
99 echo "Dropping endpoint database";
100 dropdb ${DB}
102 echo "Creating endpoint database"
103 createdb ${DB}
105 # load data
106 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -d ${DB}
107 psql ${DB} -c 'VACUUM ANALYZE '
109 echo "starting tomcat"
110 if test -z "${tomcat}"; then
111 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
112 start_tomcat.sh
113 else
114 sudo service ${tomcat} start
115 fi
117 echo "initializing database"
118 echo "S M D IF R TP" >stderr.txt
121 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
122 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
123 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
125 #echo "Continue?"
126 #read a
128 #for y in 2007 2008 2010 2011 ;do
129 for y in 2012 ;do
130 # get hotpost URLS
131 for hot in $(curl -s ${HOTSPOTS_URL}/${y}/ | grep -o '>HMSG.*\.nt' | colrm 1 1); do
132 file="${HOTSPOTS_URL}/${y}/${hot}"
134 time_status=$(echo ${hot} | egrep -o '[[:digit:]]{6}_[[:digit:]]{4}')
136 # get time information for acquisition
137 year="20$(expr substr ${time_status} 1 2)"
138 month=$(expr substr ${time_status} 3 2)
139 day=$(expr substr ${time_status} 5 2)
140 time2=$(expr substr ${time_status} 8 2)
141 time2="${time2}:$(expr substr ${time_status} 10 2)"
143 # store file
144 echo -n "storing " $file; echo; echo;
145 # ${countTime} ./strabon -db endpoint store $file
147 tmr1=$(timer)
148 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
149 tmr2=$(timer)
150 printf '%s ' $((tmr2-tmr1)) >> stderr.txt
152 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
154 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
156 # insertMunicipalities
157 echo -n "inserting Municipalities " ;echo; echo; echo;
158 # query=`echo "${insertMunicipalities}" `
159 # ${countTime} ./strabon -db endpoint update "${query}"
161 tmr1=$(timer)
163 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
164 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
165 sed "s/SENSOR/MSG2/g"`
167 ../endpoint update ${ENDPOINT} "${query}"
169 tmr2=$(timer)
170 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
171 echo;echo;echo;echo "File ${file} inserted Municipalities!"
173 # deleteSeaHotspots
174 echo -n "Going to deleteSeaHotspots ${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
175 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
176 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
177 sed "s/SENSOR/MSG2/g"`
178 # ${countTime} ./strabon -db endpoint update "${query}"
180 tmr1=$(timer)
181 ../endpoint update ${ENDPOINT} "${query}"
183 tmr2=$(timer)
184 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
185 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
187 # echo "Continue?"
188 # read a
189 # invalidForFires
190 echo -n "invalidForFires ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
191 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
192 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
193 sed "s/SENSOR/MSG2/g" |\
194 sed "s/SAT/METEOSAT9/g"`
195 # ${countTime} ./strabon -db endpoint update "${query}"
196 tmr1=$(timer)
197 ../endpoint update ${ENDPOINT} "${query}"
198 tmr2=$(timer)
199 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
200 echo "File ${file} invalidForFires done!"
202 # refinePartialSeaHotspots
203 echo -n "refinePartialSeaHotspots ${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
204 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
205 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
206 sed "s/SENSOR/MSG2/g" |\
207 sed "s/SAT/METEOSAT9/g"`
208 # ${countTime} ./strabon -db endpoint update "${query}"
209 tmr1=$(timer)
210 ../endpoint update ${ENDPOINT} "${query}"
211 tmr2=$(timer)
212 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
214 echo "File ${file} refinePartialSeaHotspots done!"
215 # echo "Continue?"
216 # read a
218 # refineTimePersistence
219 echo -n "Going to refineTimePersistence ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
220 min_acquisition_time=`date --date="${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
221 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/${year}-${month}-${day}T${time2}:00/g" | \
222 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
223 sed "s/SENSOR/MSG2/g" | \
224 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
225 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
226 sed "s/SAT/METEOSAT9/g"`
228 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
230 tmr1=$(timer)
231 ../endpoint update ${ENDPOINT} "${query}"
232 tmr2=$(timer)
233 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
234 echo;echo;echo;echo "File ${file} timePersistence done!"
235 # echo "Continue?"
236 # read a
239 # discover
240 echo -n "Going to discover ${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
241 min_acquisition_time=`date --date="${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
242 max_acquisition_time=`date --date="${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
243 query=`echo "${discover}" | \
244 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
245 sed "s/SENSOR/MSG2/g" | \
246 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
247 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
249 tmr1=$(timer)
250 ../endpoint query ${ENDPOINT} "${query}"
251 tmr2=$(timer)
252 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
253 echo;echo;echo;echo "Discovered hotspots done!"
255 done
256 done