Strabon

view scripts/v2.2/runNoaRefinementChain.sh @ 582:e614ad7569c5

polishing
author Kostis Kyzirakos <kkyzir@di.uoa.gr>
date Wed Jul 18 16:54:59 2012 +0300 (2012-07-18)
parents e9d16ae9cd00
children 86e5ba1d801a
line source
1 #!/bin/bash
2 LOC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4 ENDPOINT="http://localhost:8080/endpoint"
5 DB="endpoint"
6 #GRIDURL="http://kk.di.uoa.gr/grid_4.nt"
7 GRIDURL="http://jose.di.uoa.gr/rdf/coastline/grid_4.nt"
8 #INIT="http://jose.di.uoa.gr/rdf/Kallikratis-Coastline.ntriples"
9 INIT="../Kalli_coast.sql"
11 #CHECKDIR="/home/konstantina/allhot/"
12 #CHECKDIR="${HOME}/teleios/nkua/Hotspots/"
14 #POSTGISTEMPLATE="postgistemplate"
15 #POSTGISTEMPLATE="template_postgis"
17 #dataDir="http://localhost/noa-teleios/out_triples/"
18 #dataDir="http://kk.di.uoa.gr/out_triples/"
19 #dataDir="http://godel.di.uoa.gr/allhot/"
20 dataDir="http://jose.di.uoa.gr/rdf/hotspots/20"
21 name="HMSG2_IR_039_s7_"
22 suffix=".hotspots.nt"
24 logFile="chain.log"
25 #countWTime="/usr/bin/time -p %e"
26 #echo > ${logFile}
28 function timer()
29 {
30 if [[ $# -eq 0 ]]; then
31 t=$(date '+%s%N')
32 t=$((t/1000000))
33 echo $t
34 else
35 local stime=$1
36 etime=$(date '+%s%N')
37 etime=$((etime/1000000))
39 if [[ -z "$stime" ]]; then stime=$etime; fi
40 dt=$((etime - stime)) #dt in milliseconds
41 dM=$((dt%1000))
42 Dt=$((dt/1000)) #delta t in seconds
43 ds=$((Dt % 60))
44 dm=$(((Dt / 60) % 60))
45 dh=$((Dt / 3600))
46 printf '%d:%02d:%02d.%03d' $dh $dm $ds $dM
47 fi
48 }
50 # find out the postgres service to use
51 postgres=$(ls -1 /etc/init.d/| grep postgres | head -1)
53 tomcat=
54 function chooseTomcat()
55 {
56 if test -s /etc/fedora-release ; then
57 tomcat="tomcat"
58 #elif test -s /etc/centos-release ; then
59 #elif test -s /etc/yellowdog-release ; then
60 #elif test -s /etc/redhat-release ; then
61 #elif test -s /etc/SuSE-release ; then
62 #elif test -s /etc/gentoo-release ; then
63 elif test -s /etc/lsb-release ; then # Ubuntu
64 tomcat="tomcat7"
65 elif test -s /etc/debian_version ; then
66 tomcat="tomcat"
67 fi
69 # check for service availability
70 if ! test -e "/etc/init.d/${tomcat}"; then
71 tomcat=
72 fi
73 }
75 insertMunicipalities=`cat ${LOC}/InsertMunicipalities.sparql`
76 deleteSeaHotspots=`cat ${LOC}/DeleteInSea.sparql` # | sed 's/\"/\\\"/g'`
77 refinePartialSeaHotspots=`cat ${LOC}/Refine.sparql` # | sed 's/\"/\\\"/g'`
78 refineTimePersistence=`cat ${LOC}/TimePersistence.sparql` # | sed 's/\"/\\\"/g'`
79 invalidForFires=`cat ${LOC}/InvalidForFires.sparql`
80 discover=`cat ${LOC}/discover.sparql`
81 #InsertMunicipalities =`cat ${LOC}/InsertMunicipalities.sparql` # | sed 's/\"/\\\"/g'`
84 # Initialize (stop tomcat, restart postgres, drop/create database, start tomcat)
85 chooseTomcat
86 echo "stopping tomcat"
87 if test -z "${tomcat}"; then
88 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
89 stop_tomcat.sh
90 else
91 sudo service ${tomcat} stop
92 fi
94 sudo service ${postgres} restart
96 # get the main version of postgres
97 POSTGRES_MAIN_VERSION=$(sudo service ${postgres} status | grep -o '.\..' | cut -b 1)
99 echo "Dropping endpoint database";
100 dropdb -U postgres ${DB}
102 echo "Creating endpoint database"
103 createdb -U postgres ${DB}
105 # load data
106 echo "initializing database"
107 curl -s http://dev.strabon.di.uoa.gr/rdf/Kallikratis-Coastline-Corine-dump-postgres-${POSTGRES_MAIN_VERSION}.tgz | tar xz -O | psql -U postgres -d ${DB}
108 psql ${DB} -U postgres -c 'VACUUM ANALYZE '
110 echo "starting tomcat"
111 if test -z "${tomcat}"; then
112 # work-around for babis (standalone tomcat, with start_tomcat.sh and stop_tomcat.sh scripts)
113 start_tomcat.sh
114 else
115 sudo service ${tomcat} start
116 fi
118 #echo "IM S D R TP" >stderr.txt
119 echo "Store Municipalities DeleteInSea Corine RefineInCoast TimePersistency" >stderr.txt
121 #./scripts/endpoint query ${ENDPOINT} "SELECT (COUNT(*) AS ?C) WHERE {?s ?p ?o}"
122 #sudo -u postgres psql -d endpoint -c 'CREATE INDEX datetime_values_idx_value ON datetime_values USING btree(value)';
123 #sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
125 #echo "Continue?"
126 #read a
128 for y in 7 8 10 11 ;do
129 for mon in `seq 4 10`; do
130 for d in `seq 1 30`; do
131 for h in `seq 0 23 `; do
132 for m in `seq 0 15 45`; do
133 time=`printf "%02d%02d\n" $h $m`
134 time2=`printf "%02d:%02d\n" $h $m`
135 day=`printf "%02d" $d`
136 month=`printf "%02d" $mon`
137 year=`printf "%02d" $y`
138 file=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
139 # file=${dataUrl}${name}_${time}$suffix
141 check=${dataDir}${year}/${name}${year}${month}${day}_${time}$suffix
142 wget -q --spider $check
144 if [[ $? -ne 0 ]];
145 then echo "FILE" $check "NOT EXISTS" ; continue
146 fi
149 # store file
150 echo -n "storing " $file; echo; echo;
151 # echo "Hotspot : " $h:$m >> stderr.txt
152 # ${countTime} ./strabon -db endpoint store $file
154 tmr1=$(timer)
155 ../endpoint store ${ENDPOINT} N-Triples -u ${file}
156 tmr2=$(timer)
157 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
159 # sudo -u postgres psql -d endpoint -c 'VACUUM ANALYZE;';
161 echo;echo;echo;echo "File ${file} stored!" >> ${logFile}
163 # insertMunicipalities
164 echo -n "inserting Municipalities " ;echo; echo; echo;
165 # query=`echo "${insertMunicipalities}" `
166 # ${countTime} ./strabon -db endpoint update "${query}"
168 tmr1=$(timer)
170 query=`echo "${insertMunicipalities}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
171 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
172 sed "s/SENSOR/MSG2/g"`
174 ../endpoint update ${ENDPOINT} "${query}"
176 tmr2=$(timer)
177 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
179 echo;echo;echo;echo "File ${file} inserted Municipalities!"
181 # deleteSeaHotspots
182 echo -n "Going to deleteSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ;echo; echo; echo;
183 query=`echo "${deleteSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
184 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
185 sed "s/SENSOR/MSG2/g"`
186 # ${countTime} ./strabon -db endpoint update "${query}"
188 tmr1=$(timer)
189 ../endpoint update ${ENDPOINT} "${query}"
191 tmr2=$(timer)
192 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
193 echo;echo;echo;echo "File ${file} deleteSeaHotspots done!"
195 # echo "Continue?"
196 # read a
197 # invalidForFires
198 echo -n "invalidForFires 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
199 query=`echo "${invalidForFires}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
200 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
201 sed "s/SENSOR/MSG2/g" |\
202 sed "s/SAT/METEOSAT9/g"`
203 # ${countTime} ./strabon -db endpoint update "${query}"
204 tmr1=$(timer)
205 ../endpoint update ${ENDPOINT} "${query}"
206 tmr2=$(timer)
207 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
208 echo "File ${file} invalidForFires done!"
210 # refinePartialSeaHotspots
211 echo -n "refinePartialSeaHotspots 20${year}-${month}-${day}T${time2}:00 " ; echo; echo ; echo;
212 query=`echo "${refinePartialSeaHotspots}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
213 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
214 sed "s/SENSOR/MSG2/g" |\
215 sed "s/SAT/METEOSAT9/g"`
216 # ${countTime} ./strabon -db endpoint update "${query}"
217 tmr1=$(timer)
218 ../endpoint update ${ENDPOINT} "${query}"
219 tmr2=$(timer)
220 printf '%s ' $((tmr2-tmr1)) >>stderr.txt
222 echo "File ${file} refinePartialSeaHotspots done!"
223 # echo "Continue?"
224 # read a
226 # refineTimePersistence
227 echo -n "Going to refineTimePersistence 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
228 min_acquisition_time=`date --date="20${year}-${month}-${day} ${time2}:00 EEST -30 minutes" +%Y-%m-%dT%H:%m:00`
229 query=`echo "${refineTimePersistence}" | sed "s/TIMESTAMP/20${year}-${month}-${day}T${time2}:00/g" | \
230 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
231 sed "s/SENSOR/MSG2/g" | \
232 sed "s/ACQUISITIONS_IN_HALF_AN_HOUR/3.0/g" | \
233 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
234 sed "s/SAT/METEOSAT9/g"`
236 #sudo -u postgres psql -d ${DB} -c 'VACUUM ANALYZE;';
238 tmr1=$(timer)
239 ../endpoint update ${ENDPOINT} "${query}"
240 tmr2=$(timer)
241 printf '%s \n' $((tmr2-tmr1)) >>stderr.txt
242 echo;echo;echo;echo "File ${file} timePersistence done!"
243 # echo "Continue?"
244 # read a
245 done #minutes
246 done #hours
248 # discover
249 echo -n "Going to discover 20${year}-${month}-${day}T${time2}:00 ";echo;echo;echo;
250 min_acquisition_time=`date --date="20${year}-${month}-${day} 00:00 EEST" +%Y-%m-%dT%H:%m:00`
251 max_acquisition_time=`date --date="20${year}-${month}-${day} 23:59 EEST" +%Y-%m-%dT%H:%m:00`
252 query=`echo "${discover}" | \
253 sed "s/PROCESSING_CHAIN/DynamicThresholds/g" | \
254 sed "s/SENSOR/MSG2/g" | \
255 sed "s/MIN_ACQUISITION_TIME/${min_acquisition_time}/g" |\
256 sed "s/MAX_ACQUISITION_TIME/${max_acquisition_time}/g"`
258 tmr1=$(timer)
259 ../endpoint query ${ENDPOINT} "${query}"
260 tmr2=$(timer)
261 printf '%s \n' $((tmr2-tmr1)) >>discover.txt
262 echo;echo;echo;echo "Discovered hotspots done!"
264 done #days
265 done #months
266 done #years