/Designs/Measuring_instruments/RMDS01A/SW/Tools/README.txt
0,0 → 1,14
=== RMDS Synchronization Tools ===
 
= INFO
 
o First, nice directory structure is created from jpg files, see tidyup.sh for description
o Then complete sync of directory is done, this is repeated every 24 hours
o The script then runs infinite loop watching for new files and syncing them to server every hour
 
= USAGE
 
o You must have working ssh public key based authentication to server
o It is recommended to run the script in detachable screen
o Run ./sync.sh /path/to/folder where path is the directory to which jpg's from Spectrum Lab are saved
 
/Designs/Measuring_instruments/RMDS01A/SW/Tools/sync.sh
0,0 → 1,102
#!/usr/bin/env bash
 
[[ $# -ne 1 ]] && echo "Usage: ./sync.sh /path/to/folder" && exit 1
 
# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
exec > >(tee logfile.txt)
 
# Also redirect stderr
exec 2>&1
 
FREQUENCY=60
 
TIDYUP="./tidyup.sh"
RSYNC="./upload.sh"
 
DEFAULT="\033[00m"
RED="\033[01;31m"
BLUE="\033[01;36m"
 
function info() {
echo -en "$BLUE"; echo -n $1; echo -e "$DEFAULT"
}
 
function error() {
echo -en "$RED"; echo -n $1; echo -e "$DEFAULT"
}
 
function syncdir() {
EXIT=1
i=1
while [ $EXIT -ne 0 ]; do
info "Trying to sync $1, try number $i"
"$RSYNC" "$1"
EXIT=$?
[ $EXIT -ne 0 ] && error "sync failed"
let i++
done
return $EXIT
}
 
function tidyup() {
info "sorting $1"
"$TIDYUP" $1
EXIT=$?
if [ $EXIT -eq 1 ]; then
info "No new files to sort"
return 0
elif [ $EXIT -ne 0 ]; then
error "sorting failed, please send logfile.txt to toxygen1@gmail.com"
return 1
fi
return 0
}
 
# Change working directory
cd $1
 
# reset counter
HOURCOUNT=24
 
# Periodically tidy up and do incremental sync
while :
do
# start timer
start_time=`date +%s`
# sort
tidyup . || error "Sort failed, please send logfile.txt to toxygen1@gmail.com"
# increase counter every hour
# if 24 hour mark is hit, do daily sync
if [[ "$HOURCOUNT" -eq 24 ]]
then
info "Doing complete sync"
syncdir .
HOURCOUNT=0
# next line is important for the first run of the loop
read LAST < LAST
fi
let HOURCOUNT++
 
# read last processed day
OLD="$LAST"
read LAST < LAST
 
# sync last updated folder
syncdir "$LAST"
 
# days changed, sync yesterday too
[[ "$LAST" != "$OLD" ]] && info "syncing yesterday" && syncdir "$OLD"
 
tail -n 1000 logfile.txt > tmp.txt
mv tmp.txt logfile.txt
 
# end timer
end_time=`date +%s`
ELAPSED=`expr $end_time - $start_time`
info "execution time was $ELAPSED s"
 
# if last sync took less than TIME, sleep to make up 1 hour
[[ $ELAPSED -lt $FREQUENCY ]] && sleep `expr $FREQUENCY - $ELAPSED`
done
/Designs/Measuring_instruments/RMDS01A/SW/Tools/tidyup.sh
0,0 → 1,96
#!/usr/bin/env bash
#
# tidyup.sh
#
# This script will sort create organized directory structure
# from observation files in one directory.
#
# Takes one argument: path to the directory
#
# Returns 0 if sorting succeeds
# Returns 1 if there are no files to sort
# Returns >1 in case of error
#
# before:
#
# |
# | meteor_uflu_130128_0010.jpg
# | meteor_uflu_130128_0011.jpg
# | meteor_uflu_130128_1310.jpg
# | meteor_uflu_130129_1112.jpg
# | meteor_uflu_130129_1113.jpg
# | .
# | .
# | .
#
# after:
#
# |
# |- uflu <- observatory
# | |- 2013 <- year
# | |- 01 <- month
# | |- 28 <- day
# | | |- 00 <- hour
# | | | |- meteor_uflu_130128_0010.jpg
# | | | |- meteor_uflu_130128_0011.jpg
# | | |
# | | |- 13
# | | |- meteor_uflu_130128_1310.jpg
# | |
# | |- 29
# | | |- 11
# | |- meteor_uflu_130129_0012.jpg
# | |- meteor_uflu_130129_0012.jpg
# .
# .
# .
#
#
# filename format must be as meteor_uflu_130128_0010.jpg
 
EXT=jpg
DELIM="_"
SLASH="/"
LAST=""
 
 
# turn on debug
set -x
 
# none or 1 argument allowed
[[ "$#" -ne 1 ]] && echo "Wrong number of arguments ($#)" && exit 1
 
# directory in which to sort must exists
[[ ! -d "$1" ]] && echo "Directory doesn't exist" && exit 1
cd $1
 
# if there are no files with $EXT extension in the directory then quit
ls -f *.$EXT > /dev/null 2>&1
[[ "$?" -ne 0 ]] && exit 1
 
for i in *.$EXT; do
echo "processing " $i
PREFIX=`echo $i | cut -d $DELIM -f1,2`
OBSERVATORY=`echo $PREFIX | cut -d $DELIM -f2`
POSTFIX=`echo $i | cut -d $DELIM -f4`
 
TIMESTAMP=`echo "$i" | cut -d $DELIM -f3`
YEAR=20`echo "$TIMESTAMP" | cut -c 1-2`
MONTH=`echo "$TIMESTAMP" | cut -c 3-4`
DAY=`echo "$TIMESTAMP" | cut -c 5-6`
HOUR=`echo "$POSTFIX" | cut -c 1-2`
 
# observatory / year / month / day / hour
DAYDIR="$OBSERVATORY$SLASH$YEAR$SLASH$MONTH$SLASH$DAY$SLASH$HOUR"
 
# create directory with observatory name, year, month and day if hasn't existed before
[[ -d "$DAYDIR" ]] || mkdir -p "$DAYDIR"
mv "$i" "$DAYDIR"
done
 
echo -n "$OBSERVATORY$SLASH$YEAR$SLASH$MONTH$SLASH$DAY" > LAST
 
exit 0
 
 
/Designs/Measuring_instruments/RMDS01A/SW/Tools/upload.sh
0,0 → 1,23
#!/usr/bin/env bash
#
# sync directory to server
#
# this script should not be called manually
 
[[ "$#" -ne 1 ]] && echo "Please provide directory to upload" && exit 1
 
# debug
set -x
 
#cd `dirname $1`
#UPDIR=`basename $1`
UPDIR="$1"
 
# if exclude-list.txt is present, use it
# could as well be changed for rsync -q, but this is more clear
if [[ -f exclude-list.txt ]]
then
rsync -avtz "$UPDIR"/ --exclude-from='exclude-list.txt' meteor@neptun.avc-cvut.cz:data/"$UPDIR"
else
rsync -avtz "$UPDIR"/ meteor@neptun.avc-cvut.cz:data/"$UPDIR"
fi