4928 |
kaklik |
1 |
#!/usr/bin/python |
|
|
2 |
|
|
|
3 |
import pandas as pd |
|
|
4 |
import os |
|
|
5 |
import time |
|
|
6 |
import datetime |
|
|
7 |
|
|
|
8 |
|
|
|
9 |
|
|
|
10 |
dataSource = "/home/odroid/geozor/station/testingData/" |
|
|
11 |
dataArchive = "/home/odroid/geozor/station/testingData/archive/" |
|
|
12 |
loop = 1 |
|
|
13 |
|
|
|
14 |
|
|
|
15 |
while True: |
|
|
16 |
try: |
|
|
17 |
print("Start") |
|
|
18 |
## Create sorted list of csv files |
|
|
19 |
listOfDataFiles = list() #empty list |
|
|
20 |
listOfSpecDataFiles = list() #empty list |
|
|
21 |
files = list() #empty list |
|
|
22 |
falg = False # is computation needed |
|
|
23 |
|
|
|
24 |
files = sorted(os.listdir(dataSource)) # list of all files and folders in directory |
|
|
25 |
for idx, val in enumerate(files): #goes through files |
|
|
26 |
if val.endswith("data.csv"): # in case of *data.csv |
|
|
27 |
listOfDataFiles.append(val) #add file to listOfFiles |
|
|
28 |
|
|
|
29 |
## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day |
|
|
30 |
if len(listOfDataFiles)>=2: # if there are more than 2 data files |
|
|
31 |
first = listOfDataFiles[0] # get first of them |
|
|
32 |
last = listOfDataFiles[-1] # get last of them |
|
|
33 |
|
|
|
34 |
if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first |
|
|
35 |
flag = True # computation needed |
|
|
36 |
print("Computing...") |
|
|
37 |
print(loop) |
|
|
38 |
loop +=1 |
|
|
39 |
listOfSpecDataFiles = list() # empty list |
|
|
40 |
|
|
|
41 |
for file in listOfDataFiles: # go through data files and create lis of data files measured on same day |
|
|
42 |
# if the day is same like the first one |
|
|
43 |
if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()): |
|
|
44 |
listOfSpecDataFiles.append(file) |
|
|
45 |
|
|
|
46 |
for file in listOfSpecDataFiles: |
|
|
47 |
df=pd.read_csv(file, sep=';', header=None) # read current csv |
|
|
48 |
dim=df.shape # gets data file dimensions |
|
|
49 |
rowsInd=dim[0] # maximal index of rows |
|
|
50 |
columnsInd=dim[1] # maximal index of columns |
|
|
51 |
values=pd.DataFrame() # empty DataFrame |
|
|
52 |
|
|
|
53 |
for x in range(0,columnsInd): # for each column |
|
|
54 |
values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3 |
|
|
55 |
|
|
|
56 |
filename = first[:8]+'000000_VRTY-S1_data_mean.csv' |
|
|
57 |
outfile = open(filename, 'a') |
|
|
58 |
values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv |
|
|
59 |
outfile.close() |
|
|
60 |
|
|
|
61 |
# move files to archive structure |
|
|
62 |
for file in listOfSpecDataFiles: |
|
|
63 |
year = file[:4] |
|
|
64 |
month = file[4:6] |
|
|
65 |
day = file[6:8] |
|
|
66 |
directory = dataArchive + year + "/" + month + "/" + day + "/" |
|
|
67 |
if not os.path.exists(directory): |
|
|
68 |
os.makedirs(directory) |
|
|
69 |
os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file |
|
|
70 |
|
|
|
71 |
else: |
|
|
72 |
flag = False # computation is not needed |
|
|
73 |
else: |
|
|
74 |
flag = False # computation is not needed |
|
|
75 |
|
|
|
76 |
if flag == False: |
|
|
77 |
time.sleep(10) #long sleep, because is nothing to process |
|
|
78 |
|
|
|
79 |
except ValueError: |
|
|
80 |
print ValueError |