Rev Author Line No. Line
4928 kaklik 1 #!/usr/bin/python
2  
3 import pandas as pd
4 import os
5 import time
6 import datetime
7  
8  
9  
10 dataSource = "/home/odroid/geozor/station/testingData/"
11 dataArchive = "/home/odroid/geozor/station/testingData/archive/"
12 loop = 1
13  
14  
15 while True:
16 try:
17 print("Start")
18 ## Create sorted list of csv files
19 listOfDataFiles = list() #empty list
20 listOfSpecDataFiles = list() #empty list
21 files = list() #empty list
22 falg = False # is computation needed
23  
24 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory
25 for idx, val in enumerate(files): #goes through files
26 if val.endswith("data.csv"): # in case of *data.csv
27 listOfDataFiles.append(val) #add file to listOfFiles
28  
29 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day
30 if len(listOfDataFiles)>=2: # if there are more than 2 data files
31 first = listOfDataFiles[0] # get first of them
32 last = listOfDataFiles[-1] # get last of them
33  
34 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first
35 flag = True # computation needed
36 print("Computing...")
37 print(loop)
38 loop +=1
39 listOfSpecDataFiles = list() # empty list
40  
41 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day
42 # if the day is same like the first one
43 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()):
44 listOfSpecDataFiles.append(file)
45  
46 for file in listOfSpecDataFiles:
47 df=pd.read_csv(file, sep=';', header=None) # read current csv
48 dim=df.shape # gets data file dimensions
49 rowsInd=dim[0] # maximal index of rows
50 columnsInd=dim[1] # maximal index of columns
51 values=pd.DataFrame() # empty DataFrame
52  
53 for x in range(0,columnsInd): # for each column
54 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3
55  
56 filename = first[:8]+'000000_VRTY-S1_data_mean.csv'
57 outfile = open(filename, 'a')
58 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv
59 outfile.close()
60  
61 # move files to archive structure
62 for file in listOfSpecDataFiles:
63 year = file[:4]
64 month = file[4:6]
65 day = file[6:8]
66 directory = dataArchive + year + "/" + month + "/" + day + "/"
67 if not os.path.exists(directory):
68 os.makedirs(directory)
69 os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file
70  
71 else:
72 flag = False # computation is not needed
73 else:
74 flag = False # computation is not needed
75  
76 if flag == False:
77 time.sleep(10) #long sleep, because is nothing to process
78  
79 except ValueError:
80 print ValueError