Rev 4928 Rev 4929
Line 1... Line 1...
1 #!/usr/bin/python 1 #!/usr/bin/python
2   2  
3 import pandas as pd 3 import pandas as pd
4 import os 4 import os
5 import time 5 import time
6 import datetime 6 import datetime
7   -  
8   7  
-   8 from pymlab import config
-   9 from mlabutils import ejson
9   10  
-   11 parser = ejson.Parser()
-   12  
-   13 #### Script Arguments ###############################################
-   14  
-   15 if len(sys.argv) != 2:
-   16 sys.stderr.write("Invalid number of arguments.\n")
-   17 sys.stderr.write("Usage: %s CONFIG_FILE\n" % (sys.argv[0], ))
-   18 sys.exit(1)
-   19  
-   20 value = parser.parse_file(sys.argv[1])
10 dataSource = "/home/odroid/geozor/station/testingData/" 21 dataSource = value['data_path']
11 dataArchive = "/home/odroid/geozor/station/testingData/archive/" 22 dataArchive = value['data_archive']
12 loop = 1 23 loop = 1
13   24  
14   25  
15 while True: 26 while True:
16 try: 27 try:
17 print("Start") 28 print("Start")
18 ## Create sorted list of csv files 29 ## Create sorted list of csv files
19 listOfDataFiles = list() #empty list 30 listOfDataFiles = list() #empty list
20 listOfSpecDataFiles = list() #empty list 31 listOfSpecDataFiles = list() #empty list
21 files = list() #empty list 32 files = list() #empty list
22 falg = False # is computation needed 33 falg = False # is computation needed
23 34
24 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory 35 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory
25 for idx, val in enumerate(files): #goes through files 36 for idx, val in enumerate(files): #goes through files
26 if val.endswith("data.csv"): # in case of *data.csv 37 if val.endswith("data.csv"): # in case of *data.csv
27 listOfDataFiles.append(val) #add file to listOfFiles 38 listOfDataFiles.append(val) #add file to listOfFiles
28   39  
29 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day 40 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day
30 if len(listOfDataFiles)>=2: # if there are more than 2 data files 41 if len(listOfDataFiles)>=2: # if there are more than 2 data files
31 first = listOfDataFiles[0] # get first of them 42 first = listOfDataFiles[0] # get first of them
32 last = listOfDataFiles[-1] # get last of them 43 last = listOfDataFiles[-1] # get last of them
33 44
34 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first 45 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first
35 flag = True # computation needed 46 flag = True # computation needed
36 print("Computing...") 47 print("Computing...")
37 print(loop) 48 print(loop)
38 loop +=1 49 loop +=1
39 listOfSpecDataFiles = list() # empty list 50 listOfSpecDataFiles = list() # empty list
40 51
41 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day 52 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day
42 # if the day is same like the first one 53 # if the day is same like the first one
43 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()): 54 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()):
44 listOfSpecDataFiles.append(file) 55 listOfSpecDataFiles.append(file)
45 56
46 for file in listOfSpecDataFiles: 57 for file in listOfSpecDataFiles:
47 df=pd.read_csv(file, sep=';', header=None) # read current csv 58 df=pd.read_csv(file, sep=';', header=None) # read current csv
48 dim=df.shape # gets data file dimensions 59 dim=df.shape # gets data file dimensions
49 rowsInd=dim[0] # maximal index of rows 60 rowsInd=dim[0] # maximal index of rows
50 columnsInd=dim[1] # maximal index of columns 61 columnsInd=dim[1] # maximal index of columns
51 values=pd.DataFrame() # empty DataFrame 62 values=pd.DataFrame() # empty DataFrame
52   63  
53 for x in range(0,columnsInd): # for each column 64 for x in range(0,columnsInd): # for each column
54 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3 65 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3
55 66
56 filename = first[:8]+'000000_VRTY-S1_data_mean.csv' 67 filename = first[:8]+'000000_VRTY-S1_data_mean.csv'
57 outfile = open(filename, 'a') 68 outfile = open(filename, 'a')
58 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv 69 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv
59 outfile.close() 70 outfile.close()
60 71
61 # move files to archive structure 72 # move files to archive structure
62 for file in listOfSpecDataFiles: 73 for file in listOfSpecDataFiles:
63 year = file[:4] 74 year = file[:4]