Rev 4930 Rev 4931
1 #!/usr/bin/python 1 #!/usr/bin/python
2   2  
3 import pandas as pd 3 import pandas as pd
-   4 import sys
4 import os 5 import os
5 import time 6 import time
6 import datetime 7 import datetime
7   8  
8 from pymlab import config 9 from pymlab import config
9 from mlabutils import ejson 10 from mlabutils import ejson
10   11  
11 parser = ejson.Parser() 12 parser = ejson.Parser()
12   13  
13 #### Script Arguments ############################################### 14 #### Script Arguments ###############################################
14   15  
15 if len(sys.argv) != 2: 16 if len(sys.argv) != 2:
16 sys.stderr.write("Invalid number of arguments.\n") 17 sys.stderr.write("Invalid number of arguments.\n")
17 sys.stderr.write("Usage: %s CONFIG_FILE\n" % (sys.argv[0], )) 18 sys.stderr.write("Usage: %s CONFIG_FILE\n" % (sys.argv[0], ))
18 sys.exit(1) 19 sys.exit(1)
19   20  
20 value = parser.parse_file(sys.argv[1]) 21 value = parser.parse_file(sys.argv[1])
21 dataSource = value['data_path'] 22 dataSource = value['data_path']
22 dataArchive = value['data_archive'] 23 dataArchive = value['data_archive']
23 dataUpload = value['data_upload'] 24 dataUpload = value['data_upload']
24 stationName = value['origin'] 25 stationName = value['origin']
25   26  
26 loop = 1 27 loop = 1
27   28  
28   29  
29 while True: 30 while True:
30 try: 31 try:
31 print("Start") 32 print("Start")
32 ## Create sorted list of csv files 33 ## Create sorted list of csv files
33 listOfDataFiles = list() #empty list 34 listOfDataFiles = list() #empty list
34 listOfSpecDataFiles = list() #empty list 35 listOfSpecDataFiles = list() #empty list
35 files = list() #empty list 36 files = list() #empty list
36 falg = False # is computation needed 37 falg = False # is computation needed
37 38
38 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory 39 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory
39 for idx, val in enumerate(files): #goes through files 40 for idx, val in enumerate(files): #goes through files
40 if val.endswith("data.csv"): # in case of *data.csv 41 if val.endswith("data.csv"): # in case of *data.csv
41 listOfDataFiles.append(val) #add file to listOfFiles 42 listOfDataFiles.append(val) #add file to listOfFiles
42   43  
43 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day 44 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day
44 if len(listOfDataFiles)>=2: # if there are more than 2 data files 45 if len(listOfDataFiles)>=2: # if there are more than 2 data files
45 first = listOfDataFiles[0] # get first of them 46 first = listOfDataFiles[0] # get first of them
46 last = listOfDataFiles[-1] # get last of them 47 last = listOfDataFiles[-1] # get last of them
47 48
48 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first 49 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first
49 flag = True # computation needed 50 flag = True # computation needed
50 print("Computing...") 51 print("Computing...")
51 print(loop) 52 print(loop)
52 loop +=1 53 loop +=1
53 listOfSpecDataFiles = list() # empty list 54 listOfSpecDataFiles = list() # empty list
54 55
55 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day 56 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day
56 # if the day is same like the first one 57 # if the day is same like the first one
57 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()): 58 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()):
58 listOfSpecDataFiles.append(file) 59 listOfSpecDataFiles.append(file)
59 60
60 for file in listOfSpecDataFiles: 61 for file in listOfSpecDataFiles:
61 df=pd.read_csv(file, sep=';', header=None) # read current csv 62 df=pd.read_csv(dataSource + file, sep=';', header=None) # read current csv
62 dim=df.shape # gets data file dimensions 63 dim=df.shape # gets data file dimensions
63 rowsInd=dim[0] # maximal index of rows 64 rowsInd=dim[0] # maximal index of rows
64 columnsInd=dim[1] # maximal index of columns 65 columnsInd=dim[1] # maximal index of columns
65 values=pd.DataFrame() # empty DataFrame 66 values=pd.DataFrame() # empty DataFrame
66   67  
67 for x in range(0,columnsInd): # for each column 68 for x in range(0,columnsInd): # for each column
68 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3 69 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3
69 70
70 filename = dataUpload + first[:8]+'000000_' + stationName + '_data_mean.csv' 71 filename = dataUpload + first[:8]+'000000_' + stationName + '_data_mean.csv'
71 outfile = open(filename, 'a') 72 outfile = open(filename, 'a')
72 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv 73 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv
73 outfile.close() 74 outfile.close()
74 75
75 # move files to archive structure 76 # move files to archive structure
76 for file in listOfSpecDataFiles: 77 for file in listOfSpecDataFiles:
77 year = file[:4] 78 year = file[:4]
78 month = file[4:6] 79 month = file[4:6]
79 day = file[6:8] 80 day = file[6:8]
80 directory = dataArchive + year + "/" + month + "/" + day + "/" 81 directory = dataArchive + year + "/" + month + "/" + day + "/"
81 if not os.path.exists(directory): 82 if not os.path.exists(directory):
82 os.makedirs(directory) 83 os.makedirs(directory)
83 os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file 84 os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file
84 85
85 else: 86 else:
86 flag = False # computation is not needed 87 flag = False # computation is not needed
87 else: 88 else:
88 flag = False # computation is not needed 89 flag = False # computation is not needed
89 90
90 if flag == False: 91 if flag == False:
91 time.sleep(10) #long sleep, because is nothing to process 92 time.sleep(10) #long sleep, because is nothing to process
92 93
93 except ValueError: 94 except ValueError:
94 print ValueError 95 print ValueError