Rev 4929 Rev 4930
1 #!/usr/bin/python 1 #!/usr/bin/python
2   2  
3 import pandas as pd 3 import pandas as pd
4 import os 4 import os
5 import time 5 import time
6 import datetime 6 import datetime
7   7  
8 from pymlab import config 8 from pymlab import config
9 from mlabutils import ejson 9 from mlabutils import ejson
10   10  
11 parser = ejson.Parser() 11 parser = ejson.Parser()
12   12  
13 #### Script Arguments ############################################### 13 #### Script Arguments ###############################################
14   14  
15 if len(sys.argv) != 2: 15 if len(sys.argv) != 2:
16 sys.stderr.write("Invalid number of arguments.\n") 16 sys.stderr.write("Invalid number of arguments.\n")
17 sys.stderr.write("Usage: %s CONFIG_FILE\n" % (sys.argv[0], )) 17 sys.stderr.write("Usage: %s CONFIG_FILE\n" % (sys.argv[0], ))
18 sys.exit(1) 18 sys.exit(1)
19   19  
20 value = parser.parse_file(sys.argv[1]) 20 value = parser.parse_file(sys.argv[1])
21 dataSource = value['data_path'] 21 dataSource = value['data_path']
22 dataArchive = value['data_archive'] 22 dataArchive = value['data_archive']
-   23 dataUpload = value['data_upload']
-   24 stationName = value['origin']
-   25  
23 loop = 1 26 loop = 1
24   27  
25   28  
26 while True: 29 while True:
27 try: 30 try:
28 print("Start") 31 print("Start")
29 ## Create sorted list of csv files 32 ## Create sorted list of csv files
30 listOfDataFiles = list() #empty list 33 listOfDataFiles = list() #empty list
31 listOfSpecDataFiles = list() #empty list 34 listOfSpecDataFiles = list() #empty list
32 files = list() #empty list 35 files = list() #empty list
33 falg = False # is computation needed 36 falg = False # is computation needed
34 37
35 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory 38 files = sorted(os.listdir(dataSource)) # list of all files and folders in directory
36 for idx, val in enumerate(files): #goes through files 39 for idx, val in enumerate(files): #goes through files
37 if val.endswith("data.csv"): # in case of *data.csv 40 if val.endswith("data.csv"): # in case of *data.csv
38 listOfDataFiles.append(val) #add file to listOfFiles 41 listOfDataFiles.append(val) #add file to listOfFiles
39   42  
40 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day 43 ## Find the newest and oldest and compare them. If they are from different day, compute the average of all measurement from oldest day
41 if len(listOfDataFiles)>=2: # if there are more than 2 data files 44 if len(listOfDataFiles)>=2: # if there are more than 2 data files
42 first = listOfDataFiles[0] # get first of them 45 first = listOfDataFiles[0] # get first of them
43 last = listOfDataFiles[-1] # get last of them 46 last = listOfDataFiles[-1] # get last of them
44 47
45 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first 48 if time.mktime(datetime.datetime.strptime(last[:8], "%Y%m%d").timetuple()) > time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()): # if the last is older than first
46 flag = True # computation needed 49 flag = True # computation needed
47 print("Computing...") 50 print("Computing...")
48 print(loop) 51 print(loop)
49 loop +=1 52 loop +=1
50 listOfSpecDataFiles = list() # empty list 53 listOfSpecDataFiles = list() # empty list
51 54
52 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day 55 for file in listOfDataFiles: # go through data files and create lis of data files measured on same day
53 # if the day is same like the first one 56 # if the day is same like the first one
54 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()): 57 if time.mktime(datetime.datetime.strptime(first[:8], "%Y%m%d").timetuple()) == time.mktime(datetime.datetime.strptime(file[:8], "%Y%m%d").timetuple()):
55 listOfSpecDataFiles.append(file) 58 listOfSpecDataFiles.append(file)
56 59
57 for file in listOfSpecDataFiles: 60 for file in listOfSpecDataFiles:
58 df=pd.read_csv(file, sep=';', header=None) # read current csv 61 df=pd.read_csv(file, sep=';', header=None) # read current csv
59 dim=df.shape # gets data file dimensions 62 dim=df.shape # gets data file dimensions
60 rowsInd=dim[0] # maximal index of rows 63 rowsInd=dim[0] # maximal index of rows
61 columnsInd=dim[1] # maximal index of columns 64 columnsInd=dim[1] # maximal index of columns
62 values=pd.DataFrame() # empty DataFrame 65 values=pd.DataFrame() # empty DataFrame
63   66  
64 for x in range(0,columnsInd): # for each column 67 for x in range(0,columnsInd): # for each column
65 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3 68 values = values.set_value(0,x,round(df[x].mean(),3),0) #calculates mean value for all cloumns and round it by 3
66 69
67 filename = first[:8]+'000000_VRTY-S1_data_mean.csv' 70 filename = dataUpload + first[:8]+'000000_' + stationName + '_data_mean.csv'
68 outfile = open(filename, 'a') 71 outfile = open(filename, 'a')
69 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv 72 values.to_csv(filename, sep=';', header=None, index=False, mode='a') # save (add) DataFrame to csv
70 outfile.close() 73 outfile.close()
71 74
72 # move files to archive structure 75 # move files to archive structure
73 for file in listOfSpecDataFiles: 76 for file in listOfSpecDataFiles:
74 year = file[:4] 77 year = file[:4]
75 month = file[4:6] 78 month = file[4:6]
76 day = file[6:8] 79 day = file[6:8]
77 directory = dataArchive + year + "/" + month + "/" + day + "/" 80 directory = dataArchive + year + "/" + month + "/" + day + "/"
78 if not os.path.exists(directory): 81 if not os.path.exists(directory):
79 os.makedirs(directory) 82 os.makedirs(directory)
80 os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file 83 os.rename(dataSource + file, dataArchive + year + "/" + month + "/" + day + "/" + file) # move file
81 84
82 else: 85 else:
83 flag = False # computation is not needed 86 flag = False # computation is not needed
84 else: 87 else:
85 flag = False # computation is not needed 88 flag = False # computation is not needed
86 89
87 if flag == False: 90 if flag == False:
88 time.sleep(10) #long sleep, because is nothing to process 91 time.sleep(10) #long sleep, because is nothing to process
89 92
90 except ValueError: 93 except ValueError:
91 print ValueError 94 print ValueError