1+ from dweather_client .ipfs_errors import *
2+ from dweather_client .ipfs_queries import *
3+ from dweather_client .tests .mock_fixtures import get_patched_datasets
4+ from dweather_client .client import get_australia_station_history , get_station_history , get_gridcell_history , get_tropical_storms ,\
5+ get_yield_history , get_irrigation_data , get_power_history , get_gas_history , get_alberta_power_history , GRIDDED_DATASETS , has_dataset_updated ,\
6+ get_forecast_datasets , get_forecast , get_cme_station_history , get_european_station_history , get_hourly_station_history , get_drought_monitor_history , get_japan_station_history ,\
7+ get_afr_history , get_cwv_station_history , get_teleconnections_history , get_station_forecast_history , get_station_forecast_stations , get_eaufrance_history
8+ from dweather_client .aliases_and_units import snotel_to_ghcnd
9+ import pandas as pd
10+ from io import StringIO
11+ import datetime
12+ from astropy import units as u
13+ from astropy .units import imperial
14+ import pytest
15+ import csv
16+
17+ if __name__ == '__main__' :
18+ #cme_futures_obj = StationForecastDataset("cme_futures-daily", ipfs_timeout=ipfs_timeout)
19+ cme_futures_obj = StationForecastDataset ("cme_futures-daily" , ipfs_timeout = None )
20+ current_head = cme_futures_obj .head
21+ current_metadata = cme_futures_obj .get_metadata (current_head )
22+ station_data_dictionary = {} # Key will be the station, and the value will be a pandas dataframe with the data
23+ dataframes = {}
24+ while True : #current_metadata['previous hash'] is not None:
25+ #current_date = current_metadata["date range"][0]
26+ #current_datetime = datetime.date(int(current_date[0:4]), int(current_date[5:7]), int(current_date[8:])) #Beware current date's type changes here
27+ current_datetime = datetime .datetime .strptime (current_metadata ["date range" ][0 ], "%Y-%m-%d" ).date ()
28+ station_features = json .loads (cme_futures_obj .get_stations (current_datetime ))['features' ]
29+ for feature in station_features :
30+ station_name = feature ["properties" ]["station name" ] #this is the station name
31+
32+ csv_text = cme_futures_obj .get_data (station_name , (current_datetime ))
33+ df = pd .read_csv (StringIO (csv_text ))
34+ df ['forecast_date' ] = current_datetime
35+ if station_name not in station_data_dictionary :
36+ station_data_dictionary [station_name ] = {
37+ "hashes" : [current_head ],
38+ "previous_hashes" : [],
39+ "forecast_dates" : [current_datetime .strftime ("%Y-%m-%d" )],
40+ }
41+ else :
42+ station_data_dictionary [station_name ]["hashes" ].append (current_head )
43+ station_data_dictionary [station_name ]["forecast_dates" ].append (current_datetime .strftime ("%Y-%m-%d" ))
44+
45+ if station_name not in dataframes :
46+ dataframes [station_name ] = {}
47+ dataframes [station_name ][current_head ] = df
48+
49+ if current_metadata ["previous hash" ] is None :
50+ break
51+
52+ previous_hash = current_metadata ["previous hash" ]
53+ current_head = previous_hash
54+ current_metadata = cme_futures_obj .get_metadata (current_head )
55+
56+ for station_key , data in station_data_dictionary .items ():
57+ data ["previous_hashes" ] = data ["hashes" ][1 :]
58+ data ["hashes" ] = data ["hashes" ][:- 1 ]
59+
60+ # Print the station data dictionary in the desired format
61+ filename = 'station_table.csv'
62+ with open (filename , 'w' , newline = '' ) as file :
63+ writer = csv .writer (file )
64+ writer .writerow (["DATE" , "SETT" , "forecast_date" ])
65+ for station_key , data in station_data_dictionary .items ():
66+ for hash_value , previous_hash , forecast_date in zip (data ["hashes" ], data ["previous_hashes" ], data ["forecast_dates" ]):
67+ df = dataframes [station_key ][hash_value ]
68+ for _ , row in df .iterrows ():
69+ date = str (row ['DATE' ])
70+ sett = str (row ['SETT' ])
71+ forecast_date = str (forecast_date )
72+ writer .writerow ([date , sett , forecast_date ])
73+
74+ data = pd .read_csv (filename )
75+ data .to_csv ('station_table_new.csv' ,index = False , columns = ["DATE" , "SETT" , "forecast_date" ])
76+ data_modified = pd .read_csv ('station_table_new.csv' )
77+ import ipdb ;ipdb .set_trace ()
78+
79+
0 commit comments