CME_Rollup.py - Updated

MaanaAragula · MaanaAragula · commit 5d37c1291dc2 · 2023-06-16T09:58:04.000-05:00
Working Rollup code with changes made with the forecasted_dt column
diff --git a/CME_Rollup.py b/CME_Rollup.py
@@ -17,23 +17,24 @@
 data_directory = os.getcwd()
 
 if __name__ == '__main__':
-    #cme_futures_obj = StationForecastDataset("cme_futures-daily", ipfs_timeout=ipfs_timeout)
     cme_futures_obj = StationForecastDataset("cme_futures-daily", ipfs_timeout=None)
     current_head = cme_futures_obj.head
     current_metadata = cme_futures_obj.get_metadata(current_head)
     station_data_dictionary = {} # Key will be the station, and the value will be a pandas dataframe with the data
     dataframes = {}
-    while True: #current_metadata['previous hash'] is not None:
-        #current_date = current_metadata["date range"][0]
-        #current_datetime = datetime.date(int(current_date[0:4]), int(current_date[5:7]), int(current_date[8:])) #Beware current date's type changes here
+    while True: 
         current_datetime = datetime.datetime.strptime(current_metadata["date range"][0], "%Y-%m-%d").date()
         station_features = json.loads(cme_futures_obj.get_stations(current_datetime))['features']
         for feature in station_features: 
-            station_name = feature["properties"]["station name"] #this is the station name
-            
+            station_name = feature["properties"]["station name"] #this is the station name     
             csv_text = cme_futures_obj.get_data(station_name, (current_datetime))
             df = pd.read_csv(StringIO(csv_text))
-            df['forecasted_dt'] = current_datetime.strftime("%Y-%m")
+            try:
+                df.rename(columns={"DATE": "forecasted_dt", "value": "SETT"}, inplace=True)
+                df["dt"] = current_datetime.strftime("%Y-%m-%d")
+            except KeyError:
+                print("Required columns not found in the dataset. Please verify the column names.")
+                continue
             if station_name not in station_data_dictionary:
                 station_data_dictionary[station_name] = {
                     "hashes": [current_head],
@@ -59,31 +60,32 @@
         data["previous_hashes"] = data["hashes"][1:]
         data["hashes"] = data["hashes"][:-1]
 
-    # Print the station data dictionary in the desired format
+
     for station_key, data in station_data_dictionary.items():
-        station_filename = f"{station_key}_table.csv"  # Generate a filename based on the station name
+        station_filename = f"{station_key}.csv"  # Generate a filename based on the station name
         with open(station_filename, 'w', newline='') as file:
             writer = csv.writer(file)
             writer.writerow(["forecasted_dt", "SETT", "dt"])
             for hash_value, previous_hash, forecast_date in zip(data["hashes"], data["previous_hashes"], data["forecast_dates"]):
                 df = dataframes[station_key][hash_value]
                 for _, row in df.iterrows():
                     forecasted_dt = str(row['forecasted_dt'])
+                    forecasted_dt = forecasted_dt[:-3]
                     sett = str(row['SETT'])
                     dt = str(forecast_date)
                     writer.writerow([forecasted_dt, sett, dt])
     station_data = {}
     for file_station_key, data in station_data_dictionary.items():
-        station_filename = f"{file_station_key}_table.csv"  # Generate the filename of the CSV file
+        station_filename = f"{file_station_key}.csv"  # Generate the filename of the CSV file
         file_path = os.path.join(data_directory, station_filename)  # Create the full file path
         station_data[file_station_key] = pd.read_csv(file_path)
 
-    file_station_name = "D2X"
+    """file_station_name = "D2X"
     station_df = station_data[file_station_name]
     station_df['dt'] = pd.to_datetime(station_df['dt']).dt.strftime("%Y-%m-%d")
-    print(station_df)
+    print(station_df)"""
+    import ipdb;ipdb.set_trace()
 
 
-    import ipdb;ipdb.set_trace()
+    
 
-    
