loop running indefinitely

[shantanu97]

My elif command is running indefinitely. My elif logic is simple. I have two API. 1) USA API- I wanted this API need should work only before year 1985. 2) Australia API- This API works only for year 1985 & after. After that I merge these two dataframe into one dataframe.

To run this logic we need to write this command in the terminal. You can put any value of longitude, latitude in below command.

python test.py -latitude '88' -longitude '75' -startYear '1975' -endYear '2021

File attached.

Can anyone one help me out why this elif logic is running indefinitely??? Also, please tell me how to fixed it.

import requests
import json
import argparse
import time
import pandas as pd
import warnings
warnings.filterwarnings("ignore")
##
#   sample command: python test.py -latitude '' -longitude '' -startYear '' -endYear ''
##
parser = argparse.ArgumentParser(description="Process some integers.")
parser.add_argument("-latitude", help="Latitude(Degress)")
parser.add_argument("-longitude", help="Longitude(Degress)")
parser.add_argument("-startYear", help="Start of the Year")
parser.add_argument("-endYear", help="End of the Year")
parser.add_argument("--verbose", help="display processing information")
start = time.time()
def main(latitude,longitude,startYear,endYear,verbose):   
    parameters = {  #Australia API
        "latd": latitude, # [deg]
        "latm": 00, # [deg]
        "lats": 00, # [deg]
        "lond": longitude, # [deg]
        "lonm": 00, # [deg]
        "lons": 00, # [deg]
        "elev" : 00, # [km]
        "year" : None, # [YYYY]
        "month" : '07', # [MM]
        "day": '01', # [DD]
        "Ein": 'D'  # [Model]
    }
    parameters1 = {    #USA API
    'lat1': latitude, # [deg]
    'lon1': longitude, # [deg]
    'model': 'IGRF', # [Model]
    'startYear': None, # [year]
    'startMonth': 7, # [month]
    'startDay':1,  # [date] 
    'resultFormat': 'json', # [format] 
}
    hostname = "https://api.geomagnetism.ga.gov.au/agrf" #Australia API
    hostname1 = "http://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?%s"  #USA API
    df_1=pd.DataFrame()
    df_2=pd.DataFrame()
    for year in range(startYear, endYear): 
        if endYear < 1985:
            if  startYear < 1985:        
                print('Good, this loop working')        
        elif startYear < 1985:  # Loop is running indefinetly
            for year in range(startYear, 1985): #USA API
                try:
                    parameters1["year"] = year      #USA API
                    response = requests.get(hostname1, params= dict(parameters1, ps=str(year)))
                    # extract JSON payload of response as Python dictionary
                    json_payload = response.json()
                    # raise an Exception if we encoutnered any HTTP error codes like 404
                    response.raise_for_status()
                except requests.exceptions.ConnectionError as e:
                    # handle any typo errors in url or endpoint, or just patchy internet connection
                    print(e)
                except requests.exceptions.HTTPError as e:
                    # handle HTTP error codes in the response
                    print(e, json_payload['error'])
                except requests.exceptions.RequestException as e:
                    # general error handling
                    print(e, json_payload['error'])
                else:
                    json_payload = response.json()
                    #print(json.dumps(json_payload, indent=4, sort_keys=True))
                    df = pd.DataFrame(json_payload['result'])
                    new_row = {
                        "SourceFile": hostname1,
                        "Year": year,
                        "Magnetic Declination": df.iloc[0, 2],
                        "Latitude": -35,
                        "Longitude": 145
                    }
                    df_1 = df_1.append(new_row, ignore_index=True)
                    df_1 = df_1[['Year', 'Latitude', 'Longitude','Magnetic Declination','SourceFile']]
                    df_1.to_csv('magnetic_declination_usa.csv',index=False)  

                    for year in range(1985, endYear):
                        try:     #Australia API
                            parameters["year"] = year
                            response = requests.get(hostname, params= dict(parameters, ps=str(year)))
                            # extract JSON payload of response as Python dictionary
                            json_payload = response.json()
                            # raise an Exception if we encoutnered any HTTP error codes like 404
                            response.raise_for_status()
                        except requests.exceptions.ConnectionError as e:
                            # handle any typo errors in url or endpoint, or just patchy internet connection
                            print(e)
                        except requests.exceptions.HTTPError as e:
                            # handle HTTP error codes in the response
                            print(e, json_payload['error'])
                        except requests.exceptions.RequestException as e:
                            # general error handling
                            print(e, json_payload['error'])
                        else:
                            json_payload = response.json()
                            #print(json.dumps(json_payload, indent=4, sort_keys=True))
                            df = pd.DataFrame(json_payload)
                            new_row = {
                                "SourceFile": hostname,
                                "Year": year,
                                "Magnetic Declination": df.iloc[5, 3],
                                "Latitude": latitude,
                                "Longitude": longitude
                            }
                            df_2 = df_2.append(new_row, ignore_index=True)
                            df_2 = df_2[['Year', 'Latitude', 'Longitude','Magnetic Declination','SourceFile']]
                            df_2["Magnetic Declination"] = df_2["Magnetic Declination"].apply(lambda x: x.replace(" deg", ""))
                            df_2.to_csv('magnetic_declination_australia.csv',index=False)

                df_3 = pd.concat([df_1,df_2], axis=0)            #Merge dataframe into one    
                df_3.to_csv('Magnetic_Declination_(USA+Australia).csv',index=False) 

        else:    # Case where endYear < 1985 and startYear > 1985 (probably an input error)
            print("Invalid Year. Please check the startYear & EndYear.")
                                               
if __name__ == '__main__':
    start = time.time()
    args = parser.parse_args()
    latitude = args.latitude
    longitude = args.longitude
    startYear = int(args.startYear)
    endYear = int(args.endYear)
    verbose = args.verbose
    main(latitude,longitude,startYear,endYear,verbose)  # Calling Main Function
    print("Processed time:", time.time() - start)  # Total Time

[snippsat]

Just to comment structure of code,when you have function(main) that over 100 lines,
that dos all kind of stuff talk to API,logic,Pandas ect.
Then the whole point of using functions is gone.
The point function is that they should not do a lot stuff,but rather one thing good and return the result.
Then code it easier to read and test,now your error is in the main function so have to break it down is error from API,logic..ect or other stuff.

Just want to point this out,maybe i have some time later to dig more into the error later.

[deanhystad]

I don't see it running forever, but the logic looks odd. I think what you are trying to do is process years up to 1985 using the USA API and from 1985 onward using the Australia API. This is not what your code does. Because the Australia API loop is embedded inside the USA API loop, your code will only call the Australia API if the start year is prior to 1985, and it is called repeatedly, once for each year prior to 1985. I would write the code like this:

for year in range(start_year, end_year+1):  # do you want to include end_year?
    if year < 1985:
        # USA API
    else:
        # Australia API

[Gribouillis]

+1 for Dr snippsat's diagnosis: function is too long.

[deanhystad]

A first pass could be splitting the USA and Australia requests out into their own functions.

import requests
import json
import argparse
import time
import pandas as pd
import warnings
warnings.filterwarnings("ignore")

def get_usa_data(df, year, latitude, longitude):   
    parameters = {
        'lat1': latitude, # [deg]
        'lon1': longitude, # [deg]
        'model': 'IGRF', # [Model]
        'startYear': None, # [year]
        'startMonth': 7, # [month]
        'startDay':1,  # [date] 
        'resultFormat': 'json', # [format] 
    }
    hostname = "http://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?%s"

    for year in range(startYear, endYear): 
        try:
            parameters["year"] = year
            response = requests.get(hostname, params=dict(parameters, ps=str(year)))
            # extract JSON payload of response as Python dictionary
            json_payload = response.json()
            # raise an Exception if we encoutnered any HTTP error codes like 404
            response.raise_for_status()
        except requests.exceptions.ConnectionError as e:
            # handle any typo errors in url or endpoint, or just patchy internet connection
            print(e)
        except requests.exceptions.HTTPError as e:
            # handle HTTP error codes in the response
            print(e, json_payload['error'])
        except requests.exceptions.RequestException as e:
            # general error handling
            print(e, json_payload['error'])
        else:
            json_payload = response.json()
            #print(json.dumps(json_payload, indent=4, sort_keys=True))
            row_df = pd.DataFrame(json_payload['result'])
            new_row = {
                "SourceFile": hostname,
                "Year": year,
                "Magnetic Declination": row_df.iloc[0, 2],
                "Latitude": -35,
                "Longitude": 145
            }
            df = df.append(new_row, ignore_index=True)
        return df
 
def get_australia_data(df, year, latitude, longitude):   
    parameters = {
        "latd": latitude, # [deg]
        "latm": 00, # [deg]
        "lats": 00, # [deg]
        "lond": longitude, # [deg]
        "lonm": 00, # [deg]
        "lons": 00, # [deg]
        "elev" : 00, # [km]
        "year" : None, # [YYYY]
        "month" : '07', # [MM]
        "day": '01', # [DD]
        "Ein": 'D'  # [Model]
    }
    hostname = "https://api.geomagnetism.ga.gov.au/agrf"

    try:
        parameters["year"] = year
        response = requests.get(hostname, params=dict(parameters, ps=str(year)))
        # extract JSON payload of response as Python dictionary
        json_payload = response.json()
        # raise an Exception if we encoutnered any HTTP error codes like 404
        response.raise_for_status()
    except requests.exceptions.ConnectionError as e:
        # handle any typo errors in url or endpoint, or just patchy internet connection
        print(e)
    except requests.exceptions.HTTPError as e:
        # handle HTTP error codes in the response
        print(e, json_payload['error'])
    except requests.exceptions.RequestException as e:
        # general error handling
        print(e, json_payload['error'])
    else:
        json_payload = response.json()
        #print(json.dumps(json_payload, indent=4, sort_keys=True))
        row_df = pd.DataFrame(json_payload)
        new_row = {
            "SourceFile": hostname,
            "Year": year,
            "Magnetic Declination": row_df.iloc[5, 3],
            "Latitude": latitude,
            "Longitude": longitude
        }
        df = df.append(new_row, ignore_index=True)


if __name__ == '__main__':
    start = time.time()
    parser = argparse.ArgumentParser(description="Process some integers.")
    parser.add_argument("-latitude", help="Latitude(Degress)")
    parser.add_argument("-longitude", help="Longitude(Degress)")
    parser.add_argument("-startYear", help="Start of the Year")
    parser.add_argument("-endYear", help="End of the Year")
    parser.add_argument("--verbose", help="display processing information")
    args = parser.parse_args()
    latitude = args.latitude
    longitude = args.longitude
    startYear = int(args.startYear)
    endYear = int(args.endYear)
    verbose = args.verbose

    usa_data = pd.DataFrame()
    australia_data = pd.DataFrame()

    for year in range(startYear, endYear):
        if year < 1985:
            get_usa_data(usa_data, year, latitude, longitude) 
        else:
            get_australia_data(australia_data, year, latitude, longitude) 

    usa_data = usa_data[['Year', 'Latitude', 'Longitude','Magnetic Declination','SourceFile']]
    usa_data.to_csv('magnetic_declination_usa.csv', index=False) 

    australia_data = australia_data[['Year', 'Latitude', 'Longitude','Magnetic Declination','SourceFile']]
    australia_data.to_csv('magnetic_declination_australia.csv', index=False)

    combined_data = pd.concat([usa_data, australia_data], axis=0) 
    combined_data.to_csv('Magnetic_Declination_(USA+Australia).csv', index=False) 

    print("Processed time:", time.time() - start)  # Total Time

[Underscore]

Can u copy and paste
the error?

[deanhystad]

There is no error to paste. Original post says the loop runs indefinitely. Also last activity on this thread was June 14, 2021. Please check the date before responding to threads.