Webscrape script, add to run every day at a specific time

[jjcooper]

Hello,

I have the following code which pulls data from the web (Booking.com). Does anyone know how to add code to it which would run it every day at a specific time, say 09:00?

Many thanks in advance.

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
import time
from selenium.webdriver.support.ui import WebDriverWait
import numpy as np
from selenium.webdriver.support import expected_conditions as EC

from datetime import datetime
from datetime import timedelta

TodaysDate = time.strftime("%Y-%m-%d")

Startdate = (datetime.now() + timedelta(days=0) ).strftime('%Y-%m-%d')
Enddate = (datetime.now() + timedelta(days=1) ).strftime('%Y-%m-%d')

driver = webdriver.Chrome()
list=["https://www.booking.com/hotel/gb/clayton-hotel-cardiff.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYATG4ARfIAQzYAQHoAQH4AQKIAgGoAgO4At_V07EGwAIB0gIkZmRhNGVhYzMtYTQ2Mi00OGQxLWEyNmEtYjgzYWEzNjg0ZmIw2AIF4AIB&sid=ec1eae1807c6ef490266327d6508f518&all_sr_blocks=29091610_91471356_2_2_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=5;highlighted_blocks=29091610_91471356_2_2_0;hpos=5;matching_block_id=29091610_91471356_2_2_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=29091610_91471356_2_2_0__8910;srepoch=1714744083;srpvid=49fd6104516f02ba;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/cardiffparkplaza.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYATG4ARfIAQzYAQHoAQH4AQKIAgGoAgO4At_V07EGwAIB0gIkZmRhNGVhYzMtYTQ2Mi00OGQxLWEyNmEtYjgzYWEzNjg0ZmIw2AIF4AIB&sid=ec1eae1807c6ef490266327d6508f518&all_sr_blocks=22664804_331143887_2_2_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=6;highlighted_blocks=22664804_331143887_2_2_0;hpos=6;matching_block_id=22664804_331143887_2_2_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=22664804_331143887_2_2_0__11500;srepoch=1714744260;srpvid=49fd6104516f02ba;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/radisson-blu.en-gb.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&all_sr_blocks=4611414_380875172_2_2_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=6;highlighted_blocks=4611414_380875172_2_2_0;hpos=6;matching_block_id=4611414_380875172_2_2_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=4611414_380875172_2_2_0__12500;srepoch=1715187672;srpvid=752677a88d430184;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/indigo-cardiff.en-gb.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&all_sr_blocks=234837705_244371218_2_2_0;checkin=" + Startdate + ";checkout" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=9;highlighted_blocks=234837705_244371218_2_2_0;hpos=9;matching_block_id=234837705_244371218_2_2_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=234837705_244371218_2_2_0__12060;srepoch=1715195119;srpvid=752677a88d430184;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/parkgate-cardiff.en-gb.html?label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&aid=304142&ucfs=1&arphpl=1&checkin=" + Startdate + "&checkout=" + Enddate + "&dest_id=1233&dest_type=district&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=12&hapos=12&sr_order=popularity&srpvid=752677a88d430184&srepoch=1715195223&all_sr_blocks=724073602_310086127_2_2_0&highlighted_blocks=724073602_310086127_2_2_0&matching_block_id=724073602_310086127_2_2_0&sr_pri_blocks=724073602_310086127_2_2_0__16400&from_sustainable_property_sr=1&from=searchresults#hotelTmpl/",
    "https://www.booking.com/hotel/gb/macdonaldhollandhouse.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYATG4ARfIAQzYAQHoAQH4AQKIAgGoAgO4At_V07EGwAIB0gIkZmRhNGVhYzMtYTQ2Mi00OGQxLWEyNmEtYjgzYWEzNjg0ZmIw2AIF4AIB&sid=ec1eae1807c6ef490266327d6508f518&all_sr_blocks=22241114_95454762_2_42_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=12;highlighted_blocks=22241114_95454762_2_42_0;hpos=12;matching_block_id=22241114_95454762_2_42_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=22241114_95454762_2_42_0__9900;srepoch=1714744853;srpvid=49fd6104516f02ba;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/holiday-inn-cardiff-city.en-gb.html?label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&aid=304142&ucfs=1&arphpl=1&checkin=" + Startdate + "&checkout=" + Enddate + "&dest_id=1233&dest_type=district&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=11&hapos=11&sr_order=popularity&srpvid=0b22868a538e0117&srepoch=1715195307&all_sr_blocks=3419103_244944025_2_2_0&highlighted_blocks=3419103_244944025_2_2_0&matching_block_id=3419103_244944025_2_2_0&sr_pri_blocks=3419103_244944025_2_2_0__10900&from_sustainable_property_sr=1&from=searchresults#hotelTmpl/",
    "https://www.booking.com/hotel/gb/park-inn-cardiff-city-centre.en-gb.html?label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&aid=304142&ucfs=1&arphpl=1&checkin=" + Startdate + "&checkout=" + Enddate + "&dest_id=1233&dest_type=district&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=2&hapos=2&sr_order=popularity&srpvid=0b22868a538e0117&srepoch=1715195349&all_sr_blocks=3616318_338890992_2_2_0&highlighted_blocks=3616318_338890992_2_2_0&matching_block_id=3616318_338890992_2_2_0&sr_pri_blocks=3616318_338890992_2_2_0__8280&from_sustainable_property_sr=1&from=searchresults#hotelTmpl/",
    "https://www.booking.com/hotel/gb/cardiff-marriott.en-gb.html?label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AsDf7rEGwAIB0gIkYmQ3ZjQ4ZjktNDE5Yy00ZmY1LThjMjktYTFhMGM0MmNjMGI22AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&aid=304142&ucfs=1&arphpl=1&checkin=" + Startdate + "&checkout=" + Enddate + "&dest_id=1233&dest_type=district&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=4&hapos=4&sr_order=popularity&srpvid=85f186cb22c10193&srepoch=1715195419&all_sr_blocks=3687806_246419015_0_2_0&highlighted_blocks=3687806_246419015_0_2_0&matching_block_id=3687806_246419015_0_2_0&sr_pri_blocks=3687806_246419015_0_2_0__11000&from_sustainable_property_sr=1&from=searchresults#hotelTmpl/",
    "https://www.booking.com/hotel/gb/hanoverinternationalcardiff.en-gb.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYATG4ARfIAQzYAQHoAQH4AQKIAgGoAgO4At_V07EGwAIB0gIkZmRhNGVhYzMtYTQ2Mi00OGQxLWEyNmEtYjgzYWEzNjg0ZmIw2AIF4AIB&sid=93afbcb22bab72120df44694d6dc40f0&all_sr_blocks=21752512_94349799_2_42_0&checkin=" + Startdate + "&checkout=" + Enddate + "&dest_id=1233&dest_type=district&dist=0&group_adults=2&group_children=0&hapos=16&highlighted_blocks=21752512_94349799_2_42_0&hpos=16&keep_landing=1&matching_block_id=21752512_94349799_2_42_0&no_rooms=1&req_adults=2&req_children=0&sb_price_type=total&sr_order=popularity&sr_pri_blocks=21752512_94349799_2_42_0__10028&srepoch=1714744962&srpvid=49fd6104516f02ba&type=total&ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/leonardo-hotel-cardiff.html?label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYATG4ARfIAQzYAQHoAQH4AQKIAgGoAgO4At_V07EGwAIB0gIkZmRhNGVhYzMtYTQ2Mi00OGQxLWEyNmEtYjgzYWEzNjg0ZmIw2AIF4AIB&aid=304142&ucfs=1&arphpl=1&checkin=" + Startdate + ";checkout=" + Enddate + "&dest_id=1233&dest_type=district&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=17&hapos=17&sr_order=popularity&srpvid=49fd6104516f02ba&srepoch=1714744987&all_sr_blocks=3633717_95158809_0_2_0&highlighted_blocks=3633717_95158809_0_2_0&matching_block_id=3633717_95158809_0_2_0&sr_pri_blocks=3633717_95158809_0_2_0__10530&from_sustainable_property_sr=1&from=searchresults#hotelTmpl/",
    "https://www.booking.com/hotel/gb/hilton-cardiff.en-gb.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AtbH6bEGwAIB0gIkZjJkN2ZlMmUtOGZmYS00NGEyLTllYzQtZjYyZjgxNjI1MTNl2AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&all_sr_blocks=3387110_246006931_2_34_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=1;highlighted_blocks=3387110_246006931_2_34_0;hpos=1;matching_block_id=3387110_246006931_2_34_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=3387110_246006931_2_34_0__10900;srepoch=1715103321;srpvid=54987baa66de01b8;type=total;ucfs=1&#hotelTmpl/",
    "https://www.booking.com/hotel/gb/macdonaldhollandhouse.en-gb.html?aid=304142&label=gen173nr-1FCAEoggI46AdIM1gEaFCIAQGYAQm4ARjIAQzYAQHoAQH4AQuIAgGoAgS4AtbH6bEGwAIB0gIkZjJkN2ZlMmUtOGZmYS00NGEyLTllYzQtZjYyZjgxNjI1MTNl2AIG4AIB&sid=93afbcb22bab72120df44694d6dc40f0&all_sr_blocks=22241109_95454762_2_42_0;checkin=" + Startdate + ";checkout=" + Enddate + ";dest_id=1233;dest_type=district;dist=0;group_adults=2;group_children=0;hapos=22;highlighted_blocks=22241109_95454762_2_42_0;hpos=22;matching_block_id=22241109_95454762_2_42_0;no_rooms=1;req_adults=2;req_children=0;room1=A%2CA;sb_price_type=total;sr_order=popularity;sr_pri_blocks=22241109_95454762_2_42_0__48900;srepoch=1715103426;srpvid=54987baa66de01b8;type=total;ucfs=1&#hotelTmpl/",
    ]
all_dates = []
all_prices = []
all_hotel_names = []

for url in list:
    driver.get(url)
    time.sleep(6)
    
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    time.sleep(6)
    
    data = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '(//button[@data-testid="date-display-field-start"])[2]')))
    data.click()
    time.sleep(6)
    hotel_name=driver.find_element(By.XPATH, '//h2[@class="d2fee87262 pp-header__title"]')
    hotel_name=hotel_name.text

    
    while True:
        dates = [date.get_attribute('aria-label') for date in driver.find_elements(By.XPATH, '//span[@class="cf06f772fa ef091eb985 b6f813c05a"]')]
        prices = [price.text for price in driver.find_elements(By.XPATH, '//span[@class="cf06f772fa ef091eb985 b6f813c05a"]//div/span')]
        
        all_hotel_names.extend([hotel_name] * len(prices))
        
        all_dates.extend(dates)
        all_prices.extend(prices)
        
        try:
            next_month_button = driver.find_element(By.XPATH, '//button[@aria-label="Next month"]')
            
            if next_month_button.is_enabled():
                next_month_button.click()
                time.sleep(4)  
            else:
                break  
        except NoSuchElementException:
            break  
        
        time.sleep(4)

excelfilename = TodaysDate +".xlsx"

df = pd.DataFrame({'Hotel Name': all_hotel_names, 'Check-in': all_dates, 'Price': all_prices})
df["Price"] = df["Price"].replace("—", np.nan, regex=True)
df=df.dropna(subset="Price")
df["Check-in"]=pd.to_datetime(df["Check-in"])
df["Checkout"] = df["Check-in"] + pd.Timedelta(days=1)
print(df)
df.to_excel(excelfilename, index=False)
driver.quit()

[python]

[/python]

[Larz60+]

I just randomly searched pypi.org and found many packages.

This one looks promising.

[sawtooth500]

Are you running this on a linux server? If so you can just set a cron job to run it. There are also scheduling options on windows machines to run scripts on powershell at predetermined times.