Download data from WRDS using Python Selenium

from selenium import webdriver

from time import sleep

import numpy as np

driver = webdriver.Chrome()

driver.get('https://wrds-www.wharton.upenn.edu/')

# Login WRDS

driver.find_element_by_id('id_username').clear()

driver.find_element_by_id('id_username').send_keys(ACCOUNT)

driver.find_element_by_id('id_password').clear()

driver.find_element_by_id ('id_password').send_keys(PASSWORD)

driver.find_element_by_xpath('//button[@class="btn btn-danger"]').click()

## -->CRSP

driver.find_element_by_xpath("//div[@class='col-md-4 col-lg-4'][1]//li[10]/a").click()

### -->stock files

driver.find_element_by_xpath("//div[@class='row m-bottom0'][2]/div[@class='col-md-4 col-lg-4'][1]//li[1]/a").click()

### -->daily

driver.find_element_by_xpath("//div[@class='col-md-4 col-lg-4'][1]//li[2]/a").click()

### -->TICKER FORMAT

driver.find_element_by_id('format-TICKER').click()

### -->all stocks in database

driver.find_element_by_id('search_option_method3').click()

### -->select columns

driver.find_element_by_id('select_all_button-80DBA1CF').click()

### -->output format

driver.find_element_by_id('csv').click()

## -->build date list

year = range(1930,1962)

year =[str(i) for i in year]

month = ['01','07']

list1 = []

list2 = []

for ix in year :

for iy in month :

list1.append( ix + '-' + iy +'-01')

list2.append( ix + '-' + str((int(iy)+5)).zfill(2)+'-'+str(30+int(iy)//6))

datelist = [list1,list2]

### --> for loop (change search dates)

datelist

for i in np.arange(39,41):

### -->modify end date before begin date, otherwise the begin date won't change

driver.find_element_by_id('select_end_date').clear()

driver.find_element_by_id('select_end_date').send_keys(datelist[1][i])

driver.find_element_by_id('select_beg_date').clear()

driver.find_element_by_id('select_beg_date').send_keys(datelist[0][i])

### -->submit query

driver.find_element_by_xpath('//button[@id="form_submit"]').click()

### -->switch to new page

driver.switch_to.window(driver.window_handles[1])

### -->click until 'csv' is prepared and download

temp=driver.find_element_by_xpath('//p[2]/a')

while not ('.csv' in temp.text):

sleep(5)

temp=driver.find_element_by_xpath('//p[2]/a')

driver.find_element_by_xpath('//p[2]/a').click()

### -->close new page and switch back to original page

driver.close()

driver.switch_to.window(driver.window_handles[0])

driver.switch_to.window(driver.window_handles[0])

len(datelist[0])