import time import traceback from flask import jsonify from bs4 import BeautifulSoup from selenium import webdriver from selenium.common.exceptions import TimeoutException,NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.wait import WebDriverWait def recent_report(email, password, ssn): try: url = "https://member.identityiq.com/" chrome_path = "/usr/lib/chromium-browser/chromedriver" # chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe" chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe" option = webdriver.ChromeOptions() option.add_argument('--headless') option.add_argument('--no-sandbox') option.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome(chrome_path,options=option) #driver = webdriver.Chrome(chrome_local_path,options=option) #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs" phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe" #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any']) #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any']) driver.get(url + 'login.aspx') driver.find_element_by_id('txtUsername').send_keys(email) driver.find_element_by_id('txtPassword').send_keys(password) driver.find_element_by_id("imgBtnLogin").click() timeout = 120 time.sleep(5) try: ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header") error_text = ele_acc_lock.text driver.quit() return "ACCOUNT_LOCKED" except: element_login = driver.find_element(By.CLASS_NAME, "chakra-text") if element_login.text == 'Invalid login attempt': driver.quit() return "INVALID_LOGIN" try: wait = WebDriverWait(driver, timeout) WebDriverWait(driver, timeout).until( ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx') ) wait.until( lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx') ) if driver.current_url == url + 'security-question': element_present = ec.presence_of_element_located( (By.NAME, 'userSecurityAnswer') ) WebDriverWait(driver, timeout).until(element_present) driver.find_element_by_name('userSecurityAnswer').send_keys(ssn) driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click() #element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID')) #WebDriverWait(driver, timeout).until(element_present) driver.get(url + 'Dashboard.aspx') if driver.current_url == url + 'Dashboard.aspx': driver.get(url + 'CreditReport.aspx') element_present = ec.presence_of_element_located((By.ID, 'reportTop')) WebDriverWait(driver, timeout).until(element_present) driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") xpath_value = '' try: driver.find_element_by_xpath("//b[contains(., '***SECURITY FREEZE')]") xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[10]/div[2]/address-history/div/ng-repeat[1]' except NoSuchElementException: xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[9]/div[2]/address-history/div/ng-repeat[1]' try: element_present = ec.presence_of_element_located(( By.XPATH,xpath_value)) WebDriverWait(driver, timeout).until(element_present) except TimeoutException: pass last_height = driver.execute_script("return document.body.scrollHeight") while True: driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(5) new_height = driver.execute_script("return document.body.scrollHeight") if new_height == last_height: break last_height = new_height element = driver.find_element(By.ID, "ctrlCreditReport") soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml') driver.quit() soup.html.insert(0, soup.new_tag('head')) target = soup.find('head') css_list = ''' ''' target.insert(0, BeautifulSoup(css_list, 'lxml').find('head')) for div in soup.findAll('div', {'class': "link_header"}): div.decompose() for span in soup.findAll('span', {'class': "return_topSpan"}): span.decompose() for img in soup.findAll('img'): img['src'] = 'https://www.member.identityiq.com' + str(img['src']) return soup.prettify() driver.quit() # return traceback.format_exc() return "ERROR" except TimeoutException: traceback.print_exc() driver.quit() return traceback.format_exc() return "TIME_OUT" except Exception : traceback.print_exc() return traceback.format_exc() return "ERROR"