import sys, traceback import time from flask import jsonify from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from selenium.common.exceptions import TimeoutException from selenium.webdriver.support.wait import WebDriverWait from Basic.log_manager import get_logging def credit_report_3b(email, password): logger = get_logging() try: url = "https://www.smartcredit.com/" chrome_path = "/usr/lib/chromium-browser/chromedriver" #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe" chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe" print(chrome_local_path) option = webdriver.ChromeOptions() #option.add_argument('--headless') option.add_argument('--no-sandbox') option.add_argument('--disable-dev-sh-usage') #driver = webdriver.Chrome(chrome_path,options=option) #driver = webdriver.Chrome(chrome_local_path,options=option) phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs" phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe" driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any']) #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any']) driver.get(url + 'login/') password = str(password) driver.find_element_by_name('j_username').send_keys(email) driver.find_element_by_name('j_password').send_keys(password) driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click() timeout = 10 time.sleep(5) try: alert_div = driver.find_element_by_class_name("alert-danger") paragraph_element = alert_div.find_element_by_tag_name("p") error_text = paragraph_element.text driver.quit() return "INVALID_LOGIN" except: pass try: WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/')) if driver.current_url == url + 'member/home/': try: driver.get(url + 'member/credit-report/3b/simple.htm?format=HTML') element = driver.find_element(By.ID, "reportTop") except Exception as e: driver.get(url + 'member/credit-report/3b/') try: element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]')) WebDriverWait(driver, timeout).until(element_present) except TimeoutException: pass element = driver.find_element(By.ID, "reportTop") soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml') driver.quit() soup.html.insert(0, soup.new_tag('head')) target = soup.find('head') css_list = ''' ''' target.insert(0, BeautifulSoup(css_list, 'lxml').find('head')) alert_box = soup.find('div', {'class': "alert alert-warning fade in text-center"}) if alert_box: alert_box.decompose() alert_box1 = soup.find('div', {'class': "alert alert-warning fade in text-center mb-2"}) if alert_box1: alert_box1.decompose() try: soup.find('td', {'class': "backtoTopText"}).string = "" except Exception: pass heading_sublink = soup.find('span', {'class': "heading-sublink"}) if heading_sublink: heading_sublink.decompose() report_history_loading = soup.find('div', {'id': "report-history-loading"}) if report_history_loading: report_history_loading.decompose() for td_right in soup.findAll('td', {'align': "right"}): td_right.decompose() for img in soup.findAll('img'): img['src'] = 'https://www.smartcredit.com' + str(img['src']) return soup.prettify() driver.quit() return "ERROR" except TimeoutException: traceback.print_exc() logger.error("email: "+email+ " "+traceback.format_exc()) driver.quit() # return traceback.format_exc() return "TIME_OUT" except Exception as e: traceback.print_exc() logger.error("email: "+email+ " "+traceback.format_exc()) # return traceback.format_exc() return "ERROR"