inital commit

scrapper/webistes/smart_credit/credit_3b_scrapper.py

@@ -0,0 +1,143 @@
+import sys, traceback
+import time
+from flask import jsonify
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def credit_report_3b(email, password):
+
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+       
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+        print(chrome_local_path)
+        option = webdriver.ChromeOptions()
+        #option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+                                                           
+
+        driver.get(url + 'login/')
+        password = str(password)
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+        
+        timeout = 10
+
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+
+        try:
+                WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+                if driver.current_url == url + 'member/home/':
+
+                    try:
+                      
+                      driver.get(url + 'member/credit-report/3b/simple.htm?format=HTML')  
+                      
+                      element = driver.find_element(By.ID, "reportTop")
+
+                    except Exception as e:
+
+                      driver.get(url + 'member/credit-report/3b/')
+
+                    try:
+                        element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
+                        WebDriverWait(driver, timeout).until(element_present)
+
+                    except TimeoutException:
+                        pass
+
+                    element = driver.find_element(By.ID, "reportTop")
+
+                    soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+
+                    driver.quit()
+                    soup.html.insert(0, soup.new_tag('head'))
+                    target = soup.find('head')
+                    css_list = '''
+                    <link rel="stylesheet" type="text/css" href="https://www.smartcredit.com/resources/css/sc/pages/member/credit-report/3b/base.css"
+                    charset="utf-8">
+                    <style>
+                        td#personalInfo {
+                            width: 100%;
+                        }
+                        #reportTop table{
+                            width: 100%;
+                        }
+                    </style>
+                    '''
+                    target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
+
+                    alert_box = soup.find('div', {'class': "alert alert-warning fade in text-center"})
+                    if alert_box:
+                        alert_box.decompose()
+
+                    alert_box1 = soup.find('div', {'class': "alert alert-warning fade in text-center mb-2"})
+                    if alert_box1:
+                        alert_box1.decompose()
+                    try:
+                     soup.find('td', {'class': "backtoTopText"}).string = ""
+                    except Exception:
+                        pass
+
+                    heading_sublink = soup.find('span', {'class': "heading-sublink"})
+                    if heading_sublink:
+                        heading_sublink.decompose()
+
+                    report_history_loading = soup.find('div', {'id': "report-history-loading"})
+                    if report_history_loading:
+                        report_history_loading.decompose()
+
+                    for td_right in soup.findAll('td', {'align': "right"}):
+                        td_right.decompose()
+
+                    for img in soup.findAll('img'):
+                        img['src'] = 'https://www.smartcredit.com' + str(img['src'])
+
+                    return soup.prettify()
+                
+                
+                driver.quit()
+                return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            # return traceback.format_exc()
+            return "TIME_OUT"
+
+    except Exception as e:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        # return traceback.format_exc()
+        return "ERROR"
+