inital commit

scrapper/webistes/smart_credit/click_refresh_btn.py

@@ -0,0 +1,96 @@
+import traceback
+import time
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.log_manager import get_logging
+
+def smart_credit_click_refresh_btn(email, password):
+
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        #phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+        
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+            if driver.current_url == url + 'member/home/':
+                driver.get(url + 'member/credit-report/3b/')
+
+                try:
+                    element_present = ec.presence_of_element_located(
+                        (By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
+                    )
+                    WebDriverWait(driver, timeout).until(element_present)
+                except TimeoutException:
+                    driver.quit()
+                    return "ERROR"
+
+                driver.get(url + 'member/credit-report/3b/confirm.htm')
+                driver.maximize_window()
+
+                try:
+                    checkbox = WebDriverWait(driver, 10).until(
+                        ec.element_to_be_clickable((By.NAME, "isConfirmedTerms"))
+                    )
+                    if not checkbox.is_selected():
+                        driver.execute_script("arguments[0].click();", checkbox)
+
+                    driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').click()
+                except NoSuchElementException:
+                    return "BUTTON_NOT_FOUND"
+
+                driver.quit()
+                return "DONE"
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        #return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/smart_credit/credit_3b_scrapper.py

@@ -0,0 +1,143 @@
+import sys, traceback
+import time
+from flask import jsonify
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def credit_report_3b(email, password):
+
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+       
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+        print(chrome_local_path)
+        option = webdriver.ChromeOptions()
+        #option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+                                                           
+
+        driver.get(url + 'login/')
+        password = str(password)
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+        
+        timeout = 10
+
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+
+        try:
+                WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+                if driver.current_url == url + 'member/home/':
+
+                    try:
+                      
+                      driver.get(url + 'member/credit-report/3b/simple.htm?format=HTML')  
+                      
+                      element = driver.find_element(By.ID, "reportTop")
+
+                    except Exception as e:
+
+                      driver.get(url + 'member/credit-report/3b/')
+
+                    try:
+                        element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
+                        WebDriverWait(driver, timeout).until(element_present)
+
+                    except TimeoutException:
+                        pass
+
+                    element = driver.find_element(By.ID, "reportTop")
+
+                    soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+
+                    driver.quit()
+                    soup.html.insert(0, soup.new_tag('head'))
+                    target = soup.find('head')
+                    css_list = '''
+                    <link rel="stylesheet" type="text/css" href="https://www.smartcredit.com/resources/css/sc/pages/member/credit-report/3b/base.css"
+                    charset="utf-8">
+                    <style>
+                        td#personalInfo {
+                            width: 100%;
+                        }
+                        #reportTop table{
+                            width: 100%;
+                        }
+                    </style>
+                    '''
+                    target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
+
+                    alert_box = soup.find('div', {'class': "alert alert-warning fade in text-center"})
+                    if alert_box:
+                        alert_box.decompose()
+
+                    alert_box1 = soup.find('div', {'class': "alert alert-warning fade in text-center mb-2"})
+                    if alert_box1:
+                        alert_box1.decompose()
+                    try:
+                     soup.find('td', {'class': "backtoTopText"}).string = ""
+                    except Exception:
+                        pass
+
+                    heading_sublink = soup.find('span', {'class': "heading-sublink"})
+                    if heading_sublink:
+                        heading_sublink.decompose()
+
+                    report_history_loading = soup.find('div', {'id': "report-history-loading"})
+                    if report_history_loading:
+                        report_history_loading.decompose()
+
+                    for td_right in soup.findAll('td', {'align': "right"}):
+                        td_right.decompose()
+
+                    for img in soup.findAll('img'):
+                        img['src'] = 'https://www.smartcredit.com' + str(img['src'])
+
+                    return soup.prettify()
+                
+                
+                driver.quit()
+                return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            # return traceback.format_exc()
+            return "TIME_OUT"
+
+    except Exception as e:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        # return traceback.format_exc()
+        return "ERROR"
+        

scrapper/webistes/smart_credit/credit_3b_scrapper_new.py

@@ -0,0 +1,96 @@
+import sys, traceback
+import time
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def credit_report_3b_new(email, password):
+
+    logger = get_logging()
+    
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        #phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        password = str(password)
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+        
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+        
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+            
+            if driver.current_url == url + 'member/home/':
+
+                driver.get(url + 'member/credit-report/3b/simple.htm?format=JSON')
+
+                element = driver.find_element(By.ID, "reportTop")
+
+
+                try:
+                        element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
+                        WebDriverWait(driver, timeout).until(element_present)
+
+                except TimeoutException:
+                        pass
+
+
+                soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+
+                data = driver.find_element(By.ID, "TokenDisplay").text
+
+                driver.quit()
+
+                return  data
+ 
+              
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception as e:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        #return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/smart_credit/get_refresh_btn_status.py

@@ -0,0 +1,110 @@
+import traceback
+import time
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def smart_credit_refresh_status(email, password):
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+            if driver.current_url == url + 'member/home/':
+                driver.get(url + 'member/credit-report/3b/')
+
+                btn_text = None
+                try:
+                    element_present = ec.presence_of_element_located(
+                        (By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
+                    )
+                    WebDriverWait(driver, timeout).until(element_present)
+                    btn_text = driver.find_element(
+                        By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2'
+                    ).text
+                except TimeoutException:
+                    pass
+
+                driver.get(url + 'member/credit-report/3b/confirm.htm')
+                driver.maximize_window()
+                response = {
+                    'button_type': 0,
+                    'refresh_date': btn_text
+                }
+
+                try:
+                    driver.find_element(By.ID, "toggle-marketing")
+                    btn_text = driver.find_element(By.ID, "toggle-marketing").text
+
+                    if btn_text and str(btn_text).lower().strip() == 'order your 3-bureau report & scores':
+                        response['button_type'] = 1
+                        response['refresh_date'] = btn_text
+                except NoSuchElementException:
+                    pass
+
+                try:
+                    driver.find_element(By.NAME, "isConfirmedTerms")
+                    btn_text = driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').text
+
+                    if btn_text and str(btn_text).lower().strip() == 'place order now':
+                        response['button_type'] = 2
+                        response['refresh_date'] = btn_text
+                except NoSuchElementException:
+                    pass
+
+                driver.quit()
+                return response
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        traceback.print_exc()
+        #return traceback.format_exc()
+        return "ERROR"