inital commit

scrapper/webistes/identity_iq/click_refresh_btn.py

@@ -0,0 +1,165 @@
+import traceback
+import time
+import random
+from flask import jsonify
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.helper import read_proxies
+from Basic.log_manager import get_logging
+from Basic.add_remove_text import remove_and_insert_text
+from Basic.helper import get_current_datetime
+from Basic.helper import check_access_denied
+from Basic.send_email import send_email
+
+
+
+def click_refresh_btn(email, password, ssn):
+
+    logger = get_logging()
+    all_proxies = read_proxies()
+    proxy_address = random.choice(all_proxies)
+    print("proxy_address:---", proxy_address)
+
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument(f'--proxy-server={proxy_address}')
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+
+        driver.get(url + 'login.aspx')
+
+        driver.implicitly_wait(10)
+
+        try:
+            driver.find_element_by_id("close-pc-btn-handler").click()
+        except:
+            pass
+
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 60
+
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code": 1003, "data": "ACCOUNT_LOCKED"}
+            return jsonify(response)
+
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code":1002 , "data": "INVALID_LOGIN"}
+                return jsonify(response)
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                # element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
+
+                # WebDriverWait(driver, timeout).until(element_present)
+
+                driver.get(url + 'Dashboard.aspx')
+
+            if driver.current_url == url + 'Dashboard.aspx':
+
+                driver.get(url + 'Dashboard.aspx')
+
+                element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                try:
+                    element_click = driver.find_element(By.CLASS_NAME, 'chakra-modal__close-btn')
+                    element_click.click()
+                except NoSuchElementException:
+                            pass
+
+                try:
+
+                    element_btn = wait.until(ec.element_to_be_clickable((By.XPATH, "//button[contains(., 'Refresh Report')]")))
+                    driver.execute_script("arguments[0].click();", element_btn)
+
+                    driver.quit()
+                    response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":200, "status_code":200 , "data": "DONE"}
+                    return jsonify(response)
+                   
+
+                except NoSuchElementException:
+
+                    driver.quit()
+                    response = {"page":'Dashboard.aspx', "ip":proxy_address,"page_code":200, "status_code":400 , "data": "BUTTON_NOT_FOUND"}
+                    return jsonify(response)
+                   
+
+
+
+            driver.quit()
+            traceback.format_exc()
+            response = {"page":'security-question.aspx', "ip":proxy_address,"page_code":403, "status_code":405 , "data": "ERROR"}
+            return jsonify(response)
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            result = check_access_denied(url+ 'Dashboard.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
+            driver.quit()
+            response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":result, "status_code":408, "data": "TIME_OUT"}
+            return jsonify(response)
+
+    except Exception:
+        traceback.format_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        result = check_access_denied(url+ 'login.aspx')
+        subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
+        if(result == 403):
+            remove_and_insert_text(proxy_address)
+            send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc()) 
+        response = {"page":'login.aspx', "ip":proxy_address,"page_code":result, "status_code":404, "data": "ERROR"}
+        return jsonify(response)
+
+
+
+
+
+

scrapper/webistes/identity_iq/get_refresh_date.py

@@ -0,0 +1,190 @@
+import traceback
+import time
+import random
+
+from bs4 import BeautifulSoup
+from flask import jsonify
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.helper import read_proxies
+from Basic.log_manager import get_logging
+from Basic.add_remove_text import remove_and_insert_text
+from Basic.helper import get_current_datetime
+from Basic.helper import check_access_denied
+from Basic.send_email import send_email
+
+
+
+def get_refresh_date(email, password, ssn):
+
+    logger = get_logging()
+    all_proxies = read_proxies()
+    proxy_address = random.choice(all_proxies)
+    print("proxy_address:---", proxy_address)
+
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument(f'--proxy-server={proxy_address}')
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+
+
+        driver.get(url + 'login.aspx')
+
+        driver.implicitly_wait(10)
+
+        try:
+            driver.find_element_by_id("close-pc-btn-handler").click()
+        except:
+            pass
+
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 60
+
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code": 1003, "data": "ACCOUNT_LOCKED"}
+            return jsonify(response)
+
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                response = {"page":'login.aspx', "ip":proxy_address,"page_code":200, "status_code":1002 , "data": "INVALID_LOGIN"}
+                return jsonify(response)
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                # element_present = ec.presence_of_element_located((By.TAG_NAME, 'dashboard-layout'))
+
+                # WebDriverWait(driver, timeout).until(element_present)
+                driver.get(url + 'Dashboard.aspx')
+
+            if driver.current_url == url + 'Dashboard.aspx':
+                #driver.get(url + 'Dashboard.aspx')
+                element_present = ec.presence_of_element_located((By.ID,'chakra-toast-manager-top'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                while True:
+                    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                    driver.implicitly_wait(10)
+                    time.sleep(5)
+                    new_height = driver.execute_script("return document.body.scrollHeight")
+
+                    if new_height == last_height:
+                        break
+                    last_height = new_height
+
+                try:
+                    element_click = driver.find_element(By.CLASS_NAME,'chakra-modal__close-btn')
+                    element_click.click()
+                except NoSuchElementException:
+                            pass
+
+
+                try:
+                    element = driver.find_element(By.CLASS_NAME,'chakra-skeleton')
+                    response = {
+                        'refresh_date': element.text,
+                        'button_type': 1,
+                        "ip":proxy_address, 
+                        "status_code":200,
+                        "page":"Dashboard.aspx",
+                        "page_code":200,
+                    }
+
+                    if 'Refresh Report' in element.text:
+                        response['button_type'] = 2
+
+
+
+                except NoSuchElementException:
+                    response = {
+                        'refresh_date': 'Not Found',
+                        'button_type': 1,
+                        "ip":proxy_address, 
+                        "status_code":200,
+                        "page":"Dashboard.aspx",
+                        "page_code":200,
+                    }
+
+
+
+                driver.quit()
+                return jsonify(response)
+
+            driver.quit()
+            result = check_access_denied(url+ 'Dashboard.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email) 
+            response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":result, "status_code":405, "data": "ERROR"}
+            return jsonify(response)
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            result = check_access_denied(url+ 'Dashboard.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
+            driver.quit()
+            response = {"page":'Dashboard.aspx', "ip":proxy_address,"page_code":result, "status_code":408, "data": "TIME_OUT"}
+            return jsonify(response)
+
+    except Exception:
+        traceback.format_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        result = check_access_denied(url+ 'login.aspx')
+        subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
+        if(result == 403):
+            remove_and_insert_text(proxy_address)
+            send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
+        driver.quit()
+        response = {"page":'login.aspx', "ip":proxy_address,"page_code":result, "status_code":404, "data": "ERROR"}
+        return jsonify(response)
+
+

scrapper/webistes/identity_iq/recent_report.py

@@ -0,0 +1,230 @@
+import time
+import requests
+import traceback
+import random
+from flask import jsonify
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException,NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.helper import read_proxies
+from Basic.log_manager import get_logging
+from Basic.add_remove_text import remove_and_insert_text
+from Basic.helper import get_current_datetime
+from Basic.helper import check_access_denied
+from Basic.send_email import send_email
+
+
+
+def recent_report(email, password, ssn):
+
+    logger = get_logging()
+
+    all_proxies = read_proxies()
+    proxy_address = random.choice(all_proxies)
+    print("proxy_address:---", proxy_address)
+
+
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        # chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument(f'--proxy-server={proxy_address}')
+        option.add_argument('--headless') # Run Chrome in headless mode
+        option.add_argument('--disable-gpu')  # Disable GPU acceleration
+        option.add_argument('--no-sandbox') # Bypass OS security model
+        option.add_argument('--disable-dev-shm-usage') # Overcome limited resource problems
+        
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path)
+
+
+        driver.get(url + 'login.aspx')
+        #driver.save_screenshot("server_screenshot_login.png")
+        
+        driver.implicitly_wait(10)
+        try:
+            driver.find_element_by_id("close-pc-btn-handler").click()
+        except:
+            pass
+
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 120
+        time.sleep(5)
+       
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            response = {"page":'login.aspx', "ip":proxy_address, "status_code": 1003, "page_code":200, "data": "ACCOUNT_LOCKED"}
+            return jsonify(response)
+
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                response = {"page":'login.aspx', "ip":proxy_address, "status_code":1002 ,"page_code":200, "data": "INVALID_LOGIN"}
+                return jsonify(response)
+
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+
+            if driver.current_url == url + 'security-question':
+                #driver.save_screenshot("server_screenshot_security_question.png")
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                #element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
+                #WebDriverWait(driver, timeout).until(element_present)
+                driver.get(url + 'Dashboard.aspx')
+                #print('Dashboard success')
+                
+            if driver.current_url == url + 'Dashboard.aspx':
+
+                response = requests.get(url + 'Dashboard.aspx')
+
+                if response.status_code == 200:
+                   print("Access successful!")
+                elif response.status_code == 403:
+                    print("Access forbidden. Your IP might be blocked.")
+                elif response.status_code == 401:
+                    print("Unauthorized. You may need to log in.")
+                else:
+                    print(f"Received status code {response.status_code}. Something") 
+                        
+                #driver.save_screenshot("server_screenshot_security_Dashboard.png")
+                
+                driver.get(url + 'CreditReport.aspx')
+                #print('CreditReport success')
+                #driver.save_screenshot("server_screenshot_CreditReport.png")
+
+
+                try:
+                    element_present = ec.presence_of_element_located((By.ID, 'reportTop'))             
+                    WebDriverWait(driver, timeout).until(element_present)
+                    #driver.save_screenshot("server_screenshot_final.png")
+                    #print('element_present success')
+                except:
+                    pass
+                    #print('element not present')
+                
+                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                #driver.save_screenshot("server_screenshot_final.png")
+                xpath_value = ''
+
+                try:
+                    driver.find_element_by_xpath("//b[contains(., '***SECURITY FREEZE')]")
+                    xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[10]/div[2]/address-history/div/ng-repeat[1]'
+
+                except NoSuchElementException:
+
+                    xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[9]/div[2]/address-history/div/ng-repeat[1]'
+
+
+                try:
+                    element_present = ec.presence_of_element_located(( By.XPATH,xpath_value))
+                    WebDriverWait(driver, timeout).until(element_present)
+                except TimeoutException:
+                    pass
+
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                while True:
+                    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                    time.sleep(5)
+                    new_height = driver.execute_script("return document.body.scrollHeight")
+                    if new_height == last_height:
+                        break
+                    last_height = new_height
+
+                element = driver.find_element(By.ID, "ctrlCreditReport")
+
+                soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+                driver.quit()
+                soup.html.insert(0, soup.new_tag('head'))
+                target = soup.find('head')
+                css_list = '''
+                <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"
+                rel="stylesheet" />
+                <link href="https://member.identityiq.com/CreditReportContent/DEFAULT/style.DEFAULT.css"
+                rel="stylesheet" />
+                <link href="https://member.identityiq.com/css/CRDownload.css" rel="stylesheet" />
+                <script src="https://member.identityiq.com/Scripts/jquery-3.6.0.min.js" type="text/javascript"></script>
+                <script src="https://member.identityiq.com/Scripts/angular.js" type="text/javascript" ></script>
+                <script src="https://member.identityiq.com/Scripts/CRDownload.js" type="text/javascript" ></script>
+                '''
+                target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
+
+                for div in soup.findAll('div', {'class': "link_header"}):
+                    div.decompose()
+
+                for span in soup.findAll('span', {'class': "return_topSpan"}):
+                    span.decompose()
+
+                for img in soup.findAll('img'):
+                    img['src'] = 'https://www.member.identityiq.com' + str(img['src'])
+
+                response = { "ip":proxy_address, "page_code":200, "page":"CreditReport.aspx", "status_code":200, "data": soup.prettify()}
+                return jsonify(response)
+
+            driver.quit()
+
+            response = {"page":'security-question.aspx', "ip":proxy_address, "page_code":403, "status_code":405, "data": "ERROR"}
+            return jsonify(response)
+
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            result = check_access_denied(url+ 'CreditReport.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on CreditReport.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())  
+            driver.quit()
+            #return traceback.format_exc()
+            response = {"page":'CreditReport.aspx', "ip":proxy_address, "page_code":result, "status_code":408, "data": "TIME_OUT"}
+            return jsonify(response)
+
+    except Exception :
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        traceback.print_exc()
+        result = check_access_denied(url+ 'login.aspx')
+        subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
+        if(result == 403):
+            remove_and_insert_text(proxy_address)
+            send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc()) 
+        driver.quit()
+        #return traceback.format_exc()
+        response = {"page":'login.aspx', "ip":proxy_address, "page_code":result, "status_code":404, "data": "ERROR"}
+        return jsonify(response)
+    

scrapper/webistes/identity_iq/user_validation.py

@@ -0,0 +1,107 @@
+
+import traceback
+import random
+import time
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.helper import read_proxies
+from Basic.log_manager import get_logging
+
+
+def user_validation(email, password, ssn):
+
+    logger = get_logging()
+    all_proxies = read_proxies()
+    proxy_address = random.choice(all_proxies)
+    print("proxy_address:---", proxy_address)
+
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/bin/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument(f'--proxy-server={proxy_address}')
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+
+
+        driver.get(url + 'login.aspx')
+
+        driver.implicitly_wait(10)
+
+        try:
+            driver.find_element_by_id("close-pc-btn-handler").click()
+        except:
+            pass
+
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 10
+
+        time.sleep(5)
+        try:
+            driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            driver.quit()
+            return "ACCOUNT_LOCKED"
+
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                return "INVALID_LOGIN"
+       
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                WebDriverWait(driver, timeout).until( ec.url_to_be(url + 'Dashboard.aspx'))
+
+                if driver.current_url == url + 'Dashboard.aspx':
+           
+                    driver.quit()
+                    return "DONE"
+
+            driver.quit()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            return "ERROR"
+
+        except TimeoutException:
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        return "ERROR"

scrapper/webistes/identity_iq_old/click_refresh_btn.py

@@ -0,0 +1,114 @@
+import traceback
+import time
+
+from flask import jsonify
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def click_refresh_btn(email, password, ssn):
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+        driver.get(url + 'login.aspx')
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 120
+        
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            return "ACCOUNT_LOCKED"  
+           
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                return "INVALID_LOGIN"
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                # element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
+                
+                # WebDriverWait(driver, timeout).until(element_present)
+
+                driver.get(url + 'Dashboard.aspx')
+
+            if driver.current_url == url + 'Dashboard.aspx':
+
+                driver.get(url + 'Dashboard.aspx')
+
+                element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                try:
+                    element_click = driver.find_element(By.CLASS_NAME, 'chakra-modal__close-btn')
+                    element_click.click()  
+                except NoSuchElementException:  
+                            pass 
+               
+                try:
+
+                    element_btn = wait.until(ec.element_to_be_clickable((By.XPATH, "//button[contains(., 'Refresh Report')]")))
+                    driver.execute_script("arguments[0].click();", element_btn)
+
+                    driver.quit()
+                    return "DONE" 
+
+                except NoSuchElementException:
+
+                    driver.quit()
+                    return "BUTTON_NOT_FOUND"
+
+                
+
+            driver.quit()
+            return traceback.format_exc()
+            return "ERROR"
+
+        except TimeoutException:
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/identity_iq_old/get_refresh_date.py

@@ -0,0 +1,134 @@
+import traceback
+import time
+from bs4 import BeautifulSoup
+
+from flask import jsonify
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def get_refresh_date(email, password, ssn):
+    try:
+        url = "https://member.identityiq.com/" 
+        
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+        
+        driver.get(url + 'login.aspx')
+    
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 120
+        
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            return "ACCOUNT_LOCKED"  
+           
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                return "INVALID_LOGIN"
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                # element_present = ec.presence_of_element_located((By.TAG_NAME, 'dashboard-layout'))
+                
+                # WebDriverWait(driver, timeout).until(element_present)
+                driver.get(url + 'Dashboard.aspx')
+
+            if driver.current_url == url + 'Dashboard.aspx':
+                #driver.get(url + 'Dashboard.aspx')
+                element_present = ec.presence_of_element_located((By.ID,'chakra-toast-manager-top'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                while True:
+                    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                    driver.implicitly_wait(10)
+                    time.sleep(5)
+                    new_height = driver.execute_script("return document.body.scrollHeight")
+                    
+                    if new_height == last_height:
+                        break
+                    last_height = new_height
+
+                try:
+                    element_click = driver.find_element(By.CLASS_NAME,'chakra-modal__close-btn')
+                    element_click.click()  
+                except NoSuchElementException:  
+                            pass   
+
+                
+                try:
+                    element = driver.find_element(By.CLASS_NAME,'chakra-skeleton')
+                    response = {
+                        'refresh_date': element.text,
+                        'button_type': 1
+                    }
+
+                    if 'Refresh Report' in element.text:
+                        response['button_type'] = 2
+                       
+                       
+                    
+                except NoSuchElementException:
+                    response = {
+                        'refresh_date': 'Not Found',
+                        'button_type': 1
+                    }
+
+
+
+                driver.quit()
+                return jsonify(response)
+
+            driver.quit()
+            return traceback.format_exc()
+            return "ERROR"
+
+        except TimeoutException:
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/identity_iq_old/recent_report.py

@@ -0,0 +1,157 @@
+import time
+import traceback
+from flask import jsonify
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException,NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def recent_report(email, password, ssn):
+
+    try:
+        url = "https://member.identityiq.com/"
+        
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        # chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+        driver.get(url + 'login.aspx')
+        
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 120
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            return "ACCOUNT_LOCKED"  
+           
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                return "INVALID_LOGIN"
+        
+        
+        try:
+            wait = WebDriverWait(driver, timeout)
+            
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+            
+
+            if driver.current_url == url + 'security-question':
+                
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                #element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
+                #WebDriverWait(driver, timeout).until(element_present)
+                driver.get(url + 'Dashboard.aspx')
+
+            
+
+            if driver.current_url == url + 'Dashboard.aspx':
+                driver.get(url + 'CreditReport.aspx')
+                element_present = ec.presence_of_element_located((By.ID, 'reportTop'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+
+                xpath_value = ''
+
+                try:
+                    driver.find_element_by_xpath("//b[contains(., '***SECURITY FREEZE')]")
+                    xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[10]/div[2]/address-history/div/ng-repeat[1]'
+
+                except NoSuchElementException:
+                    
+                    xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[9]/div[2]/address-history/div/ng-repeat[1]'
+                    
+
+                try:
+                    element_present = ec.presence_of_element_located(( By.XPATH,xpath_value))
+                    WebDriverWait(driver, timeout).until(element_present)
+                except TimeoutException:
+                    pass
+
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                while True:
+                    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                    time.sleep(5)
+                    new_height = driver.execute_script("return document.body.scrollHeight")
+                    if new_height == last_height:
+                        break
+                    last_height = new_height
+
+                element = driver.find_element(By.ID, "ctrlCreditReport")
+
+                soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+                driver.quit()
+                soup.html.insert(0, soup.new_tag('head'))
+                target = soup.find('head')
+                css_list = '''
+                <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" 
+                rel="stylesheet" />
+                <link href="https://member.identityiq.com/CreditReportContent/DEFAULT/style.DEFAULT.css" 
+                rel="stylesheet" /> 
+                <link href="https://member.identityiq.com/css/CRDownload.css" rel="stylesheet" /> 
+                <script src="https://member.identityiq.com/Scripts/jquery-3.6.0.min.js" type="text/javascript"></script>
+                <script src="https://member.identityiq.com/Scripts/angular.js" type="text/javascript" ></script>
+                <script src="https://member.identityiq.com/Scripts/CRDownload.js" type="text/javascript" ></script>
+                '''
+                target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
+
+                for div in soup.findAll('div', {'class': "link_header"}):
+                    div.decompose()
+
+                for span in soup.findAll('span', {'class': "return_topSpan"}):
+                    span.decompose()
+
+                for img in soup.findAll('img'):
+                    img['src'] = 'https://www.member.identityiq.com' + str(img['src'])
+
+                return soup.prettify()
+
+            driver.quit()
+            # return traceback.format_exc()
+            return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            driver.quit()
+            return traceback.format_exc()
+            return "TIME_OUT"
+           
+    except Exception :
+        traceback.print_exc()
+        return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/identity_iq_old/user_validation.py

@@ -0,0 +1,76 @@
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def user_validation(email, password, ssn):
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login.aspx')
+        driver.find_element_by_name('txtUsername').send_keys(email)
+        driver.find_element_by_name('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 10
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'SecurityQuestions.aspx') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'SecurityQuestions.aspx') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'SecurityQuestions.aspx':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'FBfbforcechangesecurityanswer$txtSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('FBfbforcechangesecurityanswer$txtSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+            if driver.current_url == url + 'Dashboard.aspx':
+                element_present = ec.presence_of_element_located((By.ID, 'Fbscore_divDt'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                try:
+                    driver.find_element(By.ID, "Fbscore_ibtnRefresh").click()
+                except NoSuchElementException:
+                    return "BUTTON_NOT_FOUND"
+
+                driver.quit()
+                return "DONE"
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        return "ERROR"

scrapper/webistes/smart_credit/click_refresh_btn.py

@@ -0,0 +1,96 @@
+import traceback
+import time
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.log_manager import get_logging
+
+def smart_credit_click_refresh_btn(email, password):
+
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        #phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+        
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+            if driver.current_url == url + 'member/home/':
+                driver.get(url + 'member/credit-report/3b/')
+
+                try:
+                    element_present = ec.presence_of_element_located(
+                        (By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
+                    )
+                    WebDriverWait(driver, timeout).until(element_present)
+                except TimeoutException:
+                    driver.quit()
+                    return "ERROR"
+
+                driver.get(url + 'member/credit-report/3b/confirm.htm')
+                driver.maximize_window()
+
+                try:
+                    checkbox = WebDriverWait(driver, 10).until(
+                        ec.element_to_be_clickable((By.NAME, "isConfirmedTerms"))
+                    )
+                    if not checkbox.is_selected():
+                        driver.execute_script("arguments[0].click();", checkbox)
+
+                    driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').click()
+                except NoSuchElementException:
+                    return "BUTTON_NOT_FOUND"
+
+                driver.quit()
+                return "DONE"
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        #return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/smart_credit/credit_3b_scrapper.py

@@ -0,0 +1,143 @@
+import sys, traceback
+import time
+from flask import jsonify
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def credit_report_3b(email, password):
+
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+       
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+        print(chrome_local_path)
+        option = webdriver.ChromeOptions()
+        #option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+                                                           
+
+        driver.get(url + 'login/')
+        password = str(password)
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+        
+        timeout = 10
+
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+
+        try:
+                WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+                if driver.current_url == url + 'member/home/':
+
+                    try:
+                      
+                      driver.get(url + 'member/credit-report/3b/simple.htm?format=HTML')  
+                      
+                      element = driver.find_element(By.ID, "reportTop")
+
+                    except Exception as e:
+
+                      driver.get(url + 'member/credit-report/3b/')
+
+                    try:
+                        element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
+                        WebDriverWait(driver, timeout).until(element_present)
+
+                    except TimeoutException:
+                        pass
+
+                    element = driver.find_element(By.ID, "reportTop")
+
+                    soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+
+                    driver.quit()
+                    soup.html.insert(0, soup.new_tag('head'))
+                    target = soup.find('head')
+                    css_list = '''
+                    <link rel="stylesheet" type="text/css" href="https://www.smartcredit.com/resources/css/sc/pages/member/credit-report/3b/base.css"
+                    charset="utf-8">
+                    <style>
+                        td#personalInfo {
+                            width: 100%;
+                        }
+                        #reportTop table{
+                            width: 100%;
+                        }
+                    </style>
+                    '''
+                    target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
+
+                    alert_box = soup.find('div', {'class': "alert alert-warning fade in text-center"})
+                    if alert_box:
+                        alert_box.decompose()
+
+                    alert_box1 = soup.find('div', {'class': "alert alert-warning fade in text-center mb-2"})
+                    if alert_box1:
+                        alert_box1.decompose()
+                    try:
+                     soup.find('td', {'class': "backtoTopText"}).string = ""
+                    except Exception:
+                        pass
+
+                    heading_sublink = soup.find('span', {'class': "heading-sublink"})
+                    if heading_sublink:
+                        heading_sublink.decompose()
+
+                    report_history_loading = soup.find('div', {'id': "report-history-loading"})
+                    if report_history_loading:
+                        report_history_loading.decompose()
+
+                    for td_right in soup.findAll('td', {'align': "right"}):
+                        td_right.decompose()
+
+                    for img in soup.findAll('img'):
+                        img['src'] = 'https://www.smartcredit.com' + str(img['src'])
+
+                    return soup.prettify()
+                
+                
+                driver.quit()
+                return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            # return traceback.format_exc()
+            return "TIME_OUT"
+
+    except Exception as e:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        # return traceback.format_exc()
+        return "ERROR"
+        

scrapper/webistes/smart_credit/credit_3b_scrapper_new.py

@@ -0,0 +1,96 @@
+import sys, traceback
+import time
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def credit_report_3b_new(email, password):
+
+    logger = get_logging()
+    
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        #phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        password = str(password)
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+        
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            error_text = paragraph_element.text
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+        
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+            
+            if driver.current_url == url + 'member/home/':
+
+                driver.get(url + 'member/credit-report/3b/simple.htm?format=JSON')
+
+                element = driver.find_element(By.ID, "reportTop")
+
+
+                try:
+                        element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
+                        WebDriverWait(driver, timeout).until(element_present)
+
+                except TimeoutException:
+                        pass
+
+
+                soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
+
+                data = driver.find_element(By.ID, "TokenDisplay").text
+
+                driver.quit()
+
+                return  data
+ 
+              
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception as e:
+        traceback.print_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        #return traceback.format_exc()
+        return "ERROR"

scrapper/webistes/smart_credit/get_refresh_btn_status.py

@@ -0,0 +1,110 @@
+import traceback
+import time
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.support.wait import WebDriverWait
+from Basic.log_manager import get_logging
+
+
+def smart_credit_refresh_status(email, password):
+    logger = get_logging()
+    try:
+        url = "https://www.smartcredit.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-sh-usage')
+        #driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+         
+        driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+       
+
+        driver.get(url + 'login/')
+        driver.find_element_by_name('j_username').send_keys(email)
+        driver.find_element_by_name('j_password').send_keys(password)
+        driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
+
+        timeout = 10
+        time.sleep(5)
+        try:
+            alert_div = driver.find_element_by_class_name("alert-danger")
+            paragraph_element = alert_div.find_element_by_tag_name("p")
+            driver.quit()
+            return "INVALID_LOGIN"   
+           
+        except:
+             pass
+
+        try:
+            WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
+
+            if driver.current_url == url + 'member/home/':
+                driver.get(url + 'member/credit-report/3b/')
+
+                btn_text = None
+                try:
+                    element_present = ec.presence_of_element_located(
+                        (By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
+                    )
+                    WebDriverWait(driver, timeout).until(element_present)
+                    btn_text = driver.find_element(
+                        By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2'
+                    ).text
+                except TimeoutException:
+                    pass
+
+                driver.get(url + 'member/credit-report/3b/confirm.htm')
+                driver.maximize_window()
+                response = {
+                    'button_type': 0,
+                    'refresh_date': btn_text
+                }
+
+                try:
+                    driver.find_element(By.ID, "toggle-marketing")
+                    btn_text = driver.find_element(By.ID, "toggle-marketing").text
+
+                    if btn_text and str(btn_text).lower().strip() == 'order your 3-bureau report & scores':
+                        response['button_type'] = 1
+                        response['refresh_date'] = btn_text
+                except NoSuchElementException:
+                    pass
+
+                try:
+                    driver.find_element(By.NAME, "isConfirmedTerms")
+                    btn_text = driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').text
+
+                    if btn_text and str(btn_text).lower().strip() == 'place order now':
+                        response['button_type'] = 2
+                        response['refresh_date'] = btn_text
+                except NoSuchElementException:
+                    pass
+
+                driver.quit()
+                return response
+
+            driver.quit()
+            return "ERROR"
+
+        except TimeoutException:
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            driver.quit()
+            return "TIME_OUT"
+
+    except Exception:
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        traceback.print_exc()
+        #return traceback.format_exc()
+        return "ERROR"