inital commit

scrapper/webistes/identity_iq/get_refresh_date.py

@@ -0,0 +1,190 @@
+import traceback
+import time
+import random
+
+from bs4 import BeautifulSoup
+from flask import jsonify
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as ec
+from selenium.webdriver.support.wait import WebDriverWait
+
+from Basic.helper import read_proxies
+from Basic.log_manager import get_logging
+from Basic.add_remove_text import remove_and_insert_text
+from Basic.helper import get_current_datetime
+from Basic.helper import check_access_denied
+from Basic.send_email import send_email
+
+
+
+def get_refresh_date(email, password, ssn):
+
+    logger = get_logging()
+    all_proxies = read_proxies()
+    proxy_address = random.choice(all_proxies)
+    print("proxy_address:---", proxy_address)
+
+    try:
+        url = "https://member.identityiq.com/"
+
+        chrome_path = "/usr/lib/chromium-browser/chromedriver"
+        #chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
+        chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
+
+        option = webdriver.ChromeOptions()
+        option.add_argument(f'--proxy-server={proxy_address}')
+        option.add_argument('--headless')
+        option.add_argument('--no-sandbox')
+        option.add_argument('--disable-dev-shm-usage')
+        driver = webdriver.Chrome(chrome_path,options=option)
+        #driver = webdriver.Chrome(chrome_local_path,options=option)
+
+        #phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
+        phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
+
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+        #driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
+
+
+        driver.get(url + 'login.aspx')
+
+        driver.implicitly_wait(10)
+
+        try:
+            driver.find_element_by_id("close-pc-btn-handler").click()
+        except:
+            pass
+
+        driver.find_element_by_id('txtUsername').send_keys(email)
+        driver.find_element_by_id('txtPassword').send_keys(password)
+        driver.find_element_by_id("imgBtnLogin").click()
+
+        timeout = 60
+
+        time.sleep(5)
+        try:
+            ele_acc_lock =  driver.find_element(By.CLASS_NAME, "chakra-modal__header")
+            error_text = ele_acc_lock.text
+            driver.quit()
+            response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code": 1003, "data": "ACCOUNT_LOCKED"}
+            return jsonify(response)
+
+        except:
+             element_login =  driver.find_element(By.CLASS_NAME, "chakra-text")
+             if element_login.text == 'Invalid login attempt':
+                driver.quit()
+                response = {"page":'login.aspx', "ip":proxy_address,"page_code":200, "status_code":1002 , "data": "INVALID_LOGIN"}
+                return jsonify(response)
+
+        try:
+            wait = WebDriverWait(driver, timeout)
+            WebDriverWait(driver, timeout).until(
+                ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
+            )
+            wait.until(
+                lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
+            )
+
+            if driver.current_url == url + 'security-question':
+                element_present = ec.presence_of_element_located(
+                    (By.NAME, 'userSecurityAnswer')
+                )
+                WebDriverWait(driver, timeout).until(element_present)
+                driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
+                driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
+
+                # element_present = ec.presence_of_element_located((By.TAG_NAME, 'dashboard-layout'))
+
+                # WebDriverWait(driver, timeout).until(element_present)
+                driver.get(url + 'Dashboard.aspx')
+
+            if driver.current_url == url + 'Dashboard.aspx':
+                #driver.get(url + 'Dashboard.aspx')
+                element_present = ec.presence_of_element_located((By.ID,'chakra-toast-manager-top'))
+                WebDriverWait(driver, timeout).until(element_present)
+
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                while True:
+                    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                    driver.implicitly_wait(10)
+                    time.sleep(5)
+                    new_height = driver.execute_script("return document.body.scrollHeight")
+
+                    if new_height == last_height:
+                        break
+                    last_height = new_height
+
+                try:
+                    element_click = driver.find_element(By.CLASS_NAME,'chakra-modal__close-btn')
+                    element_click.click()
+                except NoSuchElementException:
+                            pass
+
+
+                try:
+                    element = driver.find_element(By.CLASS_NAME,'chakra-skeleton')
+                    response = {
+                        'refresh_date': element.text,
+                        'button_type': 1,
+                        "ip":proxy_address, 
+                        "status_code":200,
+                        "page":"Dashboard.aspx",
+                        "page_code":200,
+                    }
+
+                    if 'Refresh Report' in element.text:
+                        response['button_type'] = 2
+
+
+
+                except NoSuchElementException:
+                    response = {
+                        'refresh_date': 'Not Found',
+                        'button_type': 1,
+                        "ip":proxy_address, 
+                        "status_code":200,
+                        "page":"Dashboard.aspx",
+                        "page_code":200,
+                    }
+
+
+
+                driver.quit()
+                return jsonify(response)
+
+            driver.quit()
+            result = check_access_denied(url+ 'Dashboard.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email) 
+            response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":result, "status_code":405, "data": "ERROR"}
+            return jsonify(response)
+
+        except TimeoutException:
+            traceback.print_exc()
+            logger.error("email: "+email+ " "+traceback.format_exc())
+            result = check_access_denied(url+ 'Dashboard.aspx')
+            subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
+            if(result == 403):
+                remove_and_insert_text(proxy_address)
+                send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
+            driver.quit()
+            response = {"page":'Dashboard.aspx', "ip":proxy_address,"page_code":result, "status_code":408, "data": "TIME_OUT"}
+            return jsonify(response)
+
+    except Exception:
+        traceback.format_exc()
+        logger.error("email: "+email+ " "+traceback.format_exc())
+        result = check_access_denied(url+ 'login.aspx')
+        subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
+        if(result == 403):
+            remove_and_insert_text(proxy_address)
+            send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
+        driver.quit()
+        response = {"page":'login.aspx', "ip":proxy_address,"page_code":result, "status_code":404, "data": "ERROR"}
+        return jsonify(response)
+
+