rajib/RTG
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

inital commit

Browse Source
This commit is contained in:
Pulak
2026-06-19 20:08:01 +06:00
commit 8a5abeeae4
13128 changed files with 3192007 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
scrapper/Basic/add_remove_text.py Normal file
View File

@@ -0,0 +1,24 @@
def remove_and_insert_text(text_to_remove):
source_file = 'proxies.txt'
destination_file = "proxies_block_ips.txt"
with open(destination_file, 'a') as file:
file.write('\n' + text_to_remove)
with open(source_file, 'r') as file:
content = file.read()
if text_to_remove in content:
modified_content = content.replace(text_to_remove, '')
cleaned_content = '\n'.join(line.strip() for line in modified_content.splitlines() if line.strip())
with open(source_file, 'w') as file:
file.write(cleaned_content)
else:
print("remove text not found")

27
scrapper/Basic/helper.py Normal file
View File

@@ -0,0 +1,27 @@
from datetime import datetime
import requests
def check_access_denied(url):
try:
response = requests.get(url)
return response.status_code
except requests.exceptions.RequestException as e:
return f"Error: {e}"
def get_current_datetime():
now = datetime.now()
formatted_time = now.strftime("%Y-%m-%d %H:%M:%S")
return formatted_time
def read_proxies():
all_proxies = []
with open('proxies.txt', 'r') as file:
for line in file:
all_proxies.append(line.strip())
return all_proxies

29
scrapper/Basic/log_manager.py Normal file
View File

@@ -0,0 +1,29 @@
import logging
from datetime import datetime
import os
def get_logging():
log_dir = 'logs'
if not os.path.exists(log_dir):
os.makedirs(log_dir)
date_str = datetime.now().strftime('%Y-%m-%d')
log_filename = os.path.join(log_dir, f'app_{date_str}.log')
if not os.path.exists(log_filename):
logging.basicConfig(filename=log_filename, filemode='w', level=logging.ERROR,
format='%(asctime)s - %(levelname)s - %(message)s')
else:
logging.basicConfig(filename=log_filename, filemode='a', level=logging.ERROR,
format='%(asctime)s - %(levelname)s - %(message)s')
return logging

42
scrapper/Basic/send_email.py Normal file
View File

@@ -0,0 +1,42 @@
import smtplib
from email.mime.text import MIMEText
def send_email(subject,message):
print('-- Email Sending --')
# Configuration
port = 587
smtp_server = "smtp.gmail.com"
login = "noreply.readytogoletters@gmail.com" # Your login generated by Mailtrap
password = "fymebmopnajzwkfb"
sender_email = "test.velocity@gmail.com"
# port = 587
# smtp_server = "sandbox.smtp.mailtrap.io"
# login = "b879299bf7dd9a"
# password = "3c1ee98e9ddeb7"
# sender_email = "pulak.deb@softrobotics.com.bd"
# List of recipient email addresses
receiver_emails = ["rajibcuetcse@gmail.com","bpulakd@gmail.com"]
#receiver_emails = ["bpulakd@gmail.com"]
text = f"{message}"
message = MIMEText(text, "plain") # Create MIMEText object
message["Subject"] = subject
message["From"] = sender_email
message["To"] = ", ".join(receiver_emails) # Join the list of receiver emails into a string separated by commas
# Send the email
with smtplib.SMTP(smtp_server, port) as server:
server.starttls() # Secure the connection
server.login(login, password)
# Loop through each recipient and send the email individually
for recipient in receiver_emails:
server.sendmail(sender_email, recipient, message.as_string())
print('Sent')

284
scrapper/README.md Normal file
View File

@@ -0,0 +1,284 @@
# credit-scrapper-poc
## Server setup
####Step 1 – Installing Nginx
```
sudo apt update
sudo apt install nginx
```
####Step 2 – Adjusting the Firewall
```
sudo ufw allow 'Nginx HTTP'
sudo ufw allow 'OpenSSH'
sudo ufw enable
sudo ufw status
```
####Step 3 – Checking your Web Server
```
sudo systemctl status nginx
curl -4 icanhazip.com
```
When you have your server’s IP address, enter it into your browser’s address bar:
####Step 4 – Installing the Components from the Ubuntu Repositories
```
sudo apt update
sudo apt install python3-pip python3-dev build-essential libssl-dev libffi-dev python3-setuptools
```
####Step 5 – Clone the repository
```
git clone https://gitlab.com/sunflowerlab/Repositories/TechnologyPOCs/credit-scrapper-poc.git
```
####Step 6 – Creating a Python Virtual Environment
```
sudo apt install python3-venv
cd credit-scrapper-poc
python3.6 -m venv venv
source venv/bin/activate
```
####Step 7 – Setting Up an Application
```
pip install wheel
pip install uwsgi
pip install -r requirements.txt
```
####Step 8 – Test the setup
```
python app.py
uwsgi --socket 0.0.0.0:5000 --protocol=http -w wsgi:app
uwsgi --socket 0.0.0.0:8080 --protocol=http -w wsgi:app
```
####Step 9 – Deactivate python environment
```
deactivate
```
####Step 10 – Creating a systemd Unit File
```
sudo nano /etc/systemd/system/app.service
```
Add the following contain
```
[Unit]
Description=uWSGI instance to serve app
After=network.target
[Service]
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/credit-scrapper-poc
Environment="PATH=/home/ubuntu/credit-scrapper-poc/venv/bin"
ExecStart=/home/ubuntu/credit-scrapper-poc/venv/bin/uwsgi --ini app.ini
[Install]
WantedBy=multi-user.target
```
Update the directory paths.
We can now start the uWSGI service we created and enable it so that it starts at boot:
```
sudo systemctl start app
sudo systemctl enable app
sudo systemctl status app
```
####Step 11 – Configuring Nginx to Proxy Requests
```
sudo nano /etc/nginx/sites-available/app
```
###Step 12 - Install Javascript library
source install_phantomjs.sh
Add the following contain
```
server {
listen 80;
server_name 54.68.236.191;
location / {
proxy_read_timeout 1200s;
proxy_connect_timeout 1200s;
include uwsgi_params;
uwsgi_pass unix:/home/ubuntu/credit-scrapper-poc/app.sock;
# when a client closes the connection then keep the channel to uwsgi open. Otherwise uwsgi throws an IOError
uwsgi_ignore_client_abort on;
uwsgi_buffer_size 60M;
uwsgi_buffers 8 60M;
uwsgi_busy_buffers_size 60M;
uwsgi_read_timeout 1200;
uwsgi_send_timeout 1200;
uwsgi_connect_timeout 1200;
}
}
```
Update the directory paths and IP or domain address.
```
sudo nano /etc/nginx/conf.d/timeout.conf
```
Add the following contain
```
proxy_connect_timeout 1200;
proxy_send_timeout 1200;
proxy_read_timeout 1200;
send_timeout 1200;
```
```
sudo ln -s /etc/nginx/sites-available/app /etc/nginx/sites-enabled
sudo nginx -t
sudo systemctl restart nginx
sudo systemctl restart app
```
When you have your server’s IP address, enter it into your browser’s address bar.
## API Details
1. Smart Credit - Get Credit Report 3B
Method - `POST`
Endpoint - `/smart-credit-credit-report-3b`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password"
}`
Response - `HTML CONTENT` or `ERROR`
2. IdentityIQ - Get most recent Credit Report
Method - `POST`
Endpoint - `/identity-iq-credit-report-html`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password",
"ssn": "last-four-digit"
}`
Response - `HTML CONTENT` or `ERROR` or `TIME_OUT`
3. IdentityIQ - Get refresh date and button type
Method - `POST`
Endpoint - `/identity-iq-report-refresh-date`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password",
"ssn": "last -four-digit"
}`
Response - `{
'refresh_date': "MM/DD/YYYY",
'button_type': 1 or 2
}`
or
`ERROR`
or
`TIME_OUT`
button_type - `1` for Purchase AND `2` for Refresh
4. IdentityIQ - To click on the refresh button.
Method - `POST`
Endpoint - `/identity-iq-click-refresh-btn`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password",
"ssn": "last -four-digit"
}`
Response - `DONE` or `ERROR` or `TIME_OUT` or `BUTTON_NOT_FOUND`
5. SmartCredit - Get refresh date and button type
Method - `POST`
Endpoint - `/smart-credit-credit-refresh-status`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password"
}`
Response - `{
'button_type': 0 or 1 or 2.,
'refresh_date': TEXT
}`
or
`ERROR`
or
`TIME_OUT`
button_type - `0` for no button AND `1` for Purchase AND `2` for Refresh
6. SmartCredit - To click on the refresh button.
Method - `POST`
Endpoint - `/smart-credit-credit-click-refresh-btn`
Content-Type - `application/json`
Payload - `{
"email": "email",
"password": "password"
}`
Response - `DONE` or `ERROR` or `TIME_OUT` or `BUTTON_NOT_FOUND`
***Reference Link*** https://www.digitalocean.com/community/tutorials/how-to-serve-flask-applications-with-uswgi-and-nginx-on-ubuntu-18-04

12
scrapper/app.ini Normal file
View File

@@ -0,0 +1,12 @@
[uwsgi]
module = wsgi:app
master = true
processes = 5
socket = app.sock
chmod-socket = 660
vacuum = true
die-on-term = true

74
scrapper/app.py Normal file
View File

@@ -0,0 +1,74 @@
from flask import Flask, request, jsonify
from werkzeug.debug import DebuggedApplication
from webistes.smart_credit.click_refresh_btn import smart_credit_click_refresh_btn
from webistes.smart_credit.credit_3b_scrapper import credit_report_3b
from webistes.smart_credit.get_refresh_btn_status import smart_credit_refresh_status
from webistes.smart_credit.credit_3b_scrapper_new import credit_report_3b_new
from webistes.identity_iq.click_refresh_btn import click_refresh_btn
from webistes.identity_iq.get_refresh_date import get_refresh_date
from webistes.identity_iq.recent_report import recent_report
from webistes.identity_iq.user_validation import user_validation
app = Flask(__name__)
app.config['DEBUG'] = True
application = DebuggedApplication(app, True)
@app.route('/')
def index():
return jsonify({'message': 'Welcome to Scrapper!'})
@app.route('/smart-credit-credit-report-3b', methods=['POST'])
def get_credit_report_3b():
data = request.get_json()
return credit_report_3b(data['email'], data['password'])
@app.route('/smart-credit-credit-refresh-status', methods=['POST'])
def get_smart_credit_refresh_status():
data = request.get_json()
return smart_credit_refresh_status(data['email'], data['password'])
@app.route('/smart-credit-credit-click-refresh-btn', methods=['POST'])
def get_smart_credit_click_refresh_btn():
data = request.get_json()
return smart_credit_click_refresh_btn(data['email'], data['password'])
@app.route('/smart-credit-credit-report-3b-json', methods=['POST'])
def get_credit_report_3b_new():
data = request.get_json()
return credit_report_3b_new(data['email'], data['password'])
@app.route('/identity-iq-credit-report-html', methods=['POST'])
def get_identity_iq_credit_report_html():
data = request.get_json()
return recent_report(data['email'], data['password'], data['ssn'])
@app.route('/identity-iq-report-refresh-date', methods=['POST'])
def get_identity_iq_credit_report_refresh_date():
data = request.get_json()
return get_refresh_date(data['email'], data['password'], data['ssn'])
@app.route('/identity-iq-click-refresh-btn', methods=['POST'])
def get_identity_iq_report_click_refresh_btn():
data = request.get_json()
return click_refresh_btn(data['email'], data['password'], data['ssn'])
@app.route('/identity-iq-user-validation', methods=['POST'])
def get_identity_iq_user_validation():
data = request.get_json()
return user_validation(data['email'], data['password'], data['ssn'])
if __name__ == '__main__':
app.run(host='0.0.0.0')

0
scrapper/app.sock Normal file
View File

BIN
scrapper/chromedriver_linux64/chromedriver Normal file
View File

Binary file not shown.

16
scrapper/dateproxies.txt Normal file
View File

@@ -0,0 +1,16 @@
45.61.116.115:6793
154.29.233.242:6003
64.64.127.184:6137
45.43.82.237:6231
104.250.201.177:6722
173.245.88.218:5521
192.177.103.95:6588
173.0.9.172:5755
107.181.154.4:5682
206.41.169.26:5606
171.22.248.68:5960
45.43.83.48:6331
136.0.207.63:6640
198.37.99.152:5943
206.232.13.104:5770
216.19.217.75:6315

BIN
scrapper/images/login_screen.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 34 KiB

12
scrapper/install_phantomjs.sh Normal file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env fish
sudo apt-get update
sudo apt-get install build-essential chrpath libssl-dev libxft-dev -y
sudo apt-get install libfreetype6 libfreetype6-dev -y
sudo apt-get install libfontconfig1 libfontconfig1-dev -y
cd ~ || exit
export PHANTOM_JS="phantomjs-2.1.1-linux-x86_64"
wget https://github.com/Medium/phantomjs/releases/download/v2.1.1/$PHANTOM_JS.tar.bz2
sudo tar xvjf $PHANTOM_JS.tar.bz2
sudo mv $PHANTOM_JS /usr/local/share
sudo ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/bin
phantomjs --version

3
scrapper/proxies.txt Normal file
View File

@@ -0,0 +1,3 @@
166.88.195.82:5714
31.58.20.24:5708

1
scrapper/proxies1.txt Normal file
View File

@@ -0,0 +1 @@
206.206.124.81:6662

1
scrapper/proxies_block_ips.txt Normal file
View File

@@ -0,0 +1 @@
45.41.176.106:6404

8
scrapper/refreshproxies.txt Normal file
View File

@@ -0,0 +1,8 @@
107.181.141.230:6627
142.147.129.124:5733
173.239.237.123:5769
173.239.237.236:5882
198.23.147.141:5156
107.181.142.229:5822
192.186.172.180:9180
38.170.159.196:6787

7
scrapper/requirements.txt Normal file
View File

@@ -0,0 +1,7 @@
requests~=2.23.0
beautifulsoup4~=4.9.1
lxml
flask~=1.1.2
selenium~=3.141.0
pyvirtualdisplay
Werkzeug~=1.0.1

0
scrapper/webistes/__init__.py Normal file
View File

0
scrapper/webistes/identity_iq/__init__.py Normal file
View File

165
scrapper/webistes/identity_iq/click_refresh_btn.py Normal file
View File

@@ -0,0 +1,165 @@
import traceback
import time
import random
from flask import jsonify
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from Basic.helper import read_proxies
from Basic.log_manager import get_logging
from Basic.add_remove_text import remove_and_insert_text
from Basic.helper import get_current_datetime
from Basic.helper import check_access_denied
from Basic.send_email import send_email
def click_refresh_btn(email, password, ssn):
logger = get_logging()
all_proxies = read_proxies()
proxy_address = random.choice(all_proxies)
print("proxy_address:---", proxy_address)
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument(f'--proxy-server={proxy_address}')
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.implicitly_wait(10)
try:
driver.find_element_by_id("close-pc-btn-handler").click()
except:
pass
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 60
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code": 1003, "data": "ACCOUNT_LOCKED"}
return jsonify(response)
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code":1002 , "data": "INVALID_LOGIN"}
return jsonify(response)
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
# element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
# WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
if driver.current_url == url + 'Dashboard.aspx':
driver.get(url + 'Dashboard.aspx')
element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
WebDriverWait(driver, timeout).until(element_present)
try:
element_click = driver.find_element(By.CLASS_NAME, 'chakra-modal__close-btn')
element_click.click()
except NoSuchElementException:
pass
try:
element_btn = wait.until(ec.element_to_be_clickable((By.XPATH, "//button[contains(., 'Refresh Report')]")))
driver.execute_script("arguments[0].click();", element_btn)
driver.quit()
response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":200, "status_code":200 , "data": "DONE"}
return jsonify(response)
except NoSuchElementException:
driver.quit()
response = {"page":'Dashboard.aspx', "ip":proxy_address,"page_code":200, "status_code":400 , "data": "BUTTON_NOT_FOUND"}
return jsonify(response)
driver.quit()
traceback.format_exc()
response = {"page":'security-question.aspx', "ip":proxy_address,"page_code":403, "status_code":405 , "data": "ERROR"}
return jsonify(response)
except TimeoutException:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
result = check_access_denied(url+ 'Dashboard.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
driver.quit()
response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":result, "status_code":408, "data": "TIME_OUT"}
return jsonify(response)
except Exception:
traceback.format_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
result = check_access_denied(url+ 'login.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
response = {"page":'login.aspx', "ip":proxy_address,"page_code":result, "status_code":404, "data": "ERROR"}
return jsonify(response)

190
scrapper/webistes/identity_iq/get_refresh_date.py Normal file
View File

@@ -0,0 +1,190 @@
import traceback
import time
import random
from bs4 import BeautifulSoup
from flask import jsonify
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from Basic.helper import read_proxies
from Basic.log_manager import get_logging
from Basic.add_remove_text import remove_and_insert_text
from Basic.helper import get_current_datetime
from Basic.helper import check_access_denied
from Basic.send_email import send_email
def get_refresh_date(email, password, ssn):
logger = get_logging()
all_proxies = read_proxies()
proxy_address = random.choice(all_proxies)
print("proxy_address:---", proxy_address)
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument(f'--proxy-server={proxy_address}')
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.implicitly_wait(10)
try:
driver.find_element_by_id("close-pc-btn-handler").click()
except:
pass
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 60
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address, "page_code":200, "status_code": 1003, "data": "ACCOUNT_LOCKED"}
return jsonify(response)
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address,"page_code":200, "status_code":1002 , "data": "INVALID_LOGIN"}
return jsonify(response)
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
# element_present = ec.presence_of_element_located((By.TAG_NAME, 'dashboard-layout'))
# WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
if driver.current_url == url + 'Dashboard.aspx':
#driver.get(url + 'Dashboard.aspx')
element_present = ec.presence_of_element_located((By.ID,'chakra-toast-manager-top'))
WebDriverWait(driver, timeout).until(element_present)
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.implicitly_wait(10)
time.sleep(5)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
try:
element_click = driver.find_element(By.CLASS_NAME,'chakra-modal__close-btn')
element_click.click()
except NoSuchElementException:
pass
try:
element = driver.find_element(By.CLASS_NAME,'chakra-skeleton')
response = {
'refresh_date': element.text,
'button_type': 1,
"ip":proxy_address,
"status_code":200,
"page":"Dashboard.aspx",
"page_code":200,
}
if 'Refresh Report' in element.text:
response['button_type'] = 2
except NoSuchElementException:
response = {
'refresh_date': 'Not Found',
'button_type': 1,
"ip":proxy_address,
"status_code":200,
"page":"Dashboard.aspx",
"page_code":200,
}
driver.quit()
return jsonify(response)
driver.quit()
result = check_access_denied(url+ 'Dashboard.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email)
response = {"page":'Dashboard.aspx', "ip":proxy_address, "page_code":result, "status_code":405, "data": "ERROR"}
return jsonify(response)
except TimeoutException:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
result = check_access_denied(url+ 'Dashboard.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on Dashboard.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
driver.quit()
response = {"page":'Dashboard.aspx', "ip":proxy_address,"page_code":result, "status_code":408, "data": "TIME_OUT"}
return jsonify(response)
except Exception:
traceback.format_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
result = check_access_denied(url+ 'login.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address,"page_code":result, "status_code":404, "data": "ERROR"}
return jsonify(response)

230
scrapper/webistes/identity_iq/recent_report.py Normal file
View File

@@ -0,0 +1,230 @@
import time
import requests
import traceback
import random
from flask import jsonify
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import TimeoutException,NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from Basic.helper import read_proxies
from Basic.log_manager import get_logging
from Basic.add_remove_text import remove_and_insert_text
from Basic.helper import get_current_datetime
from Basic.helper import check_access_denied
from Basic.send_email import send_email
def recent_report(email, password, ssn):
logger = get_logging()
all_proxies = read_proxies()
proxy_address = random.choice(all_proxies)
print("proxy_address:---", proxy_address)
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
# chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument(f'--proxy-server={proxy_address}')
option.add_argument('--headless') # Run Chrome in headless mode
option.add_argument('--disable-gpu') # Disable GPU acceleration
option.add_argument('--no-sandbox') # Bypass OS security model
option.add_argument('--disable-dev-shm-usage') # Overcome limited resource problems
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path)
driver.get(url + 'login.aspx')
#driver.save_screenshot("server_screenshot_login.png")
driver.implicitly_wait(10)
try:
driver.find_element_by_id("close-pc-btn-handler").click()
except:
pass
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 120
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address, "status_code": 1003, "page_code":200, "data": "ACCOUNT_LOCKED"}
return jsonify(response)
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
response = {"page":'login.aspx', "ip":proxy_address, "status_code":1002 ,"page_code":200, "data": "INVALID_LOGIN"}
return jsonify(response)
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
#driver.save_screenshot("server_screenshot_security_question.png")
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
#element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
#WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
#print('Dashboard success')
if driver.current_url == url + 'Dashboard.aspx':
response = requests.get(url + 'Dashboard.aspx')
if response.status_code == 200:
print("Access successful!")
elif response.status_code == 403:
print("Access forbidden. Your IP might be blocked.")
elif response.status_code == 401:
print("Unauthorized. You may need to log in.")
else:
print(f"Received status code {response.status_code}. Something")
#driver.save_screenshot("server_screenshot_security_Dashboard.png")
driver.get(url + 'CreditReport.aspx')
#print('CreditReport success')
#driver.save_screenshot("server_screenshot_CreditReport.png")
try:
element_present = ec.presence_of_element_located((By.ID, 'reportTop'))
WebDriverWait(driver, timeout).until(element_present)
#driver.save_screenshot("server_screenshot_final.png")
#print('element_present success')
except:
pass
#print('element not present')
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
#driver.save_screenshot("server_screenshot_final.png")
xpath_value = ''
try:
driver.find_element_by_xpath("//b[contains(., '***SECURITY FREEZE')]")
xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[10]/div[2]/address-history/div/ng-repeat[1]'
except NoSuchElementException:
xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[9]/div[2]/address-history/div/ng-repeat[1]'
try:
element_present = ec.presence_of_element_located(( By.XPATH,xpath_value))
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
pass
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
element = driver.find_element(By.ID, "ctrlCreditReport")
soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
driver.quit()
soup.html.insert(0, soup.new_tag('head'))
target = soup.find('head')
css_list = '''
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"
rel="stylesheet" />
<link href="https://member.identityiq.com/CreditReportContent/DEFAULT/style.DEFAULT.css"
rel="stylesheet" />
<link href="https://member.identityiq.com/css/CRDownload.css" rel="stylesheet" />
<script src="https://member.identityiq.com/Scripts/jquery-3.6.0.min.js" type="text/javascript"></script>
<script src="https://member.identityiq.com/Scripts/angular.js" type="text/javascript" ></script>
<script src="https://member.identityiq.com/Scripts/CRDownload.js" type="text/javascript" ></script>
'''
target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
for div in soup.findAll('div', {'class': "link_header"}):
div.decompose()
for span in soup.findAll('span', {'class': "return_topSpan"}):
span.decompose()
for img in soup.findAll('img'):
img['src'] = 'https://www.member.identityiq.com' + str(img['src'])
response = { "ip":proxy_address, "page_code":200, "page":"CreditReport.aspx", "status_code":200, "data": soup.prettify()}
return jsonify(response)
driver.quit()
response = {"page":'security-question.aspx', "ip":proxy_address, "page_code":403, "status_code":405, "data": "ERROR"}
return jsonify(response)
except TimeoutException:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
result = check_access_denied(url+ 'CreditReport.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on CreditReport.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
driver.quit()
#return traceback.format_exc()
response = {"page":'CreditReport.aspx', "ip":proxy_address, "page_code":result, "status_code":408, "data": "TIME_OUT"}
return jsonify(response)
except Exception :
logger.error("email: "+email+ " "+traceback.format_exc())
traceback.print_exc()
result = check_access_denied(url+ 'login.aspx')
subject = f"RTG-IQ-IP {proxy_address} is Blocked on login.aspx page error_code:{result}"
if(result == 403):
remove_and_insert_text(proxy_address)
send_email(subject, get_current_datetime() + " "+ email +" "+traceback.format_exc())
driver.quit()
#return traceback.format_exc()
response = {"page":'login.aspx', "ip":proxy_address, "page_code":result, "status_code":404, "data": "ERROR"}
return jsonify(response)

107
scrapper/webistes/identity_iq/user_validation.py Normal file
View File

@@ -0,0 +1,107 @@
import traceback
import random
import time
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from Basic.helper import read_proxies
from Basic.log_manager import get_logging
def user_validation(email, password, ssn):
logger = get_logging()
all_proxies = read_proxies()
proxy_address = random.choice(all_proxies)
print("proxy_address:---", proxy_address)
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/bin/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument(f'--proxy-server={proxy_address}')
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.implicitly_wait(10)
try:
driver.find_element_by_id("close-pc-btn-handler").click()
except:
pass
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 10
time.sleep(5)
try:
driver.find_element(By.CLASS_NAME, "chakra-modal__header")
driver.quit()
return "ACCOUNT_LOCKED"
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
return "INVALID_LOGIN"
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
WebDriverWait(driver, timeout).until( ec.url_to_be(url + 'Dashboard.aspx'))
if driver.current_url == url + 'Dashboard.aspx':
driver.quit()
return "DONE"
driver.quit()
logger.error("email: "+email+ " "+traceback.format_exc())
return "ERROR"
except TimeoutException:
logger.error("email: "+email+ " "+traceback.format_exc())
driver.quit()
return "TIME_OUT"
except Exception:
logger.error("email: "+email+ " "+traceback.format_exc())
return "ERROR"

0
scrapper/webistes/identity_iq_old/__init__.py Normal file
View File

114
scrapper/webistes/identity_iq_old/click_refresh_btn.py Normal file
View File

@@ -0,0 +1,114 @@
import traceback
import time
from flask import jsonify
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
def click_refresh_btn(email, password, ssn):
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 120
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
return "ACCOUNT_LOCKED"
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
return "INVALID_LOGIN"
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
# element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
# WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
if driver.current_url == url + 'Dashboard.aspx':
driver.get(url + 'Dashboard.aspx')
element_present = ec.presence_of_element_located((By.CLASS_NAME, 'chakra-skeleton'))
WebDriverWait(driver, timeout).until(element_present)
try:
element_click = driver.find_element(By.CLASS_NAME, 'chakra-modal__close-btn')
element_click.click()
except NoSuchElementException:
pass
try:
element_btn = wait.until(ec.element_to_be_clickable((By.XPATH, "//button[contains(., 'Refresh Report')]")))
driver.execute_script("arguments[0].click();", element_btn)
driver.quit()
return "DONE"
except NoSuchElementException:
driver.quit()
return "BUTTON_NOT_FOUND"
driver.quit()
return traceback.format_exc()
return "ERROR"
except TimeoutException:
driver.quit()
return "TIME_OUT"
except Exception:
return traceback.format_exc()
return "ERROR"

134
scrapper/webistes/identity_iq_old/get_refresh_date.py Normal file
View File

@@ -0,0 +1,134 @@
import traceback
import time
from bs4 import BeautifulSoup
from flask import jsonify
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
def get_refresh_date(email, password, ssn):
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 120
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
return "ACCOUNT_LOCKED"
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
return "INVALID_LOGIN"
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
# element_present = ec.presence_of_element_located((By.TAG_NAME, 'dashboard-layout'))
# WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
if driver.current_url == url + 'Dashboard.aspx':
#driver.get(url + 'Dashboard.aspx')
element_present = ec.presence_of_element_located((By.ID,'chakra-toast-manager-top'))
WebDriverWait(driver, timeout).until(element_present)
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.implicitly_wait(10)
time.sleep(5)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
try:
element_click = driver.find_element(By.CLASS_NAME,'chakra-modal__close-btn')
element_click.click()
except NoSuchElementException:
pass
try:
element = driver.find_element(By.CLASS_NAME,'chakra-skeleton')
response = {
'refresh_date': element.text,
'button_type': 1
}
if 'Refresh Report' in element.text:
response['button_type'] = 2
except NoSuchElementException:
response = {
'refresh_date': 'Not Found',
'button_type': 1
}
driver.quit()
return jsonify(response)
driver.quit()
return traceback.format_exc()
return "ERROR"
except TimeoutException:
driver.quit()
return "TIME_OUT"
except Exception:
return traceback.format_exc()
return "ERROR"

157
scrapper/webistes/identity_iq_old/recent_report.py Normal file
View File

@@ -0,0 +1,157 @@
import time
import traceback
from flask import jsonify
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import TimeoutException,NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
def recent_report(email, password, ssn):
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
# chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.find_element_by_id('txtUsername').send_keys(email)
driver.find_element_by_id('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 120
time.sleep(5)
try:
ele_acc_lock = driver.find_element(By.CLASS_NAME, "chakra-modal__header")
error_text = ele_acc_lock.text
driver.quit()
return "ACCOUNT_LOCKED"
except:
element_login = driver.find_element(By.CLASS_NAME, "chakra-text")
if element_login.text == 'Invalid login attempt':
driver.quit()
return "INVALID_LOGIN"
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'security-question') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'security-question') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'security-question':
element_present = ec.presence_of_element_located(
(By.NAME, 'userSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('userSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
#element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
#WebDriverWait(driver, timeout).until(element_present)
driver.get(url + 'Dashboard.aspx')
if driver.current_url == url + 'Dashboard.aspx':
driver.get(url + 'CreditReport.aspx')
element_present = ec.presence_of_element_located((By.ID, 'reportTop'))
WebDriverWait(driver, timeout).until(element_present)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
xpath_value = ''
try:
driver.find_element_by_xpath("//b[contains(., '***SECURITY FREEZE')]")
xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[10]/div[2]/address-history/div/ng-repeat[1]'
except NoSuchElementException:
xpath_value = '//*[@id="ctrlCreditReport"]/transunion-report/div[2]/div[9]/div[2]/address-history/div/ng-repeat[1]'
try:
element_present = ec.presence_of_element_located(( By.XPATH,xpath_value))
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
pass
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
element = driver.find_element(By.ID, "ctrlCreditReport")
soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
driver.quit()
soup.html.insert(0, soup.new_tag('head'))
target = soup.find('head')
css_list = '''
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"
rel="stylesheet" />
<link href="https://member.identityiq.com/CreditReportContent/DEFAULT/style.DEFAULT.css"
rel="stylesheet" />
<link href="https://member.identityiq.com/css/CRDownload.css" rel="stylesheet" />
<script src="https://member.identityiq.com/Scripts/jquery-3.6.0.min.js" type="text/javascript"></script>
<script src="https://member.identityiq.com/Scripts/angular.js" type="text/javascript" ></script>
<script src="https://member.identityiq.com/Scripts/CRDownload.js" type="text/javascript" ></script>
'''
target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
for div in soup.findAll('div', {'class': "link_header"}):
div.decompose()
for span in soup.findAll('span', {'class': "return_topSpan"}):
span.decompose()
for img in soup.findAll('img'):
img['src'] = 'https://www.member.identityiq.com' + str(img['src'])
return soup.prettify()
driver.quit()
# return traceback.format_exc()
return "ERROR"
except TimeoutException:
traceback.print_exc()
driver.quit()
return traceback.format_exc()
return "TIME_OUT"
except Exception :
traceback.print_exc()
return traceback.format_exc()
return "ERROR"

76
scrapper/webistes/identity_iq_old/user_validation.py Normal file
View File

@@ -0,0 +1,76 @@
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
def user_validation(email, password, ssn):
try:
url = "https://member.identityiq.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
#driver = webdriver.Chrome(chrome_path,options=option)
driver = webdriver.Chrome(chrome_local_path,options=option)
#phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
#driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login.aspx')
driver.find_element_by_name('txtUsername').send_keys(email)
driver.find_element_by_name('txtPassword').send_keys(password)
driver.find_element_by_id("imgBtnLogin").click()
timeout = 10
try:
wait = WebDriverWait(driver, timeout)
WebDriverWait(driver, timeout).until(
ec.url_to_be(url + 'SecurityQuestions.aspx') or ec.url_to_be(url + 'Dashboard.aspx')
)
wait.until(
lambda driver: driver.current_url != (url + 'SecurityQuestions.aspx') or (url + 'Dashboard.aspx')
)
if driver.current_url == url + 'SecurityQuestions.aspx':
element_present = ec.presence_of_element_located(
(By.NAME, 'FBfbforcechangesecurityanswer$txtSecurityAnswer')
)
WebDriverWait(driver, timeout).until(element_present)
driver.find_element_by_name('FBfbforcechangesecurityanswer$txtSecurityAnswer').send_keys(ssn)
driver.find_element_by_id("FBfbforcechangesecurityanswer_ibtSubmit").click()
element_present = ec.presence_of_element_located((By.ID, 'hdnMemberID'))
WebDriverWait(driver, timeout).until(element_present)
if driver.current_url == url + 'Dashboard.aspx':
element_present = ec.presence_of_element_located((By.ID, 'Fbscore_divDt'))
WebDriverWait(driver, timeout).until(element_present)
try:
driver.find_element(By.ID, "Fbscore_ibtnRefresh").click()
except NoSuchElementException:
return "BUTTON_NOT_FOUND"
driver.quit()
return "DONE"
driver.quit()
return "ERROR"
except TimeoutException:
driver.quit()
return "TIME_OUT"
except Exception:
return "ERROR"

0
scrapper/webistes/smart_credit/__init__.py Normal file
View File

96
scrapper/webistes/smart_credit/click_refresh_btn.py Normal file
View File

@@ -0,0 +1,96 @@
import traceback
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.support.wait import WebDriverWait
from Basic.log_manager import get_logging
def smart_credit_click_refresh_btn(email, password):
logger = get_logging()
try:
url = "https://www.smartcredit.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
#driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
#phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login/')
driver.find_element_by_name('j_username').send_keys(email)
driver.find_element_by_name('j_password').send_keys(password)
driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
timeout = 10
time.sleep(5)
try:
alert_div = driver.find_element_by_class_name("alert-danger")
paragraph_element = alert_div.find_element_by_tag_name("p")
error_text = paragraph_element.text
driver.quit()
return "INVALID_LOGIN"
except:
pass
try:
WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
if driver.current_url == url + 'member/home/':
driver.get(url + 'member/credit-report/3b/')
try:
element_present = ec.presence_of_element_located(
(By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
)
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
driver.quit()
return "ERROR"
driver.get(url + 'member/credit-report/3b/confirm.htm')
driver.maximize_window()
try:
checkbox = WebDriverWait(driver, 10).until(
ec.element_to_be_clickable((By.NAME, "isConfirmedTerms"))
)
if not checkbox.is_selected():
driver.execute_script("arguments[0].click();", checkbox)
driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').click()
except NoSuchElementException:
return "BUTTON_NOT_FOUND"
driver.quit()
return "DONE"
driver.quit()
return "ERROR"
except TimeoutException:
logger.error("email: "+email+ " "+traceback.format_exc())
driver.quit()
return "TIME_OUT"
except Exception:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
#return traceback.format_exc()
return "ERROR"

143
scrapper/webistes/smart_credit/credit_3b_scrapper.py Normal file
View File

@@ -0,0 +1,143 @@
import sys, traceback
import time
from flask import jsonify
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.wait import WebDriverWait
from Basic.log_manager import get_logging
def credit_report_3b(email, password):
logger = get_logging()
try:
url = "https://www.smartcredit.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
print(chrome_local_path)
option = webdriver.ChromeOptions()
#option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
#driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login/')
password = str(password)
driver.find_element_by_name('j_username').send_keys(email)
driver.find_element_by_name('j_password').send_keys(password)
driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
timeout = 10
time.sleep(5)
try:
alert_div = driver.find_element_by_class_name("alert-danger")
paragraph_element = alert_div.find_element_by_tag_name("p")
error_text = paragraph_element.text
driver.quit()
return "INVALID_LOGIN"
except:
pass
try:
WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
if driver.current_url == url + 'member/home/':
try:
driver.get(url + 'member/credit-report/3b/simple.htm?format=HTML')
element = driver.find_element(By.ID, "reportTop")
except Exception as e:
driver.get(url + 'member/credit-report/3b/')
try:
element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
pass
element = driver.find_element(By.ID, "reportTop")
soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
driver.quit()
soup.html.insert(0, soup.new_tag('head'))
target = soup.find('head')
css_list = '''
<link rel="stylesheet" type="text/css" href="https://www.smartcredit.com/resources/css/sc/pages/member/credit-report/3b/base.css"
charset="utf-8">
<style>
td#personalInfo {
width: 100%;
}
#reportTop table{
width: 100%;
}
</style>
'''
target.insert(0, BeautifulSoup(css_list, 'lxml').find('head'))
alert_box = soup.find('div', {'class': "alert alert-warning fade in text-center"})
if alert_box:
alert_box.decompose()
alert_box1 = soup.find('div', {'class': "alert alert-warning fade in text-center mb-2"})
if alert_box1:
alert_box1.decompose()
try:
soup.find('td', {'class': "backtoTopText"}).string = ""
except Exception:
pass
heading_sublink = soup.find('span', {'class': "heading-sublink"})
if heading_sublink:
heading_sublink.decompose()
report_history_loading = soup.find('div', {'id': "report-history-loading"})
if report_history_loading:
report_history_loading.decompose()
for td_right in soup.findAll('td', {'align': "right"}):
td_right.decompose()
for img in soup.findAll('img'):
img['src'] = 'https://www.smartcredit.com' + str(img['src'])
return soup.prettify()
driver.quit()
return "ERROR"
except TimeoutException:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
driver.quit()
# return traceback.format_exc()
return "TIME_OUT"
except Exception as e:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
# return traceback.format_exc()
return "ERROR"

96
scrapper/webistes/smart_credit/credit_3b_scrapper_new.py Normal file
View File

@@ -0,0 +1,96 @@
import sys, traceback
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.wait import WebDriverWait
from Basic.log_manager import get_logging
def credit_report_3b_new(email, password):
logger = get_logging()
try:
url = "https://www.smartcredit.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
#driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
#phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login/')
password = str(password)
driver.find_element_by_name('j_username').send_keys(email)
driver.find_element_by_name('j_password').send_keys(password)
driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
timeout = 10
time.sleep(5)
try:
alert_div = driver.find_element_by_class_name("alert-danger")
paragraph_element = alert_div.find_element_by_tag_name("p")
error_text = paragraph_element.text
driver.quit()
return "INVALID_LOGIN"
except:
pass
try:
WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
if driver.current_url == url + 'member/home/':
driver.get(url + 'member/credit-report/3b/simple.htm?format=JSON')
element = driver.find_element(By.ID, "reportTop")
try:
element_present = ec.presence_of_element_located((By.XPATH, '//*[@id="TokenDisplay"]/table[1]'))
WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
pass
soup = BeautifulSoup(str(element.get_attribute('outerHTML')), 'lxml')
data = driver.find_element(By.ID, "TokenDisplay").text
driver.quit()
return data
driver.quit()
return "ERROR"
except TimeoutException:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
driver.quit()
return "TIME_OUT"
except Exception as e:
traceback.print_exc()
logger.error("email: "+email+ " "+traceback.format_exc())
#return traceback.format_exc()
return "ERROR"

110
scrapper/webistes/smart_credit/get_refresh_btn_status.py Normal file
View File

@@ -0,0 +1,110 @@
import traceback
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.support.wait import WebDriverWait
from Basic.log_manager import get_logging
def smart_credit_refresh_status(email, password):
logger = get_logging()
try:
url = "https://www.smartcredit.com/"
chrome_path = "/usr/lib/chromium-browser/chromedriver"
#chrome_local_path = "D:\\scrapper\\chromedriver_win32\\chromedriver.exe"
chrome_local_path = "D:\\scrapper\\chromedriver_win64\\chromedriver.exe"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-sh-usage')
#driver = webdriver.Chrome(chrome_path,options=option)
#driver = webdriver.Chrome(chrome_local_path,options=option)
phantomjs_server_path = "/usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"
phantomjs_local_path = "D:\\scrapper\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe"
driver = webdriver.PhantomJS(executable_path= phantomjs_server_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
#driver = webdriver.PhantomJS(executable_path= phantomjs_local_path,service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get(url + 'login/')
driver.find_element_by_name('j_username').send_keys(email)
driver.find_element_by_name('j_password').send_keys(password)
driver.find_element_by_xpath('//*[@id="login-form"]/div[1]/div/button').click()
timeout = 10
time.sleep(5)
try:
alert_div = driver.find_element_by_class_name("alert-danger")
paragraph_element = alert_div.find_element_by_tag_name("p")
driver.quit()
return "INVALID_LOGIN"
except:
pass
try:
WebDriverWait(driver, timeout).until(ec.url_to_be(url + 'member/home/'))
if driver.current_url == url + 'member/home/':
driver.get(url + 'member/credit-report/3b/')
btn_text = None
try:
element_present = ec.presence_of_element_located(
(By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2')
)
WebDriverWait(driver, timeout).until(element_present)
btn_text = driver.find_element(
By.CSS_SELECTOR, '.alert.alert-warning.fade.in.text-center.mb-2'
).text
except TimeoutException:
pass
driver.get(url + 'member/credit-report/3b/confirm.htm')
driver.maximize_window()
response = {
'button_type': 0,
'refresh_date': btn_text
}
try:
driver.find_element(By.ID, "toggle-marketing")
btn_text = driver.find_element(By.ID, "toggle-marketing").text
if btn_text and str(btn_text).lower().strip() == 'order your 3-bureau report & scores':
response['button_type'] = 1
response['refresh_date'] = btn_text
except NoSuchElementException:
pass
try:
driver.find_element(By.NAME, "isConfirmedTerms")
btn_text = driver.find_element(By.XPATH, '//*[@id="confirm-3b-form"]/div[4]/div/div/button').text
if btn_text and str(btn_text).lower().strip() == 'place order now':
response['button_type'] = 2
response['refresh_date'] = btn_text
except NoSuchElementException:
pass
driver.quit()
return response
driver.quit()
return "ERROR"
except TimeoutException:
logger.error("email: "+email+ " "+traceback.format_exc())
driver.quit()
return "TIME_OUT"
except Exception:
logger.error("email: "+email+ " "+traceback.format_exc())
traceback.print_exc()
#return traceback.format_exc()
return "ERROR"

5
scrapper/wsgi.py Normal file
View File

@@ -0,0 +1,5 @@
from app import app
if __name__ == "__main__":
app.run()