This commit is contained in:
Alessio 2025-02-01 09:25:17 +01:00
parent af5d31e2a2
commit 895337923b

View File

@ -1,5 +1,6 @@
from bs4 import BeautifulSoup as bs from bs4 import BeautifulSoup as bs
import csv, jsbeautifier, cloudscraper, os, re, smtplib import csv, jsbeautifier, cloudscraper, os, re, smtplib
from selenium import webdriver
from email.mime.text import MIMEText from email.mime.text import MIMEText
from json import loads from json import loads
from datetime import datetime from datetime import datetime
@ -18,7 +19,9 @@ def get_config():
return configs return configs
def get_soup(url): #dall'url outputta la soup della pagina def get_soup(url): #dall'url outputta la soup della pagina
r = cloudscraper.create_scraper() #driver.get(url)
#data = driver.page_source
r = cloudscraper.create_scraper(session)
page=r.get(url) page=r.get(url)
data=page.text data=page.text
return bs(data, features="html.parser") return bs(data, features="html.parser")
@ -182,5 +185,32 @@ def modulo_scarica():
new_email(subject,body,configs) new_email(subject,body,configs)
if __name__ == "__main__": if __name__ == "__main__":
########################to correct the tls error
#import cloudscraper
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.poolmanager import PoolManager
import ssl
class SSLAdapter(HTTPAdapter):
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=ssl.PROTOCOL_TLS,
ciphers='ECDHE-RSA-AES128-GCM-SHA256')
#firefox_options = webdriver.FirefoxOptions()
#firefox_options.add_argument("--profile=/opt/saturn_cli/selenium_profile")
#driver = webdriver.Chrome()
session = cloudscraper.Session()
session.mount('https://', SSLAdapter())
#response = session.get('https://example.com')
#print(response.text)
########################
modulo_scarica() modulo_scarica()