126 lines
3.9 KiB
Python
126 lines
3.9 KiB
Python
|
from Eljur.auth import Authorization
|
||
|
|
from scraper import scrape
|
||
|
|
from env_init import LOGIN, PASSWORD, DOMAIN, TG_ID, send_photo, send_document
|
||
|
|
import logging
|
||
|
|
from time import sleep
|
||
|
|
from random import randint
|
||
|
|
from os import mkdir
|
||
|
|
from difflib import HtmlDiff
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
|
||
|
|
|
||
|
|
def get_difference(old, new):
|
||
|
|
old = old.split("\n")
|
||
|
|
new = new.split("\n")
|
||
|
|
difference = HtmlDiff(wrapcolumn=81)
|
||
|
|
|
||
|
|
try:
|
||
|
|
html_file = open("data/difference.html", "w")
|
||
|
|
html_file.truncate(0)
|
||
|
|
ans = difference.make_file(fromlines=old, tolines=new, fromdesc="Было", todesc="Стало")
|
||
|
|
html_file.write(ans)
|
||
|
|
html_file.close
|
||
|
|
logging.debug("data/difference.html updated")
|
||
|
|
except:
|
||
|
|
logging.critical(f"cant update data/difference.html: {Exception}")
|
||
|
|
|
||
|
|
|
||
|
|
def update_html(val=""):
|
||
|
|
try:
|
||
|
|
html_file = open("data/table.html", "w")
|
||
|
|
html_file.truncate(0)
|
||
|
|
html_file.write(val)
|
||
|
|
html_file.close
|
||
|
|
logging.debug("data/table.html updated")
|
||
|
|
except FileNotFoundError:
|
||
|
|
mkdir("data")
|
||
|
|
logging.info(f"cant update data/table.html: {Exception}")
|
||
|
|
logging.info("retry")
|
||
|
|
update_html(val=val)
|
||
|
|
except:
|
||
|
|
logging.critical(f"cant update data/table.html: {Exception}")
|
||
|
|
exit(1)
|
||
|
|
|
||
|
|
|
||
|
|
def get_html():
|
||
|
|
val = ""
|
||
|
|
try:
|
||
|
|
with open('data/table.html') as html_file:
|
||
|
|
val = html_file.read()
|
||
|
|
logging.debug("data/table.html got")
|
||
|
|
except Exception as error:
|
||
|
|
logging.error(f"cant read data/table.html: {str(error)}")
|
||
|
|
return val
|
||
|
|
|
||
|
|
|
||
|
|
def verify_data():
|
||
|
|
authorisation = Authorization()
|
||
|
|
|
||
|
|
data = {
|
||
|
|
"username": LOGIN,
|
||
|
|
"password": PASSWORD,
|
||
|
|
}
|
||
|
|
subdomain = DOMAIN
|
||
|
|
|
||
|
|
answer = authorisation.login(subdomain, data)
|
||
|
|
if "session" not in answer:
|
||
|
|
print(answer)
|
||
|
|
logging.critical("can`t log in")
|
||
|
|
logging.error(answer)
|
||
|
|
exit(1)
|
||
|
|
elif answer['answer']['user']['username'] != LOGIN: # type: ignore
|
||
|
|
print(answer)
|
||
|
|
logging.critical("data verified. Wrong LOGIN!")
|
||
|
|
logging.error(f"Your login is {answer['answer']['user']['username']}! Update env") # type: ignore
|
||
|
|
exit(255)
|
||
|
|
logging.info(f'logged in as {LOGIN}')
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def run():
|
||
|
|
global new_table, old_table
|
||
|
|
new_table = scrape()
|
||
|
|
old_table = get_html()
|
||
|
|
old_last_block = ""
|
||
|
|
|
||
|
|
if old_table != "":
|
||
|
|
old_last_block = old_table[old_table.find("""<h1 class="page-title">Объявления</h1>"""):old_table.find("""<div class="board-item__footer">""")]
|
||
|
|
|
||
|
|
if new_table != " ":
|
||
|
|
# new_table = new_table[new_table.find("""<h1 class="page-title">Объявления</h1>"""):new_table.find("""<div class="board-item__footer">""")]
|
||
|
|
new_table = new_table[new_table.find("""<h1 class="page-title">Объявления</h1>"""):new_table.find("""<span class="control-label lgray">""")]
|
||
|
|
|
||
|
|
|
||
|
|
if old_table != new_table and new_table != " ":
|
||
|
|
new_last_block = new_table[new_table.find("""<h1 class="page-title">Объявления</h1>"""):new_table.find("""<div class="board-item__footer">""")]
|
||
|
|
|
||
|
|
update_html(new_table)
|
||
|
|
new_table_soup = BeautifulSoup(new_table, "lxml")
|
||
|
|
new_table_pretty = new_table_soup.prettify()
|
||
|
|
old_table_soup = BeautifulSoup(old_table, "lxml")
|
||
|
|
old_table_pretty = old_table_soup.prettify()
|
||
|
|
get_difference(old_table_pretty, new_table_pretty)
|
||
|
|
|
||
|
|
|
||
|
|
if old_last_block != new_last_block:
|
||
|
|
send_photo("data/screenshot.png")
|
||
|
|
else:
|
||
|
|
send_photo("sad.png")
|
||
|
|
|
||
|
|
send_document("data/difference.html")
|
||
|
|
send_document("data/table.html")
|
||
|
|
|
||
|
|
return
|
||
|
|
logging.debug("same result. nothing to do")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
verify_data()
|
||
|
|
logging.info("log in success")
|
||
|
|
|
||
|
|
while True:
|
||
|
|
run()
|
||
|
|
interval = randint(300, 500)
|
||
|
|
logging.debug(f"sleep: {interval}")
|
||
|
|
sleep(interval)
|