Add monitoring PF service
This commit is contained in:
Grendgi
0
app/__init__.py
Normal file
0
app/__init__.py
Normal file
45
app/auth.py
Normal file
45
app/auth.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Lightweight admin gate for the web UI.
|
||||
|
||||
A single shared PIN (`ADMIN_PIN` in .env) unlocks destructive actions
|
||||
(deleting projects/competitors/employees, editing employees) for the browser
|
||||
session via an HMAC-signed cookie. No per-user accounts — this is an internal
|
||||
localhost tool, not a public app.
|
||||
|
||||
If `ADMIN_PIN` is empty the gate is OPEN (nothing restricted) so the tool is
|
||||
never bricked by a missing setting; set a PIN to actually restrict.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.config import settings
|
||||
|
||||
ADMIN_COOKIE = "dld_admin"
|
||||
COOKIE_MAX_AGE = 60 * 60 * 8 # 8 hours
|
||||
|
||||
|
||||
def admin_configured() -> bool:
|
||||
return bool(settings.admin_pin)
|
||||
|
||||
|
||||
def admin_token() -> str | None:
|
||||
"""The cookie value that proves admin: HMAC(pin, marker). None if no PIN."""
|
||||
if not settings.admin_pin:
|
||||
return None
|
||||
return hmac.new(settings.admin_pin.encode(), b"dld-admin-v1", hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def pin_ok(pin: str) -> bool:
|
||||
"""Constant-time check of a submitted PIN against the configured one."""
|
||||
return bool(settings.admin_pin) and hmac.compare_digest((pin or "").strip(), settings.admin_pin)
|
||||
|
||||
|
||||
def request_is_admin(request: Request) -> bool:
|
||||
if not settings.admin_pin:
|
||||
return True # gate not configured → open
|
||||
token = request.cookies.get(ADMIN_COOKIE)
|
||||
expected = admin_token()
|
||||
return bool(token and expected and hmac.compare_digest(token, expected))
|
||||
182
app/bot.py
Normal file
182
app/bot.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""Telegram bot — registers employees by chat_id and lets them trigger checks.
|
||||
|
||||
Run as a separate process: `python -m app.bot`.
|
||||
|
||||
Bot commands (set via @BotFather → /setcommands):
|
||||
start - Подключить себя как сотрудника
|
||||
list - Список своих проектов
|
||||
check - Проверить все мои проекты сейчас
|
||||
whoami - Показать свой chat_id
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy.orm import joinedload
|
||||
from telegram import Update
|
||||
from telegram.ext import (
|
||||
Application,
|
||||
CommandHandler,
|
||||
ContextTypes,
|
||||
)
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, init_db
|
||||
from app.models import Employee, Project
|
||||
from app.services.monitor import run_check_for_project
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def cmd_start(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
|
||||
if not update.effective_user or not update.effective_chat:
|
||||
return
|
||||
user = update.effective_user
|
||||
chat_id = str(update.effective_chat.id)
|
||||
username = user.username
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
existing = (
|
||||
db.query(Employee).filter(Employee.tg_chat_id == chat_id).first()
|
||||
)
|
||||
if existing:
|
||||
await update.message.reply_text(
|
||||
f"✅ Вы уже подключены как <b>{existing.name}</b>.\n"
|
||||
f"chat_id: <code>{chat_id}</code>",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
return
|
||||
|
||||
# Try to find by username (admin pre-created employee w/o chat_id)
|
||||
if username:
|
||||
placeholder = (
|
||||
db.query(Employee)
|
||||
.filter(Employee.tg_username == username, Employee.tg_chat_id.is_(None))
|
||||
.first()
|
||||
)
|
||||
if placeholder:
|
||||
placeholder.tg_chat_id = chat_id
|
||||
db.commit()
|
||||
await update.message.reply_text(
|
||||
f"✅ Привет, <b>{placeholder.name}</b>! Вы успешно подключены.\n"
|
||||
f"Уведомления будут приходить сюда.",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
return
|
||||
|
||||
# Create a new employee record from this user
|
||||
name = (user.full_name or username or f"user_{chat_id}").strip()
|
||||
e = Employee(name=name, tg_chat_id=chat_id, tg_username=username)
|
||||
db.add(e)
|
||||
db.commit()
|
||||
await update.message.reply_text(
|
||||
f"👋 Привет, <b>{name}</b>! Вы зарегистрированы как сотрудник.\n"
|
||||
f"Откройте веб-интерфейс и создайте проекты, чтобы получать уведомления.\n"
|
||||
f"chat_id: <code>{chat_id}</code>",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def cmd_whoami(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
|
||||
if not update.effective_chat:
|
||||
return
|
||||
chat_id = str(update.effective_chat.id)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
e = db.query(Employee).filter(Employee.tg_chat_id == chat_id).first()
|
||||
if e:
|
||||
await update.message.reply_text(
|
||||
f"Вы: <b>{e.name}</b>\nchat_id: <code>{chat_id}</code>",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
else:
|
||||
await update.message.reply_text(
|
||||
f"Вы пока не подключены. Отправьте /start.\nchat_id: <code>{chat_id}</code>",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def cmd_list(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
|
||||
if not update.effective_chat:
|
||||
return
|
||||
chat_id = str(update.effective_chat.id)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
e = (
|
||||
db.query(Employee)
|
||||
.options(joinedload(Employee.projects))
|
||||
.filter(Employee.tg_chat_id == chat_id)
|
||||
.first()
|
||||
)
|
||||
if not e:
|
||||
await update.message.reply_text("Сначала /start.")
|
||||
return
|
||||
if not e.projects:
|
||||
await update.message.reply_text("У вас пока нет проектов.")
|
||||
return
|
||||
lines = [f"<b>Ваши проекты ({len(e.projects)}):</b>"]
|
||||
for p in e.projects:
|
||||
lines.append(
|
||||
f"• #{p.id} {p.title} — <code>{p.dld_permit}</code> "
|
||||
f"({p.deal_type.value})"
|
||||
)
|
||||
await update.message.reply_text("\n".join(lines), parse_mode="HTML")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def cmd_check(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
|
||||
if not update.effective_chat:
|
||||
return
|
||||
chat_id = str(update.effective_chat.id)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
e = (
|
||||
db.query(Employee)
|
||||
.options(joinedload(Employee.projects))
|
||||
.filter(Employee.tg_chat_id == chat_id)
|
||||
.first()
|
||||
)
|
||||
if not e:
|
||||
await update.message.reply_text("Сначала /start.")
|
||||
return
|
||||
if not e.projects:
|
||||
await update.message.reply_text("У вас нет проектов.")
|
||||
return
|
||||
ids = [p.id for p in e.projects]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await update.message.reply_text(f"⏳ Запускаю проверку {len(ids)} проектов…")
|
||||
total_changes = 0
|
||||
for pid in ids:
|
||||
try:
|
||||
total_changes += await asyncio.to_thread(run_check_for_project, pid)
|
||||
except Exception as ex:
|
||||
logger.exception("check failed for %s: %s", pid, ex)
|
||||
await update.message.reply_text(f"✅ Готово. Изменений: {total_changes}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not settings.tg_bot_token:
|
||||
raise SystemExit("TG_BOT_TOKEN не задан в .env")
|
||||
init_db()
|
||||
app = Application.builder().token(settings.tg_bot_token).build()
|
||||
app.add_handler(CommandHandler("start", cmd_start))
|
||||
app.add_handler(CommandHandler("whoami", cmd_whoami))
|
||||
app.add_handler(CommandHandler("list", cmd_list))
|
||||
app.add_handler(CommandHandler("check", cmd_check))
|
||||
logger.info("Bot polling…")
|
||||
app.run_polling(allowed_updates=Update.ALL_TYPES)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
49
app/config.py
Normal file
49
app/config.py
Normal file
@@ -0,0 +1,49 @@
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
DATA_DIR = BASE_DIR / "data"
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def _resolve_sqlite_url(url: str) -> str:
|
||||
"""SQLAlchemy SQLite URL with a relative path is interpreted against CWD,
|
||||
which breaks when the app is started from any directory other than the
|
||||
project root. Anchor relative SQLite paths to BASE_DIR."""
|
||||
prefix = "sqlite:///"
|
||||
if not url.startswith(prefix):
|
||||
return url
|
||||
path_part = url[len(prefix):]
|
||||
p = Path(path_part)
|
||||
if p.is_absolute():
|
||||
return url
|
||||
resolved = (BASE_DIR / p).resolve()
|
||||
resolved.parent.mkdir(parents=True, exist_ok=True)
|
||||
return f"{prefix}{resolved}"
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=BASE_DIR / ".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
tg_bot_token: str = ""
|
||||
web_host: str = "127.0.0.1"
|
||||
web_port: int = 8000
|
||||
public_base_path: str = ""
|
||||
scrape_interval_hours: int = 4
|
||||
database_url: str = f"sqlite:///{DATA_DIR / 'monitor.db'}"
|
||||
admin_chat_id: str = ""
|
||||
# Shared PIN that unlocks destructive web actions (delete/edit). Empty = gate
|
||||
# disabled (everything allowed). See app/auth.py.
|
||||
admin_pin: str = ""
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
self.database_url = _resolve_sqlite_url(self.database_url)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
31
app/db.py
Normal file
31
app/db.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import DeclarativeBase, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
connect_args={"check_same_thread": False} if settings.database_url.startswith("sqlite") else {},
|
||||
future=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def init_db():
|
||||
from app import models # noqa: F401 — registers models on Base
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
103
app/models.py
Normal file
103
app/models.py
Normal file
@@ -0,0 +1,103 @@
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db import Base
|
||||
|
||||
|
||||
class DealType(str, Enum):
|
||||
SALE = "sale"
|
||||
RENT = "rent"
|
||||
|
||||
|
||||
class Source(str, Enum):
|
||||
PROPERTYFINDER = "propertyfinder"
|
||||
BAYUT = "bayut"
|
||||
|
||||
|
||||
class ListingStatus(str, Enum):
|
||||
ACTIVE = "active"
|
||||
REMOVED = "removed"
|
||||
|
||||
|
||||
class Employee(Base):
|
||||
__tablename__ = "employees"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
tg_chat_id: Mapped[str | None] = mapped_column(String(64), unique=True, nullable=True)
|
||||
tg_username: Mapped[str | None] = mapped_column(String(200), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
|
||||
projects: Mapped[list["Project"]] = relationship(back_populates="owner")
|
||||
|
||||
|
||||
class Project(Base):
|
||||
"""Наш проект — квартира, которую агентство рекламирует."""
|
||||
|
||||
__tablename__ = "projects"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(300))
|
||||
deal_type: Mapped[DealType] = mapped_column(SAEnum(DealType))
|
||||
our_price: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# Опциональные параметры — используются для подсказок похожих объявлений
|
||||
dld_permit: Mapped[str | None] = mapped_column(String(100), index=True, nullable=True)
|
||||
building: Mapped[str | None] = mapped_column(String(300), nullable=True)
|
||||
bedrooms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
size_sqft: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
our_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
owner_id: Mapped[int] = mapped_column(ForeignKey("employees.id"))
|
||||
owner: Mapped[Employee] = relationship(back_populates="projects")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
|
||||
listings: Mapped[list["CompetitorListing"]] = relationship(
|
||||
back_populates="project", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class CompetitorListing(Base):
|
||||
"""Объявление конкурента, найденное на PF/Bayut по DLD permit нашего проекта."""
|
||||
|
||||
__tablename__ = "competitor_listings"
|
||||
__table_args__ = (UniqueConstraint("project_id", "source", "external_id", name="uq_listing"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
project_id: Mapped[int] = mapped_column(ForeignKey("projects.id"))
|
||||
project: Mapped[Project] = relationship(back_populates="listings")
|
||||
|
||||
source: Mapped[Source] = mapped_column(SAEnum(Source))
|
||||
external_id: Mapped[str] = mapped_column(String(100)) # ID на стороне PF/Bayut
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
title: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
|
||||
agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
|
||||
|
||||
current_price: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED")
|
||||
status: Mapped[ListingStatus] = mapped_column(SAEnum(ListingStatus), default=ListingStatus.ACTIVE)
|
||||
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
|
||||
price_history: Mapped[list["PriceHistory"]] = relationship(
|
||||
back_populates="listing", cascade="all, delete-orphan", order_by="PriceHistory.recorded_at.desc()"
|
||||
)
|
||||
|
||||
|
||||
class PriceHistory(Base):
|
||||
__tablename__ = "price_history"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
listing_id: Mapped[int] = mapped_column(ForeignKey("competitor_listings.id"))
|
||||
listing: Mapped[CompetitorListing] = relationship(back_populates="price_history")
|
||||
|
||||
price: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
recorded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
57
app/scheduler.py
Normal file
57
app/scheduler.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""Background scheduler — runs run_check_all() every N hours.
|
||||
|
||||
Run as a separate process: `python -m app.scheduler`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from app.config import settings
|
||||
from app.db import init_db
|
||||
from app.services.monitor import run_check_all
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def job() -> None:
|
||||
logger.info("Scheduled scan starting…")
|
||||
start = time.time()
|
||||
summary = run_check_all()
|
||||
elapsed = time.time() - start
|
||||
total_changes = sum(c for c in summary.values() if c > 0)
|
||||
logger.info(
|
||||
"Scan done in %.1fs. Projects: %d, total changes: %d",
|
||||
elapsed, len(summary), total_changes,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
hours = max(1, settings.scrape_interval_hours)
|
||||
scheduler = BlockingScheduler(timezone="UTC")
|
||||
scheduler.add_job(
|
||||
job,
|
||||
trigger=IntervalTrigger(hours=hours),
|
||||
# Omit next_run_time so APScheduler defaults the first run to now+interval
|
||||
# (i.e. don't fire immediately at startup, fire after one interval, then
|
||||
# every interval). Passing next_run_time=None instead creates the job in a
|
||||
# PAUSED state and it never fires — that was the bug.
|
||||
id="periodic-scan",
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
logger.info("Scheduler started — interval %d hour(s).", hours)
|
||||
try:
|
||||
scheduler.start()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
logger.info("Scheduler stopped.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
5
app/scrapers/__init__.py
Normal file
5
app/scrapers/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
from app.scrapers.base import ScrapedListing
|
||||
from app.scrapers.bayut import BayutScraper
|
||||
from app.scrapers.propertyfinder import PropertyFinderScraper
|
||||
|
||||
__all__ = ["ScrapedListing", "BayutScraper", "PropertyFinderScraper"]
|
||||
96
app/scrapers/base.py
Normal file
96
app/scrapers/base.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScrapedListing:
|
||||
source: str # "propertyfinder" | "bayut"
|
||||
external_id: str # listing id on the source
|
||||
url: str
|
||||
title: str | None
|
||||
price: float | None
|
||||
currency: str | None
|
||||
permit_number: str | None
|
||||
agent_name: str | None
|
||||
agency_name: str | None
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
class ScraperError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fetch_html(url: str, timeout: float = 30.0) -> str:
|
||||
"""GET a URL with browser-like headers. Raises ScraperError on non-2xx."""
|
||||
try:
|
||||
with httpx.Client(headers=DEFAULT_HEADERS, follow_redirects=True, timeout=timeout) as client:
|
||||
r = client.get(url)
|
||||
if r.status_code in (403, 429):
|
||||
raise ScraperError(f"Blocked by site ({r.status_code}) at {url}")
|
||||
if r.status_code == 404:
|
||||
return ""
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
except httpx.HTTPError as e:
|
||||
raise ScraperError(f"HTTP error for {url}: {e}") from e
|
||||
|
||||
|
||||
_NEXT_DATA_RE = re.compile(
|
||||
r'<script[^>]+id="__NEXT_DATA__"[^>]*>(.*?)</script>',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def extract_next_data(html: str) -> dict | None:
|
||||
"""Extract Next.js __NEXT_DATA__ JSON blob — both PF and Bayut are Next.js apps."""
|
||||
if not html:
|
||||
return None
|
||||
m = _NEXT_DATA_RE.search(html)
|
||||
if not m:
|
||||
# Fallback via BeautifulSoup if regex misses (rare).
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
tag = soup.find("script", id="__NEXT_DATA__")
|
||||
if not tag or not tag.string:
|
||||
return None
|
||||
raw = tag.string
|
||||
else:
|
||||
raw = m.group(1)
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("Failed to parse __NEXT_DATA__: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def parse_price(value) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = re.sub(r"[^\d.]", "", str(value))
|
||||
try:
|
||||
return float(s) if s else None
|
||||
except ValueError:
|
||||
return None
|
||||
212
app/scrapers/bayut.py
Normal file
212
app/scrapers/bayut.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""Bayut.com scraper.
|
||||
|
||||
Two operations:
|
||||
- fetch_listing(url): read a listing detail page → ScrapedListing.
|
||||
- search_similar(building, bedrooms, deal_type): search Bayut for similar candidates.
|
||||
|
||||
Bayut is a Next.js app; __NEXT_DATA__ contains the property in pageProps.
|
||||
Unlike PF, Bayut shows the permit number as text in the JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import quote_plus, urljoin
|
||||
|
||||
from app.scrapers.base import (
|
||||
ScrapedListing,
|
||||
ScraperError,
|
||||
extract_next_data,
|
||||
fetch_html,
|
||||
parse_price,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_URL = "https://www.bayut.com"
|
||||
SOURCE = "bayut"
|
||||
|
||||
|
||||
def _path_for_deal(deal_type: str) -> str:
|
||||
return "to-buy" if deal_type == "sale" else "to-rent"
|
||||
|
||||
|
||||
def _walk(node):
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
for v in node.values():
|
||||
yield from _walk(v)
|
||||
elif isinstance(node, list):
|
||||
for it in node:
|
||||
yield from _walk(it)
|
||||
|
||||
|
||||
def _extract_price(item: dict) -> tuple[float | None, str | None]:
|
||||
price = item.get("price")
|
||||
if isinstance(price, dict):
|
||||
val = price.get("value") or price.get("amount")
|
||||
cur = price.get("currency") or "AED"
|
||||
return parse_price(val), cur
|
||||
if isinstance(price, (int, float, str)):
|
||||
return parse_price(price), "AED"
|
||||
return None, "AED"
|
||||
|
||||
|
||||
def _extract_broker(item: dict) -> tuple[str | None, str | None]:
|
||||
agency = item.get("agency") or {}
|
||||
agency_name = agency.get("name") if isinstance(agency, dict) else None
|
||||
agent_name = item.get("contactName") or item.get("agentName") or item.get("ownerAgent", {}).get("name") if isinstance(item.get("ownerAgent"), dict) else item.get("contactName")
|
||||
return agent_name, agency_name
|
||||
|
||||
|
||||
def _extract_permit(item: dict) -> str | None:
|
||||
for key in ("permitNumber", "permit_number", "rera", "trakheesi", "permit"):
|
||||
v = item.get(key)
|
||||
if v:
|
||||
return str(v).strip()
|
||||
return None
|
||||
|
||||
|
||||
_ID_FROM_URL = re.compile(r"details-(\d+)\.html(?:[?#].*)?$")
|
||||
|
||||
|
||||
def _extract_id_from_url(url: str) -> str | None:
|
||||
m = _ID_FROM_URL.search(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _is_listing_dict(item: dict) -> bool:
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
has_price = "price" in item
|
||||
has_id = any(k in item for k in ("externalID", "id", "objectID"))
|
||||
return has_price and has_id
|
||||
|
||||
|
||||
class BayutScraper:
|
||||
source = SOURCE
|
||||
|
||||
def fetch_listing(self, url: str) -> ScrapedListing | None:
|
||||
try:
|
||||
html = fetch_html(url)
|
||||
except ScraperError as e:
|
||||
logger.warning("Bayut refetch failed for %s: %s", url, e)
|
||||
return None
|
||||
|
||||
if not html:
|
||||
return ScrapedListing(
|
||||
source=SOURCE, external_id=_extract_id_from_url(url) or "", url=url,
|
||||
title=None, price=None, currency=None, permit_number=None,
|
||||
agent_name=None, agency_name=None, is_active=False,
|
||||
)
|
||||
|
||||
data = extract_next_data(html)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
best = None
|
||||
best_score = -1
|
||||
for node in _walk(data):
|
||||
if not _is_listing_dict(node):
|
||||
continue
|
||||
score = 0
|
||||
if "title" in node or "name" in node:
|
||||
score += 2
|
||||
if "agency" in node or "contactName" in node:
|
||||
score += 2
|
||||
if "rooms" in node or "bedrooms" in node:
|
||||
score += 1
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = node
|
||||
|
||||
if best is None:
|
||||
logger.warning("Bayut: no listing dict found in __NEXT_DATA__ for %s", url)
|
||||
return None
|
||||
|
||||
price, currency = _extract_price(best)
|
||||
agent_name, agency_name = _extract_broker(best)
|
||||
ext_id = (
|
||||
str(best.get("externalID") or best.get("id") or "")
|
||||
or _extract_id_from_url(url)
|
||||
or ""
|
||||
)
|
||||
return ScrapedListing(
|
||||
source=SOURCE,
|
||||
external_id=ext_id,
|
||||
url=url,
|
||||
title=best.get("title") or best.get("name"),
|
||||
price=price,
|
||||
currency=currency,
|
||||
permit_number=_extract_permit(best),
|
||||
agent_name=agent_name,
|
||||
agency_name=agency_name,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
def search_similar(
|
||||
self,
|
||||
building: str | None,
|
||||
bedrooms: int | None,
|
||||
deal_type: str,
|
||||
limit: int = 20,
|
||||
location_url: str | None = None,
|
||||
) -> list[ScrapedListing]:
|
||||
if not building:
|
||||
return []
|
||||
path = _path_for_deal(deal_type)
|
||||
q = quote_plus(building.strip())
|
||||
url = f"{BASE_URL}/{path}/property/dubai/?q={q}"
|
||||
if bedrooms is not None:
|
||||
url += f"&beds_in={bedrooms}"
|
||||
logger.info("Bayut search_similar: %s", url)
|
||||
|
||||
try:
|
||||
html = fetch_html(url)
|
||||
except ScraperError as e:
|
||||
logger.warning("Bayut search failed: %s", e)
|
||||
return []
|
||||
|
||||
data = extract_next_data(html)
|
||||
if not data:
|
||||
return []
|
||||
|
||||
results: list[ScrapedListing] = []
|
||||
seen_ids: set[str] = set()
|
||||
for node in _walk(data):
|
||||
if not _is_listing_dict(node):
|
||||
continue
|
||||
ext_id = str(node.get("externalID") or node.get("id") or "")
|
||||
if not ext_id or ext_id in seen_ids:
|
||||
continue
|
||||
|
||||
title = node.get("title") or node.get("name") or ""
|
||||
if building.lower() not in (title or "").lower():
|
||||
slug = str(node.get("slug") or "").lower()
|
||||
building_token = building.lower().replace(" ", "-")
|
||||
if building_token not in slug:
|
||||
continue
|
||||
|
||||
seen_ids.add(ext_id)
|
||||
price, currency = _extract_price(node)
|
||||
agent_name, agency_name = _extract_broker(node)
|
||||
cand_url = urljoin(BASE_URL, f"/property/details-{ext_id}.html")
|
||||
|
||||
results.append(
|
||||
ScrapedListing(
|
||||
source=SOURCE,
|
||||
external_id=ext_id,
|
||||
url=cand_url,
|
||||
title=title or None,
|
||||
price=price,
|
||||
currency=currency,
|
||||
permit_number=_extract_permit(node),
|
||||
agent_name=agent_name,
|
||||
agency_name=agency_name,
|
||||
is_active=True,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
325
app/scrapers/propertyfinder.py
Normal file
325
app/scrapers/propertyfinder.py
Normal file
@@ -0,0 +1,325 @@
|
||||
"""PropertyFinder.ae scraper.
|
||||
|
||||
Two operations:
|
||||
- fetch_listing(url): read a listing detail page → ScrapedListing (title/price/agent/permit).
|
||||
- search_similar(building, bedrooms, deal_type): search PF for similar candidates
|
||||
by building name + bedrooms filter → list[ScrapedListing].
|
||||
|
||||
PF is a Next.js app — listing data sits in <script id="__NEXT_DATA__">.
|
||||
Note: PF intentionally hides the Trakheesi permit as an image on the detail page,
|
||||
so permit may come back as None — that's fine, we don't depend on it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.scrapers.base import (
|
||||
ScrapedListing,
|
||||
ScraperError,
|
||||
extract_next_data,
|
||||
fetch_html,
|
||||
parse_price,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_URL = "https://www.propertyfinder.ae"
|
||||
SOURCE = "propertyfinder"
|
||||
|
||||
# PF location hierarchy, most specific first. search_similar scopes by the most
|
||||
# specific id available on a reference listing page.
|
||||
_LOC_TYPE_PRIORITY = {
|
||||
"TOWER": 5,
|
||||
"BUILDING": 4,
|
||||
"DEVELOPMENT": 3,
|
||||
"SUBCOMMUNITY": 2,
|
||||
"COMMUNITY": 1,
|
||||
"CITY": 0,
|
||||
}
|
||||
|
||||
|
||||
def _category_for_deal(deal_type: str) -> int:
|
||||
return 1 if deal_type == "sale" else 2
|
||||
|
||||
|
||||
def _get(d, *keys, default=None):
|
||||
cur = d
|
||||
for k in keys:
|
||||
if not isinstance(cur, dict):
|
||||
return default
|
||||
cur = cur.get(k)
|
||||
if cur is None:
|
||||
return default
|
||||
return cur
|
||||
|
||||
|
||||
def _walk(node):
|
||||
"""Iterate over every dict in a nested JSON structure."""
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
for v in node.values():
|
||||
yield from _walk(v)
|
||||
elif isinstance(node, list):
|
||||
for it in node:
|
||||
yield from _walk(it)
|
||||
|
||||
|
||||
def _extract_price(item: dict) -> tuple[float | None, str | None]:
|
||||
price = item.get("price")
|
||||
if isinstance(price, dict):
|
||||
val = price.get("value") or price.get("amount") or price.get("min") or price.get("from")
|
||||
cur = price.get("currency") or "AED"
|
||||
return parse_price(val), cur
|
||||
if isinstance(price, (int, float, str)):
|
||||
return parse_price(price), item.get("currency") or "AED"
|
||||
return None, "AED"
|
||||
|
||||
|
||||
def _extract_broker(item: dict) -> tuple[str | None, str | None]:
|
||||
broker = item.get("broker") or item.get("agency") or {}
|
||||
agent = item.get("agent") or item.get("contact") or {}
|
||||
agency_name = broker.get("name") if isinstance(broker, dict) else None
|
||||
agent_name = agent.get("name") if isinstance(agent, dict) else None
|
||||
return agent_name, agency_name
|
||||
|
||||
|
||||
def _extract_permit(item: dict) -> str | None:
|
||||
for key in ("permit_number", "permitNumber", "trakheesi", "rera", "permit"):
|
||||
v = item.get(key)
|
||||
if v:
|
||||
return str(v).strip()
|
||||
reg = item.get("regulatory") or item.get("regulation") or {}
|
||||
if isinstance(reg, dict):
|
||||
for key in ("permit", "permit_number", "trakheesi", "rera"):
|
||||
v = reg.get(key)
|
||||
if v:
|
||||
return str(v).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _find_permit_on_page(data: dict) -> str | None:
|
||||
"""The DLD permit number lives in a regulatory block rendered as an image,
|
||||
but its plain value is still in __NEXT_DATA__: the dict that carries a
|
||||
`permit_validation_url` (the Trakheesi link) also has the number in
|
||||
`number`. Walk the page and pull it out."""
|
||||
for node in _walk(data):
|
||||
if isinstance(node, dict) and node.get("permit_validation_url") and node.get("number"):
|
||||
return str(node["number"]).strip()
|
||||
return None
|
||||
|
||||
|
||||
_ID_FROM_URL = re.compile(r"-(\d+)\.html(?:[?#].*)?$")
|
||||
|
||||
|
||||
def _extract_id_from_url(url: str) -> str | None:
|
||||
m = _ID_FROM_URL.search(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _is_listing_dict(item: dict) -> bool:
|
||||
"""Heuristic: a listing dict contains a price plus an id-like field."""
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
has_price = "price" in item
|
||||
has_id = any(k in item for k in ("id", "reference", "listing_id", "externalID"))
|
||||
return has_price and has_id
|
||||
|
||||
|
||||
class PropertyFinderScraper:
|
||||
source = SOURCE
|
||||
|
||||
def fetch_listing(self, url: str) -> ScrapedListing | None:
|
||||
"""Refetch a known listing URL. Returns:
|
||||
- ScrapedListing(is_active=False) if the URL returns 404 (listing removed)
|
||||
- ScrapedListing with current data if alive
|
||||
- None on network/parse failure (we won't update the DB in that case)
|
||||
"""
|
||||
try:
|
||||
html = fetch_html(url)
|
||||
except ScraperError as e:
|
||||
logger.warning("PF refetch failed for %s: %s", url, e)
|
||||
return None
|
||||
|
||||
if not html:
|
||||
return ScrapedListing(
|
||||
source=SOURCE, external_id=_extract_id_from_url(url) or "", url=url,
|
||||
title=None, price=None, currency=None, permit_number=None,
|
||||
agent_name=None, agency_name=None, is_active=False,
|
||||
)
|
||||
|
||||
data = extract_next_data(html)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
# On a PF detail page the property dict is nested in pageProps. Walk and pick
|
||||
# the dict that has both a "price" and an id, ignoring trivial nested ones.
|
||||
best = None
|
||||
best_score = -1
|
||||
for node in _walk(data):
|
||||
if not _is_listing_dict(node):
|
||||
continue
|
||||
score = 0
|
||||
if "title" in node or "name" in node:
|
||||
score += 2
|
||||
if any(k in node for k in ("broker", "agent", "agency")):
|
||||
score += 2
|
||||
if "bedrooms" in node or "rooms" in node:
|
||||
score += 1
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = node
|
||||
|
||||
if best is None:
|
||||
logger.warning("PF: no listing dict found in __NEXT_DATA__ for %s", url)
|
||||
return None
|
||||
|
||||
price, currency = _extract_price(best)
|
||||
agent_name, agency_name = _extract_broker(best)
|
||||
ext_id = (
|
||||
str(best.get("id") or best.get("reference") or best.get("listing_id") or "")
|
||||
or _extract_id_from_url(url)
|
||||
or ""
|
||||
)
|
||||
return ScrapedListing(
|
||||
source=SOURCE,
|
||||
external_id=ext_id,
|
||||
url=url,
|
||||
title=best.get("title") or best.get("name"),
|
||||
price=price,
|
||||
currency=currency,
|
||||
permit_number=_find_permit_on_page(data) or _extract_permit(best),
|
||||
agent_name=agent_name,
|
||||
agency_name=agency_name,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
def get_permit(self, url: str) -> str | None:
|
||||
"""Fetch a listing page and return only its DLD permit number (or None).
|
||||
Used to compare candidates against our own permit during suggestions."""
|
||||
try:
|
||||
html = fetch_html(url)
|
||||
except ScraperError as e:
|
||||
logger.warning("PF get_permit fetch failed for %s: %s", url, e)
|
||||
return None
|
||||
data = extract_next_data(html)
|
||||
return _find_permit_on_page(data) if data else None
|
||||
|
||||
def resolve_location_id(self, listing_url: str) -> int | None:
|
||||
"""Read a PF listing page and return the most specific location id
|
||||
(tower > building > subcommunity > community).
|
||||
|
||||
PF's search only filters by numeric location id (`l=`); the free-text
|
||||
`q=` param does NOT scope results to a building — it returns unrelated
|
||||
recommendations. So we derive the location id from a known listing that
|
||||
sits in the same building (our own listing, or an already-tracked one).
|
||||
"""
|
||||
try:
|
||||
html = fetch_html(listing_url)
|
||||
except ScraperError as e:
|
||||
logger.warning("PF resolve_location_id fetch failed for %s: %s", listing_url, e)
|
||||
return None
|
||||
data = extract_next_data(html)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
best_id: object = None
|
||||
best_rank = -1
|
||||
for node in _walk(data):
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
rank = _LOC_TYPE_PRIORITY.get(str(node.get("type", "")).upper(), -1)
|
||||
if rank > best_rank and node.get("id") and node.get("name"):
|
||||
best_rank, best_id = rank, node.get("id")
|
||||
try:
|
||||
return int(best_id) if best_id is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def search_similar(
|
||||
self,
|
||||
building: str | None,
|
||||
bedrooms: int | None,
|
||||
deal_type: str,
|
||||
limit: int = 200,
|
||||
location_url: str | None = None,
|
||||
max_pages: int = 8,
|
||||
) -> list[ScrapedListing]:
|
||||
"""Search PF for candidates in the same building, scoped by location id.
|
||||
|
||||
`location_url` is a reference listing in the target building (our own
|
||||
listing or an already-tracked competitor) — we resolve it to a PF
|
||||
location id and search by `l=`. Without it we can't reliably scope a
|
||||
building search on PF, so we return nothing rather than garbage.
|
||||
|
||||
Paginates: a same-permit competitor can sit on any results page (PF
|
||||
can't be queried by permit), so we collect across pages up to
|
||||
`max_pages`/`limit`.
|
||||
"""
|
||||
location_id = self.resolve_location_id(location_url) if location_url else None
|
||||
if location_id is None:
|
||||
logger.info(
|
||||
"PF search_similar: no location id (url=%r) — skipping (q= text search "
|
||||
"does not filter by building on PF)", location_url,
|
||||
)
|
||||
return []
|
||||
|
||||
c = _category_for_deal(deal_type)
|
||||
base = f"{BASE_URL}/en/search?c={c}&l={location_id}"
|
||||
if bedrooms is not None:
|
||||
base += f"&bf={bedrooms}&bt={bedrooms}" # PF uses bf=bedrooms-from, bt=bedrooms-to
|
||||
|
||||
results: list[ScrapedListing] = []
|
||||
seen_ids: set[str] = set()
|
||||
for page in range(1, max_pages + 1):
|
||||
page_url = base if page == 1 else f"{base}&page={page}"
|
||||
try:
|
||||
html = fetch_html(page_url)
|
||||
except ScraperError as e:
|
||||
logger.warning("PF search failed (page %d): %s", page, e)
|
||||
break
|
||||
data = extract_next_data(html)
|
||||
if not data:
|
||||
break
|
||||
|
||||
new_on_page = 0
|
||||
for node in _walk(data):
|
||||
if not _is_listing_dict(node):
|
||||
continue
|
||||
ext_id = str(node.get("id") or node.get("reference") or "")
|
||||
if not ext_id or ext_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(ext_id)
|
||||
new_on_page += 1
|
||||
|
||||
# Results are scoped to the location by l=, so no title filter.
|
||||
title = node.get("title") or node.get("name") or ""
|
||||
price, currency = _extract_price(node)
|
||||
agent_name, agency_name = _extract_broker(node)
|
||||
share = node.get("share_url") or node.get("path")
|
||||
cand_url = share if str(share).startswith("http") else urljoin(BASE_URL, str(share or ""))
|
||||
|
||||
results.append(
|
||||
ScrapedListing(
|
||||
source=SOURCE,
|
||||
external_id=ext_id,
|
||||
url=cand_url or page_url,
|
||||
title=title or None,
|
||||
price=price,
|
||||
currency=currency,
|
||||
permit_number=_extract_permit(node),
|
||||
agent_name=agent_name,
|
||||
agency_name=agency_name,
|
||||
is_active=True,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
# No new listings on this page → we've passed the last page.
|
||||
if len(results) >= limit or new_on_page == 0:
|
||||
break
|
||||
logger.info("PF search_similar: collected %d candidates (l=%s)", len(results), location_id)
|
||||
return results
|
||||
0
app/services/__init__.py
Normal file
0
app/services/__init__.py
Normal file
354
app/services/monitor.py
Normal file
354
app/services/monitor.py
Normal file
@@ -0,0 +1,354 @@
|
||||
"""Core monitoring logic.
|
||||
|
||||
Per project, for every CompetitorListing we already track:
|
||||
1. Re-fetch the listing's URL.
|
||||
2. Detect:
|
||||
- Price change → 📈📉 notify
|
||||
- URL returns 404 / removed → ❌ notify, mark removed
|
||||
- Reappeared after removal → ♻️ notify
|
||||
3. Snapshot every price into PriceHistory.
|
||||
|
||||
Adding new competitors is done via the web UI (user pastes URLs) — not here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import (
|
||||
CompetitorListing,
|
||||
Employee,
|
||||
ListingStatus,
|
||||
PriceHistory,
|
||||
Project,
|
||||
Source,
|
||||
)
|
||||
from app.scrapers import BayutScraper, PropertyFinderScraper, ScrapedListing
|
||||
from app.services.notifier import notify_admin, send_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PF = PropertyFinderScraper()
|
||||
BAYUT = BayutScraper()
|
||||
|
||||
# Same-building suggestions beyond exact permit matches are a browse heuristic —
|
||||
# cap how many we show so the page stays usable.
|
||||
_SUGGEST_OTHERS_LIMIT = 30
|
||||
|
||||
# Bayut moved to fully client-side rendering (no __NEXT_DATA__, Algolia keys
|
||||
# hidden), so it can't be scraped over plain HTTP — disabled until we add a
|
||||
# headless-browser fetcher. Flip to True once that exists.
|
||||
BAYUT_ENABLED = False
|
||||
|
||||
|
||||
def _scraper_for(source: Source):
|
||||
return PF if source == Source.PROPERTYFINDER else BAYUT
|
||||
|
||||
|
||||
def detect_source_from_url(url: str) -> Source | None:
|
||||
u = (url or "").lower()
|
||||
if "propertyfinder" in u:
|
||||
return Source.PROPERTYFINDER
|
||||
if "bayut.com" in u:
|
||||
return Source.BAYUT
|
||||
return None
|
||||
|
||||
|
||||
def _fmt_price(value: float | None, currency: str | None = "AED") -> str:
|
||||
if value is None:
|
||||
return "—"
|
||||
return f"{value:,.0f} {currency or 'AED'}".replace(",", " ")
|
||||
|
||||
|
||||
def _source_label(source: str) -> str:
|
||||
return {"propertyfinder": "PropertyFinder", "bayut": "Bayut"}.get(source, source)
|
||||
|
||||
|
||||
def add_competitor_url(db: Session, project: Project, url: str) -> tuple[CompetitorListing | None, str]:
|
||||
"""User-facing entrypoint: paste a URL → create CompetitorListing for the project.
|
||||
Returns (listing, error_message). error_message is empty on success.
|
||||
"""
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return None, "URL пустой"
|
||||
source = detect_source_from_url(url)
|
||||
if source is None:
|
||||
return None, "URL должен быть с propertyfinder.ae или bayut.com"
|
||||
if source == Source.BAYUT and not BAYUT_ENABLED:
|
||||
return None, (
|
||||
"Bayut временно не поддерживается — площадка перешла на защищённый "
|
||||
"рендеринг. Добавляйте ссылки PropertyFinder."
|
||||
)
|
||||
|
||||
scraper = _scraper_for(source)
|
||||
scraped = scraper.fetch_listing(url)
|
||||
if scraped is None:
|
||||
return None, "Не удалось загрузить страницу — сайт мог заблокировать запрос, попробуйте позже"
|
||||
if not scraped.is_active:
|
||||
return None, "Страница объявления вернула 404 — ссылка битая или объявление снято"
|
||||
|
||||
ext_id = scraped.external_id or url # fallback if id wasn't found
|
||||
existing = (
|
||||
db.query(CompetitorListing)
|
||||
.filter(
|
||||
CompetitorListing.project_id == project.id,
|
||||
CompetitorListing.source == source,
|
||||
CompetitorListing.external_id == ext_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
return None, "Это объявление уже добавлено в проект"
|
||||
|
||||
now = datetime.utcnow()
|
||||
listing = CompetitorListing(
|
||||
project_id=project.id,
|
||||
source=source,
|
||||
external_id=ext_id,
|
||||
url=url,
|
||||
title=scraped.title,
|
||||
agent_name=scraped.agent_name,
|
||||
agency_name=scraped.agency_name,
|
||||
current_price=scraped.price,
|
||||
currency=scraped.currency or "AED",
|
||||
status=ListingStatus.ACTIVE,
|
||||
first_seen_at=now,
|
||||
last_seen_at=now,
|
||||
)
|
||||
db.add(listing)
|
||||
db.flush()
|
||||
if scraped.price is not None:
|
||||
db.add(PriceHistory(listing_id=listing.id, price=scraped.price, recorded_at=now))
|
||||
db.commit()
|
||||
return listing, ""
|
||||
|
||||
|
||||
def add_competitor_urls(db: Session, project: Project, urls: list[str]) -> dict:
|
||||
"""Add several pasted/selected URLs in one go (used by the suggest page's
|
||||
multi-select). Processes them sequentially — each one re-fetches the page —
|
||||
and reports a summary. Returns {'added': int, 'skipped': int, 'errors': [..]}.
|
||||
"""
|
||||
added = 0
|
||||
skipped = 0
|
||||
errors: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in urls:
|
||||
url = (raw or "").strip()
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
listing, err = add_competitor_url(db, project, url)
|
||||
if err == "Это объявление уже добавлено в проект":
|
||||
skipped += 1
|
||||
elif err:
|
||||
errors.append(err)
|
||||
else:
|
||||
added += 1
|
||||
return {"added": added, "skipped": skipped, "errors": errors}
|
||||
|
||||
|
||||
def check_project(db: Session, project: Project) -> list[str]:
|
||||
"""Re-scan all tracked competitor listings for one project. Returns notification texts."""
|
||||
changes: list[str] = []
|
||||
now = datetime.utcnow()
|
||||
|
||||
for listing in list(project.listings):
|
||||
scraper = _scraper_for(listing.source)
|
||||
scraped = scraper.fetch_listing(listing.url)
|
||||
if scraped is None:
|
||||
# Network/parse failure — skip without changing state, try again next cycle.
|
||||
continue
|
||||
|
||||
if not scraped.is_active:
|
||||
if listing.status == ListingStatus.ACTIVE:
|
||||
listing.status = ListingStatus.REMOVED
|
||||
changes.append(
|
||||
f"❌ <b>Объявление удалено</b> — {_source_label(listing.source.value)}\n"
|
||||
f"{listing.title or 'без названия'}\n"
|
||||
f"Последняя цена: {_fmt_price(listing.current_price, listing.currency)}\n"
|
||||
f"{listing.url}"
|
||||
)
|
||||
continue
|
||||
|
||||
listing.last_seen_at = now
|
||||
|
||||
if listing.status != ListingStatus.ACTIVE:
|
||||
listing.status = ListingStatus.ACTIVE
|
||||
changes.append(
|
||||
f"♻️ <b>Объявление снова активно</b> — {_source_label(listing.source.value)}\n"
|
||||
f"{listing.title or 'без названия'}\n{listing.url}"
|
||||
)
|
||||
|
||||
# Update metadata that may have changed
|
||||
if scraped.title:
|
||||
listing.title = scraped.title
|
||||
if scraped.agent_name:
|
||||
listing.agent_name = scraped.agent_name
|
||||
if scraped.agency_name:
|
||||
listing.agency_name = scraped.agency_name
|
||||
|
||||
old_price = listing.current_price
|
||||
new_price = scraped.price
|
||||
if new_price is not None and old_price is not None and new_price != old_price:
|
||||
delta = new_price - old_price
|
||||
pct = (delta / old_price * 100.0) if old_price else 0.0
|
||||
arrow = "📈" if delta > 0 else "📉"
|
||||
changes.append(
|
||||
f"{arrow} <b>Цена изменилась</b> — {_source_label(listing.source.value)}\n"
|
||||
f"{listing.title or 'без названия'}\n"
|
||||
f"Было: {_fmt_price(old_price, listing.currency)}\n"
|
||||
f"Стало: {_fmt_price(new_price, scraped.currency or listing.currency)} "
|
||||
f"({'+' if delta > 0 else ''}{delta:,.0f} / {pct:+.1f}%)\n"
|
||||
f"{listing.url}".replace(",", " ")
|
||||
)
|
||||
listing.current_price = new_price
|
||||
listing.currency = scraped.currency or listing.currency
|
||||
db.add(PriceHistory(listing_id=listing.id, price=new_price, recorded_at=now))
|
||||
elif new_price is not None and old_price is None:
|
||||
listing.current_price = new_price
|
||||
listing.currency = scraped.currency or listing.currency
|
||||
db.add(PriceHistory(listing_id=listing.id, price=new_price, recorded_at=now))
|
||||
|
||||
project.last_checked_at = now
|
||||
db.commit()
|
||||
return changes
|
||||
|
||||
|
||||
def _notify_owner(project: Project, changes: list[str]) -> None:
|
||||
if not changes:
|
||||
return
|
||||
owner: Employee | None = project.owner
|
||||
if not owner or not owner.tg_chat_id:
|
||||
logger.warning("Project %s has no owner chat_id; skipping notification", project.id)
|
||||
notify_admin(
|
||||
f"⚠️ Проект <b>{project.title}</b> (#{project.id}) — {len(changes)} изменений, "
|
||||
f"но у владельца не задан tg_chat_id."
|
||||
)
|
||||
return
|
||||
|
||||
header = (
|
||||
f"🏠 <b>{project.title}</b>\n"
|
||||
f"Тип: {project.deal_type.value} · Изменений: {len(changes)}\n"
|
||||
f"——————————"
|
||||
)
|
||||
send_message(owner.tg_chat_id, header)
|
||||
for c in changes:
|
||||
send_message(owner.tg_chat_id, c)
|
||||
|
||||
|
||||
def _reference_url(project: Project, source: Source) -> str | None:
|
||||
"""A known listing URL in the project's building for the given source.
|
||||
|
||||
Portals (PF) scope a building search by an internal numeric location id, not
|
||||
by free text — so we hand the scraper a real listing in the same building
|
||||
(our own `our_url`, else an already-tracked competitor) to resolve that id.
|
||||
"""
|
||||
candidates: list[str] = []
|
||||
if project.our_url:
|
||||
candidates.append(project.our_url)
|
||||
candidates.extend(l.url for l in project.listings if l.source == source)
|
||||
for url in candidates:
|
||||
if detect_source_from_url(url) == source:
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def resolve_our_permit(project: Project) -> str | None:
|
||||
"""Our project's DLD permit number. Prefer the value the user typed; else
|
||||
read it off our own listing (PF exposes the number in __NEXT_DATA__)."""
|
||||
if project.dld_permit and project.dld_permit.strip():
|
||||
return project.dld_permit.strip()
|
||||
ref = _reference_url(project, Source.PROPERTYFINDER)
|
||||
return PF.get_permit(ref) if ref else None
|
||||
|
||||
|
||||
def suggest_similar(project: Project, our_permit: str | None = None) -> dict[str, list[ScrapedListing]]:
|
||||
"""Search PF + Bayut for listings in this project's building.
|
||||
|
||||
Candidates that share our DLD permit are the same physical listing under a
|
||||
different broker (rare, but it happens) — those are surfaced first. The rest
|
||||
are same-building heuristics. Returns {'propertyfinder': [...], 'bayut': [...]}.
|
||||
"""
|
||||
out: dict[str, list[ScrapedListing]] = {"propertyfinder": [], "bayut": []}
|
||||
bedrooms = project.bedrooms
|
||||
deal_type = project.deal_type.value
|
||||
|
||||
try:
|
||||
out["propertyfinder"] = PF.search_similar(
|
||||
project.building, bedrooms, deal_type,
|
||||
location_url=_reference_url(project, Source.PROPERTYFINDER),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("PF suggest failed: %s", e)
|
||||
if BAYUT_ENABLED:
|
||||
try:
|
||||
out["bayut"] = BAYUT.search_similar(
|
||||
project.building, bedrooms, deal_type,
|
||||
location_url=_reference_url(project, Source.BAYUT),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Bayut suggest failed: %s", e)
|
||||
|
||||
# Hide candidates already tracked for this project — and our own listing.
|
||||
excluded = {(l.source.value, l.external_id) for l in project.listings}
|
||||
if project.our_url:
|
||||
own_src = detect_source_from_url(project.our_url)
|
||||
m = re.search(r"(\d+)\.html", project.our_url)
|
||||
if own_src and m:
|
||||
excluded.add((own_src.value, m.group(1)))
|
||||
for src in out:
|
||||
out[src] = [c for c in out[src] if (src, c.external_id) not in excluded]
|
||||
|
||||
# Permit-first: PF can't be queried by permit and search results don't carry
|
||||
# it — so read each PF candidate's permit (concurrently) and put the ones
|
||||
# matching ours first. Keep all matches; cap the same-building remainder.
|
||||
if our_permit and out["propertyfinder"]:
|
||||
pf = out["propertyfinder"]
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=12) as ex:
|
||||
permits = list(ex.map(PF.get_permit, [c.url for c in pf]))
|
||||
for cand, permit in zip(pf, permits):
|
||||
cand.permit_number = permit
|
||||
matches = [c for c in pf if c.permit_number == our_permit]
|
||||
others = [c for c in pf if c.permit_number != our_permit]
|
||||
out["propertyfinder"] = matches + others[:_SUGGEST_OTHERS_LIMIT]
|
||||
except Exception as e:
|
||||
logger.exception("PF permit enrichment failed: %s", e)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def run_check_for_project(project_id: int) -> int:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
return 0
|
||||
changes = check_project(db, project)
|
||||
_notify_owner(project, changes)
|
||||
return len(changes)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def run_check_all() -> dict[int, int]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ids = [p.id for p in db.query(Project).all()]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
summary = {}
|
||||
for pid in ids:
|
||||
try:
|
||||
summary[pid] = run_check_for_project(pid)
|
||||
except Exception as e:
|
||||
logger.exception("Check for project %s failed: %s", pid, e)
|
||||
summary[pid] = -1
|
||||
return summary
|
||||
48
app/services/notifier.py
Normal file
48
app/services/notifier.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Telegram notification sender. Uses httpx directly — no bot framework needed
|
||||
for outbound messages, so we can call it from the scheduler thread without
|
||||
needing an event loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TG_API = "https://api.telegram.org"
|
||||
|
||||
|
||||
def send_message(chat_id: str, text: str, parse_mode: str = "HTML") -> bool:
|
||||
if not settings.tg_bot_token:
|
||||
logger.warning("TG_BOT_TOKEN not set — skipping notification to %s", chat_id)
|
||||
return False
|
||||
if not chat_id:
|
||||
logger.warning("Empty chat_id — skipping notification")
|
||||
return False
|
||||
url = f"{TG_API}/bot{settings.tg_bot_token}/sendMessage"
|
||||
try:
|
||||
with httpx.Client(timeout=15.0) as client:
|
||||
r = client.post(
|
||||
url,
|
||||
json={
|
||||
"chat_id": chat_id,
|
||||
"text": text,
|
||||
"parse_mode": parse_mode,
|
||||
"disable_web_page_preview": False,
|
||||
},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
logger.error("TG send failed: %s %s", r.status_code, r.text)
|
||||
return False
|
||||
return True
|
||||
except httpx.HTTPError as e:
|
||||
logger.error("TG send exception: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def notify_admin(text: str) -> None:
|
||||
if settings.admin_chat_id:
|
||||
send_message(settings.admin_chat_id, text)
|
||||
29
app/templates/admin_login.html
Normal file
29
app/templates/admin_login.html
Normal file
@@ -0,0 +1,29 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Вход администратора — DLD Monitor{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<div class="row justify-content-center">
|
||||
<div class="col-md-5">
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-body">
|
||||
<h5 class="mb-3">🔒 Режим администратора</h5>
|
||||
<p class="text-muted small">
|
||||
Введите PIN, чтобы разблокировать удаление проектов, конкурентов и
|
||||
редактирование/удаление сотрудников. Сессия — на 8 часов.
|
||||
</p>
|
||||
{% if error %}<div class="alert alert-danger py-2">{{ error }}</div>{% endif %}
|
||||
<form method="post" action="{{ url_path('/admin/login') }}">
|
||||
<input type="hidden" name="next" value="{{ next }}">
|
||||
<div class="mb-3">
|
||||
<input type="password" name="pin" required autofocus class="form-control"
|
||||
placeholder="PIN администратора">
|
||||
</div>
|
||||
<button class="btn btn-primary w-100">Войти</button>
|
||||
</form>
|
||||
<a href="{{ url_path(next) }}" class="btn btn-link btn-sm w-100 mt-2">← Отмена</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
88
app/templates/base.html
Normal file
88
app/templates/base.html
Normal file
@@ -0,0 +1,88 @@
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>{% block title %}DLD Monitor{% endblock %}</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<style>
|
||||
body { background: #f4f5f9; }
|
||||
.navbar-brand { letter-spacing: .3px; }
|
||||
.nav-link.active { font-weight: 600; color: #0d6efd !important; }
|
||||
.badge-sale { background: #198754; }
|
||||
.badge-rent { background: #0d6efd; }
|
||||
.card { border: 1px solid #e9ecef; }
|
||||
.listing-card { border-left: 4px solid #dee2e6; }
|
||||
.listing-card.removed { border-left-color: #dc3545; opacity: 0.65; }
|
||||
.listing-card.active { border-left-color: #198754; }
|
||||
.src-pf { color: #d63384; font-weight: 600; }
|
||||
.src-bayut { color: #0d6efd; font-weight: 600; }
|
||||
.price-up { color: #dc3545; }
|
||||
.price-down { color: #198754; }
|
||||
pre.permit { display: inline; background: #eee; padding: 2px 6px; border-radius: 4px; }
|
||||
.stat-chip { background:#fff; border:1px solid #e9ecef; border-radius:.5rem; padding:.4rem .8rem; }
|
||||
.stat-chip .n { font-size:1.25rem; font-weight:700; line-height:1; }
|
||||
footer { color:#9aa0a6; font-size:.8rem; }
|
||||
/* Mobile tweaks */
|
||||
@media (max-width: 575.98px) {
|
||||
body { background: #fff; }
|
||||
.container { padding-left: .75rem; padding-right: .75rem; }
|
||||
h3 { font-size: 1.35rem; }
|
||||
.card-body { padding: .75rem; }
|
||||
.fs-4 { font-size: 1.1rem !important; }
|
||||
}
|
||||
</style>
|
||||
{% block head %}{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
<nav class="navbar navbar-expand-lg bg-white border-bottom mb-4">
|
||||
<div class="container">
|
||||
<a class="navbar-brand fw-bold" href="{{ url_path('/') }}">🏠 DLD Monitor</a>
|
||||
<button class="navbar-toggler" type="button" data-bs-toggle="collapse"
|
||||
data-bs-target="#mainnav" aria-controls="mainnav"
|
||||
aria-expanded="false" aria-label="Меню">
|
||||
<span class="navbar-toggler-icon"></span>
|
||||
</button>
|
||||
<div class="collapse navbar-collapse" id="mainnav">
|
||||
<ul class="navbar-nav me-auto mb-2 mb-lg-0">
|
||||
<li class="nav-item">
|
||||
<a href="{{ url_path('/') }}" class="nav-link {{ 'active' if request.url.path == '/' or request.url.path.startswith('/projects') else '' }}">Проекты</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a href="{{ url_path('/employees') }}" class="nav-link {{ 'active' if request.url.path.startswith('/employees') else '' }}">Сотрудники</a>
|
||||
</li>
|
||||
</ul>
|
||||
<div class="d-flex flex-column flex-lg-row align-items-stretch align-items-lg-center gap-2">
|
||||
<a href="{{ url_path('/projects/new') }}" class="btn btn-sm btn-primary">+ Проект</a>
|
||||
<span class="vr d-none d-lg-block"></span>
|
||||
{% if request.state.is_admin %}
|
||||
{% if request.state.admin_configured %}
|
||||
<span class="badge bg-success align-self-start align-self-lg-center">🔓 админ</span>
|
||||
<form method="post" action="{{ url_path('/admin/logout') }}">
|
||||
<input type="hidden" name="next" value="{{ request.url.path }}">
|
||||
<button class="btn btn-sm btn-outline-secondary w-100">Выйти</button>
|
||||
</form>
|
||||
{% else %}
|
||||
<span class="badge bg-warning text-dark align-self-start align-self-lg-center" title="ADMIN_PIN не задан в .env — права не ограничены">⚠ без PIN</span>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<a href="{{ url_path('/admin/login?next=' ~ request.url.path) }}" class="btn btn-sm btn-outline-dark">🔒 Админ</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div class="container">
|
||||
{% if flash %}
|
||||
<div class="alert alert-info">{{ flash }}</div>
|
||||
{% endif %}
|
||||
{% block content %}{% endblock %}
|
||||
</div>
|
||||
|
||||
<footer class="container text-center py-4">DLD Monitor · HOME LIGA REAL ESTATE</footer>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
|
||||
{% block scripts %}{% endblock %}
|
||||
</body>
|
||||
</html>
|
||||
86
app/templates/employees.html
Normal file
86
app/templates/employees.html
Normal file
@@ -0,0 +1,86 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Сотрудники — DLD Monitor{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<h3>Сотрудники</h3>
|
||||
|
||||
<div class="alert alert-info">
|
||||
<b>Как подключить Telegram:</b>
|
||||
<ol class="mb-0">
|
||||
<li>Сотрудник пишет боту в Telegram команду <code>/whoami</code> — бот вернёт его <code>chat_id</code>.</li>
|
||||
<li>Вставьте этот <code>chat_id</code> в поле справа от имени и нажмите «Сохранить».</li>
|
||||
</ol>
|
||||
Если сотрудник пока не знает свой chat_id — пусть просто отправит <code>/start</code>, бот ответит и пришлёт id.
|
||||
</div>
|
||||
|
||||
<form method="post" action="{{ url_path('/employees') }}" class="row g-2 mb-4 bg-white p-3 rounded shadow-sm">
|
||||
<div class="col-md-5">
|
||||
<input name="name" required class="form-control" placeholder="Имя сотрудника">
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<input name="tg_username" class="form-control" placeholder="@username (опционально)">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<button class="btn btn-primary w-100">Добавить</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{% if not request.state.is_admin %}
|
||||
<div class="alert alert-secondary py-2 small">
|
||||
🔒 Редактирование и удаление сотрудников доступно только в режиме админа.
|
||||
<a href="{{ url_path('/admin/login?next=/employees') }}">Войти</a>.
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="table-responsive">
|
||||
<table class="table bg-white rounded shadow-sm align-middle">
|
||||
<thead class="table-light">
|
||||
<tr><th>Имя</th><th>@username</th><th>Chat ID</th><th>Проектов</th>{% if request.state.is_admin %}<th style="width:1%"></th>{% endif %}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for e in employees %}
|
||||
{% if request.state.is_admin %}
|
||||
<tr>
|
||||
<form method="post" action="{{ url_path('/employees/' ~ e.id ~ '/update') }}">
|
||||
<td>
|
||||
<input name="name" value="{{ e.name }}" class="form-control form-control-sm" required>
|
||||
</td>
|
||||
<td>
|
||||
<input name="tg_username" value="{{ e.tg_username or '' }}" class="form-control form-control-sm" placeholder="username">
|
||||
</td>
|
||||
<td>
|
||||
<div class="input-group input-group-sm">
|
||||
<input name="tg_chat_id" value="{{ e.tg_chat_id or '' }}" class="form-control"
|
||||
placeholder="например 123456789">
|
||||
{% if e.tg_chat_id %}
|
||||
<span class="input-group-text text-success">✓</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ e.projects|length }}</td>
|
||||
<td class="text-nowrap">
|
||||
<button class="btn btn-sm btn-primary">Сохранить</button>
|
||||
</form>
|
||||
<form method="post" action="{{ url_path('/employees/' ~ e.id ~ '/delete') }}" class="d-inline"
|
||||
onsubmit="return confirm('Удалить сотрудника?');">
|
||||
<button class="btn btn-sm btn-outline-danger">Удалить</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
<td>{{ e.name }}</td>
|
||||
<td>{% if e.tg_username %}@{{ e.tg_username }}{% else %}—{% endif %}</td>
|
||||
<td>
|
||||
{% if e.tg_chat_id %}<code>{{ e.tg_chat_id }}</code> <span class="text-success">✓</span>
|
||||
{% else %}<span class="text-muted">не задан</span>{% endif %}
|
||||
</td>
|
||||
<td>{{ e.projects|length }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
136
app/templates/project_detail.html
Normal file
136
app/templates/project_detail.html
Normal file
@@ -0,0 +1,136 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ project.title }} — DLD Monitor{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<div class="d-flex flex-column flex-md-row justify-content-between align-items-start gap-3 mb-3">
|
||||
<div>
|
||||
<h3 class="mb-1">{{ project.title }}</h3>
|
||||
<div class="text-muted">
|
||||
{% if project.deal_type.value == 'sale' %}
|
||||
<span class="badge badge-sale">Продажа</span>
|
||||
{% else %}
|
||||
<span class="badge badge-rent">Аренда</span>
|
||||
{% endif %}
|
||||
· Владелец: {{ project.owner.name }}
|
||||
{% if project.last_checked_at %}
|
||||
· Проверено: {{ project.last_checked_at | msk }} МСК
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="small text-muted mt-1">
|
||||
{% if project.building %}🏢 {{ project.building }}{% endif %}
|
||||
{% if project.bedrooms is not none %} · 🛏️ {{ project.bedrooms }} BR{% endif %}
|
||||
{% if project.size_sqft %} · 📐 {{ "{:,.0f}".format(project.size_sqft).replace(",", " ") }} sqft{% endif %}
|
||||
{% if project.dld_permit %} · permit: <code>{{ project.dld_permit }}</code>{% endif %}
|
||||
</div>
|
||||
{% if project.our_price %}
|
||||
<div class="mt-2">Наша цена: <b>{{ "{:,.0f}".format(project.our_price).replace(",", " ") }} AED</b></div>
|
||||
{% endif %}
|
||||
{% if project.our_url %}
|
||||
<div class="small mt-1">Наше объявление: <a href="{{ project.our_url }}" target="_blank" rel="noopener">{{ project.our_url }}</a></div>
|
||||
{% endif %}
|
||||
{% if project.notes %}<div class="mt-2 text-muted"><i>{{ project.notes }}</i></div>{% endif %}
|
||||
</div>
|
||||
<div class="d-flex gap-2 flex-shrink-0">
|
||||
<form action="{{ url_path('/projects/' ~ project.id ~ '/check') }}" method="post">
|
||||
<button class="btn btn-primary text-nowrap">Проверить сейчас</button>
|
||||
</form>
|
||||
{% if request.state.is_admin %}
|
||||
<form action="{{ url_path('/projects/' ~ project.id ~ '/delete') }}" method="post"
|
||||
onsubmit="return confirm('Удалить проект и всю историю?');">
|
||||
<button class="btn btn-outline-danger">Удалить</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if error %}
|
||||
<div class="alert alert-danger">{{ error }}</div>
|
||||
{% endif %}
|
||||
{% if message %}
|
||||
<div class="alert alert-success">{{ message }}</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="bg-white rounded shadow-sm p-3 mb-4">
|
||||
<form method="post" action="{{ url_path('/projects/' ~ project.id ~ '/listings') }}" class="row g-2">
|
||||
<div class="col-md-9">
|
||||
<input name="url" type="url" required class="form-control"
|
||||
placeholder="Вставьте URL объявления конкурента с propertyfinder.ae или bayut.com">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<button class="btn btn-primary w-100">+ Добавить конкурента</button>
|
||||
</div>
|
||||
</form>
|
||||
{% if project.our_url %}
|
||||
<div class="mt-2">
|
||||
<a href="{{ url_path('/projects/' ~ project.id ~ '/suggest') }}" class="btn btn-sm btn-outline-secondary">
|
||||
🔍 Подобрать похожие на PropertyFinder
|
||||
</a>
|
||||
<span class="text-muted small ms-2">
|
||||
— по зданию из вашего объявления{% if project.bedrooms is not none %}, {{ project.bedrooms }} BR{% endif %};
|
||||
совпадения по DLD permit — первыми. Займёт ~15–20 сек.
|
||||
</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="mt-2 text-muted small">
|
||||
🔍 «Подобрать похожие» появится, когда у проекта заполнены <b>наше объявление (URL)</b> и <b>спальни</b>.
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<h5 class="mb-3">Отслеживаемые конкуренты ({{ project.listings|length }})</h5>
|
||||
|
||||
{% if not project.listings %}
|
||||
<div class="alert alert-light border">
|
||||
Пока ничего не отслеживается. Добавьте URL объявления конкурента выше.
|
||||
</div>
|
||||
{% else %}
|
||||
{% for l in project.listings %}
|
||||
<div class="card mb-3 listing-card {{ 'removed' if l.status.value == 'removed' else 'active' }}">
|
||||
<div class="card-body">
|
||||
<div class="d-flex flex-column flex-sm-row justify-content-between gap-2">
|
||||
<div>
|
||||
<span class="src-{{ 'pf' if l.source.value == 'propertyfinder' else 'bayut' }}">
|
||||
{{ 'PropertyFinder' if l.source.value == 'propertyfinder' else 'Bayut' }}
|
||||
</span>
|
||||
{% if l.status.value == 'removed' %}<span class="badge bg-danger ms-2">Удалено</span>{% endif %}
|
||||
<h6 class="mt-1 mb-1">
|
||||
<a href="{{ l.url }}" target="_blank" rel="noopener">{{ l.title or 'без названия' }}</a>
|
||||
</h6>
|
||||
<div class="text-muted small">
|
||||
Брокер: {{ l.agent_name or '—' }} ({{ l.agency_name or '—' }})<br>
|
||||
Добавлено: {{ l.first_seen_at | msk }} ·
|
||||
Последний раз видели активным: {{ l.last_seen_at | msk }} (МСК)
|
||||
</div>
|
||||
</div>
|
||||
<div class="text-sm-end flex-shrink-0">
|
||||
<div class="fs-4 fw-bold">
|
||||
{% if l.current_price %}
|
||||
{{ "{:,.0f}".format(l.current_price).replace(",", " ") }} {{ l.currency or 'AED' }}
|
||||
{% else %}—{% endif %}
|
||||
</div>
|
||||
{% if request.state.is_admin %}
|
||||
<form method="post" action="{{ url_path('/listings/' ~ l.id ~ '/delete') }}"
|
||||
onsubmit="return confirm('Перестать отслеживать?');" class="mt-2">
|
||||
<button class="btn btn-sm btn-outline-danger">Удалить</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if l.price_history|length > 1 %}
|
||||
<details class="mt-2">
|
||||
<summary class="text-muted small">История цены ({{ l.price_history|length }} записей)</summary>
|
||||
<ul class="small mt-2">
|
||||
{% for h in l.price_history %}
|
||||
<li>{{ h.recorded_at | msk }} —
|
||||
{% if h.price %}{{ "{:,.0f}".format(h.price).replace(",", " ") }} AED{% else %}—{% endif %}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</details>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
81
app/templates/project_form.html
Normal file
81
app/templates/project_form.html
Normal file
@@ -0,0 +1,81 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Новый проект — DLD Monitor{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<h3>Новый проект</h3>
|
||||
|
||||
{% if no_employees %}
|
||||
<div class="alert alert-warning">
|
||||
Сначала добавьте хотя бы одного <a href="{{ url_path('/employees') }}">сотрудника</a> — он будет получать уведомления в Telegram.
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" action="{{ url_path('/projects/new') }}" class="bg-white p-4 rounded shadow-sm">
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Название квартиры / лота</label>
|
||||
<input name="title" required class="form-control" placeholder="Например: Aykon City Tower B, 2BR, Apt 1502">
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4 mb-3">
|
||||
<label class="form-label">Тип сделки</label>
|
||||
<select name="deal_type" class="form-select" required>
|
||||
<option value="sale">Продажа</option>
|
||||
<option value="rent">Аренда</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-md-4 mb-3">
|
||||
<label class="form-label">Наша цена (AED)</label>
|
||||
<input name="our_price" type="number" step="1" class="form-control" placeholder="например 1 720 000">
|
||||
</div>
|
||||
<div class="col-md-4 mb-3">
|
||||
<label class="form-label">Владелец (сотрудник)</label>
|
||||
<select name="owner_id" class="form-select" required {% if no_employees %}disabled{% endif %}>
|
||||
{% for e in employees %}
|
||||
<option value="{{ e.id }}">{{ e.name }}{% if e.tg_chat_id %} ✓ TG{% endif %}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
<p class="text-muted small mb-3">
|
||||
Поля ниже опциональны, но позволят системе предлагать «похожие» объявления конкурентов с PF и Bayut.
|
||||
</p>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-6 mb-3">
|
||||
<label class="form-label">Здание / проект</label>
|
||||
<input name="building" class="form-control" placeholder="Aykon City Tower B">
|
||||
</div>
|
||||
<div class="col-md-3 mb-3">
|
||||
<label class="form-label">Спальни</label>
|
||||
<input name="bedrooms" type="number" min="0" max="20" class="form-control" placeholder="2">
|
||||
</div>
|
||||
<div class="col-md-3 mb-3">
|
||||
<label class="form-label">Площадь (sqft)</label>
|
||||
<input name="size_sqft" type="number" step="1" class="form-control" placeholder="1058">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8 mb-3">
|
||||
<label class="form-label">Ссылка на наше объявление (опционально)</label>
|
||||
<input name="our_url" type="url" class="form-control" placeholder="https://www.propertyfinder.ae/...">
|
||||
</div>
|
||||
<div class="col-md-4 mb-3">
|
||||
<label class="form-label">DLD permit (опционально)</label>
|
||||
<input name="dld_permit" class="form-control" placeholder="71-1-1-...">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Заметка</label>
|
||||
<textarea name="notes" class="form-control" rows="2"></textarea>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary" {% if no_employees %}disabled{% endif %}>Создать</button>
|
||||
<a href="{{ url_path('/') }}" class="btn btn-link">Отмена</a>
|
||||
</form>
|
||||
|
||||
{% endblock %}
|
||||
67
app/templates/projects_list.html
Normal file
67
app/templates/projects_list.html
Normal file
@@ -0,0 +1,67 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Проекты — DLD Monitor{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h3 class="mb-0">Наши проекты</h3>
|
||||
<a href="{{ url_path('/projects/new') }}" class="btn btn-primary">+ Новый проект</a>
|
||||
</div>
|
||||
|
||||
{% if projects %}
|
||||
<div class="d-flex gap-2 mb-3">
|
||||
<div class="stat-chip"><div class="n">{{ projects|length }}</div><div class="small text-muted">проектов</div></div>
|
||||
<div class="stat-chip"><div class="n">{{ projects|map(attribute='listings')|map('length')|sum }}</div><div class="small text-muted">конкурентов</div></div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if not projects %}
|
||||
<div class="alert alert-light border text-center py-5">
|
||||
Пока ни одного проекта. <a href="{{ url_path('/projects/new') }}">Добавьте первый</a>.
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="table-responsive bg-white rounded shadow-sm">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Название</th>
|
||||
<th>Здание</th>
|
||||
<th>Тип</th>
|
||||
<th>Наша цена</th>
|
||||
<th>Конкуренты</th>
|
||||
<th>Владелец</th>
|
||||
<th>Последняя проверка (МСК)</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for p in projects %}
|
||||
<tr>
|
||||
<td><a href="{{ url_path('/projects/' ~ p.id) }}">{{ p.title }}</a></td>
|
||||
<td class="small">
|
||||
{% if p.building %}{{ p.building }}{% else %}—{% endif %}
|
||||
{% if p.bedrooms is not none %} · {{ p.bedrooms }}BR{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if p.deal_type.value == 'sale' %}
|
||||
<span class="badge badge-sale">Продажа</span>
|
||||
{% else %}
|
||||
<span class="badge badge-rent">Аренда</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{% if p.our_price %}{{ "{:,.0f}".format(p.our_price).replace(",", " ") }} AED{% else %}—{% endif %}</td>
|
||||
<td>{{ p.listings|length }}</td>
|
||||
<td>{{ p.owner.name if p.owner else '—' }}</td>
|
||||
<td>{{ p.last_checked_at | msk }}</td>
|
||||
<td>
|
||||
<form action="{{ url_path('/projects/' ~ p.id ~ '/check') }}" method="post" class="d-inline">
|
||||
<button class="btn btn-sm btn-outline-primary">Проверить</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
138
app/templates/suggest.html
Normal file
138
app/templates/suggest.html
Normal file
@@ -0,0 +1,138 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Похожие — {{ project.title }}{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
<style>
|
||||
.suggest-card { cursor: pointer; transition: border-color .12s, box-shadow .12s; }
|
||||
.suggest-card:hover { border-color: #adb5bd; }
|
||||
.suggest-card.selected { border-color: #0d6efd; box-shadow: 0 0 0 1px #0d6efd inset; }
|
||||
.suggest-card a { cursor: pointer; }
|
||||
/* Sticky action bar so the submit button is always reachable, esp. on mobile */
|
||||
.bulk-bar {
|
||||
position: sticky; bottom: 0; z-index: 1020;
|
||||
background: #fff; border-top: 1px solid #e9ecef;
|
||||
padding: .6rem .25rem; margin-top: 1rem;
|
||||
box-shadow: 0 -4px 12px rgba(0,0,0,.05);
|
||||
}
|
||||
@media (max-width: 575.98px) {
|
||||
.bulk-bar #bulk-submit { flex: 1; }
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
<div class="mb-3">
|
||||
<a href="{{ url_path('/projects/' ~ project.id) }}" class="btn btn-sm btn-link px-0">← к проекту</a>
|
||||
</div>
|
||||
|
||||
<h3 class="mb-1">Подсказки похожих объявлений</h3>
|
||||
<p class="text-muted">
|
||||
Ищем объявления в том же здании{% if project.bedrooms is not none %}, {{ project.bedrooms }} спален{% endif %},
|
||||
тип сделки: {{ project.deal_type.value }}.
|
||||
{% if our_permit %}Совпадение по DLD permit (<code>{{ our_permit }}</code>) — это тот же объект, такие показаны первыми. {% endif %}
|
||||
Отметьте подходящие объявления галочкой (можно несколько) и нажмите «Отслеживать выбранные» — они попадут в трекинг.
|
||||
</p>
|
||||
|
||||
<form id="bulk-form" method="post" action="{{ url_path('/projects/' ~ project.id ~ '/listings/bulk') }}">
|
||||
{% for src_key, src_label in [('propertyfinder', 'PropertyFinder'), ('bayut', 'Bayut')] %}
|
||||
<h5 class="mt-4">
|
||||
<span class="src-{{ 'pf' if src_key == 'propertyfinder' else 'bayut' }}">{{ src_label }}</span>
|
||||
{% if src_key == 'bayut' and not bayut_enabled %}
|
||||
<small class="text-muted">— ⏸ временно отключён</small>
|
||||
{% else %}
|
||||
<small class="text-muted">— найдено {{ suggestions[src_key]|length }}</small>
|
||||
{% endif %}
|
||||
</h5>
|
||||
{% if src_key == 'bayut' and not bayut_enabled %}
|
||||
<div class="text-muted small">Bayut перешёл на защищённый рендеринг — поиск и трекинг временно недоступны.</div>
|
||||
{% elif not suggestions[src_key] %}
|
||||
<div class="text-muted small">Ничего не нашли (или площадка заблокировала запрос).</div>
|
||||
{% else %}
|
||||
{% for s in suggestions[src_key] %}
|
||||
{% set same_permit = our_permit and s.permit_number and s.permit_number == our_permit %}
|
||||
<label class="card mb-2 suggest-card{% if same_permit %} border-success{% endif %}">
|
||||
<div class="card-body py-2">
|
||||
<div class="d-flex gap-2 align-items-start">
|
||||
<input class="form-check-input suggest-check flex-shrink-0 mt-1" type="checkbox"
|
||||
name="urls" value="{{ s.url }}">
|
||||
<div class="flex-grow-1">
|
||||
<div class="d-flex flex-column flex-sm-row justify-content-between gap-1">
|
||||
<div>
|
||||
{% if same_permit %}<span class="badge bg-success">🎯 тот же permit</span> {% endif %}
|
||||
<a href="{{ s.url }}" target="_blank" rel="noopener">{{ s.title or 'без названия' }}</a>
|
||||
<div class="small text-muted">
|
||||
{{ s.agent_name or '—' }} ({{ s.agency_name or '—' }})
|
||||
{% if s.permit_number %} · permit <code>{{ s.permit_number }}</code>{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="fw-bold text-sm-end text-nowrap">
|
||||
{% if s.price %}{{ "{:,.0f}".format(s.price).replace(",", " ") }} {{ s.currency or 'AED' }}{% else %}—{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</label>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
<div class="bulk-bar">
|
||||
<div class="d-flex align-items-center gap-3">
|
||||
<div class="form-check m-0">
|
||||
<input class="form-check-input" type="checkbox" id="select-all">
|
||||
<label class="form-check-label small" for="select-all">Выбрать все</label>
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary ms-auto" id="bulk-submit" disabled>
|
||||
+ Отслеживать выбранные (<span id="sel-count">0</span>)
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
(function () {
|
||||
const form = document.getElementById('bulk-form');
|
||||
if (!form) return;
|
||||
const checks = Array.from(form.querySelectorAll('.suggest-check'));
|
||||
const selectAll = document.getElementById('select-all');
|
||||
const countEl = document.getElementById('sel-count');
|
||||
const submitBtn = document.getElementById('bulk-submit');
|
||||
|
||||
function refresh() {
|
||||
let n = 0;
|
||||
checks.forEach(c => {
|
||||
const card = c.closest('.suggest-card');
|
||||
if (card) card.classList.toggle('selected', c.checked);
|
||||
if (c.checked) n++;
|
||||
});
|
||||
countEl.textContent = n;
|
||||
submitBtn.disabled = n === 0;
|
||||
if (selectAll) selectAll.checked = n > 0 && n === checks.length;
|
||||
}
|
||||
|
||||
checks.forEach(c => c.addEventListener('change', refresh));
|
||||
|
||||
if (selectAll) {
|
||||
selectAll.addEventListener('change', () => {
|
||||
checks.forEach(c => { c.checked = selectAll.checked; });
|
||||
refresh();
|
||||
});
|
||||
}
|
||||
|
||||
// Confirm before submitting a large batch — each add re-fetches the page.
|
||||
form.addEventListener('submit', e => {
|
||||
const n = checks.filter(c => c.checked).length;
|
||||
if (n === 0) { e.preventDefault(); return; }
|
||||
submitBtn.disabled = true;
|
||||
submitBtn.textContent = 'Добавляем…';
|
||||
});
|
||||
|
||||
refresh();
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
386
app/web.py
Normal file
386
app/web.py
Normal file
@@ -0,0 +1,386 @@
|
||||
"""FastAPI web app — UI for managing projects, competitor listings, employees."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import Depends, FastAPI, Form, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from urllib.parse import quote
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from app.auth import (
|
||||
ADMIN_COOKIE,
|
||||
COOKIE_MAX_AGE,
|
||||
admin_configured,
|
||||
admin_token,
|
||||
pin_ok,
|
||||
request_is_admin,
|
||||
)
|
||||
from app.config import BASE_DIR, settings
|
||||
from app.db import get_db, init_db
|
||||
from app.models import CompetitorListing, DealType, Employee, Project
|
||||
from app.services.monitor import (
|
||||
BAYUT_ENABLED,
|
||||
add_competitor_url,
|
||||
add_competitor_urls,
|
||||
resolve_our_permit,
|
||||
run_check_for_project,
|
||||
suggest_similar,
|
||||
)
|
||||
|
||||
app = FastAPI(title="DLD Monitor")
|
||||
templates = Jinja2Templates(directory=str(Path(BASE_DIR) / "app" / "templates"))
|
||||
|
||||
# Timestamps are stored as naive UTC (datetime.utcnow). Show them in Moscow time
|
||||
# (UTC+3) in the UI via the `msk` Jinja filter.
|
||||
_MSK_OFFSET = timedelta(hours=3)
|
||||
|
||||
|
||||
def _to_msk(dt, fmt: str = "%Y-%m-%d %H:%M"):
|
||||
if dt is None:
|
||||
return "—"
|
||||
return (dt + _MSK_OFFSET).strftime(fmt)
|
||||
|
||||
|
||||
templates.env.filters["msk"] = _to_msk
|
||||
|
||||
|
||||
def web_path(path: str = "/") -> str:
|
||||
base = settings.public_base_path.rstrip("/")
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
return f"{base}{path}" if base else path
|
||||
|
||||
|
||||
templates.env.globals["url_path"] = web_path
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _startup() -> None:
|
||||
init_db()
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def _admin_state(request: Request, call_next):
|
||||
"""Expose admin status to every route and template (request.state.is_admin)."""
|
||||
request.state.is_admin = request_is_admin(request)
|
||||
request.state.admin_configured = admin_configured()
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def admin_required(request: Request) -> None:
|
||||
"""Dependency: block the request unless the session is in admin mode."""
|
||||
if not request.state.is_admin:
|
||||
raise HTTPException(403, "Действие доступно только администратору. Войдите в режим админа.")
|
||||
|
||||
|
||||
def _safe_next(next_url: str) -> str:
|
||||
"""Only allow same-site relative redirects after login."""
|
||||
return next_url if next_url.startswith("/") and not next_url.startswith("//") else "/"
|
||||
|
||||
|
||||
# --- Admin session --------------------------------------------------------
|
||||
|
||||
@app.get("/admin/login", response_class=HTMLResponse)
|
||||
def admin_login_page(request: Request, next: str = "/", error: str | None = None):
|
||||
return templates.TemplateResponse(
|
||||
"admin_login.html",
|
||||
{"request": request, "next": _safe_next(next), "error": error, "flash": None},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/admin/login")
|
||||
def admin_login(request: Request, pin: str = Form(...), next: str = Form("/")):
|
||||
dest = _safe_next(next)
|
||||
if not pin_ok(pin):
|
||||
return templates.TemplateResponse(
|
||||
"admin_login.html",
|
||||
{"request": request, "next": dest, "error": "Неверный PIN", "flash": None},
|
||||
status_code=401,
|
||||
)
|
||||
resp = RedirectResponse(web_path(dest), status_code=303)
|
||||
resp.set_cookie(
|
||||
ADMIN_COOKIE, admin_token(), max_age=COOKIE_MAX_AGE,
|
||||
httponly=True, samesite="lax",
|
||||
)
|
||||
return resp
|
||||
|
||||
|
||||
@app.post("/admin/logout")
|
||||
def admin_logout(next: str = Form("/")):
|
||||
resp = RedirectResponse(web_path(_safe_next(next)), status_code=303)
|
||||
resp.delete_cookie(ADMIN_COOKIE)
|
||||
return resp
|
||||
|
||||
|
||||
def _opt_float(s: str) -> float | None:
|
||||
s = (s or "").strip()
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _opt_int(s: str) -> int | None:
|
||||
s = (s or "").strip()
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
# --- Projects -------------------------------------------------------------
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def projects_list(request: Request, db: Session = Depends(get_db)):
|
||||
projects = (
|
||||
db.query(Project)
|
||||
.options(joinedload(Project.owner), joinedload(Project.listings))
|
||||
.order_by(Project.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
"projects_list.html", {"request": request, "projects": projects, "flash": None}
|
||||
)
|
||||
|
||||
|
||||
@app.get("/projects/new", response_class=HTMLResponse)
|
||||
def new_project_form(request: Request, db: Session = Depends(get_db)):
|
||||
employees = db.query(Employee).order_by(Employee.name).all()
|
||||
return templates.TemplateResponse(
|
||||
"project_form.html",
|
||||
{
|
||||
"request": request,
|
||||
"employees": employees,
|
||||
"no_employees": not employees,
|
||||
"flash": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/projects/new")
|
||||
def create_project(
|
||||
title: str = Form(...),
|
||||
deal_type: str = Form(...),
|
||||
owner_id: int = Form(...),
|
||||
our_price: str = Form(""),
|
||||
building: str = Form(""),
|
||||
bedrooms: str = Form(""),
|
||||
size_sqft: str = Form(""),
|
||||
our_url: str = Form(""),
|
||||
dld_permit: str = Form(""),
|
||||
notes: str = Form(""),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
owner = db.get(Employee, owner_id)
|
||||
if not owner:
|
||||
raise HTTPException(404, "Employee not found")
|
||||
project = Project(
|
||||
title=title.strip(),
|
||||
deal_type=DealType(deal_type),
|
||||
our_price=_opt_float(our_price),
|
||||
building=building.strip() or None,
|
||||
bedrooms=_opt_int(bedrooms),
|
||||
size_sqft=_opt_float(size_sqft),
|
||||
our_url=our_url.strip() or None,
|
||||
dld_permit=dld_permit.strip() or None,
|
||||
notes=notes.strip() or None,
|
||||
owner_id=owner.id,
|
||||
)
|
||||
db.add(project)
|
||||
db.commit()
|
||||
return RedirectResponse(web_path(f"/projects/{project.id}"), status_code=303)
|
||||
|
||||
|
||||
@app.get("/projects/{project_id}", response_class=HTMLResponse)
|
||||
def project_detail(
|
||||
project_id: int,
|
||||
request: Request,
|
||||
error: str | None = None,
|
||||
message: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
project = (
|
||||
db.query(Project)
|
||||
.options(joinedload(Project.owner), joinedload(Project.listings))
|
||||
.filter(Project.id == project_id)
|
||||
.first()
|
||||
)
|
||||
if not project:
|
||||
raise HTTPException(404, "Project not found")
|
||||
return templates.TemplateResponse(
|
||||
"project_detail.html",
|
||||
{"request": request, "project": project, "error": error, "message": message, "flash": None},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/projects/{project_id}/check")
|
||||
def project_check_now(project_id: int, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise HTTPException(404, "Project not found")
|
||||
db.close()
|
||||
run_check_for_project(project_id)
|
||||
return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303)
|
||||
|
||||
|
||||
@app.post("/projects/{project_id}/delete")
|
||||
def project_delete(project_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
raise HTTPException(404, "Project not found")
|
||||
db.delete(project)
|
||||
db.commit()
|
||||
return RedirectResponse(web_path("/"), status_code=303)
|
||||
|
||||
|
||||
# --- Competitor listings (per project) ------------------------------------
|
||||
|
||||
@app.post("/projects/{project_id}/listings")
|
||||
def add_listing(project_id: int, url: str = Form(...), db: Session = Depends(get_db)):
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
raise HTTPException(404, "Project not found")
|
||||
listing, err = add_competitor_url(db, project, url)
|
||||
if err:
|
||||
return RedirectResponse(web_path(f"/projects/{project_id}?error={quote(err)}"), status_code=303)
|
||||
return RedirectResponse(
|
||||
web_path(f"/projects/{project_id}?message=Добавлено"), status_code=303,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/projects/{project_id}/listings/bulk")
|
||||
def add_listings_bulk(
|
||||
project_id: int,
|
||||
urls: list[str] = Form(default=[]),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
raise HTTPException(404, "Project not found")
|
||||
if not urls:
|
||||
return RedirectResponse(
|
||||
web_path(f"/projects/{project_id}?error={quote('Ничего не выбрано')}"),
|
||||
status_code=303,
|
||||
)
|
||||
result = add_competitor_urls(db, project, urls)
|
||||
parts = [f"Добавлено: {result['added']}"]
|
||||
if result["skipped"]:
|
||||
parts.append(f"уже были: {result['skipped']}")
|
||||
if result["errors"]:
|
||||
parts.append(f"ошибок: {len(result['errors'])}")
|
||||
msg = " · ".join(parts)
|
||||
return RedirectResponse(
|
||||
web_path(f"/projects/{project_id}?message={quote(msg)}"), status_code=303,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/listings/{listing_id}/delete")
|
||||
def listing_delete(listing_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
|
||||
listing = db.get(CompetitorListing, listing_id)
|
||||
if not listing:
|
||||
raise HTTPException(404, "Listing not found")
|
||||
project_id = listing.project_id
|
||||
db.delete(listing)
|
||||
db.commit()
|
||||
return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303)
|
||||
|
||||
|
||||
@app.get("/projects/{project_id}/suggest", response_class=HTMLResponse)
|
||||
def project_suggest(project_id: int, request: Request, db: Session = Depends(get_db)):
|
||||
project = (
|
||||
db.query(Project)
|
||||
.options(joinedload(Project.listings))
|
||||
.filter(Project.id == project_id)
|
||||
.first()
|
||||
)
|
||||
if not project:
|
||||
raise HTTPException(404, "Project not found")
|
||||
our_permit = resolve_our_permit(project)
|
||||
suggestions = suggest_similar(project, our_permit=our_permit)
|
||||
return templates.TemplateResponse(
|
||||
"suggest.html",
|
||||
{
|
||||
"request": request,
|
||||
"project": project,
|
||||
"suggestions": suggestions,
|
||||
"our_permit": our_permit,
|
||||
"bayut_enabled": BAYUT_ENABLED,
|
||||
"flash": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- Employees ------------------------------------------------------------
|
||||
|
||||
@app.get("/employees", response_class=HTMLResponse)
|
||||
def employees_page(request: Request, db: Session = Depends(get_db)):
|
||||
employees = (
|
||||
db.query(Employee).options(joinedload(Employee.projects)).order_by(Employee.name).all()
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
"employees.html", {"request": request, "employees": employees, "flash": None}
|
||||
)
|
||||
|
||||
|
||||
@app.post("/employees")
|
||||
def employee_create(
|
||||
name: str = Form(...),
|
||||
tg_username: str = Form(""),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
e = Employee(name=name.strip(), tg_username=tg_username.strip().lstrip("@") or None)
|
||||
db.add(e)
|
||||
db.commit()
|
||||
return RedirectResponse(web_path("/employees"), status_code=303)
|
||||
|
||||
|
||||
@app.post("/employees/{employee_id}/update")
|
||||
def employee_update(
|
||||
employee_id: int,
|
||||
name: str = Form(...),
|
||||
tg_username: str = Form(""),
|
||||
tg_chat_id: str = Form(""),
|
||||
db: Session = Depends(get_db),
|
||||
_: None = Depends(admin_required),
|
||||
):
|
||||
e = db.get(Employee, employee_id)
|
||||
if not e:
|
||||
raise HTTPException(404, "Employee not found")
|
||||
e.name = name.strip()
|
||||
e.tg_username = tg_username.strip().lstrip("@") or None
|
||||
new_chat_id = tg_chat_id.strip() or None
|
||||
if new_chat_id and new_chat_id != e.tg_chat_id:
|
||||
clash = (
|
||||
db.query(Employee)
|
||||
.filter(Employee.tg_chat_id == new_chat_id, Employee.id != e.id)
|
||||
.first()
|
||||
)
|
||||
if clash:
|
||||
raise HTTPException(400, f"chat_id {new_chat_id} уже привязан к сотруднику «{clash.name}»")
|
||||
e.tg_chat_id = new_chat_id
|
||||
db.commit()
|
||||
return RedirectResponse(web_path("/employees"), status_code=303)
|
||||
|
||||
|
||||
@app.post("/employees/{employee_id}/delete")
|
||||
def employee_delete(employee_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
|
||||
e = db.get(Employee, employee_id)
|
||||
if not e:
|
||||
raise HTTPException(404, "Employee not found")
|
||||
if e.projects:
|
||||
raise HTTPException(400, "Сотрудник связан с проектами — сначала переназначьте или удалите проекты")
|
||||
db.delete(e)
|
||||
db.commit()
|
||||
return RedirectResponse(web_path("/employees"), status_code=303)
|
||||
Reference in New Issue
Block a user