From 1b8382a6cae7d5d569b9cff0e00ef6c91adea708 Mon Sep 17 00:00:00 2001 From: Grendgi Date: Fri, 5 Jun 2026 12:36:15 +0300 Subject: [PATCH] Delay PF permit competitor removal --- app/db.py | 4 ++++ app/models.py | 1 + app/services/monitor.py | 13 +++++++++++-- app/worker.py | 2 +- internal/pf/db.go | 6 ++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/app/db.py b/app/db.py index 4211645..c58b16a 100644 --- a/app/db.py +++ b/app/db.py @@ -56,3 +56,7 @@ def _migrate_competitor_listings_auto_fields() -> None: conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN permit_number VARCHAR(100)")) if "auto_discovered" not in columns: conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN auto_discovered BOOLEAN NOT NULL DEFAULT 0")) + if "permit_missing_checks" not in columns: + conn.execute( + text("ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0") + ) diff --git a/app/models.py b/app/models.py index 0a55d45..eab7bcc 100644 --- a/app/models.py +++ b/app/models.py @@ -82,6 +82,7 @@ class CompetitorListing(Base): agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True) permit_number: Mapped[str | None] = mapped_column(String(100), nullable=True) auto_discovered: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + permit_missing_checks: Mapped[int] = mapped_column(Integer, default=0, nullable=False) current_price: Mapped[float | None] = mapped_column(Float, nullable=True) currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED") diff --git a/app/services/monitor.py b/app/services/monitor.py index 013561a..359bf48 100644 --- a/app/services/monitor.py +++ b/app/services/monitor.py @@ -40,6 +40,7 @@ BAYUT = BayutScraper() # Same-building suggestions beyond exact permit matches are a browse heuristic — # cap how many we show so the page stays usable. _SUGGEST_OTHERS_LIMIT = 30 +_PERMIT_MISSING_DELETE_THRESHOLD = 3 # Bayut moved to fully client-side rendering (no __NEXT_DATA__, Algolia keys # hidden), so it can't be scraped over plain HTTP — disabled until we add a @@ -237,12 +238,14 @@ def _hide_tracked_suggestions( def sync_permit_competitors( db: Session, project: Project, + *, + count_missing: bool = True, ) -> tuple[list[str], dict[str, list[ScrapedListing]], str | None]: """Auto-maintain competitor listings with the same DLD permit. Exact-permit matches are added automatically. Previously auto-discovered - exact-permit listings that disappear from the next permit search are - deleted. Manual competitors are never auto-deleted. + exact-permit listings are deleted only after several consecutive permit + searches miss them. Manual competitors are never auto-deleted. """ changes: list[str] = [] our_permit = resolve_our_permit(project) @@ -268,6 +271,7 @@ def sync_permit_competitors( listing = existing.get(key) if listing: listing.permit_number = item.permit_number or our_permit + listing.permit_missing_checks = 0 if item.title: listing.title = item.title if item.agent_name: @@ -287,6 +291,11 @@ def sync_permit_competitors( continue if _listing_key(listing.source, listing.external_id) in matched_keys: continue + if not count_missing: + continue + listing.permit_missing_checks = (listing.permit_missing_checks or 0) + 1 + if listing.permit_missing_checks < _PERMIT_MISSING_DELETE_THRESHOLD: + continue changes.append(_format_listing_removed(project, listing, auto=True)) db.delete(listing) diff --git a/app/worker.py b/app/worker.py index ac8c194..85425fd 100644 --- a/app/worker.py +++ b/app/worker.py @@ -115,7 +115,7 @@ def cmd_suggest(payload: dict[str, Any]) -> None: project = db.get(Project, project_id) if not project: _fail("project not found") - changes, suggestions, permit = sync_permit_competitors(db, project) + changes, suggestions, permit = sync_permit_competitors(db, project, count_missing=False) db.commit() if changes: notify_project_changes(project, changes) diff --git a/internal/pf/db.go b/internal/pf/db.go index 33a3ab2..a035255 100644 --- a/internal/pf/db.go +++ b/internal/pf/db.go @@ -177,6 +177,7 @@ func (a *App) InitDB(ctx context.Context) error { agency_name VARCHAR(300), permit_number VARCHAR(100), auto_discovered BOOLEAN NOT NULL DEFAULT 0, + permit_missing_checks INTEGER NOT NULL DEFAULT 0, current_price FLOAT, currency VARCHAR(10), status VARCHAR(7) NOT NULL, @@ -259,6 +260,11 @@ func (a *App) migrateCompetitorListings(ctx context.Context) error { return err } } + if !columns["permit_missing_checks"] { + if _, err := a.DB.ExecContext(ctx, `ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } return nil }