Delay PF permit competitor removal

app/db.py

@@ -56,3 +56,7 @@ def _migrate_competitor_listings_auto_fields() -> None:
             conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN permit_number VARCHAR(100)"))
         if "auto_discovered" not in columns:
             conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN auto_discovered BOOLEAN NOT NULL DEFAULT 0"))
+        if "permit_missing_checks" not in columns:
+            conn.execute(
+                text("ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0")
+            )

app/models.py

@@ -82,6 +82,7 @@ class CompetitorListing(Base):
     agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
     permit_number: Mapped[str | None] = mapped_column(String(100), nullable=True)
     auto_discovered: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    permit_missing_checks: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
 
     current_price: Mapped[float | None] = mapped_column(Float, nullable=True)
     currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED")

app/services/monitor.py

@@ -40,6 +40,7 @@ BAYUT = BayutScraper()
 # Same-building suggestions beyond exact permit matches are a browse heuristic —
 # cap how many we show so the page stays usable.
 _SUGGEST_OTHERS_LIMIT = 30
+_PERMIT_MISSING_DELETE_THRESHOLD = 3
 
 # Bayut moved to fully client-side rendering (no __NEXT_DATA__, Algolia keys
 # hidden), so it can't be scraped over plain HTTP — disabled until we add a
@@ -237,12 +238,14 @@ def _hide_tracked_suggestions(
 def sync_permit_competitors(
     db: Session,
     project: Project,
+    *,
+    count_missing: bool = True,
 ) -> tuple[list[str], dict[str, list[ScrapedListing]], str | None]:
     """Auto-maintain competitor listings with the same DLD permit.
 
     Exact-permit matches are added automatically. Previously auto-discovered
-    exact-permit listings that disappear from the next permit search are
+    exact-permit listings are deleted only after several consecutive permit
-    deleted. Manual competitors are never auto-deleted.
+    searches miss them. Manual competitors are never auto-deleted.
     """
     changes: list[str] = []
     our_permit = resolve_our_permit(project)
@@ -268,6 +271,7 @@ def sync_permit_competitors(
         listing = existing.get(key)
         if listing:
             listing.permit_number = item.permit_number or our_permit
+            listing.permit_missing_checks = 0
             if item.title:
                 listing.title = item.title
             if item.agent_name:
@@ -287,6 +291,11 @@ def sync_permit_competitors(
             continue
         if _listing_key(listing.source, listing.external_id) in matched_keys:
             continue
+        if not count_missing:
+            continue
+        listing.permit_missing_checks = (listing.permit_missing_checks or 0) + 1
+        if listing.permit_missing_checks < _PERMIT_MISSING_DELETE_THRESHOLD:
+            continue
         changes.append(_format_listing_removed(project, listing, auto=True))
         db.delete(listing)
 

app/worker.py

@@ -115,7 +115,7 @@ def cmd_suggest(payload: dict[str, Any]) -> None:
         project = db.get(Project, project_id)
         if not project:
             _fail("project not found")
-        changes, suggestions, permit = sync_permit_competitors(db, project)
+        changes, suggestions, permit = sync_permit_competitors(db, project, count_missing=False)
         db.commit()
         if changes:
             notify_project_changes(project, changes)

internal/pf/db.go

@@ -177,6 +177,7 @@ func (a *App) InitDB(ctx context.Context) error {
 			agency_name VARCHAR(300),
 			permit_number VARCHAR(100),
 			auto_discovered BOOLEAN NOT NULL DEFAULT 0,
+			permit_missing_checks INTEGER NOT NULL DEFAULT 0,
 			current_price FLOAT,
 			currency VARCHAR(10),
 			status VARCHAR(7) NOT NULL,
@@ -259,6 +260,11 @@ func (a *App) migrateCompetitorListings(ctx context.Context) error {
 			return err
 		}
 	}
+	if !columns["permit_missing_checks"] {
+		if _, err := a.DB.ExecContext(ctx, `ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0`); err != nil {
+			return err
+		}
+	}
 	return nil
 }