Add monitoring TG service

alembic/env.py

@@ -0,0 +1,52 @@
+import asyncio
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from parser_bot.config import settings
+from parser_bot.db.models import Base
+
+config = context.config
+config.set_main_option("sqlalchemy.url", settings.database_url)
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    context.configure(
+        url=settings.database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_migrations_online() -> None:
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+    await connectable.dispose()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    asyncio.run(run_migrations_online())

alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/0001_initial.py

@@ -0,0 +1,71 @@
+"""initial schema: channels + messages
+
+Revision ID: 0001
+Revises:
+Create Date: 2026-05-05
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0001"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "channels",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("tg_id", sa.BigInteger(), nullable=True, unique=True),
+        sa.Column("identifier", sa.String(length=255), nullable=False, unique=True),
+        sa.Column("title", sa.String(length=512), nullable=True),
+        sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")),
+        sa.Column("last_message_id", sa.BigInteger(), nullable=True),
+        sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+    )
+
+    op.create_table(
+        "messages",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "channel_id",
+            sa.Integer(),
+            sa.ForeignKey("channels.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("tg_message_id", sa.BigInteger(), nullable=False),
+        sa.Column("date", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("text", sa.Text(), nullable=True),
+        sa.Column("sender_id", sa.BigInteger(), nullable=True),
+        sa.Column("has_media", sa.Boolean(), nullable=False, server_default=sa.text("false")),
+        sa.Column("views", sa.Integer(), nullable=True),
+        sa.Column("forwards", sa.Integer(), nullable=True),
+        sa.Column("raw", postgresql.JSONB(), nullable=True),
+        sa.Column(
+            "fetched_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.UniqueConstraint("channel_id", "tg_message_id", name="uq_channel_message"),
+    )
+    op.create_index(
+        "ix_messages_channel_date", "messages", ["channel_id", "date"], unique=False
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_messages_channel_date", table_name="messages")
+    op.drop_table("messages")
+    op.drop_table("channels")

alembic/versions/0002_add_media_files.py

@@ -0,0 +1,28 @@
+"""add media_files JSONB column to messages
+
+Revision ID: 0002
+Revises: 0001
+Create Date: 2026-05-05
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0002"
+down_revision: Union[str, None] = "0001"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "messages",
+        sa.Column("media_files", postgresql.JSONB(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("messages", "media_files")

alembic/versions/0003_add_grouped_id.py

@@ -0,0 +1,39 @@
+"""add grouped_id to messages (Telegram album/media-group key)
+
+Revision ID: 0003
+Revises: 0002
+Create Date: 2026-05-05
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0003"
+down_revision: Union[str, None] = "0002"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("messages", sa.Column("grouped_id", sa.BigInteger(), nullable=True))
+    op.create_index(
+        "ix_messages_grouped_id", "messages", ["channel_id", "grouped_id"]
+    )
+    # Backfill grouped_id from the stored raw JSONB for existing rows so that
+    # albums saved before this migration are grouped retroactively.
+    op.execute(
+        """
+        UPDATE messages
+        SET grouped_id = (raw->>'grouped_id')::bigint
+        WHERE grouped_id IS NULL
+          AND raw IS NOT NULL
+          AND raw->>'grouped_id' IS NOT NULL
+        """
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_messages_grouped_id", table_name="messages")
+    op.drop_column("messages", "grouped_id")

alembic/versions/0004_add_extracted.py

@@ -0,0 +1,34 @@
+"""add extracted JSONB column to messages
+
+Revision ID: 0004
+Revises: 0003
+Create Date: 2026-05-05
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0004"
+down_revision: Union[str, None] = "0003"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "messages",
+        sa.Column("extracted", postgresql.JSONB(), nullable=True),
+    )
+    # GIN index for json queries (e.g. filter by extracted->'real_estate'->>'kind').
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_messages_extracted_gin "
+        "ON messages USING GIN (extracted)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_messages_extracted_gin")
+    op.drop_column("messages", "extracted")

alembic/versions/0005_add_sender_info.py

@@ -0,0 +1,30 @@
+"""add sender_username and sender_name to messages
+
+Revision ID: 0005
+Revises: 0004
+Create Date: 2026-05-06
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0005"
+down_revision: Union[str, None] = "0004"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "messages", sa.Column("sender_username", sa.String(length=64), nullable=True)
+    )
+    op.add_column(
+        "messages", sa.Column("sender_name", sa.String(length=255), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("messages", "sender_name")
+    op.drop_column("messages", "sender_username")

alembic/versions/0006_add_app_settings.py

@@ -0,0 +1,35 @@
+"""key/value store for runtime-editable settings (LLM prompt, etc.)
+
+Revision ID: 0006
+Revises: 0005
+Create Date: 2026-05-06
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0006"
+down_revision: Union[str, None] = "0005"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "app_settings",
+        sa.Column("key", sa.String(length=64), primary_key=True),
+        sa.Column("value", postgresql.JSONB(), nullable=False),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("app_settings")

alembic/versions/0007_add_channel_vertical.py

@@ -0,0 +1,37 @@
+"""split channels into two verticals: real_estate / hr
+
+Existing rows get `real_estate` per the migration decision — the service was
+real-estate-only before this column existed.
+
+Revision ID: 0007
+Revises: 0006
+Create Date: 2026-05-19
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0007"
+down_revision: Union[str, None] = "0006"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "channels",
+        sa.Column(
+            "vertical",
+            sa.String(length=32),
+            nullable=False,
+            server_default="real_estate",
+        ),
+    )
+    op.create_index("ix_channels_vertical", "channels", ["vertical"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_channels_vertical", table_name="channels")
+    op.drop_column("channels", "vertical")

alembic/versions/0008_add_sections.py

@@ -0,0 +1,110 @@
+"""sub-sections inside each vertical (e.g. Real Estate → Dubai / Moscow)
+
+A channel now belongs to exactly one section, and each section to exactly
+one vertical. The migration auto-creates a `Общий` section per vertical
+that has at least one channel and pins all existing channels there, so the
+service keeps working without manual reclassification after upgrade.
+
+Revision ID: 0008
+Revises: 0007
+Create Date: 2026-05-20
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0008"
+down_revision: Union[str, None] = "0007"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "sections",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("vertical", sa.String(length=32), nullable=False),
+        sa.Column("slug", sa.String(length=64), nullable=False),
+        sa.Column("title", sa.String(length=255), nullable=False),
+        sa.Column("emoji", sa.String(length=8), nullable=True),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.UniqueConstraint("vertical", "slug", name="uq_section_vertical_slug"),
+    )
+    op.create_index("ix_sections_vertical", "sections", ["vertical"])
+
+    # Auto-create a `default` section for each vertical that already has channels,
+    # so the backfill below has somewhere to point.
+    op.execute(
+        """
+        INSERT INTO sections (vertical, slug, title, emoji)
+        SELECT DISTINCT c.vertical,
+                        'default',
+                        CASE c.vertical
+                            WHEN 'hr' THEN 'Общий HR'
+                            ELSE 'Общий'
+                        END,
+                        CASE c.vertical WHEN 'hr' THEN '👥' ELSE '🏠' END
+        FROM channels c
+        ON CONFLICT (vertical, slug) DO NOTHING
+        """
+    )
+
+    # Add nullable section_id first so the backfill can populate it.
+    op.add_column(
+        "channels",
+        sa.Column("section_id", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "fk_channels_section",
+        "channels",
+        "sections",
+        ["section_id"],
+        ["id"],
+        ondelete="RESTRICT",
+    )
+    op.create_index("ix_channels_section_id", "channels", ["section_id"])
+
+    op.execute(
+        """
+        UPDATE channels c
+        SET section_id = s.id
+        FROM sections s
+        WHERE s.vertical = c.vertical AND s.slug = 'default'
+        """
+    )
+
+    # Now we can safely require section_id.
+    op.alter_column("channels", "section_id", nullable=False)
+
+    # Per-section LLM prompt keys are longer than 64 chars
+    # (`llm_system_prompt:real_estate:some-long-slug`), so widen the key column.
+    op.alter_column(
+        "app_settings",
+        "key",
+        existing_type=sa.String(length=64),
+        type_=sa.String(length=128),
+        existing_nullable=False,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "app_settings",
+        "key",
+        existing_type=sa.String(length=128),
+        type_=sa.String(length=64),
+        existing_nullable=False,
+    )
+    op.drop_index("ix_channels_section_id", table_name="channels")
+    op.drop_constraint("fk_channels_section", "channels", type_="foreignkey")
+    op.drop_column("channels", "section_id")
+    op.drop_index("ix_sections_vertical", table_name="sections")
+    op.drop_table("sections")

alembic/versions/0009_add_section_access_code.py

@@ -0,0 +1,24 @@
+"""add access code to sections
+
+Revision ID: 0009
+Revises: 0008
+Create Date: 2026-05-29
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0009"
+down_revision: Union[str, None] = "0008"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("sections", sa.Column("access_code", sa.String(length=255), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("sections", "access_code")