Self-serve team onboarding for Darwin

backend/alembic/versions/a1b2c3d4e5f6_onboarding_help_thread_ts.py

@@ -0,0 +1,29 @@
+"""onboarding_request.help_thread_ts
+
+Revision ID: a1b2c3d4e5f6
+Revises: c40f5a073fc2
+Create Date: 2026-07-18 00:00:00.000000
+
+Adds help_thread_ts: the Slack ts of the customer-facing root message posted on
+submit, so lifecycle updates (approved / complete / failed) can be posted as
+replies in that thread. Additive nullable column — no impact on existing rows.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "a1b2c3d4e5f6"
+down_revision = "c40f5a073fc2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "onboarding_request",
+        sa.Column("help_thread_ts", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("onboarding_request", "help_thread_ts")

backend/alembic/versions/c40f5a073fc2_onboarding_request.py

@@ -0,0 +1,75 @@
+"""onboarding_request
+
+Revision ID: c40f5a073fc2
+Revises: 31f30b318163
+Create Date: 2026-07-17 00:00:00.000000
+
+Adds the onboarding_request table backing the self-serve team-onboarding flow
+(form -> admin approval -> auto-provision + scrape). All columns additive/new
+table, so no impact on existing data.
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c40f5a073fc2"
+down_revision = "31f30b318163"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "onboarding_request",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "requester_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("user.id"),
+            nullable=True,
+        ),
+        sa.Column("requester_email", sa.String(), nullable=False),
+        sa.Column("status", sa.String(), nullable=False, server_default="pending"),
+        sa.Column("payload", postgresql.JSONB(), nullable=False),
+        sa.Column(
+            "approver_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("user.id"),
+            nullable=True,
+        ),
+        sa.Column("decision_reason", sa.Text(), nullable=True),
+        sa.Column("error_msg", sa.Text(), nullable=True),
+        sa.Column(
+            "persona_id", sa.Integer(), sa.ForeignKey("persona.id"), nullable=True
+        ),
+        sa.Column(
+            "document_set_id",
+            sa.Integer(),
+            sa.ForeignKey("document_set.id"),
+            nullable=True,
+        ),
+        sa.Column(
+            "slack_bot_config_id",
+            sa.Integer(),
+            sa.ForeignKey("slack_bot_config.id"),
+            nullable=True,
+        ),
+        sa.Column("cc_pair_ids", postgresql.JSONB(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("onboarding_request")

backend/danswer/background/update.py

@@ -38,6 +38,7 @@
 from danswer.db.models import IndexingStatus
 from danswer.db.models import IndexModelStatus
 from danswer.db.swap_index import check_index_swap
+from danswer.onboarding.provision import finalize_ready_onboarding_requests
 from danswer.search.search_nlp_models import warm_up_encoders
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import global_version
@@ -569,6 +570,11 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
         client_secondary = SimpleJobClient(n_workers=num_workers)
 
     existing_jobs: dict[int, Future | SimpleJob] = {}
+    # Onboarding requests are finalized (doc set + assistant + Slack config) once
+    # all their sources finish scraping. Sweep for ready ones about once a minute
+    # rather than every scheduler tick.
+    last_onboarding_sweep = 0.0
+    onboarding_sweep_interval = 60.0
 
     while True:
         start = time.time()
@@ -594,6 +600,17 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
             )
         except Exception as e:
             logger.exception(f"Failed to run update due to {e}")
+
+        # Onboarding finalization sweep (~once a minute), isolated so a failure
+        # here never disrupts indexing scheduling.
+        if start - last_onboarding_sweep >= onboarding_sweep_interval:
+            last_onboarding_sweep = start
+            try:
+                with Session(get_sqlalchemy_engine()) as db_session:
+                    finalize_ready_onboarding_requests(db_session)
+            except Exception:
+                logger.exception("Onboarding finalization sweep failed")
+
         sleep_time = delay - (time.time() - start)
         if sleep_time > 0:
             time.sleep(sleep_time)

backend/danswer/connectors/slack/connector.py

@@ -27,6 +27,7 @@
 from danswer.connectors.slack.utils import make_slack_api_call_logged
 from danswer.connectors.slack.utils import make_slack_api_call_paginated
 from danswer.connectors.slack.utils import make_slack_api_rate_limited
+from danswer.connectors.slack.utils import resolve_workspace_subdomain
 from danswer.connectors.slack.utils import SlackTextCleaner
 from danswer.utils.logger import setup_logger
 
@@ -323,6 +324,11 @@ def get_all_docs(
             client=client, channel=channel, oldest=oldest, latest=latest
         )
 
+        # Resolve the channel's true workspace subdomain (Grid-safe) lazily on
+        # the first message, then reuse it for every thread in the channel — one
+        # chat.getPermalink call per channel rather than per message.
+        channel_workspace: str | None = None
+
         seen_thread_ts: set[str] = set()
         for message_batch in channel_message_batches:
             for message in message_batch:
@@ -344,9 +350,16 @@ def get_all_docs(
                     filtered_thread = [message]
 
                 if filtered_thread:
+                    if channel_workspace is None:
+                        channel_workspace = resolve_workspace_subdomain(
+                            client=client,
+                            channel_id=channel["id"],
+                            message_ts=filtered_thread[0]["ts"],
+                            fallback=workspace,
+                        )
                     channel_docs += 1
                     yield thread_to_doc(
-                        workspace=workspace,
+                        workspace=channel_workspace,
                         channel=channel,
                         thread=filtered_thread,
                         slack_cleaner=slack_cleaner,

backend/danswer/connectors/slack/utils.py

@@ -5,6 +5,7 @@
 from functools import wraps
 from typing import Any
 from typing import cast
+from urllib.parse import urlparse
 
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
@@ -18,6 +19,23 @@
 # number of messages we request per page when fetching paginated slack messages
 _SLACK_LIMIT = 900
 
+# Mis-stored Slack workspace subdomains -> their correct URL subdomain.
+# Historically some connectors were configured with the workspace *display name*
+# ("Product") instead of the URL subdomain ("uipath-product"), so their stored
+# permalinks point at the dead host `product.slack.com`. Keys are matched case-
+# and whitespace-insensitively, so this covers both "Product" and "Product ".
+# This is an explicit allow-map on purpose: only listed subdomains are rewritten,
+# so genuinely different workspaces (uipath-customer-ops, uipath-marketing, ...)
+# and links already on a correct host are left untouched. Add an entry here if a
+# new mis-configured workspace surfaces.
+_SLACK_SUBDOMAIN_FIXES = {
+    "product": "uipath-product",
+}
+
+# Captures the subdomain in the `//<sub>.slack.com` portion of a permalink.
+# `[^/]*?` is non-greedy and tolerates a stray space ("Product .slack.com").
+_SLACK_HOST_RE = re.compile(r"(//)([^/]*?)(\.slack\.com)", re.IGNORECASE)
+
 
 def get_message_link(
     event: dict[str, Any], workspace: str, channel_id: str | None = None
@@ -34,6 +52,66 @@ def get_message_link(
     )
 
 
+def normalize_slack_link(url: str) -> str:
+    """Rewrite a mis-stored Slack workspace subdomain to the canonical one.
+
+    Existing indexed docs carry links built from a mis-configured workspace
+    ("Product" -> dead `product.slack.com`). Retrieval reads every citation
+    link back through here, so chat / search / Slack-bot citations all resolve
+    to the real workspace without re-indexing or a data migration. Only the
+    subdomains in `_SLACK_SUBDOMAIN_FIXES` are rewritten; a link on the canonical
+    host, or on a genuinely different workspace, is returned unchanged."""
+    if not url or ".slack.com" not in url.lower():
+        return url
+
+    def _fix(match: "re.Match[str]") -> str:
+        subdomain = match.group(2).strip().lower()
+        canonical = _SLACK_SUBDOMAIN_FIXES.get(subdomain)
+        if canonical is not None:
+            return f"{match.group(1)}{canonical}{match.group(3)}"
+        return match.group(0)
+
+    return _SLACK_HOST_RE.sub(_fix, url, count=1)
+
+
+def resolve_workspace_subdomain(
+    client: WebClient, channel_id: str, message_ts: str, fallback: str
+) -> str:
+    """Ask Slack for the authoritative workspace subdomain of a channel.
+
+    `chat.getPermalink` returns the real permalink for a message, with the
+    correct `<sub>.slack.com` host even under Enterprise Grid (where a single
+    bot can see channels that live in different workspaces, each with its own
+    URL). We resolve this ONCE per channel and reuse it, so the connector no
+    longer depends on a hand-typed `workspace` config being right. Falls back to
+    `fallback` (the configured workspace) if the call fails."""
+    try:
+        # Same rate-limit + logging wrapping as every other Slack call in the
+        # connector, so a 429 retries (with Retry-After) instead of falling back.
+        resp = make_slack_api_rate_limited(
+            make_slack_api_call_logged(client.chat_getPermalink)
+        )(channel=channel_id, message_ts=message_ts)
+        permalink = cast(str, resp.get("permalink") or "")
+        host = urlparse(permalink).netloc.lower()
+        if host.endswith(".slack.com"):
+            subdomain = host[: -len(".slack.com")]
+            if subdomain:
+                return subdomain
+    except SlackApiError as e:
+        logger.warning(
+            "chat.getPermalink failed for channel %s (%s); "
+            "falling back to configured workspace '%s'",
+            channel_id,
+            e.response.get("error"),
+            fallback,
+        )
+    except Exception as e:
+        logger.warning(
+            "could not resolve workspace subdomain for channel %s: %s", channel_id, e
+        )
+    return fallback
+
+
 def make_slack_api_call_logged(
     call: Callable[..., SlackResponse],
 ) -> Callable[..., SlackResponse]:

backend/danswer/connectors/web/connector.py

@@ -1,5 +1,6 @@
 import io
 import ipaddress
+import os
 import random
 import re
 import socket
@@ -161,7 +162,13 @@ def get_internal_links(
 
 def start_playwright() -> Tuple[Playwright, BrowserContext]:
     playwright = sync_playwright().start()
-    browser = playwright.chromium.launch(headless=True)
+    # Optionally launch an installed browser (e.g. system Chrome) instead of the
+    # bundled Chromium. Prod leaves this unset and uses the pinned Chromium in
+    # the Linux image; it exists only so local dev can fall back to a working
+    # browser when the pinned build is incompatible with the host OS (the
+    # chromium-1097 build for playwright 1.41.2 SIGSEGVs on recent macOS).
+    browser_channel = os.environ.get("WEB_CONNECTOR_BROWSER_CHANNEL") or None
+    browser = playwright.chromium.launch(headless=True, channel=browser_channel)
 
     context = browser.new_context(user_agent=DEFAULT_USER_AGENT)
 
@@ -314,7 +321,9 @@ def _uipath_product_prefix(path: str) -> str:
     return "/" + "/".join(segs[:cut])
 
 
-def get_uipath_docs_version_base_urls(base_url: str, max_versions: int = 2) -> list[str]:
+def get_uipath_docs_version_base_urls(
+    base_url: str, max_versions: int = 2
+) -> list[str]:
     """Expand a docs.uipath.com product URL to the base URLs of its latest N
     concrete versions.
 

backend/danswer/db/models.py

@@ -1716,3 +1716,77 @@ class ChatReferral(Base):
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
+
+
+class OnboardingStatus(str, PyEnum):
+    """Lifecycle of a team's self-serve Darwin onboarding request."""
+
+    PENDING = "pending"  # submitted, awaiting admin approval
+    REJECTED = "rejected"  # admin declined
+    CANCELLED = "cancelled"  # withdrawn by the requester while still pending
+    PROVISIONING = "provisioning"  # approved; creating connectors/persona/config
+    INDEXING = "indexing"  # resources created; sources scraping
+    COMPLETE = "complete"  # all sources indexed successfully
+    FAILED = "failed"  # provisioning or indexing failed
+
+
+class OnboardingRequest(Base):
+    """A self-serve request to onboard a team/channel onto Darwin. Anyone may
+    submit; only an admin may approve. On approval the provisioning orchestrator
+    (onboarding/provision.py) creates the connectors, document set, assistant, and
+    Slack bot config, then records their ids here so the requester can monitor the
+    per-source scrape status."""
+
+    __tablename__ = "onboarding_request"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    # Who submitted (denormalize email so it survives user deletion / is display-ready).
+    requester_id: Mapped[UUID | None] = mapped_column(
+        ForeignKey("user.id"), nullable=True
+    )
+    requester_email: Mapped[str] = mapped_column(String, nullable=False)
+    # Stored as the enum VALUE string (e.g. "pending"), NOT via SA Enum() — the
+    # repo's Enum(native_enum=False) stores the NAME (uppercase), which is a known
+    # footgun. Plain String of `.value` keeps it consistent with the migration.
+    status: Mapped[str] = mapped_column(
+        String, nullable=False, default=OnboardingStatus.PENDING.value
+    )
+    # The full validated form (channel, ordered sources, prompt, SME/oncall/jira
+    # options, docs cloud/onprem roots, etc.) — JSONB for flexibility.
+    payload: Mapped[dict] = mapped_column(postgresql.JSONB(), nullable=False)
+    # Admin who approved/rejected + optional reason.
+    approver_id: Mapped[UUID | None] = mapped_column(
+        ForeignKey("user.id"), nullable=True
+    )
+    decision_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Free-text error surfaced when status == FAILED.
+    error_msg: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Provisioned resource ids (populated on approval; drive the status monitor).
+    persona_id: Mapped[int | None] = mapped_column(
+        ForeignKey("persona.id"), nullable=True
+    )
+    document_set_id: Mapped[int | None] = mapped_column(
+        ForeignKey("document_set.id"), nullable=True
+    )
+    slack_bot_config_id: Mapped[int | None] = mapped_column(
+        ForeignKey("slack_bot_config.id"), nullable=True
+    )
+    # cc_pair ids created for this onboarding (list[int]); the monitor reads their
+    # indexing status. JSONB list rather than a join table — always used together.
+    cc_pair_ids: Mapped[list[int] | None] = mapped_column(
+        postgresql.JSONB(), nullable=True
+    )
+    # Slack message ts of the customer-facing root message posted to #help-darwin
+    # on submit. Subsequent lifecycle updates (approved / complete / failed) are
+    # posted as replies in this thread (thread_ts), so the requester tracks their
+    # request in one quiet thread. Null if the root post failed/was skipped.
+    help_thread_ts: Mapped[str | None] = mapped_column(String, nullable=True)
+    created_at: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )